in courses/machine_learning/asl/open_project/time_series_anomaly_detection/tf_anomaly_detection_model_selection/anomaly_detection_module/trainer/tune_anomaly_thresholds_supervised.py [0:0]
def tune_anomaly_thresholds_supervised_eval(
labels_norm_mask,
labels_anom_mask,
time_anom_thresh_var,
mahalanobis_dist_time,
tp_thresh_eval_time_var,
fn_thresh_eval_time_var,
fp_thresh_eval_time_var,
tn_thresh_eval_time_var,
feat_anom_thresh_var,
mahalanobis_dist_feat,
tp_thresh_eval_feat_var,
fn_thresh_eval_feat_var,
fp_thresh_eval_feat_var,
tn_thresh_eval_feat_var,
params,
mode):
"""Checks tuned anomaly thresholds during supervised evaluation mode.
Given label masks, mahalanobis distances, confusion matrices, and anomaly
thresholds, returns loss and eval_metric_ops.
Args:
labels_norm_mask: tf.bool vector mask of labels for normals.
labels_anom_mask: tf.bool vector mask of labels for anomalies.
time_anom_thresh_var: tf.float64 scalar time anomaly threshold value.
mahalanobis_dist_time: Mahalanobis distance, time major.
tp_thresh_eval_time_var: tf.int64 variable to track number of true
positives wrt thresholds for time major case for evaluation.
fn_thresh_eval_time_var: tf.int64 variable to track number of false
negatives wrt thresholds for time major case for evaluation.
fp_thresh_eval_time_var: tf.int64 variable to track number of false
positives wrt thresholds for time major case for evaluation.
tn_thresh_eval_time_var: tf.int64 variable to track number of true
negatives wrt thresholds for time major case for evaluation.
feat_anom_thresh_var: tf.float64 scalar feature anomaly threshold value.
mahalanobis_dist_feat: Mahalanobis distance, features major.
tp_thresh_eval_feat_var: tf.int64 variable to track number of true
positives wrt thresholds for feat major case for evaluation.
fn_thresh_eval_feat_var: tf.int64 variable to track number of false
negatives wrt thresholds for feat major case for evaluation.
fp_thresh_eval_feat_var: tf.int64 variable to track number of false
positives wrt thresholds for feat major case for evaluation.
tn_thresh_eval_feat_var: tf.int64 variable to track number of true
negatives wrt thresholds for feat major case for evaluation.
params: Dictionary of parameters.
mode: Estimator ModeKeys. Can take value of only EVAL.
Returns:
loss: Scalar reconstruction loss.
eval_metric_ops: Evaluation metrics of threshold tuning.
"""
with tf.variable_scope(
name_or_scope="anom_thresh_eval_vars", reuse=tf.AUTO_REUSE):
# Time based
(tp_time_update_op,
fn_time_update_op,
fp_time_update_op,
tn_time_update_op) = \
update_anom_thresh_vars(
labels_norm_mask,
labels_anom_mask,
1,
time_anom_thresh_var,
mahalanobis_dist_time,
tp_thresh_eval_time_var,
fn_thresh_eval_time_var,
fp_thresh_eval_time_var,
tn_thresh_eval_time_var,
mode)
# Features based
(tp_feat_update_op,
fn_feat_update_op,
fp_feat_update_op,
tn_feat_update_op) = \
update_anom_thresh_vars(
labels_norm_mask,
labels_anom_mask,
1,
feat_anom_thresh_var,
mahalanobis_dist_feat,
tp_thresh_eval_feat_var,
fn_thresh_eval_feat_var,
fp_thresh_eval_feat_var,
tn_thresh_eval_feat_var,
mode)
with tf.variable_scope(
name_or_scope="anom_thresh_eval_vars", reuse=tf.AUTO_REUSE):
# Time based
(acc_time_update_op,
pre_time_update_op,
rec_time_update_op,
f_beta_time_update_op) = \
calculate_composite_classification_metrics(
tp_thresh_eval_time_var,
fn_thresh_eval_time_var,
fp_thresh_eval_time_var,
tn_thresh_eval_time_var,
params["f_score_beta"])
# Features based
(acc_feat_update_op,
pre_feat_update_op,
rec_feat_update_op,
f_beta_feat_update_op) = \
calculate_composite_classification_metrics(
tp_thresh_eval_feat_var,
fn_thresh_eval_feat_var,
fp_thresh_eval_feat_var,
tn_thresh_eval_feat_var,
params["f_score_beta"])
loss = tf.zeros(shape=[], dtype=tf.float64)
# Time based
acc_trues = tf.cast(
x=tp_thresh_eval_time_var + tn_thresh_eval_time_var,
dtype=tf.float64)
acc_falses = tf.cast(
x=fp_thresh_eval_time_var + fn_thresh_eval_time_var,
dtype=tf.float64)
acc_thresh_eval_time_var = acc_trues / (acc_trues + acc_falses)
tp_float = tf.cast(x=tp_thresh_eval_time_var, dtype=tf.float64)
pre_denominator = tf.cast(
x=tp_thresh_eval_time_var + fp_thresh_eval_time_var,
dtype=tf.float64)
pre_thresh_eval_time_var = tp_float / pre_denominator
rec_denominator = tf.cast(
x=tp_thresh_eval_time_var + fn_thresh_eval_time_var,
dtype=tf.float64)
rec_thresh_eval_time_var = tp_float / rec_denominator
f_beta_numerator = (1.0 + params["f_score_beta"] ** 2)
f_beta_numerator *= pre_thresh_eval_time_var
f_beta_numerator *= rec_thresh_eval_time_var
f_beta_denominator = params["f_score_beta"] ** 2
f_beta_denominator *= pre_thresh_eval_time_var
f_beta_denominator += rec_thresh_eval_time_var
f_beta_thresh_eval_time_var = f_beta_numerator / f_beta_denominator
# Features based
acc_trues = tf.cast(
x=tp_thresh_eval_feat_var + tn_thresh_eval_feat_var,
dtype=tf.float64)
acc_falses = tf.cast(
x=fp_thresh_eval_feat_var + fn_thresh_eval_feat_var,
dtype=tf.float64)
acc_thresh_eval_feat_var = acc_trues / (acc_trues + acc_falses)
tp_float = tf.cast(x=tp_thresh_eval_feat_var, dtype=tf.float64)
pre_denominator = tf.cast(
x=tp_thresh_eval_feat_var + fp_thresh_eval_feat_var,
dtype=tf.float64)
pre_thresh_eval_feat_var = tp_float / pre_denominator
rec_denominator = tf.cast(
x=tp_thresh_eval_feat_var + fn_thresh_eval_feat_var,
dtype=tf.float64)
rec_thresh_eval_feat_var = tp_float / rec_denominator
f_beta_numerator = (1.0 + params["f_score_beta"] ** 2)
f_beta_numerator *= pre_thresh_eval_feat_var
f_beta_numerator *= rec_thresh_eval_feat_var
f_beta_denominator = params["f_score_beta"] ** 2
f_beta_denominator *= pre_thresh_eval_feat_var
f_beta_denominator += rec_thresh_eval_feat_var
f_beta_thresh_eval_feat_var = f_beta_numerator / f_beta_denominator
# Anomaly detection eval metrics
eval_metric_ops = {
# Time based
"time_anom_tp": (tp_thresh_eval_time_var, tp_time_update_op),
"time_anom_fn": (fn_thresh_eval_time_var, fn_time_update_op),
"time_anom_fp": (fp_thresh_eval_time_var, fp_time_update_op),
"time_anom_tn": (tn_thresh_eval_time_var, tn_time_update_op),
"time_anom_acc": (acc_thresh_eval_time_var, acc_time_update_op),
"time_anom_pre": (pre_thresh_eval_time_var, pre_time_update_op),
"time_anom_rec": (rec_thresh_eval_time_var, rec_time_update_op),
"time_anom_f_beta": (f_beta_thresh_eval_time_var,
f_beta_time_update_op),
# Features based
"feat_anom_tp": (tp_thresh_eval_feat_var, tp_feat_update_op),
"feat_anom_fn": (fn_thresh_eval_feat_var, fn_feat_update_op),
"feat_anom_fp": (fp_thresh_eval_feat_var, fp_feat_update_op),
"feat_anom_tn": (tn_thresh_eval_feat_var, tn_feat_update_op),
"feat_anom_acc": (acc_thresh_eval_feat_var, acc_feat_update_op),
"feat_anom_pre": (pre_thresh_eval_feat_var, pre_feat_update_op),
"feat_anom_rec": (rec_thresh_eval_feat_var, rec_feat_update_op),
"feat_anom_f_beta": (f_beta_thresh_eval_feat_var,
f_beta_feat_update_op)
}
return loss, eval_metric_ops