in prediction_generation/old-code/summarize_metrics_new_debug.py [0:0]
def process_best(method):
hyperparams = dict()
stripped_method = method.replace("best_", "")
uniq_count = set()
uniq_successful_conf = set()
uniq_fail_conf = dict()
uniq_conf = dict()
for dataset_metrics in datasets_metrics:
for unit_method in dataset_metrics["results"][method]:
conf = unit_method["args"]
conf_str = json.dumps(conf, sort_keys=True)
if conf_str in uniq_conf:
uniq_conf[conf_str] = uniq_conf[conf_str] + 1
else:
uniq_conf[conf_str] = 1
if unit_method["status"] == "SUCCESS":
uniq_successful_conf.add(conf_str)
metrics = unit_method["scores"]
f1 = metrics["f1"]
precision = metrics["precision"]
recall = metrics["recall"]
if conf_str in hyperparams:
hyperparams[conf_str]["f1"].append(f1)
hyperparams[conf_str]["precision"].append(precision)
hyperparams[conf_str]["recall"].append(recall)
else:
metrics_dict = {
"f1": [f1],
"precision": [precision],
"recall": [recall]
}
hyperparams[conf_str] = metrics_dict
# elif unit_method["status"] == "FAIL" and method != 'best_pelt' and method != 'best_amoc' and method != 'best_binseg' and method != 'best_bocpd' and method != 'best_cpnp':
elif unit_method["status"] == "FAIL":
if conf_str in uniq_fail_conf:
uniq_fail_conf[conf_str].append(dataset_metrics['dataset'])
else:
uniq_fail_conf[conf_str] = [dataset_metrics['dataset']]
#print('AAAAAAAAAAAAAAAA')
#print(method)
# else:
# print('METHOD:')
# print(method)
# print(dataset_metrics["dataset"])
# dict_f1 = {key: sum(value['f1']) / len(value['f1']) for key, value in hyperparams.items() if len(value['f1']) > nb_datasets_threshold}
# dict_precision = {key: sum(value['precision']) / len(value['precision']) for key, value in hyperparams.items() if len(value['precision']) > nb_datasets_threshold}
# dict_recall = {key: sum(value['recall']) / len(value['recall']) for key, value in hyperparams.items() if len(value['recall']) > nb_datasets_threshold}
dict_f1 = {key: sum(value['f1']) / len(value['f1']) for key, value in hyperparams.items() if len(value['f1']) > nb_datasets_threshold}
dict_precision = {key: sum(value['precision']) / len(value['precision']) for key, value in hyperparams.items() if len(value['precision']) > nb_datasets_threshold}
dict_recall = {key: sum(value['recall']) / len(value['recall']) for key, value in hyperparams.items() if len(value['recall']) > nb_datasets_threshold}
print('Method Kar lChebba thabbat fl wesfan')
print(method)
print('number of failed confs with no datasets overlap')
print(len(uniq_fail_conf))
print('number of failed confs with datasets overlap')
for key, value in uniq_fail_conf.items():
if key in uniq_successful_conf:
print(key)
print(value)
# print(uniq_successful_conf.intersection(set(uniq_fail_conf.keys())))
# for key, value in hyperparams.items():
# print(len(value['f1']))
# print(method)
# print('################')
# print('dict_f1')
# print(len(dict_f1))
# print('dict_precision')
# print(len(dict_precision))
# print('dict_recall')
# print(len(dict_recall))
# print('Hyperparamssssssss')
# print(len(hyperparams))
for key, value in hyperparams.items():
uniq_count.add(len(value['precision']))
uniq_count.add(len(value['f1']))
uniq_count.add(len(value['recall']))
# print(len(value['f1']))
# print(uniq_count)
# for key, value in hyperparams.items():
# if (len(value['precision']) < 63):
# print('3omri mensit')
# print(method)
# if (len(value['recall']) < 63):
# print('3omri mensit')
# print(method)
# if (len(value['f1']) < 63):
# print('3omri mensit')
# print(method)
# if (len(value['f1']) != len(value['precision'])) or (len(value['f1']) != len(value['recall'])):
# print('Kar lChebba thabbat fl wesfan')
all_keys = set(dict_precision.keys()).union(set(dict_recall.keys())).union(set(dict_f1.keys()))
# Save all configurations to CSV for debugging purposes
data = {
'Key': list(all_keys),
'Precision': [dict_precision.get(key, float('nan')) for key in all_keys],
'Recall': [dict_recall.get(key, float('nan')) for key in all_keys],
'F1 Score': [dict_f1.get(key, float('nan')) for key in all_keys]
}
df = pd.DataFrame(data)
df.to_csv('/TCPDBench/analysis/metrics_of_'+ method + '.csv', index=False)
try:
max_f1 = dict_f1[max(dict_f1, key=dict_f1.get)]
except Exception as e:
print('###f1####')
print(e)
max_f1 = None
try:
precision_max_f1 = dict_precision[max(dict_f1, key=dict_f1.get)]
except Exception as e:
print('###precision f1 max####')
print(e)
precision_max_f1 = None
try:
recall_max_f1 = dict_recall[max(dict_f1, key=dict_f1.get)]
except Exception as e:
print('###recall f1####')
print(e)
recall_max_f1 = None
try:
max_precision = dict_precision[max(dict_precision, key=dict_precision.get)]
except Exception as e:
print('###precision####')
print(e)
max_precision = None
try:
max_recall = dict_recall[max(dict_recall, key=dict_recall.get)]
except Exception as e:
print('###recall####')
print(e)
max_recall = None
MethodsMeasurements[stripped_method].f1_best = max_f1
MethodsMeasurements[stripped_method].precision_best = max_precision
MethodsMeasurements[stripped_method].recall_best = max_recall
MethodsMeasurements[stripped_method].precision_f1_best = precision_max_f1
MethodsMeasurements[stripped_method].recall_f1_best = recall_max_f1