def _extract_aggregate_metrics()

in automl21/scs_neural/experimentation/launcher.py [0:0]

• 57 lines of code
• 13 McCabe index (conditional complexity)

    def _extract_aggregate_metrics(self, soln_metrics, soln_type='original', index_nans=[]):
        all_data, all_metrics = [], []
        result_dict, conf_dict = {}, {}
        all_data_orig, all_metrics_orig = [], []
        max_data_len = 0
        for i in range(len(soln_metrics)):
            if i in index_nans:
                continue
            if soln_type == 'neural':
                u_diff_instance = soln_metrics[i][-1]
                u_diff_instance = [x.cpu() for x in u_diff_instance]
            else:
                raise RuntimeError("Unknown solution type")
            orig_iterate_diff = u_diff_instance[0]
            if hasattr(self.cfg.model, 'learn_init_iterate') and self.cfg.model.learn_init_iterate and soln_type == 'neural':
                slice_start = 1
            else:
                slice_start = 0
            data = [x / orig_iterate_diff for x in u_diff_instance[slice_start:]]
            all_data.append(data)
            all_data_orig.append(u_diff_instance)
            max_data_len = max(len(data), max_data_len)

        for i in range(len(all_data)):
            curr_metrics = np.array(all_data[i])
            diff = max_data_len - len(curr_metrics)
            if max_data_len > 0:
                upd_curr_metrics = np.pad(curr_metrics, (0, diff), 'constant',
                                          constant_values=(curr_metrics[-1]))
            else:
                upd_curr_metrics = curr_metrics
            all_metrics.append(upd_curr_metrics)

        for i in range(len(all_data_orig)):
            curr_metrics = np.array(all_data_orig[i])
            diff = max_data_len + 1 - len(curr_metrics)
            if max_data_len > 0:
                upd_curr_metrics = np.pad(curr_metrics, (0, diff), 'constant',
                                          constant_values=(curr_metrics[-1]))
            else:
                upd_curr_metrics = curr_metrics
            all_metrics_orig.append(upd_curr_metrics)

        final_metrics = np.stack(all_metrics)
        result_dict['u_diff'] = final_metrics.mean(axis=0)
        std_dev = final_metrics.std(axis=0)
        conf_dict['u_diff'] = std_dev

        final_metrics_orig = np.stack(all_metrics_orig)
        result_dict['u_diff_orig'] = final_metrics_orig.mean(axis=0)
        std_dev = final_metrics_orig.std(axis=0)
        conf_dict['u_diff_orig'] = std_dev

        # add data from log file:
        loss_data = pd.read_csv('log.csv', delimiter=',', header=0)
        train_loss = loss_data['train_loss'].to_numpy()
        val_loss = loss_data['val_loss'].to_numpy()
        test_loss = loss_data['test_loss'].to_numpy()
        result_dict["loss"] = {}
        result_dict["loss"]["train"] = train_loss
        result_dict["loss"]["val"] = val_loss
        result_dict["loss"]["test"] = test_loss

        return result_dict, conf_dict