def aggregate_and_normalize()

in custom/metrics.py [0:0]

• 34 lines of code
• 23 McCabe index (conditional complexity)

    def aggregate_and_normalize(logging_outputs):
        agg_output = {}
        normalizer = sum(log.get('normalizer', 0) for log in logging_outputs)
        if normalizer == 0:
            return agg_output
        for name in TrainingMetrics.METRIC_NAMES:
            # 'mean of medians' special case
            if name == 'median_target_rank':
                agg_output[name] = np.mean([log[name] for log in logging_outputs if name in log])
                continue
            metric_sum = sum(log.get(name, 0) for log in logging_outputs)
            metric = metric_sum / normalizer
            agg_output[name] = metric

        # topk and top-p metrics
        keys = set()
        for log in logging_outputs:
            for k in log:
                if 'true_topk' in k or 'true_topp' in k or 'true_full_prob' in k or 'repeat_top' in k:
                    keys.add(k)
        for k in keys:
            metric_sum = sum(log.get(k, 0) for log in logging_outputs)
            metric = metric_sum / normalizer
            agg_output[k] = metric

        unique_top_keys = set()
        for log in logging_outputs:
            for k in log:
                if 'nextunique' in k:
                    unique_top_keys.add(k)
        for k in unique_top_keys:
            unique_list_of_lists = [log.get(k, []) for log in logging_outputs]
            unique_flat_list = []
            for _sublist in unique_list_of_lists:
                unique_flat_list.extend(_sublist)

            unique_metric = len(set(unique_flat_list))
            agg_output[k] = unique_metric

        return agg_output