def _evaluate_soft_ranking()

in syne_tune/optimizer/schedulers/hyperband_pasha.py [0:0]

• 46 lines of code
• 24 McCabe index (conditional complexity)

    def _evaluate_soft_ranking(self, sorted_top_rung, sorted_previous_rung) -> bool:
        """
        Soft ranking creates groups of similarly performing configurations
        and increases the resources only if a configuration goes outside of
        its group.

        :param sorted_top_rung: list of tuple(trial_id, rank, score)
        :param sorted_previous_rung: list of tuple(sorted_top_rung, rank, score)
        :return: keep_current_budget
        """
        keep_current_budget = True
        if len(sorted_previous_rung) < 2:
            epsilon = 0.0
        elif self.ranking_criterion == 'soft_ranking_std':
            epsilon = np.std(
                [e[2] for e in sorted_previous_rung]) * self.epsilon_scaling
        elif self.ranking_criterion == 'soft_ranking_median_dst' or self.ranking_criterion == 'soft_ranking_mean_dst':
            scores = [e[2] for e in sorted_previous_rung]
            distances = [abs(e1-e2) for idx1, e1 in enumerate(scores)
                         for idx2, e2 in enumerate(scores) if idx1 != idx2]
            if self.ranking_criterion == 'soft_ranking_mean_dst':
                epsilon = np.mean(distances) * self.epsilon_scaling
            elif self.ranking_criterion == 'soft_ranking_median_dst':
                epsilon = np.median(distances) * self.epsilon_scaling
            else:
                raise ValueError('Ranking criterion {} is not supported'.format(self.ranking_criterion))
        else:
            epsilon = self.epsilon

        # create groups of configurations with similar performance
        previous_rung_groups = []
        for idx, item in enumerate(sorted_previous_rung):
            current_rung_group = [item[0]]
            # add configurations that are after the current configuration
            for idx_after in range(idx + 1, len(sorted_previous_rung)):
                new_item = sorted_previous_rung[idx_after]

                if self._mode == 'max':
                    if new_item[2] < item[2] - epsilon:
                        break
                else:
                    if new_item[2] > item[2] + epsilon:
                        break
                current_rung_group.append(new_item[0])
            # add configurations that are before the current configuration
            for idx_before in range(idx - 1, -1, -1):
                new_item = sorted_previous_rung[idx_before]
                if self._mode == 'max':
                    if new_item[2] > item[2] + epsilon:
                        break
                else:
                    if new_item[2] < item[2] - epsilon:
                        break
                current_rung_group.append(new_item[0])
            previous_rung_groups.append(set(current_rung_group))

        # evaluate if a configuration has switched its group
        for idx, item in enumerate(sorted_top_rung):
            if item[0] not in previous_rung_groups[idx]:
                keep_current_budget = False
                break

        return keep_current_budget