def error_if_failure_rate_exceeded()

in ax/service/scheduler.py [0:0]

• 28 lines of code
• 9 McCabe index (conditional complexity)

    def error_if_failure_rate_exceeded(self, force_check: bool = False) -> None:
        """Checks if the failure rate (set in scheduler options) has been exceeded.

        Args:
            force_check: Indicates whether to force a failure-rate check
                regardless of the number of trials that have been executed. If False
                (default), the check will be skipped if the sweep has fewer than five
                failed iterations. If True, the check will be performed unless there
                are 0 failures.
        """
        failed_idcs = self.experiment.trial_indices_by_status[TrialStatus.FAILED]
        # We only count failed trials with indices that came after the preexisting
        # trials on experiment before scheduler use.
        num_failed_in_scheduler = sum(
            1 for f in failed_idcs if f >= self._num_preexisting_trials
        )

        # skip check if 0 failures
        if num_failed_in_scheduler == 0:
            return

        # skip check if fewer than min_failed_trials_for_failure_rate_check failures
        # unless force_check is True
        if (
            num_failed_in_scheduler
            < self.options.min_failed_trials_for_failure_rate_check
            and not force_check
        ):
            return

        num_ran_in_scheduler = (
            len(self.experiment.trials) - self._num_preexisting_trials
        )

        failure_rate_exceeded = (
            num_failed_in_scheduler / num_ran_in_scheduler
        ) > self.options.tolerated_trial_failure_rate

        if failure_rate_exceeded:
            raise FailureRateExceededError(
                "Failure rate exceeds the tolerated trial failure rate of "
                f"{self.options.tolerated_trial_failure_rate} (at least "
                f"{num_failed_in_scheduler} out of first {num_ran_in_scheduler} trials "
                "failed). Checks are triggered both at the end of a sweep and if "
                f"at least {self.options.min_failed_trials_for_failure_rate_check} "
                "trials have failed."
            )