in ax/service/scheduler.py [0:0]
def run(self, max_new_trials: int) -> bool:
"""Schedules trial evaluation(s) if stopping criterion is not triggered,
maximum parallelism is not currently reached, and capacity allows.
Logs any failures / issues.
Args:
max_new_trials: Maximum number of new trials this function should generate
and run (useful when generating and running trials in batches). Note
that this function might also re-deploy existing ``CANDIDATE`` trials
that failed to deploy before, which will not count against this number.
Returns:
Boolean representing success status.
"""
if self.should_consider_optimization_complete():
self.logger.info(
"`completion_criterion` is `True`, not running more trials."
)
return False
if self.should_abort_optimization():
self.logger.info(
"`should_abort_optimization` is `True`, not running more trials."
)
return False
# Check if capacity allows for running new evaluations and generate as many
# trials as possible, limited by capacity and model requirements.
self._sleep_if_too_early_to_poll()
existing_trials, new_trials = self._prepare_trials(
max_new_trials=max_new_trials
)
if not existing_trials and not new_trials:
# Unable to gen. new run due to max parallelism limit or need for data
# or unable to run trials due to lack of capacity.
if self._optimization_complete:
return False
if len(self.pending_trials) < 1:
raise SchedulerInternalError( # pragma: no cover
"No trials are running but model requires more data. This is an "
"invalid state of the scheduler, as no more trials can be produced "
"but also no more data is expected as there are no running trials."
"This should be investigated."
)
self._log_next_no_trials_reason = False
return False # Nothing to run.
if existing_trials:
idcs = sorted(t.index for t in existing_trials)
self.logger.debug(f"Will run pre-existing candidate trials: {idcs}.")
all_trials = [*existing_trials, *new_trials]
idcs_str = make_indices_str(indices=(t.index for t in all_trials))
self.logger.info(f"Running trials {idcs_str}...")
# TODO: Add optional timeout between retries of `run_trial(s)`.
metadata = self.run_trials(trials=all_trials)
self.logger.debug(f"Ran trials {idcs_str}.")
if self.options.debug_log_run_metadata:
self.logger.debug(f"Run metadata: {metadata}.")
self._latest_trial_start_timestamp = current_timestamp_in_millis()
self._update_and_save_trials(
existing_trials=existing_trials, new_trials=new_trials, metadata=metadata
)
self._log_next_no_trials_reason = True
return True