in launcher/nemo/slurm_launcher.py [0:0]
def launch(self, command_groups: List[List[str]]) -> str:
# Create the custom train_script.sh before launching the real job
self._make_train_script_file()
# Same as upstream, but exposing extra control for submission through slurm_create_submission_file_only
submission_file_path = self._make_submission_file(command_groups)
logger.info(f"Job {self.job_name} submission file created at '{submission_file_path}'")
job_id = ""
if not NEMO_LAUNCHER_DEBUG and not self.slurm_create_submission_file_only:
job_id = self._submit_command(submission_file_path)
if job_id:
logger.info(f"Job {self.job_name} submitted with Job ID {job_id}")
with open(self.folder / "launcher.log", "w") as f:
f.write(f"Submitted batch job {job_id}")
else:
logger.info(f"To submit your job on Slurm, run `sbatch {submission_file_path}`")
return job_id