in launcher/nemo/launchers.py [0:0]
def _make_submission_file_text(self, command_groups: List[List[str]]) -> str:
"""
Given the command groups, generate submission script file's text.
Command groups is a list of command group. A command group is defined as:
0. Command group is a list of command strings
1. Each command group occupies one bcprun, srun or bash
2. Each command group eventually has multiple commands connected by ";"
On interactive cluster, multi-gpu python scripts are launched with `torchrun --nproc_per_node=??`
:param List[List[str]] command_groups: Command groups to launch with
:return: submission script file's text
:rtype: str
"""
# now create
lines = ["#!/bin/bash", ""]
for group_ind, command_group in enumerate(command_groups):
command = "\n".join(command_group)
lines.append(command)
return "\n".join(lines)