def torch_distributed_zero_first()

in scripts/launcher_distributed.py [0:0]

• 6 lines of code
• 3 McCabe index (conditional complexity)

def torch_distributed_zero_first(local_rank: int):
    """
    Decorator to make all processes in distributed training
    wait for each local_master to do something.
    """
    if local_rank not in [-1, 0]:
        dist.barrier(device_ids=[local_rank])
    yield
    if local_rank == 0:
        dist.barrier(device_ids=[0])