in src/nanotron/logging/timers.py [0:0]
def log_all(self, logger=None, rank: Optional[int] = 0, group=None) -> None:
"""Log all timers on the specified rank."""
if logger is None:
logger = logging.get_logger(__name__)
world_rank = dist.get_rank() if group is None else dist.get_rank(group)
if rank is not None and world_rank != rank:
return
# Sort timers by name for consistent output
sorted_timers = sorted(self._timers.items())
if sorted_timers:
logger.info("---- Timing Information ----")
for name, timer in sorted_timers:
if timer.call_count > 0:
# This will trigger synchronization for CUDA timers
avg_time = timer.average_time * 1000 # Convert to ms
total_time = timer.total_time * 1000 # Convert to ms
logger.info(
f"Timer '{name}' ({timer.timer_type.value}): {total_time:.2f}ms total, "
f"{avg_time:.2f}ms avg, {timer.call_count} calls"
)
logger.info("----------------------------")