def get_hf_config_from_name_or_path()

in src/hyperpod_nemo_adapter/utils/config_utils.py [0:0]

• 11 lines of code
• 5 McCabe index (conditional complexity)

def get_hf_config_from_name_or_path(config):
    """
    Create a HF pretrained config based on user specified name or path
    """
    access_token = config.get("hf_access_token", None)
    hf_config = None
    # When multiple ranks query for the same HF config, it might error out, adding some retry logic
    for i in range(MAX_RETRY_TIME):
        try:
            hf_config = AutoConfig.from_pretrained(config.hf_model_name_or_path, token=access_token)
        except FileNotFoundError:
            if i == MAX_RETRY_TIME - 1:
                raise RuntimeError(f"Max retry timeout when fetching config from {config.hf_model_name_or_path}")
            _logger.warning(f"Retrying to get the config of {config.hf_model_name_or_path}")
    return hf_config