def get_tokenizer()

in weak_to_strong/common.py [0:0]

• 2 lines of code
• 1 McCabe index (conditional complexity)

def get_tokenizer(model_name: str):
    """
    This function returns a tokenizer based on the model name.

    Parameters:
    model_name: The name of the model for which the tokenizer is needed.

    Returns:
    A tokenizer for the specified model.
    """
    return AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)