def get_tokenizer()

in src/co_op_translator/utils/llm/markdown_utils.py [0:0]

• 2 lines of code
• 1 McCabe index (conditional complexity)

def get_tokenizer(encoding_name: str):
    """
    Get the tokenizer based on the encoding name.

    Args:
        encoding_name (str): The name of the encoding.

    Returns:
        tiktoken.Encoding: The tokenizer for the given encoding.
    """
    return tiktoken.get_encoding(encoding_name)