def count_tokens()

in src/co_op_translator/utils/llm/markdown_utils.py [0:0]

• 2 lines of code
• 1 McCabe index (conditional complexity)

def count_tokens(text: str, tokenizer) -> int:
    """
    Count the number of tokens in a given text using the tokenizer.

    Args:
        text (str): The text to tokenize.
        tokenizer (tiktoken.Encoding): The tokenizer to use.

    Returns:
        int: The number of tokens in the text.
    """
    return len(tokenizer.encode(text))