def _num_token_from_text()

in aoai/token_count_utils.py [0:0]

• 7 lines of code
• 2 McCabe index (conditional complexity)

def _num_token_from_text(text: str, model: str = "gpt-3.5-turbo-0613"):
    """Return the number of tokens used by a string."""
    try:
        encoding = tiktoken.encoding_for_model(model)
    except KeyError:
        logger.warning(f"Model {model} not found. Using cl100k_base encoding.")
        encoding = tiktoken.get_encoding("cl100k_base")
    return len(encoding.encode(text))