def count_token()

in aoai/token_count_utils.py [0:0]

• 7 lines of code
• 4 McCabe index (conditional complexity)

def count_token(input: Union[str, List, Dict], model: str = "gpt-3.5-turbo-0613") -> int:
    """Count number of tokens used by an OpenAI model.
    Args:
        input: (str, list, dict): Input to the model.
        model: (str): Model name.

    Returns:
        int: Number of tokens from the input.
    """
    if isinstance(input, str):
        return _num_token_from_text(input, model=model)
    elif isinstance(input, list) or isinstance(input, dict):
        return _num_token_from_messages(input, model=model)
    else:
        raise ValueError(f"input must be str, list or dict, but we got {type(input)}")