def num_tokens_from_functions()

in aoai/token_count_utils.py [0:0]

• 40 lines of code
• 13 McCabe index (conditional complexity)

def num_tokens_from_functions(functions, model="gpt-3.5-turbo-0613") -> int:
    """Return the number of tokens used by a list of functions.

    Args:
        functions: (list): List of function descriptions that will be passed in model.
        model: (str): Model name.

    Returns:
        int: Number of tokens from the function descriptions.
    """
    if "gpt-4o" in model:
        encoding = tiktoken.get_encoding("o200k_base")      
    else:
        encoding = tiktoken.get_encoding("cl100k_base")

    num_tokens = 0
    for f in functions:
        if 'function' in f:
            function = f["function"]
        else:
            function = f
        function_tokens = len(encoding.encode(function["name"]))
        function_tokens += len(encoding.encode(function["description"]))
        function_tokens -= 2
        if "parameters" in function:
            parameters = function["parameters"]
            if "properties" in parameters:
                for propertiesKey in parameters["properties"]:
                    function_tokens += len(encoding.encode(propertiesKey))
                    v = parameters["properties"][propertiesKey]
                    for field in v:
                        if field == "type":
                            function_tokens += 2
                            function_tokens += len(encoding.encode(v["type"]))
                        elif field == "description":
                            function_tokens += 2
                            function_tokens += len(encoding.encode(v["description"]))
                        elif field == "enum":
                            function_tokens -= 3
                            for o in v["enum"]:
                                function_tokens += 3
                                function_tokens += len(encoding.encode(o))
                        else:
                            logger.warning(f"Not supported field {field}")
                function_tokens += 11
                if len(parameters["properties"]) == 0:
                    function_tokens -= 2

        num_tokens += function_tokens

    num_tokens += 12
    return num_tokens