def num_assistant_tokens_from_messages()

in aoai/token_count_utils.py [0:0]

• 11 lines of code
• 5 McCabe index (conditional complexity)

def num_assistant_tokens_from_messages(messages, model="gpt-3.5-turbo-0613") -> int:
    
    if "gpt-4o" in model:
        encoding = tiktoken.get_encoding("o200k_base")      
    else:
        encoding = tiktoken.get_encoding("cl100k_base")
            
    num_tokens = 0
    for message in messages:
        if message["role"] == "assistant":
            if "content" in message:
                num_tokens += len(encoding.encode(message["content"]))
    return num_tokens