def number_of_tokens()

in shared/util.py [0:0]

• 5 lines of code
• 1 McCabe index (conditional complexity)

def number_of_tokens(messages, model):
    prompt = json.dumps(messages)
    encoding = tiktoken.encoding_for_model(model.replace('gpt-35-turbo','gpt-3.5-turbo'))
    num_tokens = len(encoding.encode(prompt))
    return num_tokens