def num_tokens_from_string()

in ai-sample/TokenizeDocFunction/function_app.py [0:0]

• 6 lines of code
• 2 McCabe index (conditional complexity)

def num_tokens_from_string(string: str, encoding_name="cl100k_base") -> int:
    if not string:
        return 0
    # Returns the number of tokens in a text string
    encoding = tiktoken.get_encoding(encoding_name)
    num_tokens = len(encoding.encode(string))
    return num_tokens