def _special_token_regex()

in tiktoken/core.py [0:0]

• 3 lines of code
• 2 McCabe index (conditional complexity)

def _special_token_regex(tokens: frozenset[str]) -> "regex.Pattern[str]":
    inner = "|".join(regex.escape(token) for token in tokens)
    return regex.compile(f"({inner})")