def create_tokenizer()

in ml/eval/reward_eval.py [0:0]

• 3 lines of code
• 2 McCabe index (conditional complexity)

def create_tokenizer(model_name):
    # loads the tokenizer that pairs with the model for encoding the text data
    tokenizer = AutoTokenizer.from_pretrained(model_name, use_auth_token=True)
    return tokenizer