def tokenize_all()

in tokenizers/python/baronetokenizer.py [0:0]

• 6 lines of code
• 3 McCabe index (conditional complexity)

def tokenize_all(input_file):
    with open(input_file) as f:
        for i, line in tqdm(enumerate(f)):
            tokens = tokenize_line(line)
            if len(tokens) == 0: continue
            yield dict(filename='dummyfile%s' % i, tokens=tokens)