def tokenize_all_project_folders()

in tokenizers/python/tokenizepythoncorpus.py [0:0]

• 5 lines of code
• 3 McCabe index (conditional complexity)

def tokenize_all_project_folders(directory: str, output_folder: str, only_ids: bool=False):
    os.makedirs(output_folder, exist_ok=True)
    all_dirs = [(os.path.join(directory, d), output_folder, only_ids) for d in os.listdir(directory) if os.path.isdir(os.path.join(directory, d))]
    with mp.Pool() as p:
        p.starmap(tokenize_all_files, all_dirs)