Path	Lines of Code
ablations/evaluation/launch_evals.py	139
ablations/evaluation/launch_random_evals.py	50
ablations/tokenization/launch_tokenization.py	63
ablations/training/launch_exp.py	226
fineweb-2-pipeline.py	227
misc/precision_filtering/count_common.py	44
misc/precision_filtering/run_precision_filtering.py	106
misc/precision_filtering/slurm_count_word.py	71
misc/precision_filtering/wordlist_gen.py	56
misc/precision_filtering/wordlist_score.py	40
misc/reference_datasets/monolingual/ar/download_101b_arabicwords.py	18
misc/reference_datasets/monolingual/ar/download_arabicweb24.py	74
misc/reference_datasets/monolingual/fr/download_croissant.py	75
misc/reference_datasets/monolingual/hi/download_odaigen.py	17
misc/reference_datasets/monolingual/hi/download_sangraha.py	21
misc/reference_datasets/monolingual/te/download_sangraha.py	39
misc/reference_datasets/monolingual/th/download_sea_commoncrawl.py	17
misc/reference_datasets/monolingual/tr/download_vngrs.py	19
misc/reference_datasets/monolingual/zh/download_mapcc.py	140
misc/reference_datasets/monolingual/zh/download_mnbvc.py	19
misc/reference_datasets/monolingual/zh/download_tigerbot.py	19
misc/reference_datasets/multilingual/copy_raw_data.py	105
misc/reference_datasets/multilingual/download_cc-100.py	162
misc/reference_datasets/multilingual/download_culturax.py	24
misc/reference_datasets/multilingual/download_hplt.py	41
misc/reference_datasets/multilingual/download_hplt_split.py	20
misc/reference_datasets/multilingual/download_mc4.py	29
misc/reference_datasets/multilingual/part jsons.py	112