path	# lines of code	# active days	days since first update	days since last update	# commits	# contributors	first updated	last updated	first contributor	last contributor
pyproject.toml	152	24	729	62	32	14	2023-07-03	2025-04-30	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/formatters/base.py	17	1	466	466	1	1	2024-03-22	2024-03-22	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/formatters/__init__.py	3	2	466	441	2	1	2024-03-22	2024-04-16	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/formatters/ftfy.py	43	2	466	174	2	1	2024-03-22	2025-01-08	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/formatters/pii.py	68	2	441	425	2	1	2024-04-16	2024-05-02	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/formatters/symbol_lines_remover.py	25	3	466	207	4	1	2024-03-22	2024-12-06	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/filters/regex_filter.py	11	11	740	511	16	6	2023-06-22	2024-02-06	47321860+alexchapeaux@users.noreply.github.com	nostrumg@gmail.com
src/datatrove/pipeline/filters/gopher_quality_filter.py	76	16	740	152	20	6	2023-06-22	2025-01-30	47321860+alexchapeaux@users.noreply.github.com	nostrumg@gmail.com
src/datatrove/pipeline/filters/gopher_repetition_filter.py	90	16	740	207	21	5	2023-06-22	2024-12-06	47321860+alexchapeaux@users.noreply.github.com	nostrumg@gmail.com
src/datatrove/pipeline/filters/lambda_filter.py	11	6	740	511	11	4	2023-06-22	2024-02-06	47321860+alexchapeaux@users.noreply.github.com	nostrumg@gmail.com
src/datatrove/pipeline/filters/sampler_filter.py	17	4	680	511	5	2	2023-08-21	2024-02-06	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/filters/fineweb_quality_filter.py	47	5	437	43	6	3	2024-04-20	2025-05-19	nostrumg@gmail.com	cnhx27@gmail.com
src/datatrove/pipeline/filters/language_filter.py	43	18	740	173	23	9	2023-06-22	2025-01-09	47321860+alexchapeaux@users.noreply.github.com	sk8terbo2@gmail.com
src/datatrove/pipeline/filters/fasttext_filter.py	78	7	496	173	7	4	2024-02-21	2025-01-09	nostrumg@gmail.com	sk8terbo2@gmail.com
src/datatrove/pipeline/filters/url_filter.py	91	14	708	207	17	6	2023-07-24	2024-12-06	47321860+alexchapeaux@users.noreply.github.com	nostrumg@gmail.com
src/datatrove/pipeline/filters/c4_filters.py	197	4	433	207	5	3	2024-04-24	2024-12-06	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/filters/base_filter.py	49	15	740	361	22	5	2023-06-22	2024-07-05	47321860+alexchapeaux@users.noreply.github.com	nostrumg@gmail.com
src/datatrove/pipeline/filters/__init__.py	11	13	748	433	16	4	2023-06-14	2024-04-24	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/filters/unigram_log_probs.py	46	10	708	207	11	4	2023-07-24	2024-12-06	47321860+alexchapeaux@users.noreply.github.com	nostrumg@gmail.com
src/datatrove/pipeline/decont/n_grams.py	150	7	423	63	8	3	2024-05-04	2025-04-29	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/decont/__init__.py	1	1	423	423	1	1	2024-05-04	2024-05-04	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/base.py	35	16	747	406	23	6	2023-06-15	2024-05-21	alessandro@lighton.ai	nostrumg@gmail.com
src/datatrove/pipeline/stats/line_stats.py	57	2	382	302	3	1	2024-06-14	2024-09-02	kydlicek.hynek@gmail.com	kydlicek.hynek@gmail.com
src/datatrove/pipeline/stats/lang_stats.py	26	2	382	377	2	2	2024-06-14	2024-06-19	kydlicek.hynek@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/stats/contamination_stats.py	33	1	382	382	1	1	2024-06-14	2024-06-14	kydlicek.hynek@gmail.com	kydlicek.hynek@gmail.com
src/datatrove/pipeline/stats/perplexity_stats.py	25	1	382	382	1	1	2024-06-14	2024-06-14	kydlicek.hynek@gmail.com	kydlicek.hynek@gmail.com
src/datatrove/pipeline/stats/token_stats.py	25	2	382	173	2	2	2024-06-14	2025-01-09	kydlicek.hynek@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/stats/config.py	9	1	382	382	1	1	2024-06-14	2024-06-14	kydlicek.hynek@gmail.com	kydlicek.hynek@gmail.com
src/datatrove/pipeline/stats/word_stats.py	56	1	382	382	1	1	2024-06-14	2024-06-14	kydlicek.hynek@gmail.com	kydlicek.hynek@gmail.com
src/datatrove/pipeline/stats/paragraph_stats.py	51	1	382	382	1	1	2024-06-14	2024-06-14	kydlicek.hynek@gmail.com	kydlicek.hynek@gmail.com
src/datatrove/pipeline/stats/sentence_stats.py	47	1	382	382	1	1	2024-06-14	2024-06-14	kydlicek.hynek@gmail.com	kydlicek.hynek@gmail.com
src/datatrove/pipeline/stats/base.py	91	2	382	351	2	2	2024-06-14	2024-07-15	kydlicek.hynek@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/stats/doc_stats.py	30	1	382	382	1	1	2024-06-14	2024-06-14	kydlicek.hynek@gmail.com	kydlicek.hynek@gmail.com
src/datatrove/pipeline/stats/__init__.py	11	4	722	382	4	4	2023-07-10	2024-06-14	alessandro@lighton.ai	kydlicek.hynek@gmail.com
src/datatrove/pipeline/stats/merger.py	51	1	382	382	1	1	2024-06-14	2024-06-14	kydlicek.hynek@gmail.com	kydlicek.hynek@gmail.com
src/datatrove/pipeline/readers/huggingface.py	95	11	529	63	13	7	2024-01-19	2025-04-29	nostrumg@gmail.com	huseyinabanozis@gmail.com
src/datatrove/pipeline/readers/parquet.py	57	12	713	377	15	5	2023-07-19	2024-06-19	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/readers/jsonl.py	57	26	741	377	34	10	2023-06-21	2024-06-19	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/readers/csv.py	50	11	635	377	14	6	2023-10-05	2024-06-19	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/readers/ipc.py	64	8	557	377	11	5	2023-12-22	2024-06-19	mariosasko777@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/readers/base.py	143	28	741	63	40	10	2023-06-21	2025-04-29	nostrumg@gmail.com	vivien.cabannes@gmail.com
src/datatrove/pipeline/readers/warc.py	94	24	741	202	34	9	2023-06-21	2024-12-11	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/readers/__init__.py	6	6	748	529	8	3	2023-06-14	2024-01-19	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/writers/huggingface.py	104	6	494	152	7	2	2024-02-23	2025-01-30	nostrumg@gmail.com	2779410+bramvanroy@users.noreply.github.com
src/datatrove/pipeline/writers/parquet.py	67	5	495	152	5	3	2024-02-22	2025-01-30	nostrumg@gmail.com	2779410+bramvanroy@users.noreply.github.com
src/datatrove/pipeline/writers/jsonl.py	28	18	741	307	24	8	2023-06-21	2024-08-28	nostrumg@gmail.com	justhungryman@gmail.com
src/datatrove/pipeline/writers/__init__.py	3	4	748	494	4	1	2023-06-14	2024-02-23	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/writers/disk_base.py	83	23	741	307	30	8	2023-06-21	2024-08-28	nostrumg@gmail.com	justhungryman@gmail.com
src/datatrove/pipeline/dedup/sentence_dedup.py	370	26	722	158	38	7	2023-07-10	2025-01-24	alessandro@lighton.ai	muzzynine@gmail.com
src/datatrove/pipeline/dedup/url_dedup.py	301	6	445	63	7	3	2024-04-12	2025-04-29	kydlicek.hynek@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/dedup/exact_substrings.py	229	23	706	63	27	7	2023-07-26	2025-04-29	47321860+alexchapeaux@users.noreply.github.com	nostrumg@gmail.com
src/datatrove/pipeline/dedup/minhash.py	499	33	706	63	50	7	2023-07-26	2025-04-29	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/dedup/__init__.py	12	10	746	445	11	4	2023-06-16	2024-04-12	alessandro@lighton.ai	kydlicek.hynek@gmail.com
src/datatrove/pipeline/dedup/bloom_filter.py	143	11	635	349	15	5	2023-10-05	2024-07-17	47321860+alexchapeaux@users.noreply.github.com	nostrumg@gmail.com
src/datatrove/pipeline/tokens/context_shuffler.py	46	9	669	358	11	3	2023-09-01	2024-07-08	nostrumg@gmail.com	134823555+sippycoder@users.noreply.github.com
src/datatrove/pipeline/tokens/counter.py	36	12	589	173	14	5	2023-11-20	2025-01-09	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/tokens/megatron_tokenizer.py	107	1	173	173	3	2	2025-01-09	2025-01-09	nostrumg@gmail.com	74564958+tj-solergibert@users.noreply.github.com
src/datatrove/pipeline/tokens/tokenizer.py	298	28	736	62	42	6	2023-06-26	2025-04-30	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/tokens/__init__.py	5	5	729	173	7	5	2023-07-03	2025-01-09	anton@huggingface.co	74564958+tj-solergibert@users.noreply.github.com
src/datatrove/pipeline/tokens/merger.py	153	20	733	63	24	4	2023-06-29	2025-04-29	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/pipeline/extractors/modular.py	23	8	742	488	12	5	2023-06-20	2024-02-29	alessandro@lighton.ai	thomwolf@users.noreply.github.com
src/datatrove/pipeline/extractors/trafilatura.py	28	11	742	180	16	6	2023-06-20	2025-01-02	alessandro@lighton.ai	nostrumg@gmail.com
src/datatrove/pipeline/extractors/base.py	117	17	742	63	20	6	2023-06-20	2025-04-29	alessandro@lighton.ai	kydlicek.hynek@gmail.com
src/datatrove/pipeline/extractors/__init__.py	2	3	742	729	5	3	2023-06-20	2023-07-03	alessandro@lighton.ai	anton@huggingface.co
src/datatrove/utils/batching.py	7	1	361	361	1	1	2024-07-05	2024-07-05	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/utils/binaryio.py	54	2	466	445	2	2	2024-03-22	2024-04-12	nostrumg@gmail.com	kydlicek.hynek@gmail.com
src/datatrove/utils/text.py	254	7	466	207	7	2	2024-03-22	2024-12-06	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/utils/logging.py	37	5	595	410	5	1	2023-11-14	2024-05-17	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/utils/_import_utils.py	54	6	530	62	6	3	2024-01-18	2025-04-30	mariosasko777@gmail.com	nostrumg@gmail.com
src/datatrove/utils/dataset.py	194	4	433	62	4	2	2024-04-24	2025-04-30	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/utils/tokenization.py	58	7	469	63	8	3	2024-03-19	2025-04-29	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/utils/word_tokenizers.py	361	5	406	152	7	3	2024-05-21	2025-01-30	beme248@users.noreply.github.com	nostrumg@gmail.com
src/datatrove/utils/perplexity.py	138	2	382	63	2	2	2024-06-14	2025-04-29	kydlicek.hynek@gmail.com	nostrumg@gmail.com
src/datatrove/utils/hashing.py	37	1	406	406	1	1	2024-05-21	2024-05-21	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/utils/stats.py	297	17	734	382	25	7	2023-06-28	2024-06-14	alessandro@lighton.ai	kydlicek.hynek@gmail.com
src/datatrove/utils/hashes/xxhash.py	5	1	406	406	1	1	2024-05-21	2024-05-21	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/utils/hashes/sha1.py	6	1	406	406	1	1	2024-05-21	2024-05-21	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/utils/japanese_tokenizer.py	216	1	207	207	1	1	2024-12-06	2024-12-06	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/utils/typeshelper.py	4106	10	743	207	11	6	2023-06-19	2024-12-06	alessandro@lighton.ai	nostrumg@gmail.com
src/datatrove/utils/lid.py	45	5	382	173	5	3	2024-06-14	2025-01-09	kydlicek.hynek@gmail.com	sk8terbo2@gmail.com
src/datatrove/executor/ray.py	160	1	63	63	2	2	2025-04-29	2025-04-29	nostrumg@gmail.com	nelson-liu@users.noreply.github.com
src/datatrove/executor/slurm.py	235	39	740	63	69	9	2023-06-22	2025-04-29	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/executor/base.py	98	27	748	399	37	7	2023-06-14	2024-05-28	nostrumg@gmail.com	jun.untitled@kakaobrain.com
src/datatrove/executor/__init__.py	3	2	529	63	2	2	2024-01-19	2025-04-29	nostrumg@gmail.com	nelson-liu@users.noreply.github.com
src/datatrove/executor/local.py	101	26	748	63	35	6	2023-06-14	2025-04-29	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/tools/merge_stats.py	33	9	701	410	11	3	2023-07-31	2024-05-17	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/tools/failed_logs.py	55	5	587	63	8	2	2023-11-22	2025-04-29	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/tools/jobs_status.py	71	4	483	63	6	2	2024-03-05	2025-04-29	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/tools/check_dataset.py	58	10	701	63	11	4	2023-07-31	2025-04-29	nostrumg@gmail.com	craffel@gmail.com
src/datatrove/tools/inspect_data.py	129	7	579	468	15	4	2023-11-30	2024-03-20	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/tools/launch_pickled_pipeline.py	13	3	595	496	3	1	2023-11-14	2024-02-21	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/tools/fast_mh3/Cargo.toml	26	1	207	207	1	1	2024-12-06	2024-12-06	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/tools/fast_mh3/src/s3_union_find.rs	477	1	207	207	1	1	2024-12-06	2024-12-06	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/tools/fast_mh3/src/local_union_find.rs	272	1	207	207	1	1	2024-12-06	2024-12-06	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/tools/__init__.py	1	1	701	701	1	1	2023-07-31	2023-07-31	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/io.py	188	22	530	62	23	5	2024-01-18	2025-04-30	nostrumg@gmail.com	nostrumg@gmail.com
src/datatrove/data.py	19	6	729	487	8	4	2023-07-03	2024-03-01	nostrumg@gmail.com	thomwolf@users.noreply.github.com