Path	Lines of Code
bindings/python/py_src/tokenizers/tools/visualizer.py	209
bindings/python/scripts/convert.py	342
bindings/python/src/processors.rs	642
bindings/python/src/utils/normalization.rs	477
bindings/python/stub.py	141
tokenizers/src/models/unigram/lattice.rs	588
tokenizers/src/models/unigram/trainer.rs	685
tokenizers/src/pre_tokenizers/byte_level.rs	513
tokenizers/src/pre_tokenizers/delimiter.rs	21
tokenizers/src/tokenizer/encoding.rs	780
tokenizers/src/tokenizer/mod.rs	1185