Path	Lines of Code
bindings/node/Cargo.toml	18
bindings/node/build.rs	4
bindings/node/index.js	376
bindings/node/rustfmt.toml	1
bindings/node/src/arc_rwlock_serde.rs	15
bindings/node/src/decoders.rs	161
bindings/node/src/encoding.rs	310
bindings/node/src/lib.rs	13
bindings/node/src/models.rs	303
bindings/node/src/normalizers.rs	159
bindings/node/src/pre_tokenizers.rs	231
bindings/node/src/processors.rs	110
bindings/node/src/tasks/mod.rs	2
bindings/node/src/tasks/models.rs	67
bindings/node/src/tasks/tokenizer.rs	111
bindings/node/src/tokenizer.rs	463
bindings/node/src/trainers.rs	59
bindings/node/src/utils.rs	42
bindings/node/types.ts	6
bindings/python/Cargo.toml	26
bindings/python/MANIFEST.in	7
bindings/python/conftest.py	12
bindings/python/py_src/tokenizers/__init__.py	52
bindings/python/py_src/tokenizers/__init__.pyi	221
bindings/python/py_src/tokenizers/decoders/__init__.py	13
bindings/python/py_src/tokenizers/decoders/__init__.pyi	56
bindings/python/py_src/tokenizers/implementations/__init__.py	6
bindings/python/py_src/tokenizers/implementations/base_tokenizer.py	131
bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py	131
bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py	97
bindings/python/py_src/tokenizers/implementations/char_level_bpe.py	113
bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py	83
bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py	120
bindings/python/py_src/tokenizers/models/__init__.py	6
bindings/python/py_src/tokenizers/models/__init__.pyi	93
bindings/python/py_src/tokenizers/normalizers/__init__.py	23
bindings/python/py_src/tokenizers/normalizers/__init__.pyi	101
bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py	14
bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi	90
bindings/python/py_src/tokenizers/processors/__init__.py	7
bindings/python/py_src/tokenizers/processors/__init__.pyi	40
bindings/python/py_src/tokenizers/tools/__init__.py	1
bindings/python/py_src/tokenizers/tools/visualizer-styles.css	138
bindings/python/py_src/tokenizers/tools/visualizer.py	209
bindings/python/py_src/tokenizers/trainers/__init__.py	6
bindings/python/py_src/tokenizers/trainers/__init__.pyi	28
bindings/python/pyproject.toml	61
bindings/python/scripts/convert.py	342
bindings/python/scripts/sentencepiece_extractor.py	93
bindings/python/scripts/spm_parity_check.py	208
bindings/python/setup.cfg	51
bindings/python/src/decoders.rs	577
bindings/python/src/encoding.rs	244
bindings/python/src/error.rs	35
bindings/python/src/lib.rs	39
bindings/python/src/models.rs	785
bindings/python/src/normalizers.rs	760
bindings/python/src/pre_tokenizers.rs	791
bindings/python/src/processors.rs	642
bindings/python/src/token.rs	40
bindings/python/src/tokenizer.rs	1060
bindings/python/src/trainers.rs	836
bindings/python/src/utils/iterators.rs	90
bindings/python/src/utils/mod.rs	62
bindings/python/src/utils/normalization.rs	477
bindings/python/src/utils/pretokenization.rs	254
bindings/python/src/utils/regex.rs	20
bindings/python/src/utils/serde_pyo3.rs	454
bindings/python/stub.py	141