Path Lines of Code tokenizers/Cargo.toml 82 tokenizers/benches/bert_benchmark.rs 132 tokenizers/benches/bpe_benchmark.rs 115 tokenizers/benches/common/mod.rs 76 tokenizers/benches/layout_benchmark.rs 64 tokenizers/benches/llama3_benchmark.rs 64 tokenizers/benches/unigram_benchmark.rs 37 tokenizers/src/decoders/bpe.rs 31 tokenizers/src/decoders/byte_fallback.rs 91 tokenizers/src/decoders/fuse.rs 34 tokenizers/src/decoders/mod.rs 39 tokenizers/src/decoders/sequence.rs 48 tokenizers/src/decoders/strip.rs 67 tokenizers/src/decoders/wordpiece.rs 74 tokenizers/src/lib.rs 20 tokenizers/src/models/bpe/mod.rs 59 tokenizers/src/models/bpe/model.rs 834 tokenizers/src/models/bpe/serialization.rs 217 tokenizers/src/models/bpe/trainer.rs 598 tokenizers/src/models/bpe/word.rs 277 tokenizers/src/models/mod.rs 59 tokenizers/src/models/unigram/lattice.rs 588 tokenizers/src/models/unigram/mod.rs 8 tokenizers/src/models/unigram/model.rs 509 tokenizers/src/models/unigram/serialization.rs 98 tokenizers/src/models/unigram/trainer.rs 685 tokenizers/src/models/unigram/trie.rs 43 tokenizers/src/models/wordlevel/mod.rs 204 tokenizers/src/models/wordlevel/serialization.rs 114 tokenizers/src/models/wordlevel/trainer.rs 151 tokenizers/src/models/wordpiece/mod.rs 250 tokenizers/src/models/wordpiece/serialization.rs 133 tokenizers/src/models/wordpiece/trainer.rs 147 tokenizers/src/normalizers/bert.rs 102 tokenizers/src/normalizers/byte_level.rs 164 tokenizers/src/normalizers/mod.rs 40 tokenizers/src/normalizers/precompiled.rs 67 tokenizers/src/normalizers/prepend.rs 56 tokenizers/src/normalizers/replace.rs 131 tokenizers/src/normalizers/strip.rs 133 tokenizers/src/normalizers/unicode.rs 91 tokenizers/src/normalizers/utils.rs 48 tokenizers/src/pre_tokenizers/bert.rs 76 tokenizers/src/pre_tokenizers/byte_level.rs 513 tokenizers/src/pre_tokenizers/delimiter.rs 21 tokenizers/src/pre_tokenizers/digits.rs 93 tokenizers/src/pre_tokenizers/fixed_length.rs 111 tokenizers/src/pre_tokenizers/metaspace.rs 331 tokenizers/src/pre_tokenizers/mod.rs 258 tokenizers/src/pre_tokenizers/punctuation.rs 72 tokenizers/src/pre_tokenizers/sequence.rs 73 tokenizers/src/pre_tokenizers/split.rs 33 tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs 3 tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs 132 tokenizers/src/pre_tokenizers/whitespace.rs 94 tokenizers/src/processors/bert.rs 268 tokenizers/src/processors/mod.rs 112 tokenizers/src/processors/roberta.rs 309 tokenizers/src/processors/sequence.rs 156 tokenizers/src/processors/template.rs 854 tokenizers/src/tokenizer/added_vocabulary.rs 799 tokenizers/src/tokenizer/encoding.rs 780 tokenizers/src/tokenizer/mod.rs 1185 tokenizers/src/tokenizer/normalizer.rs 2002 tokenizers/src/tokenizer/pattern.rs 187 tokenizers/src/tokenizer/pre_tokenizer.rs 228 tokenizers/src/tokenizer/serialization.rs 215 tokenizers/src/utils/cache.rs 85 tokenizers/src/utils/fancy.rs 55 tokenizers/src/utils/from_pretrained.rs 58 tokenizers/src/utils/iter.rs 71 tokenizers/src/utils/mod.rs 112 tokenizers/src/utils/onig.rs 15 tokenizers/src/utils/padding.rs 123 tokenizers/src/utils/parallelism.rs 219 tokenizers/src/utils/progress.rs 12 tokenizers/src/utils/truncation.rs 279