id: 1
unit: fn transform_range_multiple_bytes()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1857
end line: 2281
size: 409 LOC
McCabe index: 1
number of parameters: 0

id: 2
unit: fn transform_range_single_bytes()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1511
end line: 1854
size: 327 LOC
McCabe index: 1
number of parameters: 0

id: 3
unit: fn template_processing_overflowing()
file: tokenizers/src/processors/template.rs
start line: 954
end line: 1128
size: 172 LOC
McCabe index: 1
number of parameters: 0

id: 4
unit: fn process_encodings()
file: tokenizers/src/processors/roberta.rs
start line: 66
end line: 233
size: 151 LOC
McCabe index: 5
number of parameters: 3

id: 5
unit: fn deserialize()
file: tokenizers/src/pre_tokenizers/mod.rs
start line: 65
end line: 205
size: 136 LOC
McCabe index: 1
number of parameters: 1

id: 6
unit: fn process_encodings()
file: tokenizers/src/processors/bert.rs
start line: 51
end line: 192
size: 128 LOC
McCabe index: 3
number of parameters: 3

id: 7
unit: fn test_byte_level_normalize()
file: tokenizers/src/normalizers/byte_level.rs
start line: 56
end line: 173
size: 118 LOC
McCabe index: 1
number of parameters: 0

id: 8
unit: fn prune_sentence_pieces()
file: tokenizers/src/models/unigram/trainer.rs
start line: 277
end line: 432
size: 116 LOC
McCabe index: 22
number of parameters: 4

id: 9
unit: fn test_ignore_merges()
file: tokenizers/src/models/bpe/model.rs
start line: 915
end line: 1008
size: 89 LOC
McCabe index: 1
number of parameters: 0

id: 10
unit: fn process_chain()
file: tokenizers/src/processors/sequence.rs
start line: 80
end line: 171
size: 89 LOC
McCabe index: 1
number of parameters: 0

id: 11
unit: fn encode_optimized()
file: tokenizers/src/models/unigram/model.rs
start line: 240
end line: 329
size: 84 LOC
McCabe index: 16
number of parameters: 2

id: 12
unit: fn non_legacy_meta_space()
file: tokenizers/src/pre_tokenizers/metaspace.rs
start line: 268
end line: 355
size: 82 LOC
McCabe index: 1
number of parameters: 0

id: 13
unit: fn bert_processing()
file: tokenizers/src/processors/bert.rs
start line: 211
end line: 295
size: 82 LOC
McCabe index: 1
number of parameters: 0

id: 14
unit: fn processor_trims_offsets()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 470
end line: 551
size: 80 LOC
McCabe index: 1
number of parameters: 0

id: 15
unit: fn insert_test()
file: tokenizers/src/models/unigram/lattice.rs
start line: 482
end line: 570
size: 79 LOC
McCabe index: 1
number of parameters: 0

id: 16
unit: fn mappings()
file: tokenizers/src/tokenizer/encoding.rs
start line: 792
end line: 881
size: 79 LOC
McCabe index: 1
number of parameters: 0

id: 17
unit: fn roberta_processing()
file: tokenizers/src/processors/roberta.rs
start line: 259
end line: 340
size: 79 LOC
McCabe index: 1
number of parameters: 0

id: 18
unit: fn visit_map()
file: tokenizers/src/models/bpe/serialization.rs
start line: 78
end line: 155
size: 77 LOC
McCabe index: 10
number of parameters: 2

id: 19
unit: fn merge_word()
file: tokenizers/src/models/bpe/model.rs
start line: 382
end line: 467
size: 76 LOC
McCabe index: 16
number of parameters: 2

id: 20
unit: def main()
file: bindings/python/scripts/spm_parity_check.py
start line: 0
end line: 0
size: 66 LOC
McCabe index: 4
number of parameters: 0

id: 21
unit: fn make_seed_sentence_pieces()
file: tokenizers/src/models/unigram/trainer.rs
start line: 196
end line: 276
size: 65 LOC
McCabe index: 11
number of parameters: 3

id: 22
unit: fn finalize()
file: tokenizers/src/models/unigram/trainer.rs
start line: 113
end line: 186
size: 63 LOC
McCabe index: 11
number of parameters: 3

id: 23
unit: fn can_add_special_tokens()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 678
end line: 751
size: 63 LOC
McCabe index: 1
number of parameters: 0

id: 24
unit: fn template_processing()
file: tokenizers/src/processors/template.rs
start line: 889
end line: 951
size: 62 LOC
McCabe index: 1
number of parameters: 0

id: 25
unit: fn visit_map()
file: tokenizers/src/tokenizer/serialization.rs
start line: 108
end line: 171
size: 59 LOC
McCabe index: 6
number of parameters: 2

id: 26
unit: fn test_tokenize_with_and_without_dropout()
file: tokenizers/src/models/bpe/model.rs
start line: 669
end line: 734
size: 58 LOC
McCabe index: 2
number of parameters: 0

id: 27
unit: fn test_encode_special_tokens()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 941
end line: 1000
size: 55 LOC
McCabe index: 1
number of parameters: 0

id: 28
unit: fn tokenize()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 207
end line: 266
size: 53 LOC
McCabe index: 8
number of parameters: 2

id: 29
unit: fn truncate_overflow_with_stride()
file: tokenizers/src/tokenizer/encoding.rs
start line: 693
end line: 746
size: 53 LOC
McCabe index: 1
number of parameters: 0

id: 30
unit: fn validate()
file: tokenizers/src/processors/template.rs
start line: 468
end line: 524
size: 53 LOC
McCabe index: 7
number of parameters: 1

id: 31
unit: fn enable_padding()
file: bindings/python/src/tokenizer.rs
start line: 841
end line: 895
size: 52 LOC
McCabe index: 6
number of parameters: 2

id: 32
unit: def __make_html()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 51 LOC
McCabe index: 9
number of parameters: 3

id: 33
unit: fn test_unk_token()
file: tokenizers/src/models/unigram/trainer.rs
start line: 735
end line: 792
size: 50 LOC
McCabe index: 2
number of parameters: 0

id: 34
unit: fn test_deserialization_serialization_invariant()
file: tokenizers/src/tokenizer/serialization.rs
start line: 180
end line: 231
size: 50 LOC
McCabe index: 1
number of parameters: 0

id: 35
unit: fn added_characters_alignment()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1236
end line: 1286
size: 50 LOC
McCabe index: 2
number of parameters: 0

id: 36
unit: fn enable_truncation()
file: bindings/python/src/tokenizer.rs
start line: 732
end line: 783
size: 50 LOC
McCabe index: 4
number of parameters: 3

id: 37
unit: fn test_serialization()
file: tokenizers/src/models/bpe/serialization.rs
start line: 164
end line: 216
size: 49 LOC
McCabe index: 1
number of parameters: 0

id: 38
unit: fn test_encode2()
file: tokenizers/src/models/unigram/model.rs
start line: 556
end line: 608
size: 48 LOC
McCabe index: 2
number of parameters: 0

id: 39
unit: def main()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 48 LOC
McCabe index: 2
number of parameters: 0

id: 40
unit: def check_details()
file: bindings/python/scripts/spm_parity_check.py
start line: 0
end line: 0
size: 48 LOC
McCabe index: 19
number of parameters: 5

id: 41
unit: fn visit_map()
file: tokenizers/src/models/wordpiece/serialization.rs
start line: 57
end line: 106
size: 47 LOC
McCabe index: 3
number of parameters: 2

id: 42
unit: fn compute_alphabet()
file: tokenizers/src/models/bpe/trainer.rs
start line: 271
end line: 322
size: 47 LOC
McCabe index: 8
number of parameters: 4

id: 43
unit: fn pad_to_multiple()
file: tokenizers/src/utils/padding.rs
start line: 90
end line: 141
size: 46 LOC
McCabe index: 1
number of parameters: 0

id: 44
unit: fn extract_bound()
file: bindings/python/src/tokenizer.rs
start line: 283
end line: 341
size: 46 LOC
McCabe index: 5
number of parameters: 1

id: 45
unit: fn test_train()
file: tokenizers/src/models/wordlevel/trainer.rs
start line: 134
end line: 181
size: 43 LOC
McCabe index: 1
number of parameters: 0

id: 46
unit: fn tokenize_words()
file: tokenizers/src/models/bpe/trainer.rs
start line: 325
end line: 373
size: 43 LOC
McCabe index: 13
number of parameters: 5

id: 47
unit: def check_encode()
file: bindings/python/scripts/spm_parity_check.py
start line: 0
end line: 0
size: 43 LOC
McCabe index: 8
number of parameters: 1

id: 48
unit: fn encode_single_sequence()
file: tokenizers/src/tokenizer/mod.rs
start line: 718
end line: 761
size: 42 LOC
McCabe index: 2
number of parameters: 4

id: 49
unit: fn find_matches()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 367
end line: 422
size: 42 LOC
McCabe index: 14
number of parameters: 3

id: 50
unit: fn apply_template()
file: tokenizers/src/processors/template.rs
start line: 544
end line: 643
size: 42 LOC
McCabe index: 2
number of parameters: 4

id: 51
unit: fn can_extract_added_tokens()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 754
end line: 797
size: 41 LOC
McCabe index: 1
number of parameters: 0

id: 52
unit: def from_spm()
file: bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
start line: 0
end line: 0
size: 41 LOC
McCabe index: 6
number of parameters: 1

id: 53
unit: fn get_long()
file: tokenizers/src/utils/truncation.rs
start line: 222
end line: 261
size: 40 LOC
McCabe index: 1
number of parameters: 0

id: 54
unit: fn test_unigram_chars()
file: tokenizers/src/models/unigram/trainer.rs
start line: 667
end line: 713
size: 40 LOC
McCabe index: 2
number of parameters: 0

id: 55
unit: fn truncate()
file: tokenizers/src/tokenizer/encoding.rs
start line: 609
end line: 649
size: 40 LOC
McCabe index: 1
number of parameters: 0

id: 56
unit: fn truncate_left()
file: tokenizers/src/tokenizer/encoding.rs
start line: 749
end line: 789
size: 40 LOC
McCabe index: 1
number of parameters: 0

id: 57
unit: def pyi_file()
file: bindings/python/stub.py
start line: 0
end line: 0
size: 40 LOC
McCabe index: 12
number of parameters: 2

id: 58
unit: fn deserialize()
file: tokenizers/src/pre_tokenizers/metaspace.rs
start line: 35
end line: 77
size: 39 LOC
McCabe index: 3
number of parameters: 1

id: 59
unit: fn test_bpe_with_continuing_subword_prefix()
file: tokenizers/src/models/bpe/model.rs
start line: 775
end line: 816
size: 39 LOC
McCabe index: 1
number of parameters: 0

id: 60
unit: fn piece()
file: tokenizers/src/processors/template.rs
start line: 723
end line: 761
size: 39 LOC
McCabe index: 1
number of parameters: 0

id: 61
unit: fn truncate_to_empty()
file: tokenizers/src/tokenizer/encoding.rs
start line: 652
end line: 690
size: 38 LOC
McCabe index: 1
number of parameters: 0

id: 62
unit: fn with_builder()
file: bindings/python/src/models.rs
start line: 286
end line: 324
size: 38 LOC
McCabe index: 5
number of parameters: 2

id: 63
unit: fn run_e_step()
file: tokenizers/src/models/unigram/trainer.rs
start line: 451
end line: 491
size: 37 LOC
McCabe index: 4
number of parameters: 3

id: 64
unit: fn count_pairs()
file: tokenizers/src/models/bpe/trainer.rs
start line: 375
end line: 415
size: 37 LOC
McCabe index: 8
number of parameters: 4

id: 65
unit: fn save()
file: tokenizers/src/models/bpe/model.rs
start line: 531
end line: 573
size: 37 LOC
McCabe index: 1
number of parameters: 3

id: 66
unit: fn refresh_added_tokens()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 323
end line: 361
size: 37 LOC
McCabe index: 2
number of parameters: 3

id: 67
unit: fn visit_map()
file: tokenizers/src/models/wordlevel/serialization.rs
start line: 44
end line: 81
size: 36 LOC
McCabe index: 3
number of parameters: 2

id: 68
unit: fn decode_chain()
file: tokenizers/src/decoders/byte_fallback.rs
start line: 25
end line: 62
size: 36 LOC
McCabe index: 12
number of parameters: 2

id: 69
unit: fn merge_encodings()
file: tokenizers/src/tokenizer/encoding.rs
start line: 570
end line: 606
size: 36 LOC
McCabe index: 1
number of parameters: 0

id: 70
unit: fn bench_gpt2()
file: tokenizers/benches/bpe_benchmark.rs
start line: 28
end line: 68
size: 36 LOC
McCabe index: 3
number of parameters: 1

id: 71
unit: fn new()
file: bindings/python/src/models.rs
start line: 443
end line: 480
size: 36 LOC
McCabe index: 6
number of parameters: 4

id: 72
unit: fn train_from_iterator()
file: bindings/python/src/tokenizer.rs
start line: 1330
end line: 1370
size: 36 LOC
McCabe index: 4
number of parameters: 5

id: 73
unit: def vocab()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 36 LOC
McCabe index: 2
number of parameters: 2

id: 74
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
start line: 41
end line: 76
size: 35 LOC
McCabe index: 5
number of parameters: 2

id: 75
unit: fn run_m_step()
file: tokenizers/src/models/unigram/trainer.rs
start line: 492
end line: 528
size: 35 LOC
McCabe index: 5
number of parameters: 3

id: 76
unit: fn set_sentence()
file: tokenizers/src/models/unigram/lattice.rs
start line: 439
end line: 479
size: 35 LOC
McCabe index: 1
number of parameters: 0

id: 77
unit: fn tokenize()
file: tokenizers/src/models/unigram/model.rs
start line: 419
end line: 453
size: 35 LOC
McCabe index: 5
number of parameters: 2

id: 78
unit: fn do_train()
file: tokenizers/src/models/wordlevel/trainer.rs
start line: 39
end line: 81
size: 35 LOC
McCabe index: 2
number of parameters: 3

id: 79
unit: fn lrstrip()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 801
end line: 836
size: 35 LOC
McCabe index: 7
number of parameters: 3

id: 80
unit: def write()
file: bindings/python/stub.py
start line: 0
end line: 0
size: 35 LOC
McCabe index: 9
number of parameters: 4

id: 81
unit: fn can_add_tokens()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 634
end line: 675
size: 34 LOC
McCabe index: 1
number of parameters: 0

id: 82
unit: fn options_use_cases()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 800
end line: 841
size: 34 LOC
McCabe index: 1
number of parameters: 0

id: 83
unit: fn nfd_adds_new_chars()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1044
end line: 1077
size: 34 LOC
McCabe index: 1
number of parameters: 0

id: 84
unit: fn extract_id()
file: tokenizers/src/processors/template.rs
start line: 102
end line: 137
size: 34 LOC
McCabe index: 3
number of parameters: 1

id: 85
unit: fn new()
file: bindings/python/src/models.rs
start line: 780
end line: 815
size: 34 LOC
McCabe index: 3
number of parameters: 3

id: 86
unit: fn individual_digits()
file: tokenizers/src/pre_tokenizers/digits.rs
start line: 69
end line: 101
size: 33 LOC
McCabe index: 1
number of parameters: 0

id: 87
unit: fn special_token_serde()
file: tokenizers/src/processors/template.rs
start line: 764
end line: 798
size: 33 LOC
McCabe index: 1
number of parameters: 0

id: 88
unit: fn post_processor_deserialization_no_type()
file: tokenizers/src/processors/mod.rs
start line: 92
end line: 127
size: 33 LOC
McCabe index: 1
number of parameters: 0

id: 89
unit: fn pad()
file: bindings/python/src/encoding.rs
start line: 394
end line: 427
size: 33 LOC
McCabe index: 3
number of parameters: 3

id: 90
unit: fn basic()
file: tokenizers/src/pre_tokenizers/whitespace.rs
start line: 49
end line: 80
size: 32 LOC
McCabe index: 2
number of parameters: 0

id: 91
unit: fn visit_map()
file: tokenizers/src/models/unigram/serialization.rs
start line: 45
end line: 76
size: 32 LOC
McCabe index: 2
number of parameters: 2

id: 92
unit: fn decode()
file: tokenizers/src/decoders/byte_fallback.rs
start line: 70
end line: 108
size: 32 LOC
McCabe index: 1
number of parameters: 0

id: 93
unit: fn remove_chars()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1107
end line: 1139
size: 32 LOC
McCabe index: 1
number of parameters: 0

id: 94
unit: fn slice()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1428
end line: 1469
size: 32 LOC
McCabe index: 1
number of parameters: 0

id: 95
unit: fn test_prepend()
file: tokenizers/src/normalizers/prepend.rs
start line: 31
end line: 61
size: 31 LOC
McCabe index: 1
number of parameters: 0

id: 96
unit: fn chinese_chars()
file: tokenizers/src/pre_tokenizers/bert.rs
start line: 51
end line: 81
size: 31 LOC
McCabe index: 2
number of parameters: 0

id: 97
unit: fn multiple_spaces()
file: tokenizers/src/pre_tokenizers/metaspace.rs
start line: 235
end line: 265
size: 31 LOC
McCabe index: 1
number of parameters: 0

id: 98
unit: fn test_unk_not_fused()
file: tokenizers/src/models/bpe/model.rs
start line: 602
end line: 634
size: 31 LOC
McCabe index: 1
number of parameters: 0

id: 99
unit: fn template_processing_serde()
file: tokenizers/src/processors/template.rs
start line: 842
end line: 872
size: 31 LOC
McCabe index: 1
number of parameters: 0

id: 100
unit: fn test_strip_accents_multiple()
file: tokenizers/src/normalizers/strip.rs
start line: 127
end line: 156
size: 30 LOC
McCabe index: 1
number of parameters: 0

id: 101
unit: fn offsets_when_char_split_up()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 404
end line: 434
size: 30 LOC
McCabe index: 1
number of parameters: 0

id: 102
unit: fn encode()
file: bindings/python/src/tokenizer.rs
start line: 970
end line: 1000
size: 30 LOC
McCabe index: 3
number of parameters: 5

id: 103
unit: fn add_prefix_space()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 306
end line: 334
size: 29 LOC
McCabe index: 2
number of parameters: 0

id: 104
unit: fn feed()
file: tokenizers/src/models/unigram/trainer.rs
start line: 628
end line: 657
size: 29 LOC
McCabe index: 4
number of parameters: 3

id: 105
unit: fn feed()
file: tokenizers/src/models/wordlevel/trainer.rs
start line: 97
end line: 126
size: 29 LOC
McCabe index: 4
number of parameters: 3

id: 106
unit: fn feed()
file: tokenizers/src/models/bpe/trainer.rs
start line: 609
end line: 638
size: 29 LOC
McCabe index: 4
number of parameters: 3

id: 107
unit: fn decode_chain()
file: tokenizers/src/decoders/strip.rs
start line: 28
end line: 59
size: 29 LOC
McCabe index: 5
number of parameters: 2

id: 108
unit: fn with_builder()
file: bindings/python/src/models.rs
start line: 596
end line: 625
size: 29 LOC
McCabe index: 3
number of parameters: 2

id: 109
unit: fn __setitem__()
file: bindings/python/src/pre_tokenizers.rs
start line: 627
end line: 655
size: 29 LOC
McCabe index: 1
number of parameters: 1

id: 110
unit: fn __setitem__()
file: bindings/python/src/processors.rs
start line: 779
end line: 808
size: 29 LOC
McCabe index: 1
number of parameters: 1

id: 111
unit: fn do_nmt()
file: tokenizers/src/normalizers/unicode.rs
start line: 44
end line: 73
size: 28 LOC
McCabe index: 1
number of parameters: 1

id: 112
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 120
end line: 147
size: 28 LOC
McCabe index: 5
number of parameters: 2

id: 113
unit: fn processor_trims_offsets_pre_tokenized()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 437
end line: 467
size: 28 LOC
McCabe index: 1
number of parameters: 0

id: 114
unit: fn serialize()
file: tokenizers/src/models/bpe/serialization.rs
start line: 10
end line: 43
size: 28 LOC
McCabe index: 1
number of parameters: 2

id: 115
unit: fn find_matches()
file: tokenizers/src/tokenizer/pattern.rs
start line: 89
end line: 122
size: 28 LOC
McCabe index: 5
number of parameters: 2

id: 116
unit: fn refill()
file: bindings/python/src/utils/iterators.rs
start line: 80
end line: 110
size: 28 LOC
McCabe index: 6
number of parameters: 1

id: 117
unit: fn sequence_basic()
file: tokenizers/src/pre_tokenizers/sequence.rs
start line: 55
end line: 81
size: 27 LOC
McCabe index: 1
number of parameters: 0

id: 118
unit: fn test_populate_nodes()
file: tokenizers/src/models/unigram/model.rs
start line: 505
end line: 534
size: 27 LOC
McCabe index: 1
number of parameters: 0

id: 119
unit: fn test_merge()
file: tokenizers/src/models/bpe/word.rs
start line: 276
end line: 317
size: 27 LOC
McCabe index: 1
number of parameters: 0

id: 120
unit: fn validate_range + Clone>()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 242
end line: 268
size: 27 LOC
McCabe index: 5
number of parameters: 2

id: 121
unit: fn bench_train_small()
file: tokenizers/benches/bert_benchmark.rs
start line: 75
end line: 101
size: 27 LOC
McCabe index: 1
number of parameters: 1

id: 122
unit: fn bench_train_big()
file: tokenizers/benches/bert_benchmark.rs
start line: 103
end line: 129
size: 27 LOC
McCabe index: 1
number of parameters: 1

id: 123
unit: fn serialize()
file: bindings/python/src/decoders.rs
start line: 698
end line: 725
size: 27 LOC
McCabe index: 1
number of parameters: 0

id: 124
unit: fn __setitem__()
file: bindings/python/src/normalizers.rs
start line: 444
end line: 470
size: 27 LOC
McCabe index: 1
number of parameters: 1

id: 125
unit: fn extract_bound()
file: bindings/python/src/processors.rs
start line: 559
end line: 586
size: 27 LOC
McCabe index: 7
number of parameters: 1

id: 126
unit: def check_train()
file: bindings/python/scripts/spm_parity_check.py
start line: 0
end line: 0
size: 27 LOC
McCabe index: 8
number of parameters: 1

id: 127
unit: fn normalize()
file: tokenizers/src/normalizers/precompiled.rs
start line: 34
end line: 68
size: 26 LOC
McCabe index: 6
number of parameters: 2

id: 128
unit: fn test_populate()
file: tokenizers/src/models/unigram/lattice.rs
start line: 641
end line: 669
size: 26 LOC
McCabe index: 1
number of parameters: 0

id: 129
unit: fn encode_unoptimized()
file: tokenizers/src/models/unigram/model.rs
start line: 331
end line: 356
size: 26 LOC
McCabe index: 6
number of parameters: 2

id: 130
unit: fn deserialize()
file: tokenizers/src/tokenizer/serialization.rs
start line: 58
end line: 83
size: 26 LOC
McCabe index: 1
number of parameters: 1

id: 131
unit: fn serialize()
file: bindings/python/src/normalizers.rs
start line: 840
end line: 868
size: 26 LOC
McCabe index: 1
number of parameters: 0

id: 132
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/metaspace.rs
start line: 123
end line: 147
size: 25 LOC
McCabe index: 5
number of parameters: 2

id: 133
unit: fn test_unk_get_fused()
file: tokenizers/src/models/bpe/model.rs
start line: 636
end line: 662
size: 25 LOC
McCabe index: 1
number of parameters: 0

id: 134
unit: fn test_merge_max_length()
file: tokenizers/src/models/bpe/word.rs
start line: 320
end line: 352
size: 25 LOC
McCabe index: 1
number of parameters: 0

id: 135
unit: fn padding()
file: tokenizers/src/tokenizer/encoding.rs
start line: 884
end line: 908
size: 25 LOC
McCabe index: 1
number of parameters: 0

id: 136
unit: fn test_lstrip_unicode_space()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 909
end line: 938
size: 25 LOC
McCabe index: 1
number of parameters: 0

id: 137
unit: fn byte_level_normalizer()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 1002
end line: 1028
size: 25 LOC
McCabe index: 1
number of parameters: 0

id: 138
unit: fn range_conversion()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1167
end line: 1193
size: 25 LOC
McCabe index: 2
number of parameters: 0

id: 139
unit: fn encode_batch()
file: bindings/python/src/tokenizer.rs
start line: 1037
end line: 1061
size: 25 LOC
McCabe index: 4
number of parameters: 5

id: 140
unit: fn encode_batch_fast()
file: bindings/python/src/tokenizer.rs
start line: 1096
end line: 1120
size: 25 LOC
McCabe index: 4
number of parameters: 5

id: 141
unit: fn truncate_encodings_longest_first()
file: tokenizers/src/utils/truncation.rs
start line: 280
end line: 307
size: 24 LOC
McCabe index: 1
number of parameters: 0

id: 142
unit: fn find_matches()
file: tokenizers/src/utils/fancy.rs
start line: 38
end line: 62
size: 24 LOC
McCabe index: 5
number of parameters: 2

id: 143
unit: fn pre_tokenization()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 246
end line: 269
size: 24 LOC
McCabe index: 1
number of parameters: 0

id: 144
unit: fn serialize()
file: tokenizers/src/models/mod.rs
start line: 33
end line: 58
size: 24 LOC
McCabe index: 4
number of parameters: 2

id: 145
unit: fn test_single_word_is_correct()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 851
end line: 878
size: 24 LOC
McCabe index: 1
number of parameters: 0

id: 146
unit: fn save()
file: bindings/python/src/models.rs
start line: 198
end line: 223
size: 24 LOC
McCabe index: 3
number of parameters: 5

id: 147
unit: fn new()
file: bindings/python/src/models.rs
start line: 667
end line: 692
size: 24 LOC
McCabe index: 2
number of parameters: 3

id: 148
unit: fn apply_to_bpe_builder()
file: bindings/node/src/models.rs
start line: 158
end line: 182
size: 24 LOC
McCabe index: 8
number of parameters: 2

id: 149
unit: fn basic()
file: tokenizers/src/pre_tokenizers/bert.rs
start line: 26
end line: 48
size: 23 LOC
McCabe index: 1
number of parameters: 0

id: 150
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/fixed_length.rs
start line: 25
end line: 49
size: 23 LOC
McCabe index: 3
number of parameters: 2

id: 151
unit: fn basic()
file: tokenizers/src/pre_tokenizers/fixed_length.rs
start line: 58
end line: 80
size: 23 LOC
McCabe index: 2
number of parameters: 0

id: 152
unit: fn serialization()
file: tokenizers/src/pre_tokenizers/metaspace.rs
start line: 183
end line: 209
size: 23 LOC
McCabe index: 1
number of parameters: 0

id: 153
unit: fn added_tokens()
file: tokenizers/src/tokenizer/mod.rs
start line: 94
end line: 118
size: 23 LOC
McCabe index: 2
number of parameters: 2

id: 154
unit: fn remove_chars_added_by_nfd()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1080
end line: 1104
size: 23 LOC
McCabe index: 1
number of parameters: 0

id: 155
unit: fn mixed_addition_and_removal()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1142
end line: 1164
size: 23 LOC
McCabe index: 2
number of parameters: 0

id: 156
unit: fn append()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1396
end line: 1418
size: 23 LOC
McCabe index: 1
number of parameters: 0

id: 157
unit: fn piece_serde()
file: tokenizers/src/processors/template.rs
start line: 695
end line: 720
size: 23 LOC
McCabe index: 1
number of parameters: 0

id: 158
unit: fn deserialize_bert_roberta_correctly()
file: tokenizers/src/processors/mod.rs
start line: 66
end line: 89
size: 23 LOC
McCabe index: 1
number of parameters: 0

id: 159
unit: fn set_special_tokens()
file: bindings/python/src/trainers.rs
start line: 233
end line: 255
size: 23 LOC
McCabe index: 3
number of parameters: 2

id: 160
unit: fn set_special_tokens()
file: bindings/python/src/trainers.rs
start line: 447
end line: 469
size: 23 LOC
McCabe index: 3
number of parameters: 2

id: 161
unit: fn set_special_tokens()
file: bindings/python/src/trainers.rs
start line: 639
end line: 661
size: 23 LOC
McCabe index: 3
number of parameters: 2

id: 162
unit: fn set_special_tokens()
file: bindings/python/src/trainers.rs
start line: 785
end line: 807
size: 23 LOC
McCabe index: 3
number of parameters: 2

id: 163
unit: fn deserialize_sequence()
file: bindings/python/src/normalizers.rs
start line: 871
end line: 895
size: 23 LOC
McCabe index: 1
number of parameters: 0

id: 164
unit: fn from_pretrained()
file: bindings/python/src/tokenizer.rs
start line: 595
end line: 618
size: 23 LOC
McCabe index: 2
number of parameters: 3

id: 165
unit: def check()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 23 LOC
McCabe index: 4
number of parameters: 2

id: 166
unit: fn try_from()
file: bindings/node/src/tokenizer.rs
start line: 60
end line: 82
size: 23 LOC
McCabe index: 1
number of parameters: 1

id: 167
unit: fn whitespace_split()
file: tokenizers/src/pre_tokenizers/whitespace.rs
start line: 83
end line: 104
size: 22 LOC
McCabe index: 2
number of parameters: 0

id: 168
unit: fn decode_chain()
file: tokenizers/src/pre_tokenizers/metaspace.rs
start line: 151
end line: 172
size: 22 LOC
McCabe index: 4
number of parameters: 2

id: 169
unit: fn tokenize_with_cache()
file: tokenizers/src/models/bpe/model.rs
start line: 475
end line: 496
size: 22 LOC
McCabe index: 6
number of parameters: 2

id: 170
unit: fn test_bpe_from_file_merge_token_oov()
file: tokenizers/src/models/bpe/model.rs
start line: 820
end line: 846
size: 22 LOC
McCabe index: 1
number of parameters: 0

id: 171
unit: fn bench_train()
file: tokenizers/benches/unigram_benchmark.rs
start line: 37
end line: 58
size: 22 LOC
McCabe index: 1
number of parameters: 1

id: 172
unit: fn new()
file: bindings/python/src/models.rs
start line: 888
end line: 909
size: 22 LOC
McCabe index: 1
number of parameters: 3

id: 173
unit: fn replace()
file: tokenizers/src/normalizers/precompiled.rs
start line: 6
end line: 31
size: 21 LOC
McCabe index: 2
number of parameters: 3

id: 174
unit: fn basic()
file: tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
start line: 86
end line: 106
size: 21 LOC
McCabe index: 1
number of parameters: 0

id: 175
unit: fn spaces_are_included_in_every_script()
file: tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
start line: 109
end line: 129
size: 21 LOC
McCabe index: 1
number of parameters: 0

id: 176
unit: fn numbers()
file: tokenizers/src/pre_tokenizers/digits.rs
start line: 47
end line: 67
size: 21 LOC
McCabe index: 1
number of parameters: 0

id: 177
unit: fn decode_works_on_separated_tokens()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 337
end line: 358
size: 21 LOC
McCabe index: 2
number of parameters: 0

id: 178
unit: fn basic()
file: tokenizers/src/pre_tokenizers/metaspace.rs
start line: 212
end line: 232
size: 21 LOC
McCabe index: 1
number of parameters: 0

id: 179
unit: fn deserialization_should_fail()
file: tokenizers/src/models/wordpiece/serialization.rs
start line: 129
end line: 150
size: 21 LOC
McCabe index: 1
number of parameters: 0

id: 180
unit: fn test_nbest()
file: tokenizers/src/models/unigram/lattice.rs
start line: 606
end line: 628
size: 21 LOC
McCabe index: 1
number of parameters: 0

id: 181
unit: fn deserialize()
file: tokenizers/src/models/bpe/serialization.rs
start line: 47
end line: 67
size: 21 LOC
McCabe index: 1
number of parameters: 1

id: 182
unit: fn split_with_indices()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 427
end line: 447
size: 21 LOC
McCabe index: 2
number of parameters: 3

id: 183
unit: fn test_single_word_is_unicode_correct()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 881
end line: 906
size: 21 LOC
McCabe index: 1
number of parameters: 0

id: 184
unit: fn bench_train_small()
file: tokenizers/benches/bpe_benchmark.rs
start line: 70
end line: 90
size: 21 LOC
McCabe index: 1
number of parameters: 1

id: 185
unit: fn bench_train_big()
file: tokenizers/benches/bpe_benchmark.rs
start line: 92
end line: 112
size: 21 LOC
McCabe index: 1
number of parameters: 1

id: 186
unit: fn from_file()
file: bindings/python/src/models.rs
start line: 534
end line: 554
size: 21 LOC
McCabe index: 1
number of parameters: 5

id: 187
unit: fn extract_bound()
file: bindings/python/src/tokenizer.rs
start line: 375
end line: 395
size: 21 LOC
McCabe index: 7
number of parameters: 1

id: 188
unit: fn process_encodings()
file: bindings/python/src/processors.rs
start line: 254
end line: 274
size: 21 LOC
McCabe index: 2
number of parameters: 3

id: 189
unit: fn test_strip_accents()
file: tokenizers/src/normalizers/strip.rs
start line: 67
end line: 91
size: 20 LOC
McCabe index: 1
number of parameters: 0

id: 190
unit: fn punctuation_basic()
file: tokenizers/src/pre_tokenizers/punctuation.rs
start line: 46
end line: 65
size: 20 LOC
McCabe index: 1
number of parameters: 0

id: 191
unit: fn test_bpe_from_file_bad_merges()
file: tokenizers/src/models/bpe/model.rs
start line: 851
end line: 875
size: 20 LOC
McCabe index: 1
number of parameters: 0

id: 192
unit: fn added_around_edges()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1213
end line: 1233
size: 20 LOC
McCabe index: 1
number of parameters: 0

id: 193
unit: fn prepend()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1374
end line: 1393
size: 20 LOC
McCabe index: 1
number of parameters: 0

id: 194
unit: fn serialize()
file: bindings/python/src/models.rs
start line: 964
end line: 986
size: 20 LOC
McCabe index: 1
number of parameters: 0

id: 195
unit: fn __setstate__()
file: bindings/python/src/tokenizer.rs
start line: 158
end line: 177
size: 20 LOC
McCabe index: 2
number of parameters: 3

id: 196
unit: fn extract_bound()
file: bindings/python/src/tokenizer.rs
start line: 431
end line: 450
size: 20 LOC
McCabe index: 5
number of parameters: 1

id: 197
unit: fn new()
file: bindings/python/src/processors.rs
start line: 686
end line: 707
size: 20 LOC
McCabe index: 4
number of parameters: 3

id: 198
unit: fn process_encodings()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 180
end line: 199
size: 19 LOC
McCabe index: 4
number of parameters: 3

id: 199
unit: fn handling_of_newlines()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 361
end line: 380
size: 19 LOC
McCabe index: 1
number of parameters: 0

id: 200
unit: fn utf8_characters()
file: tokenizers/src/pre_tokenizers/fixed_length.rs
start line: 103
end line: 121
size: 19 LOC
McCabe index: 1
number of parameters: 0

id: 201
unit: fn save()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 276
end line: 297
size: 19 LOC
McCabe index: 1
number of parameters: 3

id: 202
unit: fn serialize()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 515
end line: 536
size: 19 LOC
McCabe index: 2
number of parameters: 2

id: 203
unit: fn template_serde()
file: tokenizers/src/processors/template.rs
start line: 801
end line: 819
size: 19 LOC
McCabe index: 1
number of parameters: 0

id: 204
unit: def extract()
file: bindings/python/scripts/sentencepiece_extractor.py
start line: 0
end line: 0
size: 19 LOC
McCabe index: 5
number of parameters: 1

id: 205
unit: fn test_vietnamese_bug()
file: tokenizers/src/normalizers/strip.rs
start line: 94
end line: 112
size: 18 LOC
McCabe index: 1
number of parameters: 0

id: 206
unit: fn get_medium()
file: tokenizers/src/utils/truncation.rs
start line: 203
end line: 220
size: 18 LOC
McCabe index: 1
number of parameters: 0

id: 207
unit: fn handling_of_multiple_whitespaces()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 383
end line: 401
size: 18 LOC
McCabe index: 1
number of parameters: 0

id: 208
unit: fn custom_length()
file: tokenizers/src/pre_tokenizers/fixed_length.rs
start line: 83
end line: 100
size: 18 LOC
McCabe index: 1
number of parameters: 0

id: 209
unit: fn test_bpe_from_file()
file: tokenizers/src/models/bpe/model.rs
start line: 738
end line: 764
size: 18 LOC
McCabe index: 1
number of parameters: 0

id: 210
unit: fn find_matches()
file: tokenizers/src/tokenizer/pattern.rs
start line: 42
end line: 60
size: 18 LOC
McCabe index: 5
number of parameters: 2

id: 211
unit: fn find_matches()
file: tokenizers/src/tokenizer/pattern.rs
start line: 64
end line: 82
size: 18 LOC
McCabe index: 5
number of parameters: 2

id: 212
unit: fn from_iter, u32)>>()
file: tokenizers/src/tokenizer/encoding.rs
start line: 542
end line: 561
size: 18 LOC
McCabe index: 2
number of parameters: 1

id: 213
unit: fn serialize_element()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 365
end line: 382
size: 18 LOC
McCabe index: 4
number of parameters: 2

id: 214
unit: fn serialize_element()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 398
end line: 415
size: 18 LOC
McCabe index: 4
number of parameters: 2

id: 215
unit: fn serialize_field()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 430
end line: 447
size: 18 LOC
McCabe index: 4
number of parameters: 2

id: 216
unit: fn serialize_field()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 470
end line: 487
size: 18 LOC
McCabe index: 4
number of parameters: 2

id: 217
unit: fn serialize_key()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 517
end line: 534
size: 18 LOC
McCabe index: 4
number of parameters: 2

id: 218
unit: fn tokenize()
file: bindings/python/src/utils/pretokenization.rs
start line: 49
end line: 66
size: 18 LOC
McCabe index: 2
number of parameters: 2

id: 219
unit: fn get_splits()
file: bindings/python/src/utils/pretokenization.rs
start line: 102
end line: 119
size: 18 LOC
McCabe index: 1
number of parameters: 3

id: 220
unit: fn extract_bound()
file: bindings/python/src/tokenizer.rs
start line: 405
end line: 422
size: 18 LOC
McCabe index: 5
number of parameters: 1

id: 221
unit: fn get_padding) -> PyResult>>()
file: bindings/python/src/tokenizer.rs
start line: 911
end line: 930
size: 18 LOC
McCabe index: 1
number of parameters: 0

id: 222
unit: fn add_special_tokens()
file: bindings/python/src/tokenizer.rs
start line: 1260
end line: 1278
size: 18 LOC
McCabe index: 3
number of parameters: 2

id: 223
unit: fn new()
file: bindings/python/src/processors.rs
start line: 495
end line: 516
size: 18 LOC
McCabe index: 4
number of parameters: 4

id: 224
unit: fn try_from()
file: bindings/node/src/tokenizer.rs
start line: 120
end line: 137
size: 18 LOC
McCabe index: 1
number of parameters: 1

id: 225
unit: fn normalize()
file: tokenizers/src/normalizers/byte_level.rs
start line: 31
end line: 47
size: 17 LOC
McCabe index: 3
number of parameters: 2

id: 226
unit: fn decoding()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 287
end line: 303
size: 17 LOC
McCabe index: 1
number of parameters: 0

id: 227
unit: fn deserialization()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 572
end line: 592
size: 17 LOC
McCabe index: 1
number of parameters: 0

id: 228
unit: fn test_special_tokens()
file: tokenizers/src/models/unigram/trainer.rs
start line: 795
end line: 813
size: 17 LOC
McCabe index: 1
number of parameters: 0

id: 229
unit: fn fmt()
file: tokenizers/src/models/unigram/lattice.rs
start line: 64
end line: 81
size: 17 LOC
McCabe index: 1
number of parameters: 2

id: 230
unit: fn tokenize()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 162
end line: 178
size: 17 LOC
McCabe index: 3
number of parameters: 2

id: 231
unit: fn test_serialization_ignore_merges()
file: tokenizers/src/models/bpe/serialization.rs
start line: 219
end line: 237
size: 17 LOC
McCabe index: 1
number of parameters: 0

id: 232
unit: fn default()
file: tokenizers/src/models/bpe/model.rs
start line: 42
end line: 58
size: 17 LOC
McCabe index: 1
number of parameters: 0

id: 233
unit: def train_from_iterator()
file: bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
start line: 0
end line: 0
size: 17 LOC
McCabe index: 1
number of parameters: 0

id: 234
unit: fn set_behavior()
file: bindings/python/src/pre_tokenizers.rs
start line: 463
end line: 479
size: 17 LOC
McCabe index: 2
number of parameters: 2

id: 235
unit: fn set_behavior()
file: bindings/python/src/pre_tokenizers.rs
start line: 565
end line: 581
size: 17 LOC
McCabe index: 2
number of parameters: 2

id: 236
unit: fn split()
file: bindings/python/src/utils/pretokenization.rs
start line: 15
end line: 31
size: 17 LOC
McCabe index: 2
number of parameters: 2

id: 237
unit: fn __new__()
file: bindings/python/src/tokenizer.rs
start line: 134
end line: 152
size: 17 LOC
McCabe index: 3
number of parameters: 2

id: 238
unit: fn add_tokens()
file: bindings/python/src/tokenizer.rs
start line: 1224
end line: 1241
size: 17 LOC
McCabe index: 3
number of parameters: 2

id: 239
unit: fn post_process()
file: bindings/python/src/tokenizer.rs
start line: 1396
end line: 1412
size: 17 LOC
McCabe index: 1
number of parameters: 4

id: 240
unit: fn train()
file: bindings/node/src/trainers.rs
start line: 33
end line: 50
size: 17 LOC
McCabe index: 1
number of parameters: 2

id: 241
unit: fn normalize()
file: tokenizers/src/normalizers/bert.rs
start line: 120
end line: 136
size: 16 LOC
McCabe index: 7
number of parameters: 2

id: 242
unit: fn decode_chain()
file: tokenizers/src/normalizers/replace.rs
start line: 89
end line: 105
size: 16 LOC
McCabe index: 3
number of parameters: 2

id: 243
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/mod.rs
start line: 46
end line: 61
size: 16 LOC
McCabe index: 1
number of parameters: 2

id: 244
unit: fn decode_chain()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 156
end line: 171
size: 16 LOC
McCabe index: 2
number of parameters: 2

id: 245
unit: fn decode_unknown_characters()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 554
end line: 569
size: 16 LOC
McCabe index: 1
number of parameters: 0

id: 246
unit: fn deserialize()
file: tokenizers/src/models/wordpiece/serialization.rs
start line: 31
end line: 46
size: 16 LOC
McCabe index: 1
number of parameters: 1

id: 247
unit: fn test_initial_alphabet()
file: tokenizers/src/models/unigram/trainer.rs
start line: 716
end line: 732
size: 16 LOC
McCabe index: 1
number of parameters: 0

id: 248
unit: fn clone()
file: tokenizers/src/models/unigram/model.rs
start line: 42
end line: 57
size: 16 LOC
McCabe index: 1
number of parameters: 1

id: 249
unit: fn test_encode()
file: tokenizers/src/models/unigram/model.rs
start line: 537
end line: 553
size: 16 LOC
McCabe index: 1
number of parameters: 0

id: 250
unit: fn clone()
file: tokenizers/src/models/bpe/model.rs
start line: 266
end line: 281
size: 16 LOC
McCabe index: 1
number of parameters: 1

id: 251
unit: fn test_bpe_byte_fallback()
file: tokenizers/src/models/bpe/model.rs
start line: 878
end line: 895
size: 16 LOC
McCabe index: 1
number of parameters: 0

id: 252
unit: fn wordpiece_decoder()
file: tokenizers/src/decoders/wordpiece.rs
start line: 69
end line: 85
size: 16 LOC
McCabe index: 1
number of parameters: 0

id: 253
unit: fn serialize()
file: tokenizers/src/tokenizer/serialization.rs
start line: 23
end line: 47
size: 16 LOC
McCabe index: 1
number of parameters: 2

id: 254
unit: fn serde()
file: tokenizers/src/processors/roberta.rs
start line: 241
end line: 256
size: 16 LOC
McCabe index: 1
number of parameters: 0

id: 255
unit: def train()
file: bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
start line: 0
end line: 0
size: 16 LOC
McCabe index: 1
number of parameters: 0

id: 256
unit: def calculate_label_colors()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 16 LOC
McCabe index: 5
number of parameters: 1

id: 257
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 591
end line: 606
size: 16 LOC
McCabe index: 2
number of parameters: 1

id: 258
unit: fn map()
file: bindings/python/src/utils/normalization.rs
start line: 157
end line: 174
size: 16 LOC
McCabe index: 2
number of parameters: 2

id: 259
unit: fn serialize_newtype_variant()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 234
end line: 250
size: 16 LOC
McCabe index: 1
number of parameters: 5

id: 260
unit: fn serialize_field()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 566
end line: 581
size: 16 LOC
McCabe index: 3
number of parameters: 3

id: 261
unit: fn __repr__()
file: bindings/python/src/tokenizer.rs
start line: 230
end line: 246
size: 16 LOC
McCabe index: 1
number of parameters: 1

id: 262
unit: fn new()
file: bindings/python/src/processors.rs
start line: 741
end line: 756
size: 16 LOC
McCabe index: 2
number of parameters: 1

id: 263
unit: fn __getitem__()
file: bindings/python/src/processors.rs
start line: 762
end line: 777
size: 16 LOC
McCabe index: 1
number of parameters: 2

id: 264
unit: fn truncate_and_assert()
file: tokenizers/src/utils/truncation.rs
start line: 263
end line: 277
size: 15 LOC
McCabe index: 1
number of parameters: 5

id: 265
unit: fn test_maybe_parallel_iterator()
file: tokenizers/src/utils/parallelism.rs
start line: 253
end line: 268
size: 15 LOC
McCabe index: 1
number of parameters: 0

id: 266
unit: fn test_populate_nodes_unk()
file: tokenizers/src/models/unigram/model.rs
start line: 486
end line: 502
size: 15 LOC
McCabe index: 1
number of parameters: 0

id: 267
unit: fn test_tokenize_unk()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 224
end line: 239
size: 15 LOC
McCabe index: 1
number of parameters: 0

id: 268
unit: fn default()
file: tokenizers/src/models/bpe/trainer.rs
start line: 59
end line: 73
size: 15 LOC
McCabe index: 1
number of parameters: 0

id: 269
unit: fn decode_chain()
file: tokenizers/src/decoders/wordpiece.rs
start line: 47
end line: 61
size: 15 LOC
McCabe index: 5
number of parameters: 2

id: 270
unit: fn from()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1000
end line: 1014
size: 15 LOC
McCabe index: 1
number of parameters: 1

id: 271
unit: fn original_range()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1196
end line: 1210
size: 15 LOC
McCabe index: 2
number of parameters: 0

id: 272
unit: fn replace()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1472
end line: 1491
size: 15 LOC
McCabe index: 1
number of parameters: 0

id: 273
unit: fn __setstate__()
file: bindings/python/src/trainers.rs
start line: 63
end line: 77
size: 15 LOC
McCabe index: 1
number of parameters: 3

id: 274
unit: fn from_file()
file: bindings/python/src/models.rs
start line: 736
end line: 750
size: 15 LOC
McCabe index: 1
number of parameters: 4

id: 275
unit: fn from_file()
file: bindings/python/src/models.rs
start line: 859
end line: 873
size: 15 LOC
McCabe index: 1
number of parameters: 4

id: 276
unit: fn __setstate__()
file: bindings/python/src/pre_tokenizers.rs
start line: 160
end line: 174
size: 15 LOC
McCabe index: 1
number of parameters: 3

id: 277
unit: fn find_matches()
file: bindings/python/src/utils/normalization.rs
start line: 21
end line: 35
size: 15 LOC
McCabe index: 2
number of parameters: 2

id: 278
unit: fn normalize()
file: bindings/python/src/utils/pretokenization.rs
start line: 33
end line: 47
size: 15 LOC
McCabe index: 2
number of parameters: 2

id: 279
unit: fn extract_bound()
file: bindings/python/src/processors.rs
start line: 599
end line: 613
size: 15 LOC
McCabe index: 3
number of parameters: 1

id: 280
unit: fn compute()
file: bindings/node/src/tasks/tokenizer.rs
start line: 18
end line: 32
size: 15 LOC
McCabe index: 1
number of parameters: 1

id: 281
unit: fn compute()
file: bindings/node/src/tasks/tokenizer.rs
start line: 75
end line: 89
size: 15 LOC
McCabe index: 1
number of parameters: 1

id: 282
unit: fn maybe_par_chunks()
file: tokenizers/src/utils/parallelism.rs
start line: 206
end line: 223
size: 14 LOC
McCabe index: 2
number of parameters: 2

id: 283
unit: fn fixed_script()
file: tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
start line: 25
end line: 38
size: 14 LOC
McCabe index: 3
number of parameters: 1

id: 284
unit: fn test_unicode_script()
file: tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
start line: 132
end line: 145
size: 14 LOC
McCabe index: 1
number of parameters: 0

id: 285
unit: fn digamma()
file: tokenizers/src/models/unigram/trainer.rs
start line: 17
end line: 30
size: 14 LOC
McCabe index: 2
number of parameters: 1

id: 286
unit: fn incomplete_vocab()
file: tokenizers/src/models/wordlevel/serialization.rs
start line: 98
end line: 111
size: 14 LOC
McCabe index: 1
number of parameters: 0

id: 287
unit: fn save()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 196
end line: 212
size: 14 LOC
McCabe index: 1
number of parameters: 3

id: 288
unit: fn test_ordered_vocab_iter()
file: tokenizers/src/models/bpe/model.rs
start line: 586
end line: 599
size: 14 LOC
McCabe index: 1
number of parameters: 0

id: 289
unit: fn test_bpe_byte_fallback_newline()
file: tokenizers/src/models/bpe/model.rs
start line: 898
end line: 912
size: 14 LOC
McCabe index: 1
number of parameters: 0

id: 290
unit: fn fmt()
file: tokenizers/src/models/bpe/word.rs
start line: 61
end line: 74
size: 14 LOC
McCabe index: 1
number of parameters: 2

id: 291
unit: fn simplify_output()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 573
end line: 586
size: 14 LOC
McCabe index: 1
number of parameters: 2

id: 292
unit: fn __setstate__()
file: bindings/python/src/decoders.rs
start line: 117
end line: 130
size: 14 LOC
McCabe index: 1
number of parameters: 3

id: 293
unit: fn __setstate__()
file: bindings/python/src/models.rs
start line: 124
end line: 137
size: 14 LOC
McCabe index: 1
number of parameters: 3

id: 294
unit: fn __getitem__()
file: bindings/python/src/pre_tokenizers.rs
start line: 612
end line: 625
size: 14 LOC
McCabe index: 1
number of parameters: 2

id: 295
unit: fn serialize()
file: bindings/python/src/pre_tokenizers.rs
start line: 898
end line: 911
size: 14 LOC
McCabe index: 1
number of parameters: 2

id: 296
unit: fn __setstate__()
file: bindings/python/src/normalizers.rs
start line: 160
end line: 173
size: 14 LOC
McCabe index: 1
number of parameters: 3

id: 297
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 403
end line: 416
size: 14 LOC
McCabe index: 2
number of parameters: 1

id: 298
unit: fn __getitem__()
file: bindings/python/src/normalizers.rs
start line: 429
end line: 442
size: 14 LOC
McCabe index: 1
number of parameters: 2

id: 299
unit: fn serialize()
file: bindings/python/src/normalizers.rs
start line: 720
end line: 733
size: 14 LOC
McCabe index: 1
number of parameters: 2

id: 300
unit: fn extract_bound()
file: bindings/python/src/utils/normalization.rs
start line: 97
end line: 111
size: 14 LOC
McCabe index: 2
number of parameters: 1

id: 301
unit: fn filter()
file: bindings/python/src/utils/normalization.rs
start line: 126
end line: 141
size: 14 LOC
McCabe index: 2
number of parameters: 2

id: 302
unit: fn slice()
file: bindings/python/src/utils/normalization.rs
start line: 176
end line: 189
size: 14 LOC
McCabe index: 1
number of parameters: 2

id: 303
unit: fn serialize_tuple_variant()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 291
end line: 304
size: 14 LOC
McCabe index: 1
number of parameters: 5

id: 304
unit: fn __setstate__()
file: bindings/python/src/tokenizer.rs
start line: 502
end line: 515
size: 14 LOC
McCabe index: 1
number of parameters: 3

id: 305
unit: fn train()
file: bindings/python/src/tokenizer.rs
start line: 1294
end line: 1307
size: 14 LOC
McCabe index: 3
number of parameters: 3

id: 306
unit: fn __setstate__()
file: bindings/python/src/encoding.rs
start line: 43
end line: 56
size: 14 LOC
McCabe index: 1
number of parameters: 3

id: 307
unit: fn __setstate__()
file: bindings/python/src/processors.rs
start line: 123
end line: 136
size: 14 LOC
McCabe index: 1
number of parameters: 3

id: 308
unit: fn process()
file: bindings/python/src/processors.rs
start line: 167
end line: 180
size: 14 LOC
McCabe index: 1
number of parameters: 4

id: 309
unit: fn serialize()
file: bindings/python/src/processors.rs
start line: 288
end line: 301
size: 14 LOC
McCabe index: 1
number of parameters: 2

id: 310
unit: def tokenizer()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 14 LOC
McCabe index: 3
number of parameters: 2

id: 311
unit: fn feed()
file: bindings/node/src/trainers.rs
start line: 52
end line: 65
size: 14 LOC
McCabe index: 1
number of parameters: 3

id: 312
unit: fn try_from()
file: bindings/node/src/pre_tokenizers.rs
start line: 23
end line: 36
size: 14 LOC
McCabe index: 2
number of parameters: 1

id: 313
unit: fn is_chinese_char()
file: tokenizers/src/normalizers/bert.rs
start line: 36
end line: 48
size: 13 LOC
McCabe index: 1
number of parameters: 1

id: 314
unit: fn normalize()
file: tokenizers/src/normalizers/strip.rs
start line: 25
end line: 40
size: 13 LOC
McCabe index: 5
number of parameters: 2

id: 315
unit: fn get_empty()
file: tokenizers/src/utils/truncation.rs
start line: 175
end line: 187
size: 13 LOC
McCabe index: 1
number of parameters: 0

id: 316
unit: fn get_short()
file: tokenizers/src/utils/truncation.rs
start line: 189
end line: 201
size: 13 LOC
McCabe index: 1
number of parameters: 0

id: 317
unit: fn pre_tokenization_no_regex()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 272
end line: 284
size: 13 LOC
McCabe index: 1
number of parameters: 0

id: 318
unit: fn serialize()
file: tokenizers/src/models/wordpiece/serialization.rs
start line: 10
end line: 27
size: 13 LOC
McCabe index: 1
number of parameters: 2

id: 319
unit: fn setup_progress()
file: tokenizers/src/models/unigram/trainer.rs
start line: 86
end line: 98
size: 13 LOC
McCabe index: 2
number of parameters: 1

id: 320
unit: fn log_sum_exp()
file: tokenizers/src/models/unigram/lattice.rs
start line: 123
end line: 135
size: 13 LOC
McCabe index: 4
number of parameters: 3

id: 321
unit: fn test_viterbi2()
file: tokenizers/src/models/unigram/lattice.rs
start line: 586
end line: 603
size: 13 LOC
McCabe index: 1
number of parameters: 0

id: 322
unit: fn setup_progress()
file: tokenizers/src/models/bpe/trainer.rs
start line: 223
end line: 235
size: 13 LOC
McCabe index: 2
number of parameters: 1

id: 323
unit: fn add_special_tokens()
file: tokenizers/src/models/bpe/trainer.rs
start line: 256
end line: 268
size: 13 LOC
McCabe index: 3
number of parameters: 3

id: 324
unit: fn fmt()
file: tokenizers/src/models/bpe/model.rs
start line: 242
end line: 254
size: 13 LOC
McCabe index: 1
number of parameters: 2

id: 325
unit: fn sequence_basic()
file: tokenizers/src/decoders/sequence.rs
start line: 42
end line: 54
size: 13 LOC
McCabe index: 1
number of parameters: 0

id: 326
unit: fn removed_around_both_edges()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1314
end line: 1327
size: 13 LOC
McCabe index: 1
number of parameters: 0

id: 327
unit: fn split()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1494
end line: 1508
size: 13 LOC
McCabe index: 1
number of parameters: 0

id: 328
unit: fn test_append_after_clear()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 2294
end line: 2311
size: 13 LOC
McCabe index: 1
number of parameters: 0

id: 329
unit: fn try_from()
file: tokenizers/src/processors/template.rs
start line: 150
end line: 163
size: 13 LOC
McCabe index: 2
number of parameters: 1

id: 330
unit: fn process_encodings()
file: tokenizers/src/processors/template.rs
start line: 655
end line: 685
size: 13 LOC
McCabe index: 1
number of parameters: 3

id: 331
unit: fn process_encodings()
file: tokenizers/src/processors/mod.rs
start line: 40
end line: 52
size: 13 LOC
McCabe index: 1
number of parameters: 3

id: 332
unit: fn create_bert_tokenizer()
file: tokenizers/benches/bert_benchmark.rs
start line: 33
end line: 45
size: 13 LOC
McCabe index: 1
number of parameters: 1

id: 333
unit: def __init__()
file: bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
start line: 0
end line: 0
size: 13 LOC
McCabe index: 1
number of parameters: 0

id: 334
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 319
end line: 331
size: 13 LOC
McCabe index: 1
number of parameters: 3

id: 335
unit: fn from()
file: bindings/python/src/pre_tokenizers.rs
start line: 927
end line: 939
size: 13 LOC
McCabe index: 1
number of parameters: 1

id: 336
unit: fn pre_tokenize()
file: bindings/python/src/pre_tokenizers.rs
start line: 954
end line: 966
size: 13 LOC
McCabe index: 1
number of parameters: 2

id: 337
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 583
end line: 596
size: 13 LOC
McCabe index: 1
number of parameters: 1

id: 338
unit: fn from()
file: bindings/python/src/normalizers.rs
start line: 749
end line: 761
size: 13 LOC
McCabe index: 1
number of parameters: 1

id: 339
unit: fn normalize()
file: bindings/python/src/normalizers.rs
start line: 776
end line: 788
size: 13 LOC
McCabe index: 1
number of parameters: 2

id: 340
unit: fn serialize_struct_variant()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 335
end line: 348
size: 13 LOC
McCabe index: 1
number of parameters: 5

id: 341
unit: fn to_encoding()
file: bindings/python/src/utils/pretokenization.rs
start line: 121
end line: 133
size: 13 LOC
McCabe index: 1
number of parameters: 3

id: 342
unit: fn extract_bound()
file: bindings/python/src/tokenizer.rs
start line: 352
end line: 365
size: 13 LOC
McCabe index: 1
number of parameters: 1

id: 343
unit: fn added_tokens()
file: bindings/python/src/processors.rs
start line: 240
end line: 252
size: 13 LOC
McCabe index: 1
number of parameters: 2

id: 344
unit: def extract()
file: bindings/python/scripts/sentencepiece_extractor.py
start line: 0
end line: 0
size: 13 LOC
McCabe index: 6
number of parameters: 1

id: 345
unit: def function()
file: bindings/python/stub.py
start line: 0
end line: 0
size: 13 LOC
McCabe index: 2
number of parameters: 3

id: 346
unit: function isMusl()
file: bindings/node/index.js
start line: 16
end line: 29
size: 13 LOC
McCabe index: 4
number of parameters: 0

id: 347
unit: fn process_encodings()
file: bindings/node/src/processors.rs
start line: 29
end line: 41
size: 13 LOC
McCabe index: 1
number of parameters: 3

id: 348
unit: fn decode()
file: tokenizers/src/pre_tokenizers/metaspace.rs
start line: 357
end line: 369
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 349
unit: fn serde()
file: tokenizers/src/models/wordpiece/serialization.rs
start line: 114
end line: 126
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 350
unit: fn save()
file: tokenizers/src/models/unigram/model.rs
start line: 463
end line: 474
size: 12 LOC
McCabe index: 1
number of parameters: 3

id: 351
unit: fn deserialization_should_fail()
file: tokenizers/src/models/wordlevel/serialization.rs
start line: 114
end line: 126
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 352
unit: fn decode()
file: tokenizers/src/decoders/strip.rs
start line: 67
end line: 79
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 353
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 473
end line: 484
size: 12 LOC
McCabe index: 1
number of parameters: 1

id: 354
unit: fn str()
file: tokenizers/src/tokenizer/pattern.rs
start line: 171
end line: 182
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 355
unit: fn regex()
file: tokenizers/src/tokenizer/pattern.rs
start line: 195
end line: 206
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 356
unit: fn sys_regex()
file: tokenizers/src/tokenizer/pattern.rs
start line: 209
end line: 220
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 357
unit: fn remove_at_beginning()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1289
end line: 1300
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 358
unit: fn count_added()
file: tokenizers/src/processors/template.rs
start line: 443
end line: 454
size: 12 LOC
McCabe index: 1
number of parameters: 2

id: 359
unit: fn pair_must_use_both_sequences()
file: tokenizers/src/processors/template.rs
start line: 1130
end line: 1141
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 360
unit: fn expect_wrong_error_message()
file: tokenizers/src/processors/template.rs
start line: 1144
end line: 1155
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 361
unit: def __init__()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 362
unit: def __make_char_states()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 12 LOC
McCabe index: 7
number of parameters: 3

id: 363
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 500
end line: 511
size: 12 LOC
McCabe index: 2
number of parameters: 1

id: 364
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 428
end line: 439
size: 12 LOC
McCabe index: 1
number of parameters: 3

id: 365
unit: fn serialize_value()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 539
end line: 550
size: 12 LOC
McCabe index: 2
number of parameters: 2

id: 366
unit: fn set_single()
file: bindings/python/src/processors.rs
start line: 715
end line: 726
size: 12 LOC
McCabe index: 3
number of parameters: 2

id: 367
unit: def get_proto()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 12 LOC
McCabe index: 2
number of parameters: 1

id: 368
unit: def converted()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 12 LOC
McCabe index: 2
number of parameters: 1

id: 369
unit: fn apply_to_wordpiece_builder()
file: bindings/node/src/models.rs
start line: 194
end line: 206
size: 12 LOC
McCabe index: 4
number of parameters: 2

id: 370
unit: fn do_handle_chinese_chars()
file: tokenizers/src/normalizers/bert.rs
start line: 98
end line: 108
size: 11 LOC
McCabe index: 2
number of parameters: 2

id: 371
unit: fn test_nfkc()
file: tokenizers/src/normalizers/unicode.rs
start line: 90
end line: 102
size: 11 LOC
McCabe index: 1
number of parameters: 0

id: 372
unit: fn truncate_encodings_empty()
file: tokenizers/src/utils/truncation.rs
start line: 310
end line: 321
size: 11 LOC
McCabe index: 1
number of parameters: 0

id: 373
unit: fn maybe_par_chunks()
file: tokenizers/src/utils/parallelism.rs
start line: 224
end line: 234
size: 11 LOC
McCabe index: 2
number of parameters: 2

id: 374
unit: fn maybe_par_chunks_cond()
file: tokenizers/src/utils/parallelism.rs
start line: 235
end line: 245
size: 11 LOC
McCabe index: 2
number of parameters: 3

id: 375
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/digits.rs
start line: 28
end line: 38
size: 11 LOC
McCabe index: 2
number of parameters: 2

id: 376
unit: fn default()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 43
end line: 53
size: 11 LOC
McCabe index: 1
number of parameters: 0

id: 377
unit: fn serialize()
file: tokenizers/src/models/unigram/serialization.rs
start line: 9
end line: 21
size: 11 LOC
McCabe index: 1
number of parameters: 2

id: 378
unit: fn serialize()
file: tokenizers/src/models/wordlevel/serialization.rs
start line: 10
end line: 20
size: 11 LOC
McCabe index: 1
number of parameters: 2

id: 379
unit: fn tokenize()
file: tokenizers/src/models/bpe/model.rs
start line: 510
end line: 521
size: 11 LOC
McCabe index: 4
number of parameters: 2

id: 380
unit: fn decode_chain()
file: tokenizers/src/decoders/bpe.rs
start line: 27
end line: 37
size: 11 LOC
McCabe index: 2
number of parameters: 2

id: 381
unit: fn do_tokenize>()
file: tokenizers/src/tokenizer/mod.rs
start line: 1118
end line: 1128
size: 11 LOC
McCabe index: 1
number of parameters: 5

id: 382
unit: fn from()
file: tokenizers/src/processors/template.rs
start line: 429
end line: 439
size: 11 LOC
McCabe index: 1
number of parameters: 1

id: 383
unit: fn missing_special_tokens()
file: tokenizers/src/processors/template.rs
start line: 875
end line: 886
size: 11 LOC
McCabe index: 2
number of parameters: 0

id: 384
unit: def __init__()
file: bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
start line: 0
end line: 0
size: 11 LOC
McCabe index: 1
number of parameters: 0

id: 385
unit: def __init__()
file: bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
start line: 0
end line: 0
size: 11 LOC
McCabe index: 1
number of parameters: 0

id: 386
unit: def train_from_iterator()
file: bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
start line: 0
end line: 0
size: 11 LOC
McCabe index: 1
number of parameters: 0

id: 387
unit: fn step()
file: bindings/python/src/decoders.rs
start line: 665
end line: 675
size: 11 LOC
McCabe index: 1
number of parameters: 3

id: 388
unit: fn set_continuing_subword_prefix()
file: bindings/python/src/models.rs
start line: 389
end line: 399
size: 11 LOC
McCabe index: 1
number of parameters: 2

id: 389
unit: fn for_each()
file: bindings/python/src/utils/normalization.rs
start line: 143
end line: 155
size: 11 LOC
McCabe index: 2
number of parameters: 2

id: 390
unit: fn split()
file: bindings/python/src/utils/normalization.rs
start line: 338
end line: 348
size: 11 LOC
McCabe index: 1
number of parameters: 3

id: 391
unit: fn serialize_str()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 150
end line: 160
size: 11 LOC
McCabe index: 2
number of parameters: 2

id: 392
unit: fn serialize_struct()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 319
end line: 331
size: 11 LOC
McCabe index: 2
number of parameters: 3

id: 393
unit: fn serialize_field()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 597
end line: 608
size: 11 LOC
McCabe index: 2
number of parameters: 3

id: 394
unit: fn decode_batch()
file: bindings/python/src/tokenizer.rs
start line: 1154
end line: 1164
size: 11 LOC
McCabe index: 2
number of parameters: 4

id: 395
unit: fn new()
file: bindings/python/src/processors.rs
start line: 410
end line: 420
size: 11 LOC
McCabe index: 1
number of parameters: 2

id: 396
unit: def normalizer()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 11 LOC
McCabe index: 3
number of parameters: 2

id: 397
unit: def normalizer()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 11 LOC
McCabe index: 3
number of parameters: 2

id: 398
unit: fn test_replace_regex()
file: tokenizers/src/normalizers/replace.rs
start line: 124
end line: 135
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 399
unit: fn serialization()
file: tokenizers/src/normalizers/replace.rs
start line: 138
end line: 148
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 400
unit: fn test_thai_bug()
file: tokenizers/src/normalizers/strip.rs
start line: 115
end line: 124
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 401
unit: fn default()
file: tokenizers/src/utils/padding.rs
start line: 32
end line: 41
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 402
unit: fn next()
file: tokenizers/src/utils/iter.rs
start line: 50
end line: 59
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 403
unit: fn next()
file: tokenizers/src/utils/iter.rs
start line: 83
end line: 98
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 404
unit: fn maybe_par_iter()
file: tokenizers/src/utils/parallelism.rs
start line: 116
end line: 126
size: 10 LOC
McCabe index: 2
number of parameters: 1

id: 405
unit: fn maybe_par_iter_mut()
file: tokenizers/src/utils/parallelism.rs
start line: 144
end line: 154
size: 10 LOC
McCabe index: 2
number of parameters: 1

id: 406
unit: fn deserialize()
file: tokenizers/src/models/unigram/serialization.rs
start line: 25
end line: 34
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 407
unit: fn next()
file: tokenizers/src/models/unigram/model.rs
start line: 396
end line: 405
size: 10 LOC
McCabe index: 2
number of parameters: 1

id: 408
unit: fn deserialize()
file: tokenizers/src/models/wordlevel/serialization.rs
start line: 24
end line: 33
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 409
unit: fn should_show_progress()
file: tokenizers/src/tokenizer/mod.rs
start line: 166
end line: 180
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 410
unit: fn do_pre_tokenize>()
file: tokenizers/src/tokenizer/mod.rs
start line: 1172
end line: 1182
size: 10 LOC
McCabe index: 2
number of parameters: 2

id: 411
unit: fn default()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 78
end line: 87
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 412
unit: fn from()
file: tokenizers/src/processors/template.rs
start line: 300
end line: 309
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 413
unit: fn default_added()
file: tokenizers/src/processors/template.rs
start line: 457
end line: 466
size: 10 LOC
McCabe index: 2
number of parameters: 2

id: 414
unit: fn get_bert_template()
file: tokenizers/src/processors/template.rs
start line: 830
end line: 839
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 415
unit: fn process_encodings()
file: tokenizers/src/processors/sequence.rs
start line: 59
end line: 68
size: 10 LOC
McCabe index: 2
number of parameters: 3

id: 416
unit: fn create_processor()
file: tokenizers/benches/layout_benchmark.rs
start line: 15
end line: 24
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 417
unit: def train_from_iterator()
file: bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
start line: 0
end line: 0
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 418
unit: def train()
file: bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
start line: 0
end line: 0
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 419
unit: def __init__()
file: bindings/python/py_src/tokenizers/trainers/__init__.pyi
start line: 0
end line: 0
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 420
unit: fn get_special_tokens()
file: bindings/python/src/trainers.rs
start line: 221
end line: 230
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 421
unit: fn get_special_tokens()
file: bindings/python/src/trainers.rs
start line: 435
end line: 444
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 422
unit: fn get_special_tokens()
file: bindings/python/src/trainers.rs
start line: 627
end line: 636
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 423
unit: fn get_special_tokens()
file: bindings/python/src/trainers.rs
start line: 773
end line: 782
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 424
unit: fn read_file()
file: bindings/python/src/models.rs
start line: 500
end line: 509
size: 10 LOC
McCabe index: 1
number of parameters: 2

id: 425
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 330
end line: 339
size: 10 LOC
McCabe index: 1
number of parameters: 4

id: 426
unit: fn serialize_unit_variant()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 209
end line: 218
size: 10 LOC
McCabe index: 1
number of parameters: 4

id: 427
unit: fn extract_bound()
file: bindings/python/src/utils/pretokenization.rs
start line: 72
end line: 82
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 428
unit: fn extract_bound()
file: bindings/python/src/utils/pretokenization.rs
start line: 88
end line: 98
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 429
unit: fn from()
file: bindings/python/src/tokenizer.rs
start line: 118
end line: 127
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 430
unit: fn get_truncation) -> PyResult>>()
file: bindings/python/src/tokenizer.rs
start line: 801
end line: 812
size: 10 LOC
McCabe index: 1
number of parameters: 0

id: 431
unit: def check_diff()
file: bindings/python/scripts/spm_parity_check.py
start line: 0
end line: 0
size: 10 LOC
McCabe index: 6
number of parameters: 4

id: 432
unit: fn get_trainer()
file: bindings/node/src/models.rs
start line: 134
end line: 143
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 433
unit: fn pre_tokenize()
file: bindings/node/src/pre_tokenizers.rs
start line: 60
end line: 70
size: 10 LOC
McCabe index: 1
number of parameters: 2

id: 434
unit: fn normalize()
file: bindings/node/src/normalizers.rs
start line: 35
end line: 44
size: 10 LOC
McCabe index: 1
number of parameters: 2

id: 435
unit: fn resolve()
file: bindings/node/src/tasks/tokenizer.rs
start line: 91
end line: 100
size: 10 LOC
McCabe index: 1
number of parameters: 3

id: 436
unit: fn compute()
file: bindings/node/src/tasks/tokenizer.rs
start line: 113
end line: 122
size: 10 LOC
McCabe index: 1
number of parameters: 1

id: 437
unit: fn expansion_followed_by_removal()
file: tokenizers/src/normalizers/precompiled.rs
start line: 76
end line: 88
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 438
unit: fn get_env_parallelism()
file: tokenizers/src/utils/parallelism.rs
start line: 41
end line: 49
size: 9 LOC
McCabe index: 2
number of parameters: 0

id: 439
unit: fn into_maybe_par_iter()
file: tokenizers/src/utils/parallelism.rs
start line: 78
end line: 90
size: 9 LOC
McCabe index: 2
number of parameters: 1

id: 440
unit: fn maybe_par_bridge()
file: tokenizers/src/utils/parallelism.rs
start line: 179
end line: 188
size: 9 LOC
McCabe index: 2
number of parameters: 1

id: 441
unit: fn default()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 144
end line: 152
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 442
unit: fn test_viterbi()
file: tokenizers/src/models/unigram/lattice.rs
start line: 573
end line: 583
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 443
unit: fn default()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 39
end line: 47
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 444
unit: fn from()
file: tokenizers/src/tokenizer/pre_tokenizer.rs
start line: 242
end line: 250
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 445
unit: fn tokenize()
file: tokenizers/src/tokenizer/mod.rs
start line: 74
end line: 92
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 446
unit: fn new()
file: tokenizers/src/tokenizer/mod.rs
start line: 1056
end line: 1064
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 447
unit: fn expand_alignments()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 906
end line: 914
size: 9 LOC
McCabe index: 2
number of parameters: 2

id: 448
unit: fn remove_at_end()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1303
end line: 1311
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 449
unit: fn lstrip()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1330
end line: 1338
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 450
unit: fn rstrip()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1341
end line: 1349
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 451
unit: fn strip()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1352
end line: 1360
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 452
unit: fn strip_unicode()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1363
end line: 1371
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 453
unit: fn default()
file: tokenizers/src/processors/template.rs
start line: 528
end line: 536
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 454
unit: fn serde()
file: tokenizers/src/processors/bert.rs
start line: 200
end line: 208
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 455
unit: fn added_tokens()
file: tokenizers/src/processors/mod.rs
start line: 30
end line: 38
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 456
unit: def __init__()
file: bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 457
unit: def train()
file: bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 458
unit: def train_from_iterator()
file: bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 459
unit: def __init__()
file: bindings/python/py_src/tokenizers/trainers/__init__.pyi
start line: 0
end line: 0
size: 9 LOC
McCabe index: 1
number of parameters: 0

id: 460
unit: fn __getstate__()
file: bindings/python/src/trainers.rs
start line: 53
end line: 61
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 461
unit: fn __getstate__()
file: bindings/python/src/decoders.rs
start line: 107
end line: 115
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 462
unit: fn get_prepend_scheme()
file: bindings/python/src/decoders.rs
start line: 374
end line: 383
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 463
unit: fn decode()
file: bindings/python/src/decoders.rs
start line: 529
end line: 537
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 464
unit: fn decode_chain()
file: bindings/python/src/decoders.rs
start line: 539
end line: 547
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 465
unit: fn __getstate__()
file: bindings/python/src/models.rs
start line: 114
end line: 122
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 466
unit: fn __getstate__()
file: bindings/python/src/pre_tokenizers.rs
start line: 150
end line: 158
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 467
unit: fn pre_tokenize_str()
file: bindings/python/src/pre_tokenizers.rs
start line: 209
end line: 219
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 468
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 728
end line: 737
size: 9 LOC
McCabe index: 1
number of parameters: 3

id: 469
unit: fn serialize()
file: bindings/python/src/pre_tokenizers.rs
start line: 869
end line: 877
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 470
unit: fn __getstate__()
file: bindings/python/src/normalizers.rs
start line: 150
end line: 158
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 471
unit: fn serialize()
file: bindings/python/src/normalizers.rs
start line: 687
end line: 695
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 472
unit: fn __getstate__()
file: bindings/python/src/tokenizer.rs
start line: 492
end line: 500
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 473
unit: fn from_buffer()
file: bindings/python/src/tokenizer.rs
start line: 567
end line: 575
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 474
unit: fn __getstate__()
file: bindings/python/src/encoding.rs
start line: 33
end line: 41
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 475
unit: fn __getstate__()
file: bindings/python/src/processors.rs
start line: 113
end line: 121
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 476
unit: fn from()
file: bindings/python/src/processors.rs
start line: 308
end line: 316
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 477
unit: def post_processor()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 478
unit: def vocab()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 2
number of parameters: 2

id: 479
unit: def post_processor()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 480
unit: def post_processor()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 481
unit: def vocab()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 2
number of parameters: 2

id: 482
unit: def post_processor()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 483
unit: def post_processor()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 484
unit: def py_file()
file: bindings/python/stub.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 2
number of parameters: 2

id: 485
unit: fn should_show_progress()
file: bindings/node/src/trainers.rs
start line: 23
end line: 31
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 486
unit: fn decode_chain()
file: bindings/node/src/decoders.rs
start line: 37
end line: 45
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 487
unit: fn tokenize()
file: bindings/node/src/models.rs
start line: 86
end line: 94
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 488
unit: fn get_vocab()
file: bindings/node/src/models.rs
start line: 104
end line: 112
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 489
unit: fn get_vocab_size()
file: bindings/node/src/models.rs
start line: 114
end line: 122
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 490
unit: fn save()
file: bindings/node/src/models.rs
start line: 124
end line: 132
size: 9 LOC
McCabe index: 1
number of parameters: 3

id: 491
unit: fn from()
file: bindings/node/src/pre_tokenizers.rs
start line: 40
end line: 48
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 492
unit: fn try_from()
file: bindings/node/src/tokenizer.rs
start line: 36
end line: 44
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 493
unit: fn compute()
file: bindings/node/src/tasks/tokenizer.rs
start line: 51
end line: 59
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 494
unit: fn try_from()
file: bindings/node/src/encoding.rs
start line: 48
end line: 56
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 495
unit: fn added_tokens()
file: bindings/node/src/processors.rs
start line: 19
end line: 27
size: 9 LOC
McCabe index: 1
number of parameters: 2

id: 496
unit: fn default()
file: tokenizers/src/normalizers/bert.rs
start line: 67
end line: 74
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 497
unit: fn test_replace_decode()
file: tokenizers/src/normalizers/replace.rs
start line: 151
end line: 158
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 498
unit: fn default()
file: tokenizers/src/utils/truncation.rs
start line: 32
end line: 39
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 499
unit: fn get_override_parallelism()
file: tokenizers/src/utils/parallelism.rs
start line: 31
end line: 38
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 500
unit: fn maybe_par_bridge()
file: tokenizers/src/utils/parallelism.rs
start line: 170
end line: 178
size: 8 LOC
McCabe index: 2
number of parameters: 1

id: 501
unit: fn deserialization()
file: tokenizers/src/pre_tokenizers/punctuation.rs
start line: 68
end line: 75
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 502
unit: fn feed()
file: tokenizers/src/models/wordpiece/trainer.rs
start line: 191
end line: 198
size: 8 LOC
McCabe index: 1
number of parameters: 3

id: 503
unit: fn fmt()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 133
end line: 140
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 504
unit: fn required_chars()
file: tokenizers/src/models/unigram/trainer.rs
start line: 188
end line: 195
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 505
unit: fn test_log_sum_exp()
file: tokenizers/src/models/unigram/lattice.rs
start line: 630
end line: 638
size: 8 LOC
McCabe index: 2
number of parameters: 0

id: 506
unit: fn test_tokenize_missing_unk_token()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 242
end line: 250
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 507
unit: fn test_from_pretrained()
file: tokenizers/src/tokenizer/serialization.rs
start line: 235
end line: 242
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 508
unit: fn find_matches()
file: tokenizers/src/tokenizer/pattern.rs
start line: 130
end line: 137
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 509
unit: fn functions()
file: tokenizers/src/tokenizer/pattern.rs
start line: 185
end line: 192
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 510
unit: fn feed()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 597
end line: 604
size: 8 LOC
McCabe index: 1
number of parameters: 3

id: 511
unit: fn transform_check()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 2284
end line: 2291
size: 8 LOC
McCabe index: 2
number of parameters: 0

id: 512
unit: fn default()
file: tokenizers/src/processors/roberta.rs
start line: 17
end line: 24
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 513
unit: fn create_gpt2_tokenizer()
file: tokenizers/benches/bpe_benchmark.rs
start line: 19
end line: 26
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 514
unit: def train_from_iterator()
file: bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
start line: 0
end line: 0
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 515
unit: def train()
file: bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
start line: 0
end line: 0
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 516
unit: def enable_padding()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 517
unit: fn feed()
file: bindings/python/src/trainers.rs
start line: 104
end line: 111
size: 8 LOC
McCabe index: 1
number of parameters: 3

id: 518
unit: fn set_initial_alphabet()
file: bindings/python/src/trainers.rs
start line: 287
end line: 294
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 519
unit: fn set_initial_alphabet()
file: bindings/python/src/trainers.rs
start line: 491
end line: 498
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 520
unit: fn set_initial_alphabet()
file: bindings/python/src/trainers.rs
start line: 819
end line: 826
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 521
unit: fn serialize()
file: bindings/python/src/decoders.rs
start line: 551
end line: 558
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 522
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 655
end line: 662
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 523
unit: fn _clear_cache()
file: bindings/python/src/models.rs
start line: 559
end line: 566
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 524
unit: fn _resize_cache()
file: bindings/python/src/models.rs
start line: 571
end line: 578
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 525
unit: fn set_continuing_subword_prefix()
file: bindings/python/src/models.rs
start line: 646
end line: 653
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 526
unit: fn _clear_cache()
file: bindings/python/src/models.rs
start line: 914
end line: 921
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 527
unit: fn _resize_cache()
file: bindings/python/src/models.rs
start line: 926
end line: 933
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 528
unit: fn pre_tokenize()
file: bindings/python/src/pre_tokenizers.rs
start line: 829
end line: 836
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 529
unit: fn serialize()
file: bindings/python/src/pre_tokenizers.rs
start line: 840
end line: 847
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 530
unit: fn deserialize()
file: bindings/python/src/pre_tokenizers.rs
start line: 851
end line: 858
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 531
unit: fn deserialize()
file: bindings/python/src/pre_tokenizers.rs
start line: 887
end line: 894
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 532
unit: fn set_handle_chinese_chars()
file: bindings/python/src/normalizers.rs
start line: 293
end line: 300
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 533
unit: fn normalize()
file: bindings/python/src/normalizers.rs
start line: 647
end line: 654
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 534
unit: fn serialize()
file: bindings/python/src/normalizers.rs
start line: 658
end line: 665
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 535
unit: fn deserialize()
file: bindings/python/src/normalizers.rs
start line: 669
end line: 676
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 536
unit: fn deserialize()
file: bindings/python/src/normalizers.rs
start line: 709
end line: 716
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 537
unit: fn nfd()
file: bindings/python/src/utils/normalization.rs
start line: 446
end line: 453
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 538
unit: fn nfkd()
file: bindings/python/src/utils/normalization.rs
start line: 455
end line: 462
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 539
unit: fn nfc()
file: bindings/python/src/utils/normalization.rs
start line: 464
end line: 471
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 540
unit: fn nfkc()
file: bindings/python/src/utils/normalization.rs
start line: 473
end line: 480
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 541
unit: fn lowercase()
file: bindings/python/src/utils/normalization.rs
start line: 482
end line: 489
size: 8 LOC
McCabe index: 3
number of parameters: 1

id: 542
unit: fn uppercase()
file: bindings/python/src/utils/normalization.rs
start line: 491
end line: 498
size: 8 LOC
McCabe index: 3
number of parameters: 1

id: 543
unit: fn prepend()
file: bindings/python/src/utils/normalization.rs
start line: 500
end line: 507
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 544
unit: fn append()
file: bindings/python/src/utils/normalization.rs
start line: 509
end line: 516
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 545
unit: fn lstrip()
file: bindings/python/src/utils/normalization.rs
start line: 518
end line: 525
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 546
unit: fn rstrip()
file: bindings/python/src/utils/normalization.rs
start line: 527
end line: 534
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 547
unit: fn strip()
file: bindings/python/src/utils/normalization.rs
start line: 536
end line: 543
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 548
unit: fn clear()
file: bindings/python/src/utils/normalization.rs
start line: 545
end line: 552
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 549
unit: fn serialize_bytes()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 165
end line: 172
size: 8 LOC
McCabe index: 2
number of parameters: 2

id: 550
unit: fn __richcmp__()
file: bindings/python/src/tokenizer.rs
start line: 248
end line: 255
size: 8 LOC
McCabe index: 1
number of parameters: 3

id: 551
unit: fn extract_bound()
file: bindings/python/src/tokenizer.rs
start line: 266
end line: 273
size: 8 LOC
McCabe index: 2
number of parameters: 1

id: 552
unit: fn get_words()
file: bindings/python/src/encoding.rs
start line: 150
end line: 157
size: 8 LOC
McCabe index: 1
number of parameters: 2

id: 553
unit: fn get_overflowing()
file: bindings/python/src/encoding.rs
start line: 247
end line: 254
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 554
unit: fn process_encodings()
file: bindings/python/src/processors.rs
start line: 101
end line: 108
size: 8 LOC
McCabe index: 1
number of parameters: 3

id: 555
unit: fn get_sep()
file: bindings/python/src/processors.rs
start line: 346
end line: 353
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 556
unit: fn get_cls()
file: bindings/python/src/processors.rs
start line: 363
end line: 370
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 557
unit: fn get_sep()
file: bindings/python/src/processors.rs
start line: 427
end line: 434
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 558
unit: fn get_cls()
file: bindings/python/src/processors.rs
start line: 444
end line: 451
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 559
unit: def vocab()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 8 LOC
McCabe index: 3
number of parameters: 2

id: 560
unit: def get_module_members()
file: bindings/python/stub.py
start line: 0
end line: 0
size: 8 LOC
McCabe index: 4
number of parameters: 1

id: 561
unit: fn compute()
file: bindings/node/src/tasks/models.rs
start line: 18
end line: 25
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 562
unit: fn compute()
file: bindings/node/src/tasks/models.rs
start line: 42
end line: 49
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 563
unit: fn compute()
file: bindings/node/src/tasks/models.rs
start line: 65
end line: 72
size: 8 LOC
McCabe index: 1
number of parameters: 1

id: 564
unit: fn test_replace()
file: tokenizers/src/normalizers/replace.rs
start line: 113
end line: 121
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 565
unit: fn default()
file: tokenizers/src/utils/from_pretrained.rs
start line: 15
end line: 21
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 566
unit: fn as_ref()
file: tokenizers/src/utils/truncation.rs
start line: 66
end line: 72
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 567
unit: fn next()
file: tokenizers/src/utils/fancy.rs
start line: 28
end line: 34
size: 7 LOC
McCabe index: 2
number of parameters: 1

id: 568
unit: fn into_maybe_par_iter()
file: tokenizers/src/utils/parallelism.rs
start line: 91
end line: 97
size: 7 LOC
McCabe index: 2
number of parameters: 1

id: 569
unit: fn into_maybe_par_iter_cond()
file: tokenizers/src/utils/parallelism.rs
start line: 99
end line: 105
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 570
unit: fn maybe_par_bridge_cond()
file: tokenizers/src/utils/parallelism.rs
start line: 190
end line: 196
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 571
unit: fn default()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 75
end line: 81
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 572
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/whitespace.rs
start line: 21
end line: 28
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 573
unit: fn to_log_prob()
file: tokenizers/src/models/unigram/trainer.rs
start line: 38
end line: 44
size: 7 LOC
McCabe index: 2
number of parameters: 1

id: 574
unit: fn is_valid_sentencepiece()
file: tokenizers/src/models/unigram/trainer.rs
start line: 100
end line: 111
size: 7 LOC
McCabe index: 3
number of parameters: 2

id: 575
unit: fn update_progress()
file: tokenizers/src/models/unigram/trainer.rs
start line: 435
end line: 441
size: 7 LOC
McCabe index: 2
number of parameters: 4

id: 576
unit: fn finalize_progress()
file: tokenizers/src/models/unigram/trainer.rs
start line: 443
end line: 449
size: 7 LOC
McCabe index: 2
number of parameters: 3

id: 577
unit: fn test_to_log_prob()
file: tokenizers/src/models/unigram/trainer.rs
start line: 816
end line: 824
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 578
unit: fn test_serialization()
file: tokenizers/src/models/unigram/serialization.rs
start line: 84
end line: 92
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 579
unit: fn test_serialization_unk_id_not_zero()
file: tokenizers/src/models/unigram/serialization.rs
start line: 95
end line: 103
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 580
unit: fn test_serialization_no_unk_id()
file: tokenizers/src/models/unigram/serialization.rs
start line: 106
end line: 114
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 581
unit: fn cmp()
file: tokenizers/src/models/unigram/lattice.rs
start line: 41
end line: 47
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 582
unit: fn fmt()
file: tokenizers/src/models/unigram/model.rs
start line: 61
end line: 67
size: 7 LOC
McCabe index: 1
number of parameters: 2

id: 583
unit: fn default()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 150
end line: 156
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 584
unit: fn cmp()
file: tokenizers/src/models/bpe/trainer.rs
start line: 30
end line: 37
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 585
unit: fn finalize_progress()
file: tokenizers/src/models/bpe/trainer.rs
start line: 238
end line: 244
size: 7 LOC
McCabe index: 2
number of parameters: 3

id: 586
unit: fn update_progress()
file: tokenizers/src/models/bpe/trainer.rs
start line: 247
end line: 253
size: 7 LOC
McCabe index: 2
number of parameters: 4

id: 587
unit: fn cmp()
file: tokenizers/src/models/bpe/word.rs
start line: 29
end line: 35
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 588
unit: fn decode()
file: tokenizers/src/decoders/fuse.rs
start line: 36
end line: 42
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 589
unit: fn process_encodings()
file: tokenizers/src/tokenizer/mod.rs
start line: 121
end line: 127
size: 7 LOC
McCabe index: 1
number of parameters: 3

id: 590
unit: fn do_normalize>()
file: tokenizers/src/tokenizer/mod.rs
start line: 1137
end line: 1145
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 591
unit: fn get_n_added_tokens()
file: tokenizers/src/tokenizer/mod.rs
start line: 1244
end line: 1250
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 592
unit: fn find_matches()
file: tokenizers/src/tokenizer/pattern.rs
start line: 23
end line: 31
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 593
unit: fn char()
file: tokenizers/src/tokenizer/pattern.rs
start line: 162
end line: 168
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 594
unit: fn space_leftmost_at_end()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 111
end line: 117
size: 7 LOC
McCabe index: 2
number of parameters: 1

id: 595
unit: fn space_rightmost_at_start()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 118
end line: 124
size: 7 LOC
McCabe index: 2
number of parameters: 1

id: 596
unit: fn from()
file: tokenizers/src/processors/template.rs
start line: 203
end line: 209
size: 7 LOC
McCabe index: 1
number of parameters: 2

id: 597
unit: fn try_from()
file: tokenizers/src/processors/template.rs
start line: 263
end line: 269
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 598
unit: fn added_tokens()
file: tokenizers/src/processors/template.rs
start line: 647
end line: 653
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 599
unit: fn tokens_serde()
file: tokenizers/src/processors/template.rs
start line: 822
end line: 828
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 600
unit: fn added_tokens()
file: tokenizers/src/processors/bert.rs
start line: 43
end line: 49
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 601
unit: fn added_tokens()
file: tokenizers/src/processors/roberta.rs
start line: 58
end line: 64
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 602
unit: def train()
file: bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
start line: 0
end line: 0
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 603
unit: fn get_initial_alphabet()
file: bindings/python/src/trainers.rs
start line: 278
end line: 284
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 604
unit: fn get_initial_alphabet()
file: bindings/python/src/trainers.rs
start line: 482
end line: 488
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 605
unit: fn get_initial_alphabet()
file: bindings/python/src/trainers.rs
start line: 810
end line: 816
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 606
unit: fn get_subtype()
file: bindings/python/src/trainers.rs
start line: 907
end line: 913
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 607
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 394
end line: 400
size: 7 LOC
McCabe index: 1
number of parameters: 3

id: 608
unit: fn get_subtype()
file: bindings/python/src/decoders.rs
start line: 689
end line: 695
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 609
unit: fn tokenize()
file: bindings/python/src/models.rs
start line: 148
end line: 154
size: 7 LOC
McCabe index: 1
number of parameters: 2

id: 610
unit: fn read_file()
file: bindings/python/src/models.rs
start line: 831
end line: 837
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 611
unit: fn get_subtype()
file: bindings/python/src/models.rs
start line: 955
end line: 961
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 612
unit: fn get_subtype()
file: bindings/python/src/normalizers.rs
start line: 831
end line: 837
size: 7 LOC
McCabe index: 1
number of parameters: 0

id: 613
unit: fn __repr__()
file: bindings/python/src/utils/normalization.rs
start line: 363
end line: 369
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 614
unit: fn new()
file: bindings/python/src/utils/regex.rs
start line: 16
end line: 22
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 615
unit: fn serialize_tuple_struct()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 281
end line: 287
size: 7 LOC
McCabe index: 1
number of parameters: 3

id: 616
unit: fn get_splits()
file: bindings/python/src/utils/pretokenization.rs
start line: 253
end line: 259
size: 7 LOC
McCabe index: 1
number of parameters: 3

id: 617
unit: fn __getnewargs__) -> PyResult>()
file: bindings/python/src/tokenizer.rs
start line: 517
end line: 523
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 618
unit: fn get_added_tokens_decoder()
file: bindings/python/src/tokenizer.rs
start line: 688
end line: 696
size: 7 LOC
McCabe index: 2
number of parameters: 1

id: 619
unit: fn get_normalizer()
file: bindings/python/src/tokenizer.rs
start line: 1428
end line: 1434
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 620
unit: fn get_pre_tokenizer()
file: bindings/python/src/tokenizer.rs
start line: 1445
end line: 1451
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 621
unit: fn get_post_processor()
file: bindings/python/src/tokenizer.rs
start line: 1462
end line: 1468
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 622
unit: fn __repr__()
file: bindings/python/src/encoding.rs
start line: 58
end line: 64
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 623
unit: fn merge()
file: bindings/python/src/encoding.rs
start line: 84
end line: 90
size: 7 LOC
McCabe index: 1
number of parameters: 2

id: 624
unit: fn deserialize()
file: bindings/python/src/processors.rs
start line: 278
end line: 284
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 625
unit: def post_processor()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 7 LOC
McCabe index: 1
number of parameters: 2

id: 626
unit: def pytest_collection_modifyitems()
file: bindings/python/conftest.py
start line: 0
end line: 0
size: 7 LOC
McCabe index: 4
number of parameters: 2

id: 627
unit: def fn_predicate()
file: bindings/python/stub.py
start line: 0
end line: 0
size: 7 LOC
McCabe index: 7
number of parameters: 1

id: 628
unit: def do_ruff()
file: bindings/python/stub.py
start line: 0
end line: 0
size: 7 LOC
McCabe index: 2
number of parameters: 2

id: 629
unit: fn from()
file: bindings/node/src/encoding.rs
start line: 67
end line: 73
size: 7 LOC
McCabe index: 1
number of parameters: 1

id: 630
unit: fn normalize()
file: tokenizers/src/normalizers/utils.rs
start line: 43
end line: 48
size: 6 LOC
McCabe index: 2
number of parameters: 2

id: 631
unit: fn is_whitespace()
file: tokenizers/src/normalizers/bert.rs
start line: 7
end line: 13
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 632
unit: fn is_control()
file: tokenizers/src/normalizers/bert.rs
start line: 16
end line: 25
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 633
unit: fn normalize()
file: tokenizers/src/normalizers/prepend.rs
start line: 18
end line: 23
size: 6 LOC
McCabe index: 2
number of parameters: 2

id: 634
unit: fn as_ref()
file: tokenizers/src/utils/truncation.rs
start line: 14
end line: 19
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 635
unit: fn as_ref()
file: tokenizers/src/utils/padding.rs
start line: 13
end line: 18
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 636
unit: fn reconstruct()
file: tokenizers/src/utils/iter.rs
start line: 36
end line: 41
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 637
unit: fn lines_with_ending()
file: tokenizers/src/utils/iter.rs
start line: 69
end line: 75
size: 6 LOC
McCabe index: 2
number of parameters: 1

id: 638
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/sequence.rs
start line: 40
end line: 45
size: 6 LOC
McCabe index: 2
number of parameters: 2

id: 639
unit: fn serde()
file: tokenizers/src/models/wordlevel/serialization.rs
start line: 89
end line: 95
size: 6 LOC
McCabe index: 1
number of parameters: 0

id: 640
unit: fn fmt()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 106
end line: 111
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 641
unit: fn with_first_and_last()
file: tokenizers/src/models/bpe/mod.rs
start line: 40
end line: 46
size: 6 LOC
McCabe index: 2
number of parameters: 1

id: 642
unit: fn with_first_and_last()
file: tokenizers/src/models/bpe/mod.rs
start line: 47
end line: 52
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 643
unit: fn next()
file: tokenizers/src/models/bpe/mod.rs
start line: 71
end line: 76
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 644
unit: fn default()
file: tokenizers/src/decoders/wordpiece.rs
start line: 24
end line: 29
size: 6 LOC
McCabe index: 1
number of parameters: 0

id: 645
unit: fn decode_chain()
file: tokenizers/src/decoders/sequence.rs
start line: 27
end line: 32
size: 6 LOC
McCabe index: 2
number of parameters: 2

id: 646
unit: fn from()
file: tokenizers/src/tokenizer/pre_tokenizer.rs
start line: 30
end line: 35
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 647
unit: fn from()
file: tokenizers/src/tokenizer/pre_tokenizer.rs
start line: 39
end line: 44
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 648
unit: fn sequence_range()
file: tokenizers/src/tokenizer/encoding.rs
start line: 204
end line: 209
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 649
unit: fn with_type_id()
file: tokenizers/src/processors/template.rs
start line: 139
end line: 144
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 650
unit: fn default()
file: tokenizers/src/processors/bert.rs
start line: 14
end line: 19
size: 6 LOC
McCabe index: 1
number of parameters: 0

id: 651
unit: fn added_tokens()
file: tokenizers/src/processors/sequence.rs
start line: 52
end line: 57
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 652
unit: def encode()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 1
number of parameters: 0

id: 653
unit: def unicode_normalizer_from_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 2
number of parameters: 1

id: 654
unit: def __make_anno_map()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 3
number of parameters: 2

id: 655
unit: fn train()
file: bindings/python/src/trainers.rs
start line: 97
end line: 102
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 656
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 215
end line: 220
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 657
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 481
end line: 486
size: 6 LOC
McCabe index: 1
number of parameters: 3

id: 658
unit: fn deserialize()
file: bindings/python/src/decoders.rs
start line: 562
end line: 567
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 659
unit: fn decode_chain()
file: bindings/python/src/decoders.rs
start line: 598
end line: 603
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 660
unit: fn __new__()
file: bindings/python/src/models.rs
start line: 106
end line: 112
size: 6 LOC
McCabe index: 1
number of parameters: 0

id: 661
unit: fn read_file()
file: bindings/python/src/models.rs
start line: 709
end line: 714
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 662
unit: fn alphabet()
file: bindings/python/src/pre_tokenizers.rs
start line: 343
end line: 348
size: 6 LOC
McCabe index: 1
number of parameters: 0

id: 663
unit: fn pre_tokenize()
file: bindings/python/src/pre_tokenizers.rs
start line: 970
end line: 975
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 664
unit: fn __len__()
file: bindings/python/src/normalizers.rs
start line: 422
end line: 427
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 665
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 606
end line: 611
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 666
unit: fn normalize()
file: bindings/python/src/normalizers.rs
start line: 792
end line: 797
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 667
unit: fn from()
file: bindings/python/src/utils/normalization.rs
start line: 39
end line: 44
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 668
unit: fn from()
file: bindings/python/src/utils/normalization.rs
start line: 48
end line: 53
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 669
unit: fn filter()
file: bindings/python/src/utils/normalization.rs
start line: 560
end line: 565
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 670
unit: fn for_each()
file: bindings/python/src/utils/normalization.rs
start line: 567
end line: 572
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 671
unit: fn serialize_some()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 184
end line: 189
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 672
unit: fn serialize_newtype_struct()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 222
end line: 227
size: 6 LOC
McCabe index: 1
number of parameters: 3

id: 673
unit: fn serialize_seq()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 262
end line: 267
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 674
unit: fn serialize_tuple()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 273
end line: 278
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 675
unit: fn serialize_map()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 307
end line: 312
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 676
unit: fn end()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 385
end line: 390
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 677
unit: fn end()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 417
end line: 422
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 678
unit: fn end()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 449
end line: 454
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 679
unit: fn end()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 489
end line: 494
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 680
unit: fn end()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 552
end line: 557
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 681
unit: fn end()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 583
end line: 588
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 682
unit: fn end()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 610
end line: 615
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 683
unit: def post_processor()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 684
unit: def member_sort()
file: bindings/python/stub.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 2
number of parameters: 1

id: 685
unit: fn apply_to_wordlevel_builder()
file: bindings/node/src/models.rs
start line: 258
end line: 264
size: 6 LOC
McCabe index: 2
number of parameters: 2

id: 686
unit: fn from()
file: bindings/node/src/tokenizer.rs
start line: 26
end line: 31
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 687
unit: fn from()
file: bindings/node/src/encoding.rs
start line: 38
end line: 43
size: 6 LOC
McCabe index: 1
number of parameters: 1

id: 688
unit: fn do_clean_text()
file: tokenizers/src/normalizers/bert.rs
start line: 92
end line: 96
size: 5 LOC
McCabe index: 4
number of parameters: 2

id: 689
unit: fn test_deserialize_defaults()
file: tokenizers/src/utils/truncation.rs
start line: 324
end line: 330
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 690
unit: fn serialize()
file: tokenizers/src/utils/mod.rs
start line: 190
end line: 194
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 691
unit: fn test_maybe_parallel_slice()
file: tokenizers/src/utils/parallelism.rs
start line: 271
end line: 276
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 692
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/delimiter.rs
start line: 20
end line: 25
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 693
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/whitespace.rs
start line: 36
end line: 40
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 694
unit: fn default()
file: tokenizers/src/models/wordpiece/trainer.rs
start line: 14
end line: 18
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 695
unit: fn word_to_tokens<''a self, word: &''a()
file: tokenizers/src/models/bpe/model.rs
start line: 469
end line: 473
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 696
unit: fn empty_matches()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 844
end line: 848
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 697
unit: fn test_len_range_inclusive()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1030
end line: 1034
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 698
unit: fn test_len_range_exclusive()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1037
end line: 1041
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 699
unit: fn get_range()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1421
end line: 1425
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 700
unit: def __init__()
file: bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
start line: 0
end line: 0
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 701
unit: def __repr__()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 5 LOC
McCabe index: 2
number of parameters: 1

id: 702
unit: def encode_batch()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 703
unit: def partition_key()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 704
unit: def __init__()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 705
unit: def __call__()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 706
unit: fn from()
file: bindings/python/src/trainers.rs
start line: 118
end line: 122
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 707
unit: fn set_prepend_scheme()
file: bindings/python/src/decoders.rs
start line: 386
end line: 390
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 708
unit: fn from()
file: bindings/python/src/decoders.rs
start line: 590
end line: 594
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 709
unit: fn from()
file: bindings/python/src/models.rs
start line: 95
end line: 99
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 710
unit: fn custom()
file: bindings/python/src/pre_tokenizers.rs
start line: 144
end line: 148
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 711
unit: fn set_pattern()
file: bindings/python/src/pre_tokenizers.rs
start line: 451
end line: 455
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 712
unit: fn get_behavior()
file: bindings/python/src/pre_tokenizers.rs
start line: 558
end line: 562
size: 5 LOC
McCabe index: 2
number of parameters: 1

id: 713
unit: fn set_prepend_scheme()
file: bindings/python/src/pre_tokenizers.rs
start line: 720
end line: 724
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 714
unit: fn from()
file: bindings/python/src/pre_tokenizers.rs
start line: 946
end line: 950
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 715
unit: fn custom()
file: bindings/python/src/normalizers.rs
start line: 144
end line: 148
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 716
unit: fn normalize_str()
file: bindings/python/src/normalizers.rs
start line: 205
end line: 209
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 717
unit: fn set_pattern()
file: bindings/python/src/normalizers.rs
start line: 619
end line: 623
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 718
unit: fn from()
file: bindings/python/src/normalizers.rs
start line: 768
end line: 772
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 719
unit: fn get_normalized()
file: bindings/python/src/utils/normalization.rs
start line: 433
end line: 437
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 720
unit: fn get_original()
file: bindings/python/src/utils/normalization.rs
start line: 440
end line: 444
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 721
unit: fn slice()
file: bindings/python/src/utils/normalization.rs
start line: 554
end line: 558
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 722
unit: fn test_basic()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 621
end line: 625
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 723
unit: fn split()
file: bindings/python/src/utils/pretokenization.rs
start line: 292
end line: 296
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 724
unit: fn normalize()
file: bindings/python/src/utils/pretokenization.rs
start line: 298
end line: 302
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 725
unit: fn __hash__()
file: bindings/python/src/tokenizer.rs
start line: 257
end line: 261
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 726
unit: fn num_special_tokens_to_add()
file: bindings/python/src/tokenizer.rs
start line: 662
end line: 666
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 727
unit: fn no_truncation()
file: bindings/python/src/tokenizer.rs
start line: 787
end line: 791
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 728
unit: fn new()
file: bindings/python/src/encoding.rs
start line: 27
end line: 31
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 729
unit: fn from()
file: bindings/python/src/processors.rs
start line: 43
end line: 47
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 730
unit: fn set_sep()
file: bindings/python/src/processors.rs
start line: 356
end line: 360
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 731
unit: fn set_cls()
file: bindings/python/src/processors.rs
start line: 373
end line: 377
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 732
unit: fn set_sep()
file: bindings/python/src/processors.rs
start line: 437
end line: 441
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 733
unit: fn set_cls()
file: bindings/python/src/processors.rs
start line: 454
end line: 458
size: 5 LOC
McCabe index: 1
number of parameters: 2

id: 734
unit: def vocab()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 5 LOC
McCabe index: 3
number of parameters: 2

id: 735
unit: def vocab()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 5 LOC
McCabe index: 3
number of parameters: 2

id: 736
unit: fn from()
file: bindings/node/src/trainers.rs
start line: 13
end line: 17
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 737
unit: fn from()
file: bindings/node/src/models.rs
start line: 27
end line: 31
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 738
unit: fn from()
file: bindings/node/src/tokenizer.rs
start line: 100
end line: 106
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 739
unit: fn resolve()
file: bindings/node/src/tasks/models.rs
start line: 27
end line: 31
size: 5 LOC
McCabe index: 1
number of parameters: 3

id: 740
unit: fn resolve()
file: bindings/node/src/tasks/models.rs
start line: 51
end line: 55
size: 5 LOC
McCabe index: 1
number of parameters: 3

id: 741
unit: fn resolve()
file: bindings/node/src/tasks/models.rs
start line: 74
end line: 78
size: 5 LOC
McCabe index: 1
number of parameters: 3

id: 742
unit: fn resolve()
file: bindings/node/src/tasks/tokenizer.rs
start line: 34
end line: 38
size: 5 LOC
McCabe index: 1
number of parameters: 3

id: 743
unit: fn from()
file: bindings/node/src/encoding.rs
start line: 14
end line: 18
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 744
unit: fn try_from()
file: bindings/node/src/encoding.rs
start line: 24
end line: 28
size: 5 LOC
McCabe index: 1
number of parameters: 1

id: 745
unit: fn normalize()
file: tokenizers/src/normalizers/utils.rs
start line: 56
end line: 59
size: 4 LOC
McCabe index: 2
number of parameters: 2

id: 746
unit: fn normalize()
file: tokenizers/src/normalizers/unicode.rs
start line: 8
end line: 11
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 747
unit: fn normalize()
file: tokenizers/src/normalizers/unicode.rs
start line: 18
end line: 21
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 748
unit: fn normalize()
file: tokenizers/src/normalizers/unicode.rs
start line: 28
end line: 31
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 749
unit: fn normalize()
file: tokenizers/src/normalizers/unicode.rs
start line: 38
end line: 41
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 750
unit: fn normalize()
file: tokenizers/src/normalizers/unicode.rs
start line: 79
end line: 82
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 751
unit: fn normalize()
file: tokenizers/src/normalizers/strip.rs
start line: 52
end line: 55
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 752
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/bert.rs
start line: 14
end line: 17
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 753
unit: fn deserialization_erroneous()
file: tokenizers/src/pre_tokenizers/punctuation.rs
start line: 79
end line: 82
size: 4 LOC
McCabe index: 1
number of parameters: 0

id: 754
unit: fn train()
file: tokenizers/src/models/unigram/trainer.rs
start line: 618
end line: 621
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 755
unit: fn default()
file: tokenizers/src/models/unigram/model.rs
start line: 83
end line: 86
size: 4 LOC
McCabe index: 1
number of parameters: 0

id: 756
unit: fn test_bpe_with_dropout_0()
file: tokenizers/src/models/bpe/model.rs
start line: 768
end line: 771
size: 4 LOC
McCabe index: 1
number of parameters: 0

id: 757
unit: fn decode_chain()
file: tokenizers/src/decoders/fuse.rs
start line: 25
end line: 28
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 758
unit: fn from()
file: tokenizers/src/tokenizer/pre_tokenizer.rs
start line: 254
end line: 257
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 759
unit: fn from()
file: tokenizers/src/tokenizer/pre_tokenizer.rs
start line: 261
end line: 264
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 760
unit: fn decode()
file: tokenizers/src/tokenizer/mod.rs
start line: 154
end line: 157
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 761
unit: fn find_matches()
file: tokenizers/src/tokenizer/pattern.rs
start line: 16
end line: 19
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 762
unit: fn find_matches()
file: tokenizers/src/tokenizer/pattern.rs
start line: 35
end line: 38
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 763
unit: def decode()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 4 LOC
McCabe index: 2
number of parameters: 3

id: 764
unit: def decode_batch()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 4 LOC
McCabe index: 2
number of parameters: 3

id: 765
unit: def __init__()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 4 LOC
McCabe index: 1
number of parameters: 4

id: 766
unit: def __init__()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 767
unit: def consecutive_chars_to_html()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 4 LOC
McCabe index: 1
number of parameters: 0

id: 768
unit: fn __repr__()
file: bindings/python/src/trainers.rs
start line: 79
end line: 82
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 769
unit: fn __str__()
file: bindings/python/src/trainers.rs
start line: 84
end line: 87
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 770
unit: fn custom()
file: bindings/python/src/decoders.rs
start line: 102
end line: 105
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 771
unit: fn __repr__()
file: bindings/python/src/decoders.rs
start line: 145
end line: 148
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 772
unit: fn __str__()
file: bindings/python/src/decoders.rs
start line: 150
end line: 153
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 773
unit: fn __repr__()
file: bindings/python/src/models.rs
start line: 237
end line: 240
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 774
unit: fn __str__()
file: bindings/python/src/models.rs
start line: 242
end line: 245
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 775
unit: fn __repr__()
file: bindings/python/src/pre_tokenizers.rs
start line: 221
end line: 224
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 776
unit: fn __str__()
file: bindings/python/src/pre_tokenizers.rs
start line: 226
end line: 229
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 777
unit: fn __repr__()
file: bindings/python/src/normalizers.rs
start line: 211
end line: 214
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 778
unit: fn __str__()
file: bindings/python/src/normalizers.rs
start line: 216
end line: 219
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 779
unit: fn serialize_bool()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 87
end line: 90
size: 4 LOC
McCabe index: 2
number of parameters: 2

id: 780
unit: fn serialize_i64()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 110
end line: 113
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 781
unit: fn serialize_u64()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 127
end line: 130
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 782
unit: fn serialize_f64()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 136
end line: 139
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 783
unit: fn serialize_unit()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 193
end line: 196
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 784
unit: fn from_str()
file: bindings/python/src/tokenizer.rs
start line: 536
end line: 539
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 785
unit: fn from_file()
file: bindings/python/src/tokenizer.rs
start line: 552
end line: 555
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 786
unit: fn __repr__()
file: bindings/python/src/tokenizer.rs
start line: 648
end line: 651
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 787
unit: fn __str__()
file: bindings/python/src/tokenizer.rs
start line: 653
end line: 656
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 788
unit: fn set_normalizer()
file: bindings/python/src/tokenizer.rs
start line: 1438
end line: 1441
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 789
unit: fn set_pre_tokenizer()
file: bindings/python/src/tokenizer.rs
start line: 1455
end line: 1458
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 790
unit: fn set_post_processor()
file: bindings/python/src/tokenizer.rs
start line: 1472
end line: 1475
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 791
unit: fn token_to_chars()
file: bindings/python/src/encoding.rs
start line: 318
end line: 321
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 792
unit: fn token_to_word()
file: bindings/python/src/encoding.rs
start line: 336
end line: 339
size: 4 LOC
McCabe index: 1
number of parameters: 2

id: 793
unit: fn __repr__()
file: bindings/python/src/processors.rs
start line: 182
end line: 185
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 794
unit: fn __str__()
file: bindings/python/src/processors.rs
start line: 187
end line: 190
size: 4 LOC
McCabe index: 1
number of parameters: 1

id: 795
unit: fn as_ref()
file: tokenizers/src/normalizers/utils.rs
start line: 22
end line: 24
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 796
unit: fn as_mut()
file: tokenizers/src/normalizers/utils.rs
start line: 28
end line: 30
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 797
unit: fn into_iter()
file: tokenizers/src/normalizers/utils.rs
start line: 37
end line: 39
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 798
unit: fn do_strip_accents()
file: tokenizers/src/normalizers/bert.rs
start line: 110
end line: 112
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 799
unit: fn do_lowercase()
file: tokenizers/src/normalizers/bert.rs
start line: 114
end line: 116
size: 3 LOC
McCabe index: 3
number of parameters: 2

id: 800
unit: fn default()
file: tokenizers/src/normalizers/byte_level.rs
start line: 14
end line: 16
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 801
unit: fn from()
file: tokenizers/src/normalizers/replace.rs
start line: 15
end line: 17
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 802
unit: fn from()
file: tokenizers/src/normalizers/replace.rs
start line: 21
end line: 23
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 803
unit: fn try_from()
file: tokenizers/src/normalizers/replace.rs
start line: 38
end line: 40
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 804
unit: fn clone()
file: tokenizers/src/normalizers/replace.rs
start line: 55
end line: 57
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 805
unit: fn eq()
file: tokenizers/src/normalizers/replace.rs
start line: 61
end line: 63
size: 3 LOC
McCabe index: 2
number of parameters: 2

id: 806
unit: fn normalize()
file: tokenizers/src/normalizers/replace.rs
start line: 83
end line: 85
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 807
unit: fn default()
file: tokenizers/src/utils/truncation.rs
start line: 60
end line: 62
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 808
unit: fn eq()
file: tokenizers/src/utils/cache.rs
start line: 32
end line: 34
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 809
unit: fn default()
file: tokenizers/src/utils/cache.rs
start line: 42
end line: 44
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 810
unit: fn from()
file: tokenizers/src/utils/mod.rs
start line: 43
end line: 45
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 811
unit: fn from()
file: tokenizers/src/utils/mod.rs
start line: 175
end line: 177
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 812
unit: fn new()
file: tokenizers/src/utils/iter.rs
start line: 29
end line: 31
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 813
unit: fn lines_with_ending()
file: tokenizers/src/utils/iter.rs
start line: 76
end line: 78
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 814
unit: fn maybe_par_iter()
file: tokenizers/src/utils/parallelism.rs
start line: 127
end line: 129
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 815
unit: fn maybe_par_iter_cond()
file: tokenizers/src/utils/parallelism.rs
start line: 131
end line: 133
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 816
unit: fn maybe_par_iter_mut()
file: tokenizers/src/utils/parallelism.rs
start line: 155
end line: 157
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 817
unit: fn maybe_par_iter_mut_cond()
file: tokenizers/src/utils/parallelism.rs
start line: 159
end line: 161
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 818
unit: fn from()
file: tokenizers/src/pre_tokenizers/split.rs
start line: 16
end line: 18
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 819
unit: fn from()
file: tokenizers/src/pre_tokenizers/split.rs
start line: 22
end line: 24
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 820
unit: fn default()
file: tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
start line: 16
end line: 18
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 821
unit: fn is_bert_punc()
file: tokenizers/src/pre_tokenizers/bert.rs
start line: 5
end line: 7
size: 3 LOC
McCabe index: 2
number of parameters: 1

id: 822
unit: fn default()
file: tokenizers/src/pre_tokenizers/digits.rs
start line: 22
end line: 24
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 823
unit: fn default_true()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 70
end line: 72
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 824
unit: fn added_tokens()
file: tokenizers/src/pre_tokenizers/byte_level.rs
start line: 176
end line: 178
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 825
unit: fn default()
file: tokenizers/src/pre_tokenizers/whitespace.rs
start line: 15
end line: 17
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 826
unit: fn is_punc()
file: tokenizers/src/pre_tokenizers/punctuation.rs
start line: 7
end line: 9
size: 3 LOC
McCabe index: 2
number of parameters: 1

id: 827
unit: fn default_split()
file: tokenizers/src/pre_tokenizers/punctuation.rs
start line: 18
end line: 20
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 828
unit: fn default()
file: tokenizers/src/pre_tokenizers/punctuation.rs
start line: 29
end line: 31
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 829
unit: fn pre_tokenize()
file: tokenizers/src/pre_tokenizers/punctuation.rs
start line: 35
end line: 37
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 830
unit: fn default_length()
file: tokenizers/src/pre_tokenizers/fixed_length.rs
start line: 20
end line: 22
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 831
unit: fn fmt()
file: tokenizers/src/pre_tokenizers/metaspace.rs
start line: 17
end line: 19
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 832
unit: fn default()
file: tokenizers/src/pre_tokenizers/metaspace.rs
start line: 117
end line: 119
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 833
unit: fn as_ref()
file: tokenizers/src/pre_tokenizers/sequence.rs
start line: 19
end line: 21
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 834
unit: fn as_mut()
file: tokenizers/src/pre_tokenizers/sequence.rs
start line: 25
end line: 27
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 835
unit: fn into_iter()
file: tokenizers/src/pre_tokenizers/sequence.rs
start line: 34
end line: 36
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 836
unit: fn train()
file: tokenizers/src/models/wordpiece/trainer.rs
start line: 183
end line: 185
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 837
unit: fn should_show_progress()
file: tokenizers/src/models/wordpiece/trainer.rs
start line: 187
end line: 189
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 838
unit: fn expecting()
file: tokenizers/src/models/wordpiece/serialization.rs
start line: 53
end line: 55
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 839
unit: fn get_vocab()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 199
end line: 201
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 840
unit: fn get_vocab_size()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 203
end line: 205
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 841
unit: fn token_to_id()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 268
end line: 270
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 842
unit: fn id_to_token()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 272
end line: 274
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 843
unit: fn get_trainer()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 299
end line: 301
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 844
unit: fn test_error_display()
file: tokenizers/src/models/wordpiece/mod.rs
start line: 309
end line: 311
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 845
unit: fn new()
file: tokenizers/src/models/mod.rs
start line: 27
end line: 29
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 846
unit: fn default()
file: tokenizers/src/models/unigram/trainer.rs
start line: 75
end line: 77
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 847
unit: fn should_show_progress()
file: tokenizers/src/models/unigram/trainer.rs
start line: 624
end line: 626
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 848
unit: fn expecting()
file: tokenizers/src/models/unigram/serialization.rs
start line: 41
end line: 43
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 849
unit: fn eq()
file: tokenizers/src/models/unigram/lattice.rs
start line: 29
end line: 31
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 850
unit: fn partial_cmp()
file: tokenizers/src/models/unigram/lattice.rs
start line: 35
end line: 37
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 851
unit: fn eq()
file: tokenizers/src/models/unigram/lattice.rs
start line: 99
end line: 101
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 852
unit: fn eq()
file: tokenizers/src/models/unigram/model.rs
start line: 34
end line: 36
size: 3 LOC
McCabe index: 2
number of parameters: 2

id: 853
unit: fn get_vocab()
file: tokenizers/src/models/unigram/model.rs
start line: 411
end line: 413
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 854
unit: fn get_vocab_size()
file: tokenizers/src/models/unigram/model.rs
start line: 415
end line: 417
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 855
unit: fn token_to_id()
file: tokenizers/src/models/unigram/model.rs
start line: 455
end line: 457
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 856
unit: fn id_to_token()
file: tokenizers/src/models/unigram/model.rs
start line: 459
end line: 461
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 857
unit: fn get_trainer()
file: tokenizers/src/models/unigram/model.rs
start line: 476
end line: 478
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 858
unit: fn default()
file: tokenizers/src/models/wordlevel/trainer.rs
start line: 29
end line: 31
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 859
unit: fn train()
file: tokenizers/src/models/wordlevel/trainer.rs
start line: 88
end line: 90
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 860
unit: fn should_show_progress()
file: tokenizers/src/models/wordlevel/trainer.rs
start line: 93
end line: 95
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 861
unit: fn expecting()
file: tokenizers/src/models/wordlevel/serialization.rs
start line: 40
end line: 42
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 862
unit: fn token_to_id()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 180
end line: 182
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 863
unit: fn id_to_token()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 184
end line: 186
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 864
unit: fn get_vocab()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 188
end line: 190
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 865
unit: fn get_vocab_size()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 192
end line: 194
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 866
unit: fn get_trainer()
file: tokenizers/src/models/wordlevel/mod.rs
start line: 214
end line: 216
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 867
unit: fn eq()
file: tokenizers/src/models/bpe/trainer.rs
start line: 20
end line: 22
size: 3 LOC
McCabe index: 2
number of parameters: 2

id: 868
unit: fn partial_cmp()
file: tokenizers/src/models/bpe/trainer.rs
start line: 25
end line: 27
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 869
unit: fn default()
file: tokenizers/src/models/bpe/trainer.rs
start line: 204
end line: 206
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 870
unit: fn train()
file: tokenizers/src/models/bpe/trainer.rs
start line: 600
end line: 602
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 871
unit: fn should_show_progress()
file: tokenizers/src/models/bpe/trainer.rs
start line: 605
end line: 607
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 872
unit: fn expecting()
file: tokenizers/src/models/bpe/serialization.rs
start line: 74
end line: 76
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 873
unit: fn default()
file: tokenizers/src/models/bpe/model.rs
start line: 258
end line: 260
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 874
unit: fn get_vocab()
file: tokenizers/src/models/bpe/model.rs
start line: 502
end line: 504
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 875
unit: fn get_vocab_size()
file: tokenizers/src/models/bpe/model.rs
start line: 506
end line: 508
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 876
unit: fn token_to_id()
file: tokenizers/src/models/bpe/model.rs
start line: 523
end line: 525
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 877
unit: fn id_to_token()
file: tokenizers/src/models/bpe/model.rs
start line: 527
end line: 529
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 878
unit: fn get_trainer()
file: tokenizers/src/models/bpe/model.rs
start line: 575
end line: 577
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 879
unit: fn eq()
file: tokenizers/src/models/bpe/word.rs
start line: 15
end line: 17
size: 3 LOC
McCabe index: 2
number of parameters: 2

id: 880
unit: fn partial_cmp()
file: tokenizers/src/models/bpe/word.rs
start line: 21
end line: 25
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 881
unit: fn default()
file: tokenizers/src/decoders/bpe.rs
start line: 21
end line: 23
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 882
unit: fn expecting()
file: tokenizers/src/tokenizer/serialization.rs
start line: 104
end line: 106
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 883
unit: fn normalize()
file: tokenizers/src/tokenizer/mod.rs
start line: 57
end line: 65
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 884
unit: fn pre_tokenize()
file: tokenizers/src/tokenizer/mod.rs
start line: 66
end line: 70
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 885
unit: fn decode_chain()
file: tokenizers/src/tokenizer/mod.rs
start line: 158
end line: 163
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 886
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 203
end line: 205
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 887
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 209
end line: 211
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 888
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 215
end line: 217
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 889
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 221
end line: 223
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 890
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 227
end line: 229
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 891
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 233
end line: 235
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 892
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 239
end line: 241
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 893
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 245
end line: 247
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 894
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 251
end line: 253
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 895
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 263
end line: 265
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 896
unit: fn from()
file: tokenizers/src/tokenizer/mod.rs
start line: 273
end line: 275
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 897
unit: fn default()
file: tokenizers/src/tokenizer/mod.rs
start line: 306
end line: 308
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 898
unit: fn from_str()
file: tokenizers/src/tokenizer/mod.rs
start line: 460
end line: 462
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 899
unit: fn deref()
file: tokenizers/src/tokenizer/mod.rs
start line: 496
end line: 498
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 900
unit: fn deref_mut()
file: tokenizers/src/tokenizer/mod.rs
start line: 502
end line: 504
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 901
unit: fn from_str()
file: tokenizers/src/tokenizer/mod.rs
start line: 1477
end line: 1479
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 902
unit: fn find_matches()
file: tokenizers/src/tokenizer/pattern.rs
start line: 12
end line: 15
size: 3 LOC
McCabe index: 2
number of parameters: 2

id: 903
unit: fn from_iter>()
file: tokenizers/src/tokenizer/encoding.rs
start line: 536
end line: 538
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 904
unit: fn hash()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 91
end line: 93
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 905
unit: fn ends_with_word()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 103
end line: 105
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 906
unit: fn starts_with_word()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 107
end line: 109
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 907
unit: fn default()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 500
end line: 502
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 908
unit: fn should_show_progress()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 591
end line: 593
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 909
unit: fn train()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 594
end line: 596
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 910
unit: fn tokenize()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 610
end line: 612
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 911
unit: fn token_to_id()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 613
end line: 615
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 912
unit: fn id_to_token()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 616
end line: 618
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 913
unit: fn get_vocab()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 619
end line: 621
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 914
unit: fn get_vocab_size()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 622
end line: 624
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 915
unit: fn save()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 625
end line: 627
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 916
unit: fn get_trainer()
file: tokenizers/src/tokenizer/added_vocabulary.rs
start line: 628
end line: 630
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 917
unit: fn fmt()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 91
end line: 93
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 918
unit: fn from()
file: tokenizers/src/tokenizer/normalizer.rs
start line: 1018
end line: 1020
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 919
unit: fn try_from()
file: tokenizers/src/processors/template.rs
start line: 169
end line: 171
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 920
unit: fn from()
file: tokenizers/src/processors/template.rs
start line: 212
end line: 214
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 921
unit: fn from()
file: tokenizers/src/processors/template.rs
start line: 217
end line: 219
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 922
unit: fn from()
file: tokenizers/src/processors/template.rs
start line: 222
end line: 224
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 923
unit: fn try_from()
file: tokenizers/src/processors/template.rs
start line: 275
end line: 277
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 924
unit: fn try_from()
file: tokenizers/src/processors/template.rs
start line: 283
end line: 285
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 925
unit: fn from()
file: tokenizers/src/processors/template.rs
start line: 313
end line: 315
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 926
unit: fn from()
file: tokenizers/src/processors/template.rs
start line: 407
end line: 409
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 927
unit: fn eq()
file: tokenizers/src/processors/template.rs
start line: 413
end line: 415
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 928
unit: fn as_ref()
file: tokenizers/src/processors/sequence.rs
start line: 31
end line: 33
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 929
unit: fn as_mut()
file: tokenizers/src/processors/sequence.rs
start line: 37
end line: 39
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 930
unit: fn into_iter()
file: tokenizers/src/processors/sequence.rs
start line: 46
end line: 48
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 931
unit: def from_file()
file: bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 932
unit: def from_file()
file: bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 933
unit: def from_file()
file: bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 934
unit: def __init__()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 2
number of parameters: 3

id: 935
unit: def from_file()
file: bindings/python/py_src/tokenizers/implementations/char_level_bpe.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 936
unit: def HTMLBody()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 937
unit: fn should_show_progress()
file: bindings/python/src/trainers.rs
start line: 93
end line: 95
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 938
unit: fn get_vocab_size()
file: bindings/python/src/trainers.rs
start line: 191
end line: 193
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 939
unit: fn set_vocab_size()
file: bindings/python/src/trainers.rs
start line: 196
end line: 198
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 940
unit: fn get_min_frequency()
file: bindings/python/src/trainers.rs
start line: 201
end line: 203
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 941
unit: fn set_min_frequency()
file: bindings/python/src/trainers.rs
start line: 206
end line: 208
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 942
unit: fn get_show_progress()
file: bindings/python/src/trainers.rs
start line: 211
end line: 213
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 943
unit: fn set_show_progress()
file: bindings/python/src/trainers.rs
start line: 216
end line: 218
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 944
unit: fn get_limit_alphabet()
file: bindings/python/src/trainers.rs
start line: 258
end line: 260
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 945
unit: fn set_limit_alphabet()
file: bindings/python/src/trainers.rs
start line: 263
end line: 265
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 946
unit: fn get_max_token_length()
file: bindings/python/src/trainers.rs
start line: 268
end line: 270
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 947
unit: fn set_max_token_length()
file: bindings/python/src/trainers.rs
start line: 273
end line: 275
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 948
unit: fn get_continuing_subword_prefix()
file: bindings/python/src/trainers.rs
start line: 297
end line: 299
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 949
unit: fn set_continuing_subword_prefix()
file: bindings/python/src/trainers.rs
start line: 302
end line: 304
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 950
unit: fn get_end_of_word_suffix()
file: bindings/python/src/trainers.rs
start line: 307
end line: 309
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 951
unit: fn set_end_of_word_suffix()
file: bindings/python/src/trainers.rs
start line: 312
end line: 314
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 952
unit: fn get_vocab_size()
file: bindings/python/src/trainers.rs
start line: 405
end line: 407
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 953
unit: fn set_vocab_size()
file: bindings/python/src/trainers.rs
start line: 410
end line: 412
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 954
unit: fn get_min_frequency()
file: bindings/python/src/trainers.rs
start line: 415
end line: 417
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 955
unit: fn set_min_frequency()
file: bindings/python/src/trainers.rs
start line: 420
end line: 422
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 956
unit: fn get_show_progress()
file: bindings/python/src/trainers.rs
start line: 425
end line: 427
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 957
unit: fn set_show_progress()
file: bindings/python/src/trainers.rs
start line: 430
end line: 432
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 958
unit: fn get_limit_alphabet()
file: bindings/python/src/trainers.rs
start line: 472
end line: 474
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 959
unit: fn set_limit_alphabet()
file: bindings/python/src/trainers.rs
start line: 477
end line: 479
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 960
unit: fn get_continuing_subword_prefix()
file: bindings/python/src/trainers.rs
start line: 501
end line: 503
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 961
unit: fn set_continuing_subword_prefix()
file: bindings/python/src/trainers.rs
start line: 506
end line: 508
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 962
unit: fn get_end_of_word_suffix()
file: bindings/python/src/trainers.rs
start line: 511
end line: 513
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 963
unit: fn set_end_of_word_suffix()
file: bindings/python/src/trainers.rs
start line: 516
end line: 518
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 964
unit: fn get_vocab_size()
file: bindings/python/src/trainers.rs
start line: 597
end line: 599
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 965
unit: fn set_vocab_size()
file: bindings/python/src/trainers.rs
start line: 602
end line: 604
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 966
unit: fn get_min_frequency()
file: bindings/python/src/trainers.rs
start line: 607
end line: 609
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 967
unit: fn set_min_frequency()
file: bindings/python/src/trainers.rs
start line: 612
end line: 614
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 968
unit: fn get_show_progress()
file: bindings/python/src/trainers.rs
start line: 617
end line: 619
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 969
unit: fn set_show_progress()
file: bindings/python/src/trainers.rs
start line: 622
end line: 624
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 970
unit: fn get_vocab_size()
file: bindings/python/src/trainers.rs
start line: 753
end line: 755
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 971
unit: fn set_vocab_size()
file: bindings/python/src/trainers.rs
start line: 758
end line: 760
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 972
unit: fn get_show_progress()
file: bindings/python/src/trainers.rs
start line: 763
end line: 765
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 973
unit: fn set_show_progress()
file: bindings/python/src/trainers.rs
start line: 768
end line: 770
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 974
unit: fn decode_chain()
file: bindings/python/src/decoders.rs
start line: 94
end line: 96
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 975
unit: fn decode()
file: bindings/python/src/decoders.rs
start line: 141
end line: 143
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 976
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 200
end line: 202
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 977
unit: fn get_prefix()
file: bindings/python/src/decoders.rs
start line: 237
end line: 239
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 978
unit: fn set_prefix()
file: bindings/python/src/decoders.rs
start line: 242
end line: 244
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 979
unit: fn get_cleanup()
file: bindings/python/src/decoders.rs
start line: 247
end line: 249
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 980
unit: fn set_cleanup()
file: bindings/python/src/decoders.rs
start line: 252
end line: 254
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 981
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 258
end line: 260
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 982
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 274
end line: 276
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 983
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 289
end line: 291
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 984
unit: fn get_start()
file: bindings/python/src/decoders.rs
start line: 301
end line: 303
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 985
unit: fn set_start()
file: bindings/python/src/decoders.rs
start line: 306
end line: 308
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 986
unit: fn get_stop()
file: bindings/python/src/decoders.rs
start line: 311
end line: 313
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 987
unit: fn set_stop()
file: bindings/python/src/decoders.rs
start line: 316
end line: 318
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 988
unit: fn get_content()
file: bindings/python/src/decoders.rs
start line: 321
end line: 323
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 989
unit: fn set_content()
file: bindings/python/src/decoders.rs
start line: 326
end line: 328
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 990
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 332
end line: 334
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 991
unit: fn get_replacement()
file: bindings/python/src/decoders.rs
start line: 354
end line: 356
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 992
unit: fn set_replacement()
file: bindings/python/src/decoders.rs
start line: 359
end line: 361
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 993
unit: fn get_split()
file: bindings/python/src/decoders.rs
start line: 364
end line: 366
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 994
unit: fn set_split()
file: bindings/python/src/decoders.rs
start line: 369
end line: 371
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 995
unit: fn get_suffix()
file: bindings/python/src/decoders.rs
start line: 414
end line: 416
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 996
unit: fn set_suffix()
file: bindings/python/src/decoders.rs
start line: 419
end line: 421
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 997
unit: fn new()
file: bindings/python/src/decoders.rs
start line: 425
end line: 427
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 998
unit: fn get_pad_token()
file: bindings/python/src/decoders.rs
start line: 445
end line: 447
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 999
unit: fn set_pad_token()
file: bindings/python/src/decoders.rs
start line: 450
end line: 452
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1000
unit: fn get_word_delimiter_token()
file: bindings/python/src/decoders.rs
start line: 455
end line: 457
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1001
unit: fn set_word_delimiter_token()
file: bindings/python/src/decoders.rs
start line: 460
end line: 462
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1002
unit: fn get_cleanup()
file: bindings/python/src/decoders.rs
start line: 465
end line: 467
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1003
unit: fn set_cleanup()
file: bindings/python/src/decoders.rs
start line: 470
end line: 472
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1004
unit: fn __getnewargs__) -> PyResult>()
file: bindings/python/src/decoders.rs
start line: 513
end line: 515
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1005
unit: fn from()
file: bindings/python/src/decoders.rs
start line: 581
end line: 583
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1006
unit: fn tokenize()
file: bindings/python/src/models.rs
start line: 62
end line: 64
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1007
unit: fn token_to_id()
file: bindings/python/src/models.rs
start line: 66
end line: 68
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1008
unit: fn id_to_token()
file: bindings/python/src/models.rs
start line: 70
end line: 72
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1009
unit: fn get_vocab()
file: bindings/python/src/models.rs
start line: 74
end line: 76
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1010
unit: fn get_vocab_size()
file: bindings/python/src/models.rs
start line: 78
end line: 80
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1011
unit: fn save()
file: bindings/python/src/models.rs
start line: 82
end line: 84
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1012
unit: fn get_trainer()
file: bindings/python/src/models.rs
start line: 86
end line: 88
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1013
unit: fn token_to_id()
file: bindings/python/src/models.rs
start line: 165
end line: 167
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1014
unit: fn id_to_token()
file: bindings/python/src/models.rs
start line: 178
end line: 180
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1015
unit: fn get_trainer()
file: bindings/python/src/models.rs
start line: 233
end line: 235
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1016
unit: fn get_dropout()
file: bindings/python/src/models.rs
start line: 364
end line: 366
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1017
unit: fn set_dropout()
file: bindings/python/src/models.rs
start line: 369
end line: 371
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1018
unit: fn get_unk_token()
file: bindings/python/src/models.rs
start line: 374
end line: 376
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1019
unit: fn set_unk_token()
file: bindings/python/src/models.rs
start line: 379
end line: 381
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1020
unit: fn get_continuing_subword_prefix()
file: bindings/python/src/models.rs
start line: 384
end line: 386
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1021
unit: fn get_end_of_word_suffix()
file: bindings/python/src/models.rs
start line: 402
end line: 404
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1022
unit: fn set_end_of_word_suffix()
file: bindings/python/src/models.rs
start line: 407
end line: 409
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1023
unit: fn get_fuse_unk()
file: bindings/python/src/models.rs
start line: 412
end line: 414
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1024
unit: fn set_fuse_unk()
file: bindings/python/src/models.rs
start line: 417
end line: 419
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1025
unit: fn get_byte_fallback()
file: bindings/python/src/models.rs
start line: 422
end line: 424
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1026
unit: fn set_byte_fallback()
file: bindings/python/src/models.rs
start line: 427
end line: 429
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1027
unit: fn get_ignore_merges()
file: bindings/python/src/models.rs
start line: 431
end line: 433
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1028
unit: fn set_ignore_merges()
file: bindings/python/src/models.rs
start line: 436
end line: 438
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1029
unit: fn get_unk_token()
file: bindings/python/src/models.rs
start line: 631
end line: 633
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1030
unit: fn set_unk_token()
file: bindings/python/src/models.rs
start line: 636
end line: 638
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1031
unit: fn get_continuing_subword_prefix()
file: bindings/python/src/models.rs
start line: 641
end line: 643
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1032
unit: fn get_max_input_chars_per_word()
file: bindings/python/src/models.rs
start line: 656
end line: 658
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1033
unit: fn set_max_input_chars_per_word()
file: bindings/python/src/models.rs
start line: 661
end line: 663
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1034
unit: fn get_unk_token()
file: bindings/python/src/models.rs
start line: 769
end line: 771
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1035
unit: fn set_unk_token()
file: bindings/python/src/models.rs
start line: 774
end line: 776
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1036
unit: fn pre_tokenize()
file: bindings/python/src/pre_tokenizers.rs
start line: 136
end line: 138
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1037
unit: fn pre_tokenize()
file: bindings/python/src/pre_tokenizers.rs
start line: 189
end line: 191
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1038
unit: fn get_add_prefix_space()
file: bindings/python/src/pre_tokenizers.rs
start line: 288
end line: 290
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1039
unit: fn set_add_prefix_space()
file: bindings/python/src/pre_tokenizers.rs
start line: 293
end line: 295
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1040
unit: fn get_use_regex()
file: bindings/python/src/pre_tokenizers.rs
start line: 298
end line: 300
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1041
unit: fn set_use_regex()
file: bindings/python/src/pre_tokenizers.rs
start line: 303
end line: 305
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1042
unit: fn get_trim_offsets()
file: bindings/python/src/pre_tokenizers.rs
start line: 308
end line: 310
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1043
unit: fn set_trim_offsets()
file: bindings/python/src/pre_tokenizers.rs
start line: 313
end line: 315
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1044
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 385
end line: 387
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1045
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 397
end line: 399
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1046
unit: fn __getnewargs__) -> PyResult>()
file: bindings/python/src/pre_tokenizers.rs
start line: 441
end line: 443
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1047
unit: fn get_pattern()
file: bindings/python/src/pre_tokenizers.rs
start line: 446
end line: 448
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1048
unit: fn get_behavior()
file: bindings/python/src/pre_tokenizers.rs
start line: 458
end line: 460
size: 3 LOC
McCabe index: 2
number of parameters: 1

id: 1049
unit: fn get_invert()
file: bindings/python/src/pre_tokenizers.rs
start line: 482
end line: 484
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1050
unit: fn set_invert()
file: bindings/python/src/pre_tokenizers.rs
start line: 487
end line: 489
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1051
unit: fn get_delimiter()
file: bindings/python/src/pre_tokenizers.rs
start line: 502
end line: 504
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1052
unit: fn set_delimiter()
file: bindings/python/src/pre_tokenizers.rs
start line: 507
end line: 509
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1053
unit: fn __getnewargs__) -> PyResult>()
file: bindings/python/src/pre_tokenizers.rs
start line: 520
end line: 522
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1054
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 535
end line: 537
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1055
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 553
end line: 555
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1056
unit: fn __getnewargs__) -> PyResult>()
file: bindings/python/src/pre_tokenizers.rs
start line: 608
end line: 610
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1057
unit: fn get_replacement()
file: bindings/python/src/pre_tokenizers.rs
start line: 694
end line: 696
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1058
unit: fn set_replacement()
file: bindings/python/src/pre_tokenizers.rs
start line: 699
end line: 701
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1059
unit: fn get_split()
file: bindings/python/src/pre_tokenizers.rs
start line: 704
end line: 706
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1060
unit: fn set_split()
file: bindings/python/src/pre_tokenizers.rs
start line: 709
end line: 711
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1061
unit: fn get_prepend_scheme()
file: bindings/python/src/pre_tokenizers.rs
start line: 714
end line: 717
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1062
unit: fn get_individual_digits()
file: bindings/python/src/pre_tokenizers.rs
start line: 756
end line: 758
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1063
unit: fn set_individual_digits()
file: bindings/python/src/pre_tokenizers.rs
start line: 761
end line: 763
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1064
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 767
end line: 769
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1065
unit: fn get_length()
file: bindings/python/src/pre_tokenizers.rs
start line: 786
end line: 788
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1066
unit: fn set_length()
file: bindings/python/src/pre_tokenizers.rs
start line: 791
end line: 793
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1067
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 797
end line: 799
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1068
unit: fn new()
file: bindings/python/src/pre_tokenizers.rs
start line: 812
end line: 814
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1069
unit: fn from()
file: bindings/python/src/pre_tokenizers.rs
start line: 918
end line: 920
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1070
unit: fn fmt()
file: bindings/python/src/error.rs
start line: 20
end line: 22
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1071
unit: fn from()
file: bindings/python/src/error.rs
start line: 28
end line: 30
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1072
unit: fn normalize()
file: bindings/python/src/normalizers.rs
start line: 136
end line: 138
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1073
unit: fn normalize()
file: bindings/python/src/normalizers.rs
start line: 187
end line: 189
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1074
unit: fn get_clean_text()
file: bindings/python/src/normalizers.rs
start line: 278
end line: 280
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1075
unit: fn set_clean_text()
file: bindings/python/src/normalizers.rs
start line: 283
end line: 285
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1076
unit: fn get_handle_chinese_chars()
file: bindings/python/src/normalizers.rs
start line: 288
end line: 290
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1077
unit: fn get_strip_accents()
file: bindings/python/src/normalizers.rs
start line: 303
end line: 305
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1078
unit: fn set_strip_accents()
file: bindings/python/src/normalizers.rs
start line: 308
end line: 310
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1079
unit: fn get_lowercase()
file: bindings/python/src/normalizers.rs
start line: 313
end line: 315
size: 3 LOC
McCabe index: 2
number of parameters: 1

id: 1080
unit: fn set_lowercase()
file: bindings/python/src/normalizers.rs
start line: 318
end line: 320
size: 3 LOC
McCabe index: 2
number of parameters: 2

id: 1081
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 349
end line: 351
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1082
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 361
end line: 363
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1083
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 373
end line: 375
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1084
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 385
end line: 387
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1085
unit: fn __getnewargs__) -> PyResult>()
file: bindings/python/src/normalizers.rs
start line: 418
end line: 420
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1086
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 480
end line: 482
size: 3 LOC
McCabe index: 2
number of parameters: 0

id: 1087
unit: fn get_left()
file: bindings/python/src/normalizers.rs
start line: 491
end line: 493
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1088
unit: fn set_left()
file: bindings/python/src/normalizers.rs
start line: 496
end line: 498
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1089
unit: fn get_right()
file: bindings/python/src/normalizers.rs
start line: 501
end line: 503
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1090
unit: fn set_right()
file: bindings/python/src/normalizers.rs
start line: 506
end line: 508
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1091
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 512
end line: 514
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1092
unit: fn get_prepend()
file: bindings/python/src/normalizers.rs
start line: 523
end line: 525
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1093
unit: fn set_prepend()
file: bindings/python/src/normalizers.rs
start line: 528
end line: 530
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1094
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 534
end line: 536
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1095
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 546
end line: 548
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1096
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 558
end line: 560
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1097
unit: fn new()
file: bindings/python/src/normalizers.rs
start line: 570
end line: 572
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1098
unit: fn get_pattern()
file: bindings/python/src/normalizers.rs
start line: 614
end line: 616
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1099
unit: fn get_content()
file: bindings/python/src/normalizers.rs
start line: 626
end line: 628
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1100
unit: fn set_content()
file: bindings/python/src/normalizers.rs
start line: 631
end line: 633
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1101
unit: fn from()
file: bindings/python/src/normalizers.rs
start line: 740
end line: 742
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1102
unit: fn destroy()
file: bindings/python/src/utils/mod.rs
start line: 18
end line: 21
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1103
unit: fn drop()
file: bindings/python/src/utils/mod.rs
start line: 39
end line: 41
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1104
unit: fn destroy()
file: bindings/python/src/utils/mod.rs
start line: 69
end line: 71
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1105
unit: fn from()
file: bindings/python/src/utils/normalization.rs
start line: 115
end line: 117
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1106
unit: fn from()
file: bindings/python/src/utils/normalization.rs
start line: 121
end line: 123
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1107
unit: fn new()
file: bindings/python/src/utils/normalization.rs
start line: 210
end line: 212
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1108
unit: fn get_normalized()
file: bindings/python/src/utils/normalization.rs
start line: 216
end line: 218
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1109
unit: fn get_original()
file: bindings/python/src/utils/normalization.rs
start line: 221
end line: 223
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1110
unit: fn nfd()
file: bindings/python/src/utils/normalization.rs
start line: 227
end line: 229
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1111
unit: fn nfkd()
file: bindings/python/src/utils/normalization.rs
start line: 233
end line: 235
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1112
unit: fn nfc()
file: bindings/python/src/utils/normalization.rs
start line: 239
end line: 241
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1113
unit: fn nfkc()
file: bindings/python/src/utils/normalization.rs
start line: 245
end line: 247
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1114
unit: fn lowercase()
file: bindings/python/src/utils/normalization.rs
start line: 251
end line: 253
size: 3 LOC
McCabe index: 3
number of parameters: 1

id: 1115
unit: fn uppercase()
file: bindings/python/src/utils/normalization.rs
start line: 257
end line: 259
size: 3 LOC
McCabe index: 3
number of parameters: 1

id: 1116
unit: fn prepend()
file: bindings/python/src/utils/normalization.rs
start line: 263
end line: 265
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1117
unit: fn append()
file: bindings/python/src/utils/normalization.rs
start line: 269
end line: 271
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1118
unit: fn lstrip()
file: bindings/python/src/utils/normalization.rs
start line: 275
end line: 277
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1119
unit: fn rstrip()
file: bindings/python/src/utils/normalization.rs
start line: 281
end line: 283
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1120
unit: fn strip()
file: bindings/python/src/utils/normalization.rs
start line: 287
end line: 289
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1121
unit: fn clear()
file: bindings/python/src/utils/normalization.rs
start line: 293
end line: 295
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1122
unit: fn slice()
file: bindings/python/src/utils/normalization.rs
start line: 299
end line: 301
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1123
unit: fn filter()
file: bindings/python/src/utils/normalization.rs
start line: 305
end line: 307
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1124
unit: fn for_each()
file: bindings/python/src/utils/normalization.rs
start line: 311
end line: 313
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1125
unit: fn map()
file: bindings/python/src/utils/normalization.rs
start line: 320
end line: 322
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1126
unit: fn replace()
file: bindings/python/src/utils/normalization.rs
start line: 359
end line: 361
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1127
unit: fn __str__()
file: bindings/python/src/utils/normalization.rs
start line: 371
end line: 373
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1128
unit: fn __getitem__()
file: bindings/python/src/utils/normalization.rs
start line: 375
end line: 377
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1129
unit: fn from()
file: bindings/python/src/utils/normalization.rs
start line: 381
end line: 383
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1130
unit: fn from()
file: bindings/python/src/utils/normalization.rs
start line: 387
end line: 389
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1131
unit: fn destroy()
file: bindings/python/src/utils/normalization.rs
start line: 399
end line: 401
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1132
unit: fn next()
file: bindings/python/src/utils/iterators.rs
start line: 27
end line: 29
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1133
unit: fn size_hint()
file: bindings/python/src/utils/iterators.rs
start line: 31
end line: 33
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1134
unit: fn serialize_i8()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 96
end line: 98
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1135
unit: fn serialize_i16()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 100
end line: 102
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1136
unit: fn serialize_i32()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 104
end line: 106
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1137
unit: fn serialize_u8()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 115
end line: 117
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1138
unit: fn serialize_u16()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 119
end line: 121
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1139
unit: fn serialize_u32()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 123
end line: 125
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1140
unit: fn serialize_f32()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 132
end line: 134
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1141
unit: fn serialize_char()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 143
end line: 145
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1142
unit: fn serialize_none()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 175
end line: 177
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1143
unit: fn serialize_unit_struct()
file: bindings/python/src/utils/serde_pyo3.rs
start line: 201
end line: 203
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1144
unit: fn from()
file: bindings/python/src/utils/pretokenization.rs
start line: 156
end line: 158
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1145
unit: fn from()
file: bindings/python/src/utils/pretokenization.rs
start line: 162
end line: 164
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1146
unit: fn new()
file: bindings/python/src/utils/pretokenization.rs
start line: 171
end line: 173
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1147
unit: fn split()
file: bindings/python/src/utils/pretokenization.rs
start line: 186
end line: 188
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1148
unit: fn normalize()
file: bindings/python/src/utils/pretokenization.rs
start line: 198
end line: 200
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1149
unit: fn tokenize()
file: bindings/python/src/utils/pretokenization.rs
start line: 209
end line: 211
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1150
unit: fn to_encoding()
file: bindings/python/src/utils/pretokenization.rs
start line: 228
end line: 230
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1151
unit: fn destroy()
file: bindings/python/src/utils/pretokenization.rs
start line: 269
end line: 271
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1152
unit: fn __getstate__) -> PyResult>()
file: bindings/python/src/tokenizer.rs
start line: 154
end line: 156
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1153
unit: fn get_content()
file: bindings/python/src/tokenizer.rs
start line: 181
end line: 183
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1154
unit: fn set_content()
file: bindings/python/src/tokenizer.rs
start line: 187
end line: 189
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1155
unit: fn get_rstrip()
file: bindings/python/src/tokenizer.rs
start line: 193
end line: 195
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1156
unit: fn get_lstrip()
file: bindings/python/src/tokenizer.rs
start line: 199
end line: 201
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1157
unit: fn get_single_word()
file: bindings/python/src/tokenizer.rs
start line: 205
end line: 207
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1158
unit: fn get_normalized()
file: bindings/python/src/tokenizer.rs
start line: 211
end line: 213
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1159
unit: fn get_special()
file: bindings/python/src/tokenizer.rs
start line: 216
end line: 218
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1160
unit: fn set_special()
file: bindings/python/src/tokenizer.rs
start line: 222
end line: 224
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1161
unit: fn __str__()
file: bindings/python/src/tokenizer.rs
start line: 226
end line: 228
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1162
unit: fn from()
file: bindings/python/src/tokenizer.rs
start line: 276
end line: 278
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1163
unit: fn from()
file: bindings/python/src/tokenizer.rs
start line: 344
end line: 346
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1164
unit: fn from()
file: bindings/python/src/tokenizer.rs
start line: 368
end line: 370
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1165
unit: fn from()
file: bindings/python/src/tokenizer.rs
start line: 398
end line: 400
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1166
unit: fn from()
file: bindings/python/src/tokenizer.rs
start line: 425
end line: 427
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1167
unit: fn from()
file: bindings/python/src/tokenizer.rs
start line: 453
end line: 455
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1168
unit: fn new()
file: bindings/python/src/tokenizer.rs
start line: 475
end line: 477
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1169
unit: fn from_model()
file: bindings/python/src/tokenizer.rs
start line: 479
end line: 481
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1170
unit: fn __new__()
file: bindings/python/src/tokenizer.rs
start line: 488
end line: 490
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1171
unit: fn to_str()
file: bindings/python/src/tokenizer.rs
start line: 630
end line: 632
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1172
unit: fn save()
file: bindings/python/src/tokenizer.rs
start line: 644
end line: 646
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1173
unit: fn get_vocab()
file: bindings/python/src/tokenizer.rs
start line: 678
end line: 680
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1174
unit: fn get_vocab_size()
file: bindings/python/src/tokenizer.rs
start line: 708
end line: 710
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1175
unit: fn no_padding()
file: bindings/python/src/tokenizer.rs
start line: 899
end line: 901
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1176
unit: fn decode()
file: bindings/python/src/tokenizer.rs
start line: 1137
end line: 1139
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1177
unit: fn token_to_id()
file: bindings/python/src/tokenizer.rs
start line: 1175
end line: 1177
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1178
unit: fn id_to_token()
file: bindings/python/src/tokenizer.rs
start line: 1188
end line: 1190
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1179
unit: fn set_encode_special_tokens()
file: bindings/python/src/tokenizer.rs
start line: 1200
end line: 1202
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1180
unit: fn get_encode_special_tokens()
file: bindings/python/src/tokenizer.rs
start line: 1208
end line: 1210
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1181
unit: fn get_model()
file: bindings/python/src/tokenizer.rs
start line: 1416
end line: 1418
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1182
unit: fn set_model()
file: bindings/python/src/tokenizer.rs
start line: 1422
end line: 1424
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1183
unit: fn from()
file: bindings/python/src/encoding.rs
start line: 18
end line: 20
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1184
unit: fn __len__()
file: bindings/python/src/encoding.rs
start line: 66
end line: 68
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1185
unit: fn get_n_sequences()
file: bindings/python/src/encoding.rs
start line: 97
end line: 99
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1186
unit: fn set_sequence_id()
file: bindings/python/src/encoding.rs
start line: 106
end line: 108
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1187
unit: fn get_ids()
file: bindings/python/src/encoding.rs
start line: 118
end line: 120
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1188
unit: fn get_tokens()
file: bindings/python/src/encoding.rs
start line: 129
end line: 131
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1189
unit: fn get_word_ids()
file: bindings/python/src/encoding.rs
start line: 172
end line: 174
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1190
unit: fn get_sequence_ids()
file: bindings/python/src/encoding.rs
start line: 185
end line: 187
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1191
unit: fn get_type_ids()
file: bindings/python/src/encoding.rs
start line: 197
end line: 199
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1192
unit: fn get_offsets()
file: bindings/python/src/encoding.rs
start line: 209
end line: 211
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1193
unit: fn get_special_tokens_mask()
file: bindings/python/src/encoding.rs
start line: 220
end line: 222
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1194
unit: fn get_attention_mask()
file: bindings/python/src/encoding.rs
start line: 233
end line: 235
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1195
unit: fn word_to_tokens()
file: bindings/python/src/encoding.rs
start line: 269
end line: 271
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1196
unit: fn word_to_chars()
file: bindings/python/src/encoding.rs
start line: 285
end line: 287
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1197
unit: fn token_to_sequence()
file: bindings/python/src/encoding.rs
start line: 301
end line: 303
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1198
unit: fn char_to_token()
file: bindings/python/src/encoding.rs
start line: 353
end line: 355
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1199
unit: fn char_to_word()
file: bindings/python/src/encoding.rs
start line: 369
end line: 371
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1200
unit: fn added_tokens()
file: bindings/python/src/processors.rs
start line: 97
end line: 99
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1201
unit: fn num_special_tokens_to_add()
file: bindings/python/src/processors.rs
start line: 147
end line: 149
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1202
unit: fn new()
file: bindings/python/src/processors.rs
start line: 337
end line: 339
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1203
unit: fn __getnewargs__) -> PyResult>()
file: bindings/python/src/processors.rs
start line: 341
end line: 343
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1204
unit: fn __getnewargs__) -> PyResult>()
file: bindings/python/src/processors.rs
start line: 422
end line: 424
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1205
unit: fn get_trim_offsets()
file: bindings/python/src/processors.rs
start line: 461
end line: 463
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1206
unit: fn set_trim_offsets()
file: bindings/python/src/processors.rs
start line: 466
end line: 468
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1207
unit: fn get_add_prefix_space()
file: bindings/python/src/processors.rs
start line: 471
end line: 473
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1208
unit: fn set_add_prefix_space()
file: bindings/python/src/processors.rs
start line: 476
end line: 478
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1209
unit: fn get_add_prefix_space()
file: bindings/python/src/processors.rs
start line: 519
end line: 521
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1210
unit: fn set_add_prefix_space()
file: bindings/python/src/processors.rs
start line: 524
end line: 526
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1211
unit: fn get_trim_offsets()
file: bindings/python/src/processors.rs
start line: 529
end line: 531
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1212
unit: fn set_trim_offsets()
file: bindings/python/src/processors.rs
start line: 534
end line: 536
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1213
unit: fn get_use_regex()
file: bindings/python/src/processors.rs
start line: 539
end line: 541
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1214
unit: fn set_use_regex()
file: bindings/python/src/processors.rs
start line: 544
end line: 546
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1215
unit: fn from()
file: bindings/python/src/processors.rs
start line: 553
end line: 555
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1216
unit: fn from()
file: bindings/python/src/processors.rs
start line: 593
end line: 595
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1217
unit: fn get_single()
file: bindings/python/src/processors.rs
start line: 710
end line: 712
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1218
unit: fn __getnewargs__) -> PyResult>()
file: bindings/python/src/processors.rs
start line: 758
end line: 760
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1219
unit: fn from()
file: bindings/python/src/token.rs
start line: 10
end line: 12
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1220
unit: fn from()
file: bindings/python/src/token.rs
start line: 15
end line: 17
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1221
unit: fn new()
file: bindings/python/src/token.rs
start line: 24
end line: 26
size: 3 LOC
McCabe index: 1
number of parameters: 4

id: 1222
unit: fn get_id()
file: bindings/python/src/token.rs
start line: 29
end line: 31
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1223
unit: fn get_value()
file: bindings/python/src/token.rs
start line: 34
end line: 36
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1224
unit: fn get_offsets()
file: bindings/python/src/token.rs
start line: 39
end line: 41
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1225
unit: fn as_tuple()
file: bindings/python/src/token.rs
start line: 43
end line: 45
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1226
unit: def __init__()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1227
unit: def normalizer()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1228
unit: def unk_id()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1229
unit: def __init__()
file: bindings/python/scripts/sentencepiece_extractor.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1230
unit: fn token_to_id()
file: bindings/node/src/models.rs
start line: 96
end line: 98
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1231
unit: fn id_to_token()
file: bindings/node/src/models.rs
start line: 100
end line: 102
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 1232
unit: fn from()
file: bindings/node/src/tokenizer.rs
start line: 183
end line: 185
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 1233
unit: fn resolve()
file: bindings/node/src/tasks/tokenizer.rs
start line: 61
end line: 63
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1234
unit: fn resolve()
file: bindings/node/src/tasks/tokenizer.rs
start line: 124
end line: 126
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 1235
unit: fn main()
file: bindings/node/build.rs
start line: 3
end line: 5
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 1236
unit: def num_special_tokens_to_add()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1237
unit: def get_vocab()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1238
unit: def get_added_tokens_decoder()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1239
unit: def get_vocab_size()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1240
unit: def no_padding()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1241
unit: def padding()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1242
unit: def enable_truncation()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1243
unit: def no_truncation()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1244
unit: def truncation()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1245
unit: def add_tokens()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1246
unit: def add_special_tokens()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1247
unit: def normalize()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1248
unit: def token_to_id()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1249
unit: def id_to_token()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1250
unit: def save_model()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1251
unit: def save()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1252
unit: def to_str()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1253
unit: def post_process()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 0

id: 1254
unit: def model()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1255
unit: def model()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1256
unit: def normalizer()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1257
unit: def normalizer()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1258
unit: def pre_tokenizer()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1259
unit: def pre_tokenizer()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1260
unit: def post_processor()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1261
unit: def post_processor()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1262
unit: def decoder()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1263
unit: def decoder()
file: bindings/python/py_src/tokenizers/implementations/base_tokenizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1264
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1265
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1266
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 5

id: 1267
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1268
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1269
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1270
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1271
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1272
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1273
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1274
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1275
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1276
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1277
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1278
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1279
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1280
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1281
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1282
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1283
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1284
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1285
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1286
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1287
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1288
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1289
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1290
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1291
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1292
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1293
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1294
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1295
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1296
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1297
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1298
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1299
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1300
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1301
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1302
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1303
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1304
unit: def __init__()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1305
unit: def normalize()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1306
unit: def normalize_str()
file: bindings/python/py_src/tokenizers/normalizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1307
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1308
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1309
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1310
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1311
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1312
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1313
unit: def alphabet()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 0

id: 1314
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1315
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1316
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1317
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1318
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1319
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1320
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1321
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1322
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1323
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1324
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1325
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1326
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1327
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1328
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1329
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1330
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1331
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1332
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1333
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1334
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1335
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1336
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1337
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1338
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1339
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1340
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1341
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1342
unit: def __init__()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1343
unit: def pre_tokenize()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1344
unit: def pre_tokenize_str()
file: bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1345
unit: def get_trainer()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1346
unit: def id_to_token()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1347
unit: def save()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1348
unit: def token_to_id()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1349
unit: def tokenize()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1350
unit: def from_file()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1351
unit: def get_trainer()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1352
unit: def id_to_token()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1353
unit: def read_file()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1354
unit: def save()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1355
unit: def token_to_id()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1356
unit: def tokenize()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1357
unit: def __init__()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1358
unit: def get_trainer()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1359
unit: def id_to_token()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1360
unit: def save()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1361
unit: def token_to_id()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1362
unit: def tokenize()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1363
unit: def __init__()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1364
unit: def from_file()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1365
unit: def get_trainer()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1366
unit: def id_to_token()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1367
unit: def read_file()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1368
unit: def save()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1369
unit: def token_to_id()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1370
unit: def tokenize()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1371
unit: def __init__()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1372
unit: def from_file()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1373
unit: def get_trainer()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1374
unit: def id_to_token()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1375
unit: def read_file()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1376
unit: def save()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1377
unit: def token_to_id()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1378
unit: def tokenize()
file: bindings/python/py_src/tokenizers/models/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1379
unit: def __init__()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 7

id: 1380
unit: def content()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1381
unit: def lstrip()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1382
unit: def normalized()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1383
unit: def rstrip()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1384
unit: def single_word()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1385
unit: def special()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1386
unit: def attention_mask()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1387
unit: def char_to_token()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1388
unit: def char_to_word()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1389
unit: def ids()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1390
unit: def merge()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1391
unit: def n_sequences()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1392
unit: def offsets()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1393
unit: def overflowing()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1394
unit: def pad()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 6

id: 1395
unit: def sequence_ids()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1396
unit: def set_sequence_id()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1397
unit: def special_tokens_mask()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1398
unit: def token_to_chars()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1399
unit: def token_to_sequence()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1400
unit: def token_to_word()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1401
unit: def tokens()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1402
unit: def truncate()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1403
unit: def type_ids()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1404
unit: def word_ids()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1405
unit: def word_to_chars()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1406
unit: def word_to_tokens()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1407
unit: def words()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1408
unit: def append()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1409
unit: def clear()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1410
unit: def filter()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1411
unit: def for_each()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1412
unit: def lowercase()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1413
unit: def lstrip()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1414
unit: def map()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1415
unit: def nfc()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1416
unit: def nfd()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1417
unit: def nfkc()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1418
unit: def nfkd()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1419
unit: def normalized()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1420
unit: def prepend()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1421
unit: def replace()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1422
unit: def rstrip()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1423
unit: def slice()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1424
unit: def split()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1425
unit: def strip()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1426
unit: def uppercase()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1427
unit: def __init__()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1428
unit: def get_splits()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1429
unit: def normalize()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1430
unit: def split()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1431
unit: def to_encoding()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1432
unit: def tokenize()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1433
unit: def __init__()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1434
unit: def __init__()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1435
unit: def add_special_tokens()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1436
unit: def add_tokens()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1437
unit: def decode()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1438
unit: def decode_batch()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1439
unit: def decoder()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1440
unit: def enable_padding()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 0

id: 1441
unit: def enable_truncation()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 5

id: 1442
unit: def encode()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 5

id: 1443
unit: def encode_batch()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1444
unit: def encode_batch_fast()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1445
unit: def encode_special_tokens()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1446
unit: def from_buffer()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1447
unit: def from_file()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1448
unit: def from_pretrained()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1449
unit: def from_str()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1450
unit: def get_added_tokens_decoder()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1451
unit: def get_vocab()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1452
unit: def get_vocab_size()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1453
unit: def id_to_token()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1454
unit: def model()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1455
unit: def no_padding()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1456
unit: def no_truncation()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1457
unit: def normalizer()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1458
unit: def num_special_tokens_to_add()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1459
unit: def padding()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1460
unit: def post_process()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1461
unit: def post_processor()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1462
unit: def pre_tokenizer()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1463
unit: def save()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1464
unit: def to_str()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1465
unit: def token_to_id()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1466
unit: def train()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1467
unit: def train_from_iterator()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1468
unit: def truncation()
file: bindings/python/py_src/tokenizers/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1469
unit: def token_ix()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 2
number of parameters: 1

id: 1470
unit: def is_multitoken()
file: bindings/python/py_src/tokenizers/tools/visualizer.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1471
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1472
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1473
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1474
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1475
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1476
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1477
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1478
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1479
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1480
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1481
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1482
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1483
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1484
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1485
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1486
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1487
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1488
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1489
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1490
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1491
unit: def __init__()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1492
unit: def decode()
file: bindings/python/py_src/tokenizers/decoders/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1493
unit: def num_special_tokens_to_add()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1494
unit: def process()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1495
unit: def __init__()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 1496
unit: def num_special_tokens_to_add()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1497
unit: def process()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1498
unit: def __init__()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1499
unit: def num_special_tokens_to_add()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1500
unit: def process()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1501
unit: def __init__()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 5

id: 1502
unit: def num_special_tokens_to_add()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1503
unit: def process()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1504
unit: def __init__()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1505
unit: def num_special_tokens_to_add()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1506
unit: def process()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1507
unit: def __init__()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1508
unit: def num_special_tokens_to_add()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1509
unit: def process()
file: bindings/python/py_src/tokenizers/processors/__init__.pyi
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 4

id: 1510
unit: def check_number_comma()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 3
number of parameters: 1

id: 1511
unit: def __init__()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1512
unit: def converted()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1513
unit: def vocab()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 2
number of parameters: 2

id: 1514
unit: def unk_id()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1515
unit: def post_processor()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1516
unit: def unk_id()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1517
unit: def unk_id()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1518
unit: def unk_id()
file: bindings/python/scripts/convert.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1519
unit: def __init__()
file: bindings/python/scripts/sentencepiece_extractor.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 1520
unit: def pytest_addoption()
file: bindings/python/conftest.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1521
unit: def pytest_configure()
file: bindings/python/conftest.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 1522
unit: def do_indent()
file: bindings/python/stub.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2