id: 1
unit: def read_file_cached()
file: tiktoken/load.py
start line: 0
end line: 0
size: 40 LOC
McCabe index: 13
number of parameters: 2

id: 2
unit: fn _byte_pair_merge()
file: src/lib.rs
start line: 17
end line: 73
size: 40 LOC
McCabe index: 8
number of parameters: 3

id: 3
unit: fn new_internal()
file: src/lib.rs
start line: 464
end line: 507
size: 37 LOC
McCabe index: 1
number of parameters: 4

id: 4
unit: fn _encode_bytes()
file: src/py.rs
start line: 68
end line: 110
size: 32 LOC
McCabe index: 5
number of parameters: 3

id: 5
unit: fn _increase_last_piece_token_len()
file: src/lib.rs
start line: 290
end line: 327
size: 30 LOC
McCabe index: 5
number of parameters: 3

id: 6
unit: def redact_file()
file: scripts/redact.py
start line: 0
end line: 0
size: 28 LOC
McCabe index: 9
number of parameters: 2

id: 7
unit: def _find_constructors()
file: tiktoken/registry.py
start line: 0
end line: 0
size: 24 LOC
McCabe index: 7
number of parameters: 0

id: 8
unit: def o200k_base()
file: tiktoken_ext/openai_public.py
start line: 0
end line: 0
size: 23 LOC
McCabe index: 1
number of parameters: 0

id: 9
unit: def get_encoding()
file: tiktoken/registry.py
start line: 0
end line: 0
size: 21 LOC
McCabe index: 6
number of parameters: 1

id: 10
unit: fn py_encode_with_unstable()
file: src/py.rs
start line: 113
end line: 131
size: 19 LOC
McCabe index: 2
number of parameters: 4

id: 11
unit: def cl100k_base()
file: tiktoken_ext/openai_public.py
start line: 0
end line: 0
size: 18 LOC
McCabe index: 1
number of parameters: 0

id: 12
unit: def benchmark_batch()
file: scripts/benchmark.py
start line: 0
end line: 0
size: 18 LOC
McCabe index: 1
number of parameters: 1

id: 13
unit: def gpt2()
file: tiktoken_ext/openai_public.py
start line: 0
end line: 0
size: 14 LOC
McCabe index: 1
number of parameters: 0

id: 14
unit: def visualise_tokens()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 14 LOC
McCabe index: 5
number of parameters: 1

id: 15
unit: def encoding_name_for_model()
file: tiktoken/model.py
start line: 0
end line: 0
size: 14 LOC
McCabe index: 7
number of parameters: 1

id: 16
unit: def read_file()
file: tiktoken/load.py
start line: 0
end line: 0
size: 14 LOC
McCabe index: 5
number of parameters: 1

id: 17
unit: fn encode_to_tiktoken_buffer()
file: src/py.rs
start line: 53
end line: 66
size: 14 LOC
McCabe index: 2
number of parameters: 4

id: 18
unit: fn decode_bytes()
file: src/lib.rs
start line: 204
end line: 217
size: 14 LOC
McCabe index: 2
number of parameters: 2

id: 19
unit: def train_simple_encoding()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 13 LOC
McCabe index: 1
number of parameters: 0

id: 20
unit: def r50k_base()
file: tiktoken_ext/openai_public.py
start line: 0
end line: 0
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 21
unit: def p50k_base()
file: tiktoken_ext/openai_public.py
start line: 0
end line: 0
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 22
unit: def p50k_edit()
file: tiktoken_ext/openai_public.py
start line: 0
end line: 0
size: 12 LOC
McCabe index: 1
number of parameters: 0

id: 23
unit: def load_tiktoken_bpe()
file: tiktoken/load.py
start line: 0
end line: 0
size: 12 LOC
McCabe index: 4
number of parameters: 2

id: 24
unit: fn py_new()
file: src/py.rs
start line: 17
end line: 28
size: 12 LOC
McCabe index: 1
number of parameters: 4

id: 25
unit: fn py_encode()
file: src/py.rs
start line: 40
end line: 51
size: 12 LOC
McCabe index: 2
number of parameters: 4

id: 26
unit: def redact()
file: scripts/redact.py
start line: 0
end line: 0
size: 11 LOC
McCabe index: 4
number of parameters: 1

id: 27
unit: fn encode_single_token()
file: src/py.rs
start line: 133
end line: 143
size: 11 LOC
McCabe index: 4
number of parameters: 2

id: 28
unit: def __getstate__()
file: tiktoken/core.py
start line: 0
end line: 0
size: 10 LOC
McCabe index: 2
number of parameters: 1

id: 29
unit: def dump_tiktoken_bpe()
file: tiktoken/load.py
start line: 0
end line: 0
size: 10 LOC
McCabe index: 3
number of parameters: 3

id: 30
unit: def decode_with_offsets()
file: tiktoken/core.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 6
number of parameters: 2

id: 31
unit: def raise_disallowed_special_token()
file: tiktoken/core.py
start line: 0
end line: 0
size: 9 LOC
McCabe index: 1
number of parameters: 1

id: 32
unit: fn decode_single_token_bytes()
file: src/py.rs
start line: 164
end line: 172
size: 9 LOC
McCabe index: 3
number of parameters: 3

id: 33
unit: def encode()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 8 LOC
McCabe index: 2
number of parameters: 3

id: 34
unit: def __init__()
file: tiktoken/core.py
start line: 0
end line: 0
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 35
unit: fn hash_current_thread()
file: src/lib.rs
start line: 136
end line: 147
size: 8 LOC
McCabe index: 1
number of parameters: 0

id: 36
unit: def encode_batch()
file: tiktoken/core.py
start line: 0
end line: 0
size: 7 LOC
McCabe index: 1
number of parameters: 5

id: 37
unit: def main()
file: scripts/redact.py
start line: 0
end line: 0
size: 7 LOC
McCabe index: 3
number of parameters: 0

id: 38
unit: def _available_plugin_modules()
file: tiktoken/registry.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 3
number of parameters: 0

id: 39
unit: def list_encoding_names()
file: tiktoken/registry.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 2
number of parameters: 0

id: 40
unit: def from_tiktoken()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 2
number of parameters: 1

id: 41
unit: def encode_ordinary()
file: tiktoken/core.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 2
number of parameters: 2

id: 42
unit: def encode()
file: tiktoken/core.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 1
number of parameters: 4

id: 43
unit: def encode_to_numpy()
file: tiktoken/core.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 1
number of parameters: 4

id: 44
unit: def encode_with_unstable()
file: tiktoken/core.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 1
number of parameters: 4

id: 45
unit: def _encode_only_native_bpe()
file: tiktoken/core.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 2
number of parameters: 2

id: 46
unit: def __setstate__()
file: tiktoken/core.py
start line: 0
end line: 0
size: 6 LOC
McCabe index: 2
number of parameters: 2

id: 47
unit: fn encode_single_piece()
file: src/py.rs
start line: 145
end line: 150
size: 6 LOC
McCabe index: 2
number of parameters: 2

id: 48
unit: fn py_decode_bytes()
file: src/py.rs
start line: 157
end line: 162
size: 6 LOC
McCabe index: 2
number of parameters: 3

id: 49
unit: fn token_byte_values()
file: src/py.rs
start line: 178
end line: 183
size: 6 LOC
McCabe index: 1
number of parameters: 2

id: 50
unit: def __init__()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 5 LOC
McCabe index: 2
number of parameters: 5

id: 51
unit: def data_gym_to_mergeable_bpe_ranks()
file: tiktoken/load.py
start line: 0
end line: 0
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 52
unit: fn test_simple_characters()
file: src/lib.rs
start line: 534
end line: 538
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 53
unit: fn test_repeated_characters()
file: src/lib.rs
start line: 541
end line: 545
size: 5 LOC
McCabe index: 1
number of parameters: 0

id: 54
unit: def encode_ordinary_batch()
file: tiktoken/core.py
start line: 0
end line: 0
size: 4 LOC
McCabe index: 1
number of parameters: 4

id: 55
unit: def encode_single_token()
file: tiktoken/core.py
start line: 0
end line: 0
size: 4 LOC
McCabe index: 2
number of parameters: 2

id: 56
unit: def _encode_single_piece()
file: tiktoken/core.py
start line: 0
end line: 0
size: 4 LOC
McCabe index: 2
number of parameters: 2

id: 57
unit: def train()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 3

id: 58
unit: def is_special_token()
file: tiktoken/core.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 59
unit: def _special_token_regex()
file: tiktoken/core.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 2
number of parameters: 1

id: 60
unit: def check_hash()
file: tiktoken/load.py
start line: 0
end line: 0
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 61
unit: fn py_encode_ordinary()
file: src/py.rs
start line: 35
end line: 37
size: 3 LOC
McCabe index: 2
number of parameters: 3

id: 62
unit: fn fmt()
file: src/lib.rs
start line: 155
end line: 157
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 63
unit: fn fmt()
file: src/lib.rs
start line: 168
end line: 170
size: 3 LOC
McCabe index: 1
number of parameters: 2

id: 64
unit: fn _get_tl_regex()
file: src/lib.rs
start line: 190
end line: 195
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 65
unit: fn _get_tl_special_regex()
file: src/lib.rs
start line: 197
end line: 199
size: 3 LOC
McCabe index: 1
number of parameters: 1

id: 66
unit: fn setup_ranks()
file: src/lib.rs
start line: 529
end line: 531
size: 3 LOC
McCabe index: 1
number of parameters: 0

id: 67
unit: def decode_bytes()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 2
number of parameters: 2

id: 68
unit: def decode()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 69
unit: def decode_tokens_bytes()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 2
number of parameters: 2

id: 70
unit: def bpe_encode()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 0

id: 71
unit: def bpe_train()
file: tiktoken/_educational.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 0

id: 72
unit: def __repr__()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 73
unit: def decode_bytes()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 74
unit: def decode()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 3

id: 75
unit: def decode_single_token_bytes()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 76
unit: def decode_tokens_bytes()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 2
number of parameters: 2

id: 77
unit: def decode_batch()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 0

id: 78
unit: def decode_bytes_batch()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 0

id: 79
unit: def token_byte_values()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 80
unit: def eot_token()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 81
unit: def special_tokens_set()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 82
unit: def n_vocab()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 83
unit: def _encode_bytes()
file: tiktoken/core.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 2

id: 84
unit: def encoding_for_model()
file: tiktoken/model.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 1
number of parameters: 1

id: 85
unit: def decode_data_gym()
file: tiktoken/load.py
start line: 0
end line: 0
size: 2 LOC
McCabe index: 2
number of parameters: 1