Path	Lines of Code
oss_scripts/build_docs.py	54
oss_scripts/pip_package/MANIFEST.in	1
oss_scripts/pip_package/setup.nightly.py	62
oss_scripts/pip_package/setup.py	65
tensorflow_text/__init__.py	68
tensorflow_text/core/kernels/constrained_sequence.cc	329
tensorflow_text/core/kernels/constrained_sequence.h	44
tensorflow_text/core/kernels/constrained_sequence_kernel.cc	189
tensorflow_text/core/kernels/darts_clone_trie_builder.cc	64
tensorflow_text/core/kernels/darts_clone_trie_builder.h	17
tensorflow_text/core/kernels/darts_clone_trie_wrapper.h	88
tensorflow_text/core/kernels/disjoint_set_forest.h	98
tensorflow_text/core/kernels/edit_changes.proto	11
tensorflow_text/core/kernels/fast_wordpiece_tokenizer.cc	439
tensorflow_text/core/kernels/fast_wordpiece_tokenizer.h	92
tensorflow_text/core/kernels/fast_wordpiece_tokenizer_kernel.cc	12
tensorflow_text/core/kernels/fast_wordpiece_tokenizer_kernel.h	19
tensorflow_text/core/kernels/fast_wordpiece_tokenizer_kernel_template.h	271
tensorflow_text/core/kernels/fast_wordpiece_tokenizer_model_builder.cc	617
tensorflow_text/core/kernels/fast_wordpiece_tokenizer_model_builder.h	14
tensorflow_text/core/kernels/fast_wordpiece_tokenizer_tflite.cc	22
tensorflow_text/core/kernels/fast_wordpiece_tokenizer_tflite.h	15
tensorflow_text/core/kernels/fast_wordpiece_tokenizer_utils.h	84
tensorflow_text/core/kernels/mst_op_kernels.cc	138
tensorflow_text/core/kernels/mst_solver.h	285
tensorflow_text/core/kernels/ngrams_kernel.cc	9
tensorflow_text/core/kernels/ngrams_kernel.h	13
tensorflow_text/core/kernels/ngrams_kernel_template.h	197
tensorflow_text/core/kernels/ngrams_tflite.cc	19
tensorflow_text/core/kernels/ngrams_tflite.h	15
tensorflow_text/core/kernels/normalize_kernels.cc	310
tensorflow_text/core/kernels/ragged_tensor_to_tensor_tflite.cc	565
tensorflow_text/core/kernels/ragged_tensor_to_tensor_tflite.h	14
tensorflow_text/core/kernels/regex_split.cc	59
tensorflow_text/core/kernels/regex_split.h	21
tensorflow_text/core/kernels/regex_split_kernels.cc	148
tensorflow_text/core/kernels/rouge_l_kernel.cc	160
tensorflow_text/core/kernels/sentence_breaking_kernels.cc	196
tensorflow_text/core/kernels/sentence_breaking_kernels_v2.cc	55
tensorflow_text/core/kernels/sentence_breaking_utils.cc	183
tensorflow_text/core/kernels/sentence_breaking_utils.h	34
tensorflow_text/core/kernels/sentence_fragmenter.cc	278
tensorflow_text/core/kernels/sentence_fragmenter.h	101
tensorflow_text/core/kernels/sentence_fragmenter_v2.cc	522
tensorflow_text/core/kernels/sentence_fragmenter_v2.h	77
tensorflow_text/core/kernels/sentencepiece_kernels.cc	610
tensorflow_text/core/kernels/spanning_tree_iterator.cc	60
tensorflow_text/core/kernels/spanning_tree_iterator.h	29
tensorflow_text/core/kernels/split_merge_tokenize_kernel.cc	164
tensorflow_text/core/kernels/text_kernels_test_util.cc	47
tensorflow_text/core/kernels/text_kernels_test_util.h	61
tensorflow_text/core/kernels/tokenizer_from_logits_kernel.cc	174
tensorflow_text/core/kernels/unicode_script_tokenize_kernel.cc	125
tensorflow_text/core/kernels/whitespace_tokenize_kernel.cc	101
tensorflow_text/core/kernels/whitespace_tokenizer.cc	65
tensorflow_text/core/kernels/whitespace_tokenizer.h	38
tensorflow_text/core/kernels/whitespace_tokenizer_config_builder.cc	53
tensorflow_text/core/kernels/whitespace_tokenizer_config_builder.h	10
tensorflow_text/core/kernels/whitespace_tokenizer_kernel.cc	9
tensorflow_text/core/kernels/whitespace_tokenizer_kernel.h	14
tensorflow_text/core/kernels/whitespace_tokenizer_kernel_template.h	159
tensorflow_text/core/kernels/whitespace_tokenizer_tflite.cc	16
tensorflow_text/core/kernels/whitespace_tokenizer_tflite.h	14
tensorflow_text/core/kernels/wordpiece_kernel.cc	244
tensorflow_text/core/kernels/wordpiece_tokenizer.cc	203
tensorflow_text/core/kernels/wordpiece_tokenizer.h	36
tensorflow_text/core/ops/constrained_sequence_op.cc	68
tensorflow_text/core/ops/fast_wordpiece_tokenizer_op.cc	11
tensorflow_text/core/ops/mst_ops.cc	60
tensorflow_text/core/ops/normalize_ops.cc	67
tensorflow_text/core/ops/regex_split_ops.cc	31
tensorflow_text/core/ops/rouge_l_op.cc	66
tensorflow_text/core/ops/sentence_breaking_ops.cc	31
tensorflow_text/core/ops/sentence_breaking_ops_v2.cc	25
tensorflow_text/core/ops/sentencepiece_ops.cc	145
tensorflow_text/core/ops/split_merge_tokenize_op.cc	72
tensorflow_text/core/ops/tokenizer_from_logits_op.cc	98
tensorflow_text/core/ops/unicode_script_tokenize_op.cc	34
tensorflow_text/core/ops/whitespace_tokenize_op.cc	33
tensorflow_text/core/ops/whitespace_tokenizer_op.cc	10
tensorflow_text/core/ops/wordpiece_op.cc	90
tensorflow_text/core/pybinds/pywrap_fast_wordpiece_tokenizer_model_builder.cc	23
tensorflow_text/core/pybinds/pywrap_whitespace_tokenizer_config_builder.cc	17
tensorflow_text/core/pybinds/tflite_registrar.cc	64
tensorflow_text/python/__init__.py	1
tensorflow_text/python/benchmarks/__init__.py	3
tensorflow_text/python/benchmarks/benchmark_utils.py	182
tensorflow_text/python/benchmarks/ops_benchmarks.py	139
tensorflow_text/python/benchmarks/tokenizers_benchmarks.py	166
tensorflow_text/python/keras/__init__.py	6
tensorflow_text/python/keras/layers/__init__.py	10
tensorflow_text/python/keras/layers/todense.py	41
tensorflow_text/python/keras/layers/tokenization_layers.py	153
tensorflow_text/python/metrics/__init__.py	6
tensorflow_text/python/metrics/text_similarity_metric_ops.py	28
tensorflow_text/python/numpy/__init__.py	1
tensorflow_text/python/numpy/viterbi_decode.py	88
tensorflow_text/python/ops/__init__.py	54
tensorflow_text/python/ops/bert_tokenizer.py	128
tensorflow_text/python/ops/create_feature_bitmask_op.py	59
tensorflow_text/python/ops/fast_wordpiece_tokenizer.py	107
tensorflow_text/python/ops/greedy_constrained_sequence_op.py	51
tensorflow_text/python/ops/hub_module_splitter.py	58
tensorflow_text/python/ops/hub_module_tokenizer.py	15
tensorflow_text/python/ops/item_selector_ops.py	179
tensorflow_text/python/ops/masking_ops.py	117
tensorflow_text/python/ops/mst_ops.py	24
tensorflow_text/python/ops/ngrams_op.py	51
tensorflow_text/python/ops/normalize_ops.py	78
tensorflow_text/python/ops/pad_along_dimension_op.py	82
tensorflow_text/python/ops/pad_model_inputs_ops.py	17
tensorflow_text/python/ops/pointer_ops.py	247
tensorflow_text/python/ops/regex_split_ops.py	71
tensorflow_text/python/ops/segment_combiner_ops.py	37
tensorflow_text/python/ops/sentence_breaking_ops.py	46
tensorflow_text/python/ops/sentencepiece_tokenizer.py	185
tensorflow_text/python/ops/sliding_window_op.py	30
tensorflow_text/python/ops/split_merge_from_logits_tokenizer.py	40
tensorflow_text/python/ops/split_merge_tokenizer.py	83
tensorflow_text/python/ops/splitter.py	15
tensorflow_text/python/ops/state_based_sentence_breaker_op.py	54
tensorflow_text/python/ops/string_ops.py	20
tensorflow_text/python/ops/tokenization.py	25
tensorflow_text/python/ops/trimmer_ops.py	138
tensorflow_text/python/ops/unicode_char_tokenizer.py	42
tensorflow_text/python/ops/unicode_script_tokenizer.py	92
tensorflow_text/python/ops/viterbi_constrained_sequence_op.py	51
tensorflow_text/python/ops/whitespace_tokenizer.py	72
tensorflow_text/python/ops/wordpiece_tokenizer.py	160
tensorflow_text/python/ops/wordshape_ops.py	188
tensorflow_text/tftext.bzl	128
tensorflow_text/tools/__init__.py	1
tensorflow_text/tools/wordpiece_vocab/__init__.py	1
tensorflow_text/tools/wordpiece_vocab/bert_vocab_from_dataset.py	23
tensorflow_text/tools/wordpiece_vocab/generate_vocab.py	160
tensorflow_text/tools/wordpiece_vocab/generate_word_counts.py	74
tensorflow_text/tools/wordpiece_vocab/measure_wordpiece_stats.py	76
tensorflow_text/tools/wordpiece_vocab/utils.py	249
tensorflow_text/tools/wordpiece_vocab/wordpiece_tokenizer_learner.py	62
tensorflow_text/tools/wordpiece_vocab/wordpiece_tokenizer_learner_lib.py	214