def create_tokenizer_from_hub_module()

in code/source/bert_preprocessing.py [0:0]

• 8 lines of code
• 1 McCabe index (conditional complexity)

def create_tokenizer_from_hub_module(bert_path):
    """
    Get the vocab file and casing info from the Hub module.
    :param bert_path: (str) path to either internet address or s3 location of bert
    :return:
    """
    BertTokenizer = bert_tokenization.FullTokenizer
    bert_layer = hub.KerasLayer(bert_path,
                                trainable=False)

    vocabulary_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
    to_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
    tokenizer = BertTokenizer(vocabulary_file, to_lower_case)

    return tokenizer