def get_encoder()

in glide_text2im/tokenizer/bpe.py [0:0]

• 11 lines of code
• 2 McCabe index (conditional complexity)

def get_encoder():
    root_dir = os.path.dirname(os.path.abspath(__file__))
    with gzip.open(os.path.join(root_dir, "encoder.json.gz"), "r") as f:
        encoder = json.load(f)
    with gzip.open(os.path.join(root_dir, "vocab.bpe.gz"), "r") as f:
        bpe_data = str(f.read(), "utf-8")
    bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split("\n")[1:-1]]
    return Encoder(
        encoder=encoder,
        bpe_merges=bpe_merges,
    )