def __init__()

in lm_human_preferences/language/encodings.py [0:0]

• 12 lines of code
• 5 McCabe index (conditional complexity)

    def __init__(self, encoder, bpe_merges, errors="replace", eot_token=None):
        self.encoder = encoder
        self.decoder = {v: k for k, v in self.encoder.items()}
        self.errors = errors  # how to handle errors in decoding
        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
        self.eot_token = eot_token
        self.cache = {}
        self.padding_token = len(encoder) + 2 # +2 unnecessary, for historical reasons
        self.decoder[self.padding_token] = ''

        # Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")