def get_encoder()

in lm_human_preferences/language/encodings.py [0:0]

• 20 lines of code
• 6 McCabe index (conditional complexity)

    def get_encoder(self):
        if self.name == "test":
            vocab = "abcdefghijklmnopqrstuvwxyz."
            assert len(vocab) == self.n_vocab

            class TestEncoder(ReversibleEncoder):
                def __init__(self):
                    super().__init__(encoder={w: i for i, w in enumerate(vocab)}, bpe_merges=list())
                    self.padding_token = len(vocab)
                def encode(self, text):
                    return [self.encoder.get(x, len(vocab) - 1) for x in text]
                def decode(self, tokens, pretty=False):
                    return ''.join([self.decoder.get(t, '<unk>') for t in tokens])

            return TestEncoder()

        encoder_dict = json.loads(read_file(self.encoder_path).decode())
        bpe_data = read_file(self.bpe_path).decode()
        bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split("\n")[1:-1]]
        assert len(encoder_dict) == self.n_vocab
        encoder = ReversibleEncoder(encoder=encoder_dict, bpe_merges=bpe_merges, eot_token=self.eot_token)
        assert encoder.padding_token >= self.n_vocab
        return encoder