def _load()

in summarize_from_feedback/datasets/encodings.py [0:0]

• 21 lines of code
• 7 McCabe index (conditional complexity)

    def _load(self):
        if self._token_str_to_idx is not None:
            return

        self._token_str_to_idx = json.loads(read_file(self.full_encoder_path).decode())
        bpe_data = read_file(self.full_bpe_path).decode()
        bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split("\n")[1:-1]]

        assert self.eot_token == self._token_str_to_idx["<|endoftext|>"]

        # Add an <|end_of_prefix|> token
        if self.eoprefix_token is not None:
            assert not self._token_str_to_idx.get("<|endofprefix|>")
            self._token_str_to_idx["<|endofprefix|>"] = self.eoprefix_token

        # Add denoise sentinel tokens like <|dn_1|> <|dn_2|> etc.
        # These tokens are added to the end of the vocabulary range
        for denoise_sentinel_idx in range(self.n_denoise_sentinels):
            str_repr = f"<|dn_{denoise_sentinel_idx}|>"
            assert not self._token_str_to_idx.get(str_repr)

            n_non_sentinel_tokens = self.n_vocab - self.n_denoise_sentinels
            sentinel_token = n_non_sentinel_tokens + denoise_sentinel_idx

            self._token_str_to_idx[str_repr] = sentinel_token

        assert len(self._token_str_to_idx) == self.n_vocab

        self._token_idx_to_str = {v: k for k, v in self._token_str_to_idx.items()}
        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))