def bpe()

in summarize_from_feedback/datasets/encodings.py [0:0]

• 35 lines of code
• 14 McCabe index (conditional complexity)

    def bpe(self, token):  # pylint: disable=method-hidden
        word = tuple(self.byte_encoder[b] for b in token.encode("utf-8"))

        if len(word) == 1:
            return word

        while True:
            min_pair = None
            min_idxs = []
            min_rank = None
            for i, pair in enumerate(zip(word[:-1], word[1:])):
                try:
                    rank = self.bpe_ranks[pair]
                    if min_rank is None or rank < min_rank:
                        min_rank = rank
                        min_pair = pair
                        del min_idxs[::]
                        min_idxs.append(i)
                    elif min_rank == rank and i > min_idxs[-1] + 1:
                        min_idxs.append(i)
                except KeyError:
                    pass

            if min_pair is None:
                break

            new_word = []
            i_start = 0
            for i in min_idxs:
                new_word.extend(word[i_start:i])
                new_word.append(min_pair[0] + min_pair[1])
                i_start = i + 2

            if i_start < len(word):
                new_word.extend(word[i_start:])

            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break

        return word