in utils.py [0:0]
def pack_replabels(tokens, num_replabels):
if all(isinstance(t, list) for t in tokens):
return [pack_replabels(t, num_replabels) for t in tokens]
assert isinstance(tokens, list)
new_tokens = []
L = len(tokens)
num = 0
prev_token = -1
for token in tokens:
if token == prev_token and num < num_replabels:
num += 1
else:
if num > 0:
new_tokens.append(num - 1)
num = 0
new_tokens.append(token + num_replabels)
prev_token = token
if num > 0:
new_tokens.append(num - 1)
return new_tokens