in utils.py [0:0]
def unpack_replabels(tokens, num_replabels):
if all(isinstance(t, list) for t in tokens):
return [unpack_replabels(t, num_replabels) for t in tokens]
assert isinstance(tokens, list)
new_tokens = []
prev_token = -1
for token in tokens:
if token >= num_replabels:
new_tokens.append(token - num_replabels)
prev_token = token
elif prev_token != -1:
for i in range(token + 1):
new_tokens.append(prev_token - num_replabels)
prev_token = -1
return new_tokens