in torchaudio/csrc/decoder/src/dictionary/Utils.cpp [118:144]
std::vector<int> unpackReplabels(
const std::vector<int>& tokens,
const Dictionary& dict,
int maxReps) {
if (tokens.empty() || maxReps <= 0) {
return tokens;
}
std::unordered_map<int, int> replabelIdxToValue;
for (int i = 1; i <= maxReps; ++i) {
replabelIdxToValue.emplace(dict.getIndex("<" + std::to_string(i) + ">"), i);
}
std::vector<int> result;
int prevToken = -1;
for (int token : tokens) {
auto it = replabelIdxToValue.find(token);
if (it == replabelIdxToValue.end()) {
result.push_back(token);
prevToken = token;
} else if (prevToken != -1) {
result.insert(result.end(), it->second, prevToken);
prevToken = -1;
}
}
return result;
}