in Runtime/Tokenizers/Decoders/Decoders.cs [280:315]
public override List<string> Decode(List<string> tokens)
{
return tokens.Select(token =>
{
int startCut = 0;
for (int i = 0; i < start; ++i)
{
if (token[i] == content[0])
{
startCut = i + 1;
continue;
}
else
{
break;
}
}
int stopCut = token.Length;
for (int i = 0; i < stop; ++i)
{
int index = token.Length - i - 1;
if (token[index] == content[0])
{
stopCut = index;
continue;
}
else
{
break;
}
}
return token.Substring(startCut, stopCut - startCut);
}).ToList();
}