public override List Decode()

in Runtime/Tokenizers/Decoders/Decoders.cs [280:315]


        public override List<string> Decode(List<string> tokens)
        {
            return tokens.Select(token =>
            {
                int startCut = 0;
                for (int i = 0; i < start; ++i)
                {
                    if (token[i] == content[0])
                    {
                        startCut = i + 1;
                        continue;
                    }
                    else
                    {
                        break;
                    }
                }

                int stopCut = token.Length;
                for (int i = 0; i < stop; ++i)
                {
                    int index = token.Length - i - 1;
                    if (token[index] == content[0])
                    {
                        stopCut = index;
                        continue;
                    }
                    else
                    {
                        break;
                    }
                }

                return token.Substring(startCut, stopCut - startCut);
            }).ToList();
        }