public List ConvertTokensToIds()

in Runtime/Tokenizers/Tokenizers/Tokenizers.cs [307:324]

• 10 lines of code
• 3 McCabe index (conditional complexity)

        public List<int> ConvertTokensToIds(List<string> tokens)
        {
            // Create an array of token IDs by mapping each token to its corresponding ID
            List<int> ids = new List<int>();

            // token.Select: applies an operation to each token in tokens
            ids = tokens.Select(t => TokensToIds.TryGetValue(t, out int id) ? id : UnknownTokenId).ToList();



            if (FuseUnk == false)
            {
                //ids = Fuse(ids, UnkTokenId);
                ids = Fuse(ids, UnkTokenId ?? -1);
            }

            return ids;
        }