public List ConvertTokensToIds()

in Runtime/Tokenizers/Tokenizers/Tokenizers.cs [116:139]

• 14 lines of code
• 4 McCabe index (conditional complexity)

        public List<int> ConvertTokensToIds(List<string> tokens)
        {
            // Create an array of token IDs by mapping each token to its corresponding ID
            List<int> ids = new List<int>();

            //Debug.Log("Dictionary Contents:");
            foreach (var kvp in TokensToIds)
            {
                Debug.Log($"Key: {kvp.Key}, Value: {kvp.Value}");
            }

            // token.Select: applies an operation to each token in tokens
            ids = tokens.Select(t => TokensToIds.TryGetValue(t, out int id) ? id : UnknownTokenId).ToList();



            if (FuseUnk == false)
            {
                //ids = Fuse(ids, UnkTokenId);
                ids = TokenizersUtils.Utils.Fuse(ids, UnkTokenId ?? -1);
            }

            return ids;
        }