public List PreTokenize()

in Runtime/Tokenizers/PreTokenizers/PreTokenizers.cs [68:92]

• 17 lines of code
• 3 McCabe index (conditional complexity)

        public List<string> PreTokenize(object text)
        {

            List<string> result = new List<string>();

            if (text is string textString)
            {
                // Else if 'text' is not a list, tokenize it directly using 'pre_tokenize_text'
                //result.Add(PreTokenizeText(textString));
                result.AddRange(PreTokenizeText(textString));
            }
            else if (text is List<string> textList)
            {
                // If 'text' is List select each element and tokenize it using 'pre_tokenize_text'
                //result = textList.Select(x => PreTokenizeText(x)).ToList();
                result = textList.SelectMany(x => PreTokenizeText(x)).ToList();
            }
            else
            {
                throw new ArgumentException("Unsupported parameter type");
            }
            // Flatten the 'result' list of lists and return a flat list
            //return result.SelectMany(x => x).ToList();
            return result;
        }