public override List PreTokenizeText()

in Runtime/Tokenizers/PreTokenizers/PreTokenizers.cs [133:144]

• 10 lines of code
• 2 McCabe index (conditional complexity)

        public override List<string> PreTokenizeText(string text)
        {
            var matches = this.pattern.Matches(text.Trim());
            var tokens = new List<string>();

            foreach (Match match in matches)
            {
                tokens.Add(match.Value);
            }

            return tokens;
        }