function tokenize()

in src/PatternParser/tokenize.php [15:31]

• 17 lines of code
• 4 McCabe index (conditional complexity)

function tokenize(string $pattern): vec<Token> {
  $tokens = vec[];
  $buffer = '';
  foreach (Str\split($pattern, '') as $byte) {
    if (TokenType::isValid($byte)) {
      $tokens[] = tuple(TokenType::STRING, $buffer);
      $buffer = '';
      $tokens[] = tuple(TokenType::assert($byte), $byte);
    } else {
      $buffer .= $byte;
    }
  }
  if ($buffer !== '') {
    $tokens[] = tuple(TokenType::STRING, $buffer);
  }
  return Vec\filter($tokens, $t ==> $t !== tuple(TokenType::STRING, ''));
}