public function getTokens()

in src/lexer/PhutilLexer.php [241:329]
76 lines of code
16 McCabe index (conditional complexity)

  public function getTokens($input, $initial_state = 'start') {
    if (empty($this->processedRules)) {
      $this->processedRules = $this->getRules();
    }
    $rules = $this->processedRules;

    $this->lastState = null;

    $position = 0;
    $length   = strlen($input);

    $tokens = array();
    $states = array();
    $states[] = 'start';
    if ($initial_state != 'start') {
      $states[] = $initial_state;
    }
    $context = array();
    while ($position < $length) {
      $state_rules = idx($rules, end($states), array());
      foreach ($state_rules as $rule) {

        $matches = null;
        if (!preg_match($rule[0], $input, $matches, 0, $position)) {
          continue;
        }

        list($regexp, $token_type, $next_state, $options) = $rule;

        $match_length = strlen($matches[0]);
        if (!$match_length) {
          if ($next_state === null) {
            throw new UnexpectedValueException(
              pht(
                "Rule '%s' matched a zero-length token and causes no ".
                "state transition.",
                $regexp));
          }
        } else {
          $position += $match_length;
          $token = array($token_type, $matches[0]);

          $copt = idx($options, 'context');
          if ($copt == 'push') {
            $context[] = $matches[0];
            $token[] = null;
          } else if ($copt == 'pop') {
            if (empty($context)) {
              throw new UnexpectedValueException(
                pht("Rule '%s' popped empty context!", $regexp));
            }
            $token[] = array_pop($context);
          } else if ($copt == 'discard') {
            if (empty($context)) {
              throw new UnexpectedValueException(
                pht("Rule '%s' discarded empty context!", $regexp));
            }
            array_pop($context);
            $token[] = null;
          } else {
            $token[] = null;
          }

          $tokens[] = $token;
        }

        if ($next_state !== null) {
          if ($next_state == '!pop') {
            array_pop($states);
            if (empty($states)) {
              throw new UnexpectedValueException(
                pht("Rule '%s' popped off the last state.", $regexp));
            }
          } else {
            $states[] = $next_state;
          }
        }

        continue 2;
      }

      throw new UnexpectedValueException(
        pht('No lexer rule matched input at char %d.', $position));
    }

    $this->lastState = $states;

    return $tokens;
  }