in hphp/hack/src/parser/core/lexer.rs [1365:1715]
fn scan_token(&mut self, in_type: bool) -> TokenKind {
let ch0 = self.peek_char(0);
match ch0 {
'[' => {
self.advance(1);
TokenKind::LeftBracket
}
']' => {
self.advance(1);
TokenKind::RightBracket
}
'(' => {
self.advance(1);
TokenKind::LeftParen
}
')' => {
self.advance(1);
TokenKind::RightParen
}
'{' => {
self.advance(1);
TokenKind::LeftBrace
}
'}' => {
self.advance(1);
TokenKind::RightBrace
}
'.' => match self.peek_char(1) {
'=' => {
self.advance(2);
TokenKind::DotEqual
}
ch if ('0'..='9').contains(&ch) => self.scan_after_decimal_point(),
'.' => {
if (self.peek_char(2)) == '.' {
self.advance(3);
TokenKind::DotDotDot
} else {
self.advance(1);
TokenKind::Dot
}
}
_ => {
self.advance(1);
TokenKind::Dot
}
},
'-' => match self.peek_char(1) {
'=' => {
self.advance(2);
TokenKind::MinusEqual
}
'-' => {
self.advance(2);
TokenKind::MinusMinus
}
'>' => {
self.advance(2);
TokenKind::MinusGreaterThan
}
_ => {
self.advance(1);
TokenKind::Minus
}
},
'+' => match self.peek_char(1) {
'=' => {
self.advance(2);
TokenKind::PlusEqual
}
'+' => {
self.advance(2);
TokenKind::PlusPlus
}
_ => {
self.advance(1);
TokenKind::Plus
}
},
'*' => match (self.peek_char(1), self.peek_char(2)) {
('=', _) => {
self.advance(2);
TokenKind::StarEqual
}
('*', '=') => {
self.advance(3);
TokenKind::StarStarEqual
}
('*', _) => {
self.advance(2);
TokenKind::StarStar
}
_ => {
self.advance(1);
TokenKind::Star
}
},
'~' => {
self.advance(1);
TokenKind::Tilde
}
'!' => match (self.peek_char(1), self.peek_char(2)) {
('=', '=') => {
self.advance(3);
TokenKind::ExclamationEqualEqual
}
('=', _) => {
self.advance(2);
TokenKind::ExclamationEqual
}
_ => {
self.advance(1);
TokenKind::Exclamation
}
},
'$' => self.scan_dollar_token(),
'/' => {
if (self.peek_char(1)) == '=' {
self.advance(2);
TokenKind::SlashEqual
} else {
self.advance(1);
TokenKind::Slash
}
}
'%' => {
if (self.peek_char(1)) == '=' {
self.advance(2);
TokenKind::PercentEqual
} else {
self.advance(1);
TokenKind::Percent
}
}
'<' => {
match (self.peek_char(1), self.peek_char(2)) {
('<', '<') => self.scan_docstring_literal(),
('<', '=') => {
self.advance(3);
TokenKind::LessThanLessThanEqual
}
// TODO: We lex and parse the spaceship operator.
// TODO: This is not in the spec at present. We should either make it an
// TODO: error, or add it to the specification.
('=', '>') => {
self.advance(3);
TokenKind::LessThanEqualGreaterThan
}
('=', _) => {
self.advance(2);
TokenKind::LessThanEqual
}
('<', _) => {
self.advance(2);
TokenKind::LessThanLessThan
}
_ => {
self.advance(1);
TokenKind::LessThan
}
}
}
'>' => {
match (self.peek_char(1), self.peek_char(2)) {
// If we are parsing a generic type argument list then we might be at the >>
// in `List<List<int>>``, or at the >= of `let x:vec<int>=...`. In that case
// we want to lex two >'s instead of >> / one > and one = instead of >=.
(ch, _) if (ch == '>' || ch == '=') && in_type => {
self.advance(1);
TokenKind::GreaterThan
}
('>', '=') => {
self.advance(3);
TokenKind::GreaterThanGreaterThanEqual
}
('>', _) => {
self.advance(2);
TokenKind::GreaterThanGreaterThan
}
('=', _) => {
self.advance(2);
TokenKind::GreaterThanEqual
}
_ => {
self.advance(1);
TokenKind::GreaterThan
}
}
}
'=' => match (self.peek_char(1), self.peek_char(2)) {
('=', '=') => {
self.advance(3);
TokenKind::EqualEqualEqual
}
('=', '>') => {
self.advance(3);
TokenKind::EqualEqualGreaterThan
}
('=', _) => {
self.advance(2);
TokenKind::EqualEqual
}
('>', _) => {
self.advance(2);
TokenKind::EqualGreaterThan
}
_ => {
self.advance(1);
TokenKind::Equal
}
},
'^' => {
if (self.peek_char(1)) == '=' {
self.advance(2);
TokenKind::CaratEqual
} else {
self.advance(1);
TokenKind::Carat
}
}
'|' => match self.peek_char(1) {
'=' => {
self.advance(2);
TokenKind::BarEqual
}
'>' => {
self.advance(2);
TokenKind::BarGreaterThan
}
'|' => {
self.advance(2);
TokenKind::BarBar
}
_ => {
self.advance(1);
TokenKind::Bar
}
},
'&' => match self.peek_char(1) {
'=' => {
self.advance(2);
TokenKind::AmpersandEqual
}
'&' => {
self.advance(2);
TokenKind::AmpersandAmpersand
}
_ => {
self.advance(1);
TokenKind::Ampersand
}
},
'?' => match (self.peek_char(1), self.peek_char(2)) {
(':', _) if !in_type => {
self.advance(2);
TokenKind::QuestionColon
}
('-', '>') => {
self.advance(3);
TokenKind::QuestionMinusGreaterThan
}
('?', '=') => {
self.advance(3);
TokenKind::QuestionQuestionEqual
}
('?', _) => {
self.advance(2);
TokenKind::QuestionQuestion
}
('a', 's') if !Self::is_name_nondigit(self.peek_char(3)) => {
self.advance(3);
TokenKind::QuestionAs
}
_ => {
self.advance(1);
TokenKind::Question
}
},
':' => {
let ch1 = self.peek_char(1);
if ch1 == ':' {
self.advance(2);
TokenKind::ColonColon
} else {
self.advance(1);
TokenKind::Colon
}
}
';' => {
self.advance(1);
TokenKind::Semicolon
}
',' => {
self.advance(1);
TokenKind::Comma
}
'@' => {
self.advance(1);
TokenKind::At
}
'0' => match self.peek_char(1) {
'x' | 'X' => {
self.advance(2);
self.scan_hex_literal()
}
'b' | 'B' => {
self.advance(2);
self.scan_binary_literal()
}
_ => self.scan_octal_or_float(),
},
ch if ('1'..='9').contains(&ch) => self.scan_decimal_or_float(),
'\'' => self.scan_single_quote_string_literal(),
'"' => self.scan_double_quote_like_string_literal_from_start(),
'`' => {
self.advance(1);
TokenKind::Backtick
}
'\\' => {
self.advance(1);
TokenKind::Backslash
}
'#' => {
self.advance(1);
TokenKind::Hash
}
'b' if {
let c1 = self.peek_char(1);
let c2 = self.peek_char(2);
let c3 = self.peek_char(3);
c1 == '"' || c1 == '\'' || (c1 == '<' && c2 == '<' && c3 == '<')
} =>
{
self.advance(1);
self.scan_token(in_type)
}
// Names
_ => {
if ch0 == INVALID && self.at_end() {
TokenKind::EndOfFile
} else if Self::is_name_nondigit(ch0) {
self.scan_name()
} else {
self.with_error(Errors::error0006);
self.advance(1);
TokenKind::ErrorToken
}
}
}
}