in src/powerquery-parser/lexer/lexer.ts [789:920]
function tokenizeDefault(locale: string, line: TLine, lineNumber: number, positionStart: number): LineModeAlteringRead {
const text: string = line.text;
const chr1: string = text[positionStart];
let token: Token.LineToken;
let lineMode: LineMode = LineMode.Default;
if (chr1 === "!") {
token = readConstant(Token.LineTokenKind.Bang, text, positionStart, 1);
} else if (chr1 === "&") {
token = readConstant(Token.LineTokenKind.Ampersand, text, positionStart, 1);
} else if (chr1 === "(") {
token = readConstant(Token.LineTokenKind.LeftParenthesis, text, positionStart, 1);
} else if (chr1 === ")") {
token = readConstant(Token.LineTokenKind.RightParenthesis, text, positionStart, 1);
} else if (chr1 === "*") {
token = readConstant(Token.LineTokenKind.Asterisk, text, positionStart, 1);
} else if (chr1 === "+") {
token = readConstant(Token.LineTokenKind.Plus, text, positionStart, 1);
} else if (chr1 === ",") {
token = readConstant(Token.LineTokenKind.Comma, text, positionStart, 1);
} else if (chr1 === "-") {
token = readConstant(Token.LineTokenKind.Minus, text, positionStart, 1);
} else if (chr1 === ";") {
token = readConstant(Token.LineTokenKind.Semicolon, text, positionStart, 1);
} else if (chr1 === "?") {
const chr2: string | undefined = text[positionStart + 1];
if (chr2 === "?") {
token = readConstant(Token.LineTokenKind.NullCoalescingOperator, text, positionStart, 2);
} else {
token = readConstant(Token.LineTokenKind.QuestionMark, text, positionStart, 1);
}
} else if (chr1 === "@") {
token = readConstant(Token.LineTokenKind.AtSign, text, positionStart, 1);
} else if (chr1 === "[") {
token = readConstant(Token.LineTokenKind.LeftBracket, text, positionStart, 1);
} else if (chr1 === "]") {
token = readConstant(Token.LineTokenKind.RightBracket, text, positionStart, 1);
} else if (chr1 === "{") {
token = readConstant(Token.LineTokenKind.LeftBrace, text, positionStart, 1);
} else if (chr1 === "}") {
token = readConstant(Token.LineTokenKind.RightBrace, text, positionStart, 1);
} else if (chr1 === '"') {
const read: LineModeAlteringRead = readOrStartTextLiteral(text, positionStart);
token = read.token;
lineMode = read.lineMode;
} else if (chr1 === "0") {
const chr2: string | undefined = text[positionStart + 1];
if (chr2 === "x" || chr2 === "X") {
token = readHexLiteral(locale, text, lineNumber, positionStart);
} else {
token = readNumericLiteral(locale, text, lineNumber, positionStart);
}
} else if ("1" <= chr1 && chr1 <= "9") {
token = readNumericLiteral(locale, text, lineNumber, positionStart);
} else if (chr1 === ".") {
const chr2: string | undefined = text[positionStart + 1];
if (chr2 === undefined) {
throw new LexError.UnexpectedEofError(locale, graphemePositionFrom(text, lineNumber, positionStart));
} else if ("1" <= chr2 && chr2 <= "9") {
token = readNumericLiteral(locale, text, lineNumber, positionStart);
} else if (chr2 === ".") {
const chr3: string = text[positionStart + 2];
if (chr3 === ".") {
token = readConstant(Token.LineTokenKind.Ellipsis, text, positionStart, 3);
} else {
token = readConstant(Token.LineTokenKind.DotDot, text, positionStart, 2);
}
} else {
throw unexpectedReadError(locale, text, lineNumber, positionStart);
}
} else if (chr1 === ">") {
const chr2: string | undefined = text[positionStart + 1];
if (chr2 === "=") {
token = readConstant(Token.LineTokenKind.GreaterThanEqualTo, text, positionStart, 2);
} else {
token = readConstant(Token.LineTokenKind.GreaterThan, text, positionStart, 1);
}
} else if (chr1 === "<") {
const chr2: string | undefined = text[positionStart + 1];
if (chr2 === "=") {
token = readConstant(Token.LineTokenKind.LessThanEqualTo, text, positionStart, 2);
} else if (chr2 === ">") {
token = readConstant(Token.LineTokenKind.NotEqual, text, positionStart, 2);
} else {
token = readConstant(Token.LineTokenKind.LessThan, text, positionStart, 1);
}
} else if (chr1 === "=") {
const chr2: string | undefined = text[positionStart + 1];
if (chr2 === ">") {
token = readConstant(Token.LineTokenKind.FatArrow, text, positionStart, 2);
} else {
token = readConstant(Token.LineTokenKind.Equal, text, positionStart, 1);
}
} else if (chr1 === "/") {
const chr2: string | undefined = text[positionStart + 1];
if (chr2 === "/") {
token = readLineComment(text, positionStart);
} else if (chr2 === "*") {
const read: LineModeAlteringRead = readOrStartMultilineComment(text, positionStart);
token = read.token;
lineMode = read.lineMode;
} else {
token = readConstant(Token.LineTokenKind.Division, text, positionStart, 1);
}
} else if (chr1 === "#") {
const chr2: string | undefined = text[positionStart + 1];
if (chr2 === '"') {
const read: LineModeAlteringRead = readOrStartQuotedIdentifier(text, positionStart);
token = read.token;
lineMode = read.lineMode;
} else {
token = readKeyword(locale, text, lineNumber, positionStart);
}
} else {
token = readKeywordOrIdentifier(locale, text, lineNumber, positionStart);
}
return {
token,
lineMode,
};
}