function tokenizeDefault()

in src/powerquery-parser/lexer/lexer.ts [789:920]


function tokenizeDefault(locale: string, line: TLine, lineNumber: number, positionStart: number): LineModeAlteringRead {
    const text: string = line.text;

    const chr1: string = text[positionStart];
    let token: Token.LineToken;
    let lineMode: LineMode = LineMode.Default;

    if (chr1 === "!") {
        token = readConstant(Token.LineTokenKind.Bang, text, positionStart, 1);
    } else if (chr1 === "&") {
        token = readConstant(Token.LineTokenKind.Ampersand, text, positionStart, 1);
    } else if (chr1 === "(") {
        token = readConstant(Token.LineTokenKind.LeftParenthesis, text, positionStart, 1);
    } else if (chr1 === ")") {
        token = readConstant(Token.LineTokenKind.RightParenthesis, text, positionStart, 1);
    } else if (chr1 === "*") {
        token = readConstant(Token.LineTokenKind.Asterisk, text, positionStart, 1);
    } else if (chr1 === "+") {
        token = readConstant(Token.LineTokenKind.Plus, text, positionStart, 1);
    } else if (chr1 === ",") {
        token = readConstant(Token.LineTokenKind.Comma, text, positionStart, 1);
    } else if (chr1 === "-") {
        token = readConstant(Token.LineTokenKind.Minus, text, positionStart, 1);
    } else if (chr1 === ";") {
        token = readConstant(Token.LineTokenKind.Semicolon, text, positionStart, 1);
    } else if (chr1 === "?") {
        const chr2: string | undefined = text[positionStart + 1];

        if (chr2 === "?") {
            token = readConstant(Token.LineTokenKind.NullCoalescingOperator, text, positionStart, 2);
        } else {
            token = readConstant(Token.LineTokenKind.QuestionMark, text, positionStart, 1);
        }
    } else if (chr1 === "@") {
        token = readConstant(Token.LineTokenKind.AtSign, text, positionStart, 1);
    } else if (chr1 === "[") {
        token = readConstant(Token.LineTokenKind.LeftBracket, text, positionStart, 1);
    } else if (chr1 === "]") {
        token = readConstant(Token.LineTokenKind.RightBracket, text, positionStart, 1);
    } else if (chr1 === "{") {
        token = readConstant(Token.LineTokenKind.LeftBrace, text, positionStart, 1);
    } else if (chr1 === "}") {
        token = readConstant(Token.LineTokenKind.RightBrace, text, positionStart, 1);
    } else if (chr1 === '"') {
        const read: LineModeAlteringRead = readOrStartTextLiteral(text, positionStart);
        token = read.token;
        lineMode = read.lineMode;
    } else if (chr1 === "0") {
        const chr2: string | undefined = text[positionStart + 1];

        if (chr2 === "x" || chr2 === "X") {
            token = readHexLiteral(locale, text, lineNumber, positionStart);
        } else {
            token = readNumericLiteral(locale, text, lineNumber, positionStart);
        }
    } else if ("1" <= chr1 && chr1 <= "9") {
        token = readNumericLiteral(locale, text, lineNumber, positionStart);
    } else if (chr1 === ".") {
        const chr2: string | undefined = text[positionStart + 1];

        if (chr2 === undefined) {
            throw new LexError.UnexpectedEofError(locale, graphemePositionFrom(text, lineNumber, positionStart));
        } else if ("1" <= chr2 && chr2 <= "9") {
            token = readNumericLiteral(locale, text, lineNumber, positionStart);
        } else if (chr2 === ".") {
            const chr3: string = text[positionStart + 2];

            if (chr3 === ".") {
                token = readConstant(Token.LineTokenKind.Ellipsis, text, positionStart, 3);
            } else {
                token = readConstant(Token.LineTokenKind.DotDot, text, positionStart, 2);
            }
        } else {
            throw unexpectedReadError(locale, text, lineNumber, positionStart);
        }
    } else if (chr1 === ">") {
        const chr2: string | undefined = text[positionStart + 1];

        if (chr2 === "=") {
            token = readConstant(Token.LineTokenKind.GreaterThanEqualTo, text, positionStart, 2);
        } else {
            token = readConstant(Token.LineTokenKind.GreaterThan, text, positionStart, 1);
        }
    } else if (chr1 === "<") {
        const chr2: string | undefined = text[positionStart + 1];

        if (chr2 === "=") {
            token = readConstant(Token.LineTokenKind.LessThanEqualTo, text, positionStart, 2);
        } else if (chr2 === ">") {
            token = readConstant(Token.LineTokenKind.NotEqual, text, positionStart, 2);
        } else {
            token = readConstant(Token.LineTokenKind.LessThan, text, positionStart, 1);
        }
    } else if (chr1 === "=") {
        const chr2: string | undefined = text[positionStart + 1];

        if (chr2 === ">") {
            token = readConstant(Token.LineTokenKind.FatArrow, text, positionStart, 2);
        } else {
            token = readConstant(Token.LineTokenKind.Equal, text, positionStart, 1);
        }
    } else if (chr1 === "/") {
        const chr2: string | undefined = text[positionStart + 1];

        if (chr2 === "/") {
            token = readLineComment(text, positionStart);
        } else if (chr2 === "*") {
            const read: LineModeAlteringRead = readOrStartMultilineComment(text, positionStart);
            token = read.token;
            lineMode = read.lineMode;
        } else {
            token = readConstant(Token.LineTokenKind.Division, text, positionStart, 1);
        }
    } else if (chr1 === "#") {
        const chr2: string | undefined = text[positionStart + 1];

        if (chr2 === '"') {
            const read: LineModeAlteringRead = readOrStartQuotedIdentifier(text, positionStart);
            token = read.token;
            lineMode = read.lineMode;
        } else {
            token = readKeyword(locale, text, lineNumber, positionStart);
        }
    } else {
        token = readKeywordOrIdentifier(locale, text, lineNumber, positionStart);
    }

    return {
        token,
        lineMode,
    };
}