in packages/jinja/src/lexer.ts [189:324]
main: while (cursorPosition < src.length) {
// First, consume all text that is outside of a Jinja statement or expression
const lastTokenType = tokens.at(-1)?.type;
if (
lastTokenType === undefined ||
lastTokenType === TOKEN_TYPES.CloseStatement ||
lastTokenType === TOKEN_TYPES.CloseExpression ||
lastTokenType === TOKEN_TYPES.Comment
) {
let text = "";
while (
cursorPosition < src.length &&
// Keep going until we hit the next Jinja statement or expression
!(
src[cursorPosition] === "{" &&
(src[cursorPosition + 1] === "%" || src[cursorPosition + 1] === "{" || src[cursorPosition + 1] === "#")
)
) {
// Consume text
text += src[cursorPosition++];
}
// There is some text to add
if (text.length > 0) {
tokens.push(new Token(text, TOKEN_TYPES.Text));
continue;
}
}
// Possibly consume a comment
if (src[cursorPosition] === "{" && src[cursorPosition + 1] === "#") {
cursorPosition += 2; // Skip the opening {#
let comment = "";
while (src[cursorPosition] !== "#" || src[cursorPosition + 1] !== "}") {
// Check for end of input
if (cursorPosition + 2 >= src.length) {
throw new SyntaxError("Missing end of comment tag");
}
comment += src[cursorPosition++];
}
tokens.push(new Token(comment, TOKEN_TYPES.Comment));
cursorPosition += 2; // Skip the closing #}
continue;
}
// Consume (and ignore) all whitespace inside Jinja statements or expressions
consumeWhile((char) => /\s/.test(char));
// Handle multi-character tokens
const char = src[cursorPosition];
// Check for unary operators
if (char === "-" || char === "+") {
const lastTokenType = tokens.at(-1)?.type;
if (lastTokenType === TOKEN_TYPES.Text || lastTokenType === undefined) {
throw new SyntaxError(`Unexpected character: ${char}`);
}
switch (lastTokenType) {
case TOKEN_TYPES.Identifier:
case TOKEN_TYPES.NumericLiteral:
case TOKEN_TYPES.StringLiteral:
case TOKEN_TYPES.CloseParen:
case TOKEN_TYPES.CloseSquareBracket:
// Part of a binary operator
// a - 1, 1 - 1, true - 1, "apple" - 1, (1) - 1, a[1] - 1
// Continue parsing normally
break;
default: {
// Is part of a unary operator
// (-1), [-1], (1 + -1), not -1, -apple
++cursorPosition; // consume the unary operator
// Check for numbers following the unary operator
const num = consumeWhile(isInteger);
tokens.push(
new Token(`${char}${num}`, num.length > 0 ? TOKEN_TYPES.NumericLiteral : TOKEN_TYPES.UnaryOperator)
);
continue;
}
}
}
// Try to match one of the tokens in the mapping table
for (const [seq, type] of ORDERED_MAPPING_TABLE) {
// inside an object literal, don't treat "}}" as expression-end
if (seq === "}}" && curlyBracketDepth > 0) {
continue;
}
const slice = src.slice(cursorPosition, cursorPosition + seq.length);
if (slice === seq) {
tokens.push(new Token(seq, type));
// possibly adjust the curly bracket depth
if (type === TOKEN_TYPES.OpenExpression) {
curlyBracketDepth = 0;
} else if (type === TOKEN_TYPES.OpenCurlyBracket) {
++curlyBracketDepth;
} else if (type === TOKEN_TYPES.CloseCurlyBracket) {
--curlyBracketDepth;
}
cursorPosition += seq.length;
continue main;
}
}
if (char === "'" || char === '"') {
++cursorPosition; // Skip the opening quote
const str = consumeWhile((c) => c !== char);
tokens.push(new Token(str, TOKEN_TYPES.StringLiteral));
++cursorPosition; // Skip the closing quote
continue;
}
if (isInteger(char)) {
// Consume integer part
let num = consumeWhile(isInteger);
// Possibly, consume fractional part
if (src[cursorPosition] === "." && isInteger(src[cursorPosition + 1])) {
++cursorPosition; // consume '.'
const frac = consumeWhile(isInteger);
num = `${num}.${frac}`;
}
tokens.push(new Token(num, TOKEN_TYPES.NumericLiteral));
continue;
}
if (isWord(char)) {
// consume any word characters and always classify as Identifier
const word = consumeWhile(isWord);
tokens.push(new Token(word, TOKEN_TYPES.Identifier));
continue;
}
throw new SyntaxError(`Unexpected character: ${char}`);
}