in src/vs/base/common/json.ts [362:517]
function scanNext(): SyntaxKind {
value = '';
scanError = ScanError.None;
tokenOffset = pos;
if (pos >= len) {
// at the end
tokenOffset = len;
return token = SyntaxKind.EOF;
}
let code = text.charCodeAt(pos);
// trivia: whitespace
if (isWhiteSpace(code)) {
do {
pos++;
value += String.fromCharCode(code);
code = text.charCodeAt(pos);
} while (isWhiteSpace(code));
return token = SyntaxKind.Trivia;
}
// trivia: newlines
if (isLineBreak(code)) {
pos++;
value += String.fromCharCode(code);
if (code === CharacterCodes.carriageReturn && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
pos++;
value += '\n';
}
return token = SyntaxKind.LineBreakTrivia;
}
switch (code) {
// tokens: []{}:,
case CharacterCodes.openBrace:
pos++;
return token = SyntaxKind.OpenBraceToken;
case CharacterCodes.closeBrace:
pos++;
return token = SyntaxKind.CloseBraceToken;
case CharacterCodes.openBracket:
pos++;
return token = SyntaxKind.OpenBracketToken;
case CharacterCodes.closeBracket:
pos++;
return token = SyntaxKind.CloseBracketToken;
case CharacterCodes.colon:
pos++;
return token = SyntaxKind.ColonToken;
case CharacterCodes.comma:
pos++;
return token = SyntaxKind.CommaToken;
// strings
case CharacterCodes.doubleQuote:
pos++;
value = scanString();
return token = SyntaxKind.StringLiteral;
// comments
case CharacterCodes.slash:
const start = pos - 1;
// Single-line comment
if (text.charCodeAt(pos + 1) === CharacterCodes.slash) {
pos += 2;
while (pos < len) {
if (isLineBreak(text.charCodeAt(pos))) {
break;
}
pos++;
}
value = text.substring(start, pos);
return token = SyntaxKind.LineCommentTrivia;
}
// Multi-line comment
if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) {
pos += 2;
const safeLength = len - 1; // For lookahead.
let commentClosed = false;
while (pos < safeLength) {
const ch = text.charCodeAt(pos);
if (ch === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) {
pos += 2;
commentClosed = true;
break;
}
pos++;
}
if (!commentClosed) {
pos++;
scanError = ScanError.UnexpectedEndOfComment;
}
value = text.substring(start, pos);
return token = SyntaxKind.BlockCommentTrivia;
}
// just a single slash
value += String.fromCharCode(code);
pos++;
return token = SyntaxKind.Unknown;
// numbers
case CharacterCodes.minus:
value += String.fromCharCode(code);
pos++;
if (pos === len || !isDigit(text.charCodeAt(pos))) {
return token = SyntaxKind.Unknown;
}
// found a minus, followed by a number so
// we fall through to proceed with scanning
// numbers
case CharacterCodes._0:
case CharacterCodes._1:
case CharacterCodes._2:
case CharacterCodes._3:
case CharacterCodes._4:
case CharacterCodes._5:
case CharacterCodes._6:
case CharacterCodes._7:
case CharacterCodes._8:
case CharacterCodes._9:
value += scanNumber();
return token = SyntaxKind.NumericLiteral;
// literals and unknown symbols
default:
// is a literal? Read the full word.
while (pos < len && isUnknownContentCharacter(code)) {
pos++;
code = text.charCodeAt(pos);
}
if (tokenOffset !== pos) {
value = text.substring(tokenOffset, pos);
// keywords: true, false, null
switch (value) {
case 'true': return token = SyntaxKind.TrueKeyword;
case 'false': return token = SyntaxKind.FalseKeyword;
case 'null': return token = SyntaxKind.NullKeyword;
}
return token = SyntaxKind.Unknown;
}
// some
value += String.fromCharCode(code);
pos++;
return token = SyntaxKind.Unknown;
}
}