in src/Compilers/CSharp/Portable/Parser/Lexer.cs [422:907]
private void ScanSyntaxToken(ref TokenInfo info)
{
// Initialize for new token scan
info.Kind = SyntaxKind.None;
info.ContextualKind = SyntaxKind.None;
info.Text = null;
char character;
char surrogateCharacter = SlidingTextWindow.InvalidCharacter;
bool isEscaped = false;
int startingPosition = TextWindow.Position;
// Start scanning the token
character = TextWindow.PeekChar();
switch (character)
{
case '\"':
case '\'':
this.ScanStringLiteral(ref info);
break;
case '/':
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.SlashEqualsToken;
}
else
{
info.Kind = SyntaxKind.SlashToken;
}
break;
case '.':
if (!this.ScanNumericLiteral(ref info))
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.DotToken;
}
break;
case ',':
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.CommaToken;
break;
case ':':
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == ':')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.ColonColonToken;
}
else
{
info.Kind = SyntaxKind.ColonToken;
}
break;
case ';':
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.SemicolonToken;
break;
case '~':
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.TildeToken;
break;
case '!':
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.ExclamationEqualsToken;
}
else
{
info.Kind = SyntaxKind.ExclamationToken;
}
break;
case '=':
TextWindow.AdvanceChar();
if ((character = TextWindow.PeekChar()) == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.EqualsEqualsToken;
}
else if (character == '>')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.EqualsGreaterThanToken;
}
else
{
info.Kind = SyntaxKind.EqualsToken;
}
break;
case '*':
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.AsteriskEqualsToken;
}
else
{
info.Kind = SyntaxKind.AsteriskToken;
}
break;
case '(':
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.OpenParenToken;
break;
case ')':
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.CloseParenToken;
break;
case '{':
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.OpenBraceToken;
break;
case '}':
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.CloseBraceToken;
break;
case '[':
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.OpenBracketToken;
break;
case ']':
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.CloseBracketToken;
break;
case '?':
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == '?')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.QuestionQuestionToken;
}
else
{
info.Kind = SyntaxKind.QuestionToken;
}
break;
case '+':
TextWindow.AdvanceChar();
if ((character = TextWindow.PeekChar()) == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.PlusEqualsToken;
}
else if (character == '+')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.PlusPlusToken;
}
else
{
info.Kind = SyntaxKind.PlusToken;
}
break;
case '-':
TextWindow.AdvanceChar();
if ((character = TextWindow.PeekChar()) == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.MinusEqualsToken;
}
else if (character == '-')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.MinusMinusToken;
}
else if (character == '>')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.MinusGreaterThanToken;
}
else
{
info.Kind = SyntaxKind.MinusToken;
}
break;
case '%':
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.PercentEqualsToken;
}
else
{
info.Kind = SyntaxKind.PercentToken;
}
break;
case '&':
TextWindow.AdvanceChar();
if ((character = TextWindow.PeekChar()) == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.AmpersandEqualsToken;
}
else if (TextWindow.PeekChar() == '&')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.AmpersandAmpersandToken;
}
else
{
info.Kind = SyntaxKind.AmpersandToken;
}
break;
case '^':
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.CaretEqualsToken;
}
else
{
info.Kind = SyntaxKind.CaretToken;
}
break;
case '|':
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.BarEqualsToken;
}
else if (TextWindow.PeekChar() == '|')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.BarBarToken;
}
else
{
info.Kind = SyntaxKind.BarToken;
}
break;
case '<':
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.LessThanEqualsToken;
}
else if (TextWindow.PeekChar() == '<')
{
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.LessThanLessThanEqualsToken;
}
else
{
info.Kind = SyntaxKind.LessThanLessThanToken;
}
}
else
{
info.Kind = SyntaxKind.LessThanToken;
}
break;
case '>':
TextWindow.AdvanceChar();
if (TextWindow.PeekChar() == '=')
{
TextWindow.AdvanceChar();
info.Kind = SyntaxKind.GreaterThanEqualsToken;
}
else
{
info.Kind = SyntaxKind.GreaterThanToken;
}
break;
case '@':
if (TextWindow.PeekChar(1) == '"')
{
this.ScanVerbatimStringLiteral(ref info);
}
else if (!this.ScanIdentifierOrKeyword(ref info))
{
TextWindow.AdvanceChar();
info.Text = TextWindow.GetText(intern: true);
this.AddError(ErrorCode.ERR_ExpectedVerbatimLiteral);
}
break;
case '$':
if (TextWindow.PeekChar(1) == '"')
{
this.ScanInterpolatedStringLiteral(false, ref info);
CheckFeatureAvailability(MessageID.IDS_FeatureInterpolatedStrings);
break;
}
else if (TextWindow.PeekChar(1) == '@' && TextWindow.PeekChar(2) == '"')
{
this.ScanInterpolatedStringLiteral(true, ref info);
CheckFeatureAvailability(MessageID.IDS_FeatureInterpolatedStrings);
break;
}
else if (this.ModeIs(LexerMode.DebuggerSyntax))
{
goto case 'a';
}
goto default;
// All the 'common' identifier characters are represented directly in
// these switch cases for optimal perf. Calling IsIdentifierChar() functions is relatively
// expensive.
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
case 'G':
case 'H':
case 'I':
case 'J':
case 'K':
case 'L':
case 'M':
case 'N':
case 'O':
case 'P':
case 'Q':
case 'R':
case 'S':
case 'T':
case 'U':
case 'V':
case 'W':
case 'X':
case 'Y':
case 'Z':
case '_':
this.ScanIdentifierOrKeyword(ref info);
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
this.ScanNumericLiteral(ref info);
break;
case '\\':
{
// Could be unicode escape. Try that.
character = TextWindow.PeekCharOrUnicodeEscape(out surrogateCharacter);
isEscaped = true;
if (SyntaxFacts.IsIdentifierStartCharacter(character))
{
goto case 'a';
}
goto default;
}
case SlidingTextWindow.InvalidCharacter:
if (!TextWindow.IsReallyAtEnd())
{
goto default;
}
if (_directives.HasUnfinishedIf())
{
this.AddError(ErrorCode.ERR_EndifDirectiveExpected);
}
if (_directives.HasUnfinishedRegion())
{
this.AddError(ErrorCode.ERR_EndRegionDirectiveExpected);
}
info.Kind = SyntaxKind.EndOfFileToken;
break;
default:
if (SyntaxFacts.IsIdentifierStartCharacter(character))
{
goto case 'a';
}
if (isEscaped)
{
SyntaxDiagnosticInfo error;
TextWindow.NextCharOrUnicodeEscape(out surrogateCharacter, out error);
AddError(error);
}
else
{
TextWindow.AdvanceChar();
}
if (_badTokenCount++ > 200)
{
// If we get too many characters that we cannot make sense of, absorb the rest of the input.
int end = TextWindow.Text.Length;
int width = end - startingPosition;
info.Text = TextWindow.Text.ToString(new TextSpan(startingPosition, width));
TextWindow.Reset(end);
}
else
{
info.Text = TextWindow.GetText(intern: true);
}
this.AddError(ErrorCode.ERR_UnexpectedCharacter, info.Text);
break;
}
}