in src/Compilers/VisualBasic/Portable/Scanner/QuickTokenAccumulator.vb [168:347]
Public Function QuickScanToken(allowLeadingMultilineTrivia As Boolean) As QuickScanResult
Dim state As AccumulatorState = If(allowLeadingMultilineTrivia, AccumulatorState.InitialAllowLeadingMultilineTrivia, AccumulatorState.Initial)
Dim offset = _lineBufferOffset
Dim page = _curPage
If page Is Nothing OrElse
page._pageStart <> (offset And s_NOT_PAGE_MASK) Then
page = GetPage(offset)
End If
Dim pageArr As Char() = page._arr
Dim qtChars = pageArr
Dim index = _lineBufferOffset And s_PAGE_MASK
Dim qtStart = index
Dim limit = index + Math.Min(MAXTOKENSIZE, _bufferLen - offset)
limit = Math.Min(limit, pageArr.Length)
Dim hashCode As Integer = Hash.FnvOffsetBias
Dim terminatorLength As Byte = 0
Dim unicodeValue As Integer = 0
While index < limit
Dim c = pageArr(index)
unicodeValue = AscW(c)
If unicodeValue >= s_CHARPROP_LENGTH Then
Exit While
End If
Dim flags = s_charProperties(unicodeValue)
Select Case state
Case AccumulatorState.InitialAllowLeadingMultilineTrivia
If flags = CharFlags.Letter Then
state = AccumulatorState.Ident
ElseIf flags = CharFlags.Punct Then
state = AccumulatorState.Punctuation
ElseIf flags = CharFlags.CompoundPunctStart Then
state = AccumulatorState.CompoundPunctStart
ElseIf (flags And (CharFlags.White Or CharFlags.CR Or CharFlags.LF)) <> 0 Then
Else
state = AccumulatorState.Bad
Exit While
End If
Case AccumulatorState.Initial
If flags = CharFlags.Letter Then
state = AccumulatorState.Ident
ElseIf flags = CharFlags.Punct Then
state = AccumulatorState.Punctuation
ElseIf flags = CharFlags.CompoundPunctStart Then
state = AccumulatorState.CompoundPunctStart
ElseIf flags = CharFlags.White Then
Else
state = AccumulatorState.Bad
Exit While
End If
Case AccumulatorState.Ident
If (flags And (CharFlags.Letter Or CharFlags.IdentOnly Or CharFlags.Digit)) <> 0 Then
ElseIf flags = CharFlags.White Then
state = AccumulatorState.FollowingWhite
ElseIf flags = CharFlags.CR Then
state = AccumulatorState.CR
ElseIf flags = CharFlags.LF Then
terminatorLength = 1
state = AccumulatorState.Done
Exit While
ElseIf flags = CharFlags.TypeChar Then
state = AccumulatorState.TypeChar
ElseIf flags = CharFlags.Punct Then
state = AccumulatorState.Done
Exit While
Else
state = AccumulatorState.Bad
Exit While
End If
Case AccumulatorState.TypeChar
If flags = CharFlags.White Then
state = AccumulatorState.FollowingWhite
ElseIf flags = CharFlags.CR Then
state = AccumulatorState.CR
ElseIf flags = CharFlags.LF Then
terminatorLength = 1
state = AccumulatorState.Done
Exit While
ElseIf (flags And (CharFlags.Punct Or CharFlags.Digit Or CharFlags.TypeChar)) <> 0 Then
state = AccumulatorState.Done
Exit While
Else
state = AccumulatorState.Bad
Exit While
End If
Case AccumulatorState.FollowingWhite
If flags = CharFlags.White Then
ElseIf flags = CharFlags.CR Then
state = AccumulatorState.CR
ElseIf flags = CharFlags.LF Then
terminatorLength = 1
state = AccumulatorState.Done
Exit While
ElseIf (flags And (CharFlags.Complex Or CharFlags.IdentOnly)) <> 0 Then
state = AccumulatorState.Bad
Exit While
Else
state = AccumulatorState.Done
Exit While
End If
Case AccumulatorState.Punctuation
If flags = CharFlags.White Then
state = AccumulatorState.FollowingWhite
ElseIf flags = CharFlags.CR Then
state = AccumulatorState.CR
ElseIf flags = CharFlags.LF Then
terminatorLength = 1
state = AccumulatorState.Done
Exit While
ElseIf (flags And (CharFlags.Letter Or CharFlags.Punct)) <> 0 Then
state = AccumulatorState.Done
Exit While
Else
state = AccumulatorState.Bad
Exit While
End If
Case AccumulatorState.CompoundPunctStart
If flags = CharFlags.White Then
ElseIf (flags And (CharFlags.Letter Or CharFlags.Digit)) <> 0 Then
state = AccumulatorState.Done
Exit While
Else
state = AccumulatorState.Bad
Exit While
End If
Case AccumulatorState.CR
If flags = CharFlags.LF Then
terminatorLength = 2
state = AccumulatorState.Done
Exit While
Else
state = AccumulatorState.Bad
End If
Exit While
Case Else
Debug.Assert(False, "should not get here")
End Select
index += 1
hashCode = (hashCode Xor unicodeValue) * Hash.FnvPrime
End While
If state = AccumulatorState.Done AndAlso (terminatorLength = 0 OrElse Not Me._IsScanningXmlDoc) Then
If terminatorLength <> 0 Then
index += 1
hashCode = (hashCode Xor unicodeValue) * Hash.FnvPrime
End If
Return New QuickScanResult(qtStart, index - qtStart, qtChars, hashCode, terminatorLength)
Else
Return Nothing
End If
End Function