in src/Parsing/Impl/Tokens/Tokenizer.cs [1961:2077]
private bool ReadIndentationAfterNewLine(NewLineKind startingKind) {
// Keep track of the indentation format for the current line
StringBuilder sb = null; // the white space we've encounted after the new line if it's mixed tabs/spaces or is an unreasonable size.
var noAllocWhiteSpace = String.Empty; // the white space we've encountered after the newline assuming it's a reasonable sized run of all spaces or tabs
bool? isSpace = null; // the current mix of whitespace, null = nothing yet, true = space, false = tab
var spaces = 0;
var indentStart = CurrentIndex;
while (true) {
var ch = NextChar();
switch (ch) {
case ' ':
if (Verbatim) {
_state.NextWhiteSpace.Append((char)ch);
}
spaces += 1;
AppendSpace(ref noAllocWhiteSpace, ref sb, ref isSpace);
break;
case '\t':
if (Verbatim) {
_state.NextWhiteSpace.Append((char)ch);
}
spaces += 8 - (spaces % 8);
AppendTab(ref noAllocWhiteSpace, ref sb, ref isSpace);
break;
case '\f':
if (Verbatim) {
_state.NextWhiteSpace.Append((char)ch);
}
spaces = 0;
if (sb == null) {
sb = new StringBuilder();
sb.Append(noAllocWhiteSpace);
}
sb.Append('\f');
break;
case '#':
_commentLocations.Add(CurrentPosition.AddColumns(-1));
if ((_options & TokenizerOptions.VerbatimCommentsAndLineJoins) != 0) {
BufferBack();
MarkTokenEnd();
return true;
} else {
BufferBack();
DiscardToken();
var commentRes = ReadSingleLineComment(out ch);
if ((_options & TokenizerOptions.Verbatim) != 0) {
_state.NextWhiteSpace.Append(commentRes.VerbatimImage);
}
DiscardToken();
}
break;
default:
BufferBack();
if (GroupingLevel > 0) {
var startingWhiteSpace = 0;
if (Verbatim) {
// we're not producing a new line after all... All of the white space
// we collected goes to the current token, including the new line token
// that we're not producing.
startingWhiteSpace = _state.CurWhiteSpace.Length;
_state.CurWhiteSpace.Append(startingKind.GetString());
_state.CurWhiteSpace.Append(_state.NextWhiteSpace);
_state.NextWhiteSpace.Clear();
}
if ((_options & TokenizerOptions.GroupingRecovery) != 0) {
var tokenEnd = Math.Min(_position, _end);
var tokenLength = tokenEnd - _start;
_state.GroupingRecovery = new GroupingRecovery(
startingKind,
noAllocWhiteSpace,
spaces,
sb,
_tokenStartIndex,
startingWhiteSpace,
_tokenStartIndex + tokenLength
);
}
return false;
}
_state.GroupingRecovery = null;
MarkTokenEnd();
if (_tokenEndIndex != _tokenStartIndex) {
// We've captured a line of significant indentation
// (i.e. not pure whitespace or comment). Check that
// any of this indentation that's in common with the
// current indent level is constructed in exactly
// the same way (i.e. has the same mix of spaces and
// tabs etc.).
CheckIndent(sb, noAllocWhiteSpace, _tokenStartIndex + startingKind.GetSize());
}
// if there's a blank line then we don't want to mess w/ the
// indentation level - Python says that blank lines are ignored.
// And if we're the last blank line in a file we don't want to
// increase the new indentation level.
if (ch == EOF) {
if (spaces < _state.Indent[_state.IndentLevel]) {
if (_kind == SourceCodeKind.InteractiveCode ||
_kind == SourceCodeKind.Statements) {
SetIndent(spaces, sb, noAllocWhiteSpace, indentStart);
} else {
DoDedent(spaces, _state.Indent[_state.IndentLevel]);
}
}
} else if (ch != '\n' && ch != '\r') {
SetIndent(spaces, sb, noAllocWhiteSpace, indentStart);
}
return true;
}
}
}