in src/dotnet/APIView/APIView/Model/CodeFile.cs [172:348]
public void ConvertToTreeTokenModel()
{
Dictionary<string, string> navigationItems = new Dictionary<string, string>();
ReviewLine reviewLine = new ReviewLine();
ReviewLine previousLine = null;
bool isDocumentation = false;
bool isHidden = false;
bool skipDiff = false;
bool isDeprecated = false;
bool skipIndent = false;
string className = "";
//Process all navigation items in old model to generate a map
GetNavigationMap(navigationItems, Navigation);
List<ReviewToken> currentLineTokens = new List<ReviewToken>();
foreach(var oldToken in Tokens)
{
//Don't include documentation in converted code file due to incorrect documentation formatting used in previous model.
if (isDocumentation && oldToken.Kind != CodeFileTokenKind.DocumentRangeEnd)
continue;
ReviewToken token = null;
switch(oldToken.Kind)
{
case CodeFileTokenKind.DocumentRangeStart:
isDocumentation = true; break;
case CodeFileTokenKind.DocumentRangeEnd:
isDocumentation = false; break;
case CodeFileTokenKind.DeprecatedRangeStart:
isDeprecated = true; break;
case CodeFileTokenKind.DeprecatedRangeEnd:
isDeprecated = false; break;
case CodeFileTokenKind.SkipDiffRangeStart:
skipDiff = true; break;
case CodeFileTokenKind.SkipDiffRangeEnd:
skipDiff = false; break;
case CodeFileTokenKind.HiddenApiRangeStart:
isHidden = true; break;
case CodeFileTokenKind.HiddenApiRangeEnd:
isHidden = false; break;
case CodeFileTokenKind.Keyword:
token = ReviewToken.CreateKeywordToken(oldToken.Value, false);
var keywordValue = oldToken.Value.ToLower();
if (keywordValue == "class" || keywordValue == "enum" || keywordValue == "struct" || keywordValue == "interface" || keywordValue == "type" || keywordValue == "namespace")
className = keywordValue;
break;
case CodeFileTokenKind.Comment:
token = ReviewToken.CreateCommentToken(oldToken.Value, false);
break;
case CodeFileTokenKind.Text:
token = ReviewToken.CreateTextToken(oldToken.Value, oldToken.NavigateToId, false);
break;
case CodeFileTokenKind.Punctuation:
token = ReviewToken.CreatePunctuationToken(oldToken.Value, false);
break;
case CodeFileTokenKind.TypeName:
token = ReviewToken.CreateTypeNameToken(oldToken.Value, false);
if (currentLineTokens.Any(t => t.Kind == TokenKind.Keyword && t.Value.ToLower() == className))
token.RenderClasses.Add(className);
className = "";
break;
case CodeFileTokenKind.MemberName:
token = ReviewToken.CreateMemberNameToken(oldToken.Value, false);
break;
case CodeFileTokenKind.StringLiteral:
token = ReviewToken.CreateStringLiteralToken(oldToken.Value, false);
break;
case CodeFileTokenKind.Literal:
token = ReviewToken.CreateLiteralToken(oldToken.Value, false);
break;
case CodeFileTokenKind.ExternalLinkStart:
token = ReviewToken.CreateStringLiteralToken(oldToken.Value, false);
break;
case CodeFileTokenKind.Whitespace:
if (currentLineTokens.Count > 0) {
currentLineTokens.Last().HasSuffixSpace = true;
}
else if (!skipIndent) {
reviewLine.Indent += oldToken.Value.Length;
}
break;
case CodeFileTokenKind.Newline:
var parent = previousLine;
skipIndent = false;
if (currentLineTokens.Count > 0)
{
while (parent != null && parent.Indent >= reviewLine.Indent)
parent = parent.parentLine;
}
else
{
//If current line is empty line then add it as an empty line under previous line's parent
parent = previousLine?.parentLine;
}
if (parent == null)
{
this.ReviewLines.Add(reviewLine);
}
else
{
parent.Children.Add(reviewLine);
reviewLine.parentLine = parent;
}
//Handle specific cases for C++ line with 'public:' and '{' to mark related line
if ((currentLineTokens.Count == 1 && currentLineTokens.First().Value == "{") ||
(currentLineTokens.Count == 2 && currentLineTokens.Any(t => t.Kind == TokenKind.Keyword && t.Value == "public")))
{
reviewLine.RelatedToLine = previousLine?.LineId;
}
if (currentLineTokens.Count == 0)
{
//Empty line. So just add previous line as related line
reviewLine.RelatedToLine = previousLine?.LineId;
}
else
{
reviewLine.Tokens = currentLineTokens;
previousLine = reviewLine;
}
reviewLine = new ReviewLine();
// If previous line ends with "," then next line will be sub line to show split content in multiple lines.
// Set next line's indent same as current line
// This is required to convert C++ tokens correctly
if (previousLine != null && previousLine.Tokens.LastOrDefault()?.Value == "," && Language == "C++")
{
reviewLine.Indent = previousLine.Indent;
skipIndent = true;
}
currentLineTokens = new List<ReviewToken>();
break;
case CodeFileTokenKind.LineIdMarker:
if (string.IsNullOrEmpty(reviewLine.LineId))
reviewLine.LineId = oldToken.Value;
break;
default:
Console.WriteLine($"Unsupported token kind to convert to new model, Kind: {oldToken.Kind}, value: {oldToken.Value}, Line Id: {oldToken.DefinitionId}");
break;
}
if (token != null)
{
currentLineTokens.Add(token);
if (oldToken.Equals("}") || oldToken.Equals("};"))
reviewLine.IsContextEndLine = true;
if (isHidden)
reviewLine.IsHidden = true;
if (oldToken.DefinitionId != null)
reviewLine.LineId = oldToken.DefinitionId;
if (oldToken.CrossLanguageDefinitionId != null)
reviewLine.CrossLanguageId = oldToken.CrossLanguageDefinitionId;
if (isDeprecated)
token.IsDeprecated = true;
if (skipDiff)
token.SkipDiff = true;
if (isDocumentation)
token.IsDocumentation = true;
}
}
//Process last line
if (currentLineTokens.Count > 0)
{
reviewLine.Tokens = currentLineTokens;
var parent = previousLine;
while (parent != null && parent.Indent >= reviewLine.Indent)
parent = parent.parentLine;
if (parent == null)
this.ReviewLines.Add(reviewLine);
else
parent.Children.Add(reviewLine);
}
}