in package/src/syntaxHighlighting/SemanticTokensProvider.ts [51:72]
function toSemanticTokens(classification: ClassificationRange, model: editor.ITextModel): DocumentSemanticToken[] {
const { line, character, length, kind } = classification;
const tokens: DocumentSemanticToken[] = [];
let remainingLength = length;
let currentLine = line;
let currentChar = character;
while (remainingLength > 0 && currentLine < model.getLineCount()) {
const lineLength = model.getLineLength(currentLine + 1);
const available = lineLength - currentChar + 1;
const tokenLength = Math.min(remainingLength, available);
tokens.push([currentLine, currentChar, tokenLength, kind, 0]);
remainingLength -= tokenLength;
currentLine++;
currentChar = 0; // reset for next line
}
return tokens;
}