in src/grammar.ts [1660:1715]
public produceFromScopes(scopesList: ScopeListElement, endIndex: number): void {
if (this._lastTokenEndIndex >= endIndex) {
return;
}
if (this._emitBinaryTokens) {
let metadata = scopesList.metadata;
if (this._tokenTypeOverrides.length > 0) {
const scopes = scopesList.generateScopes();
for (const tokenType of this._tokenTypeOverrides) {
if (tokenType.matcher(scopes)) {
metadata = StackElementMetadata.set(metadata, 0, toTemporaryType(tokenType.type), FontStyle.NotSet, 0, 0);
}
}
}
if (this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 1] === metadata) {
// no need to push a token with the same metadata
this._lastTokenEndIndex = endIndex;
return;
}
if (DebugFlags.InDebugMode) {
const scopes = scopesList.generateScopes();
console.log(' token: |' + this._lineText!.substring(this._lastTokenEndIndex, endIndex).replace(/\n$/, '\\n') + '|');
for (let k = 0; k < scopes.length; k++) {
console.log(' * ' + scopes[k]);
}
}
this._binaryTokens.push(this._lastTokenEndIndex);
this._binaryTokens.push(metadata);
this._lastTokenEndIndex = endIndex;
return;
}
const scopes = scopesList.generateScopes();
if (DebugFlags.InDebugMode) {
console.log(' token: |' + this._lineText!.substring(this._lastTokenEndIndex, endIndex).replace(/\n$/, '\\n') + '|');
for (let k = 0; k < scopes.length; k++) {
console.log(' * ' + scopes[k]);
}
}
this._tokens.push({
startIndex: this._lastTokenEndIndex,
endIndex: endIndex,
// value: lineText.substring(lastTokenEndIndex, endIndex),
scopes: scopes
});
this._lastTokenEndIndex = endIndex;
}