in Sources/Tokenizers/PreTokenizer.swift [365:377]
func mergedWithPrevious(ranges: [Range<String.Index>]) -> [Range<String.Index>] {
var merged: [Range<String.Index>] = []
var currentStart = startIndex
for range in ranges {
let mergedRange = currentStart..<range.upperBound
currentStart = range.upperBound
merged.append(mergedRange)
}
if currentStart < endIndex {
merged.append(currentStart..<endIndex)
}
return merged
}