in Sources/Tokenizers/PreTokenizer.swift [350:363]
func mergedWithNext(ranges: [Range<String.Index>]) -> [Range<String.Index>] {
var merged: [Range<String.Index>] = []
var currentStart = startIndex
for range in ranges {
if range.lowerBound == startIndex { continue }
let mergedRange = currentStart..<range.lowerBound
currentStart = range.lowerBound
merged.append(mergedRange)
}
if currentStart < endIndex {
merged.append(currentStart..<endIndex)
}
return merged
}