in spark-load/spark-load-common/src/main/java/org/apache/doris/common/io/Roaring64Map.java [630:694]
protected int ensureCumulatives(int high) {
if (allValid) {
// the whole array is valid (up-to its actual length, not its capacity)
return highToBitmap.size();
} else if (compare(high, firstHighNotValid) < 0) {
// The high is strictly below the first not valid: it is valid
// sortedHighs may have only a subset of valid values on the right. However, these invalid
// values have been set to maxValue, and we are here as high < firstHighNotValid ==> high <
// maxHigh()
int position = binarySearch(sortedHighs, high);
if (position >= 0) {
// This high has a bitmap: +1 as this index will be used as right (excluded) bound in a
// binary-search
return position + 1;
} else {
// This high has no bitmap: it could be between 2 highs with bitmaps
int insertionPosition = -position - 1;
return insertionPosition;
}
} else {
// For each deprecated buckets
SortedMap<Integer, BitmapDataProvider> tailMap =
highToBitmap.tailMap(firstHighNotValid, true);
// TODO .size on tailMap make an iterator: arg
int indexOk = highToBitmap.size() - tailMap.size();
// TODO: It should be possible to compute indexOk based on sortedHighs array
// assert indexOk == binarySearch(sortedHighs, firstHighNotValid);
Iterator<Map.Entry<Integer, BitmapDataProvider>> it = tailMap.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<Integer, BitmapDataProvider> e = it.next();
int currentHigh = e.getKey();
if (compare(currentHigh, high) > 0) {
// No need to compute more than needed
break;
} else if (e.getValue().isEmpty()) {
// highToBitmap can not be modified as we iterate over it
if (latestAddedHigh != null && latestAddedHigh.getKey().intValue() == currentHigh) {
// Dismiss the cached bitmap as it is removed from the NavigableMap
latestAddedHigh = null;
}
it.remove();
} else {
ensureOne(e, currentHigh, indexOk);
// We have added one valid cardinality
indexOk++;
}
}
if (highToBitmap.isEmpty() || indexOk == highToBitmap.size()) {
// We have compute all cardinalities
allValid = true;
}
return indexOk;
}
}