protected int ensureCumulatives()

in spark-load/spark-load-common/src/main/java/org/apache/doris/common/io/Roaring64Map.java [630:694]


    protected int ensureCumulatives(int high) {
        if (allValid) {
            // the whole array is valid (up-to its actual length, not its capacity)
            return highToBitmap.size();
        } else if (compare(high, firstHighNotValid) < 0) {
            // The high is strictly below the first not valid: it is valid

            // sortedHighs may have only a subset of valid values on the right. However, these invalid
            // values have been set to maxValue, and we are here as high < firstHighNotValid ==> high <
            // maxHigh()
            int position = binarySearch(sortedHighs, high);

            if (position >= 0) {
                // This high has a bitmap: +1 as this index will be used as right (excluded) bound in a
                // binary-search
                return position + 1;
            } else {
                // This high has no bitmap: it could be between 2 highs with bitmaps
                int insertionPosition = -position - 1;
                return insertionPosition;
            }
        } else {

            // For each deprecated buckets
            SortedMap<Integer, BitmapDataProvider> tailMap =
                    highToBitmap.tailMap(firstHighNotValid, true);

            // TODO .size on tailMap make an iterator: arg
            int indexOk = highToBitmap.size() - tailMap.size();

            // TODO: It should be possible to compute indexOk based on sortedHighs array
            // assert indexOk == binarySearch(sortedHighs, firstHighNotValid);

            Iterator<Map.Entry<Integer, BitmapDataProvider>> it = tailMap.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry<Integer, BitmapDataProvider> e = it.next();
                int currentHigh = e.getKey();

                if (compare(currentHigh, high) > 0) {
                    // No need to compute more than needed
                    break;
                } else if (e.getValue().isEmpty()) {
                    // highToBitmap can not be modified as we iterate over it
                    if (latestAddedHigh != null && latestAddedHigh.getKey().intValue() == currentHigh) {
                        // Dismiss the cached bitmap as it is removed from the NavigableMap
                        latestAddedHigh = null;
                    }
                    it.remove();
                } else {
                    ensureOne(e, currentHigh, indexOk);

                    // We have added one valid cardinality
                    indexOk++;
                }

            }

            if (highToBitmap.isEmpty() || indexOk == highToBitmap.size()) {
                // We have compute all cardinalities
                allValid = true;
            }

            return indexOk;
        }
    }