in pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/InvertedIndexAndDictionaryBasedForwardIndexCreator.java [420:575]
private void writeToForwardIndex(Dictionary dictionary, IndexCreationContext context)
throws IOException {
try (ForwardIndexCreator creator = StandardIndexes.forward().createIndexCreator(context, _forwardIndexConfig)) {
if (_dictionaryEnabled) {
if (_singleValue) {
for (int docId = 0; docId < _numDocs; docId++) {
creator.putDictId(getInt(_forwardIndexValueBuffer, docId));
}
} else {
int startIdx = 0;
for (int docId = 0; docId < _numDocs; docId++) {
int endIdx = getInt(_forwardIndexLengthBuffer, docId);
int[] values = new int[endIdx - startIdx];
int valuesIdx = 0;
for (int i = startIdx; i < endIdx; i++) {
values[valuesIdx++] = getInt(_forwardIndexValueBuffer, i);
}
creator.putDictIdMV(values);
startIdx = endIdx;
}
}
} else {
switch (creator.getValueType()) {
case INT:
if (_singleValue) {
for (int docId = 0; docId < _numDocs; docId++) {
creator.putInt(dictionary.getIntValue(getInt(_forwardIndexValueBuffer, docId)));
}
} else {
int startIdx = 0;
for (int docId = 0; docId < _numDocs; docId++) {
int endIdx = getInt(_forwardIndexLengthBuffer, docId);
int[] values = new int[endIdx - startIdx];
int valuesIdx = 0;
for (int i = startIdx; i < endIdx; i++) {
values[valuesIdx++] = dictionary.getIntValue(getInt(_forwardIndexValueBuffer, i));
}
creator.putIntMV(values);
startIdx = endIdx;
}
}
break;
case LONG:
if (_singleValue) {
for (int docId = 0; docId < _numDocs; docId++) {
creator.putLong(dictionary.getLongValue(getInt(_forwardIndexValueBuffer, docId)));
}
} else {
int startIdx = 0;
for (int docId = 0; docId < _numDocs; docId++) {
int endIdx = getInt(_forwardIndexLengthBuffer, docId);
long[] values = new long[endIdx - startIdx];
int valuesIdx = 0;
for (int i = startIdx; i < endIdx; i++) {
values[valuesIdx++] = dictionary.getLongValue(getInt(_forwardIndexValueBuffer, i));
}
creator.putLongMV(values);
startIdx = endIdx;
}
}
break;
case FLOAT:
if (_singleValue) {
for (int docId = 0; docId < _numDocs; docId++) {
creator.putFloat(dictionary.getFloatValue(getInt(_forwardIndexValueBuffer, docId)));
}
} else {
int startIdx = 0;
for (int docId = 0; docId < _numDocs; docId++) {
int endIdx = getInt(_forwardIndexLengthBuffer, docId);
float[] values = new float[endIdx - startIdx];
int valuesIdx = 0;
for (int i = startIdx; i < endIdx; i++) {
values[valuesIdx++] = dictionary.getFloatValue(getInt(_forwardIndexValueBuffer, i));
}
creator.putFloatMV(values);
startIdx = endIdx;
}
}
break;
case DOUBLE:
if (_singleValue) {
for (int docId = 0; docId < _numDocs; docId++) {
creator.putDouble(dictionary.getDoubleValue(getInt(_forwardIndexValueBuffer, docId)));
}
} else {
int startIdx = 0;
for (int docId = 0; docId < _numDocs; docId++) {
int endIdx = getInt(_forwardIndexLengthBuffer, docId);
double[] values = new double[endIdx - startIdx];
int valuesIdx = 0;
for (int i = startIdx; i < endIdx; i++) {
values[valuesIdx++] = dictionary.getDoubleValue(getInt(_forwardIndexValueBuffer, i));
}
creator.putDoubleMV(values);
startIdx = endIdx;
}
}
break;
case STRING:
if (_singleValue) {
for (int docId = 0; docId < _numDocs; docId++) {
creator.putString(dictionary.getStringValue(getInt(_forwardIndexValueBuffer, docId)));
}
} else {
int startIdx = 0;
for (int docId = 0; docId < _numDocs; docId++) {
int endIdx = getInt(_forwardIndexLengthBuffer, docId);
String[] values = new String[endIdx - startIdx];
int valuesIdx = 0;
for (int i = startIdx; i < endIdx; i++) {
values[valuesIdx++] = dictionary.getStringValue(getInt(_forwardIndexValueBuffer, i));
}
creator.putStringMV(values);
startIdx = endIdx;
}
}
break;
case BYTES:
if (_singleValue) {
for (int docId = 0; docId < _numDocs; docId++) {
creator.putBytes(dictionary.getBytesValue(getInt(_forwardIndexValueBuffer, docId)));
}
} else {
int startIdx = 0;
for (int docId = 0; docId < _numDocs; docId++) {
int endIdx = getInt(_forwardIndexLengthBuffer, docId);
byte[][] values = new byte[endIdx - startIdx][];
int valuesIdx = 0;
for (int i = startIdx; i < endIdx; i++) {
values[valuesIdx++] = dictionary.getBytesValue(getInt(_forwardIndexValueBuffer, i));
}
creator.putBytesMV(values);
startIdx = endIdx;
}
}
break;
case BIG_DECIMAL:
Preconditions.checkState(_singleValue, "BIG_DECIMAL type not supported for multi-value columns");
for (int docId = 0; docId < _numDocs; docId++) {
creator.putBigDecimal(dictionary.getBigDecimalValue(getInt(_forwardIndexValueBuffer, docId)));
}
break;
default:
throw new IllegalStateException("Invalid type" + creator.getValueType() + " cannot create forward index");
}
}
} catch (Exception e) {
throw new IOException(String.format(
"Cannot create the forward index from inverted index for column %s", _columnName), e);
} finally {
destroyBuffer(_forwardIndexValueBuffer, _forwardIndexValueBufferFile);
destroyBuffer(_forwardIndexLengthBuffer, _forwardIndexLengthBufferFile);
destroyBuffer(_forwardIndexMaxSizeBuffer, _forwardIndexMaxSizeBufferFile);
}
}