in vector/src/main/java/org/apache/arrow/vector/VectorLoader.java [102:164]
private void loadBuffers(
FieldVector vector,
Field field,
Iterator<ArrowBuf> buffers,
Iterator<ArrowFieldNode> nodes,
CompressionCodec codec,
Iterator<Long> variadicBufferCounts) {
checkArgument(nodes.hasNext(), "no more field nodes for field %s and vector %s", field, vector);
ArrowFieldNode fieldNode = nodes.next();
// variadicBufferLayoutCount will be 0 for vectors of a type except BaseVariableWidthViewVector
long variadicBufferLayoutCount = 0;
if (vector instanceof BaseVariableWidthViewVector) {
if (variadicBufferCounts.hasNext()) {
variadicBufferLayoutCount = variadicBufferCounts.next();
} else {
throw new IllegalStateException(
"No variadicBufferCounts available for BaseVariableWidthViewVector");
}
}
int bufferLayoutCount =
(int) (variadicBufferLayoutCount + TypeLayout.getTypeBufferCount(field.getType()));
List<ArrowBuf> ownBuffers = new ArrayList<>(bufferLayoutCount);
for (int j = 0; j < bufferLayoutCount; j++) {
if (!buffers.hasNext()) {
throw new IllegalArgumentException(
"no more buffers for field " + field + ". Expected " + bufferLayoutCount);
}
ArrowBuf nextBuf = buffers.next();
// for vectors without nulls, the buffer is empty, so there is no need to decompress it.
ArrowBuf bufferToAdd =
nextBuf.writerIndex() > 0 ? codec.decompress(vector.getAllocator(), nextBuf) : nextBuf;
ownBuffers.add(bufferToAdd);
if (decompressionNeeded) {
// decompression performed
nextBuf.getReferenceManager().retain();
}
}
try {
vector.loadFieldBuffers(fieldNode, ownBuffers);
if (decompressionNeeded) {
for (ArrowBuf buf : ownBuffers) {
buf.close();
}
}
} catch (RuntimeException e) {
throw new IllegalArgumentException(
"Could not load buffers for field " + field + ". error message: " + e.getMessage(), e);
}
List<Field> children = field.getChildren();
if (children.size() > 0) {
List<FieldVector> childrenFromFields = vector.getChildrenFromFields();
checkArgument(
children.size() == childrenFromFields.size(),
"should have as many children as in the schema: found %s expected %s",
childrenFromFields.size(),
children.size());
for (int i = 0; i < childrenFromFields.size(); i++) {
Field child = children.get(i);
FieldVector fieldVector = childrenFromFields.get(i);
loadBuffers(fieldVector, child, buffers, nodes, codec, variadicBufferCounts);
}
}
}