in vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java [850:950]
private void readFromJsonIntoVector(Field field, FieldVector vector) throws IOException {
ArrowType type = field.getType();
TypeLayout typeLayout = TypeLayout.getTypeLayout(type);
List<BufferType> vectorTypes = typeLayout.getBufferTypes();
List<ArrowBuf> vectorBuffers = new ArrayList<>(vectorTypes.size());
List<Integer> variadicBufferIndices = new ArrayList<>();
if (!typeLayout.isFixedBufferCount()) {
vectorTypes.add(VARIADIC_DATA_BUFFERS);
}
/*
* The order of inner buffers is:
* Fixed width vector:
* -- validity buffer
* -- data buffer
* Variable width vector:
* -- validity buffer
* -- offset buffer
* -- data buffer
*
* This is similar to what getFieldInnerVectors() used to give but now that we don't have
* inner vectors anymore, we will work directly at the buffer level -- populate buffers
* locally as we read from Json parser and do loadFieldBuffers on the vector followed by
* releasing the local buffers.
*/
readToken(START_OBJECT);
{
// If currently reading dictionaries, field name is not important so don't check
String name = readNextField("name", String.class);
if (started && !Objects.equals(field.getName(), name)) {
throw new IllegalArgumentException(
"Expected field " + field.getName() + " but got " + name);
}
/* Initialize the vector with required capacity but don't allocateNew since we would
* be doing loadFieldBuffers.
*/
int valueCount = readNextField("count", Integer.class);
vector.setInitialCapacity(valueCount);
for (int v = 0; v < vectorTypes.size(); v++) {
BufferType bufferType = vectorTypes.get(v);
nextFieldIs(bufferType.getName());
int innerBufferValueCount = valueCount;
if (bufferType.equals(OFFSET)
&& !(type instanceof Union)
&& !(type instanceof ListView)
&& !(type instanceof LargeListView)) {
/* offset buffer has 1 additional value capacity except for dense unions and ListView */
innerBufferValueCount = valueCount + 1;
}
vectorBuffers.addAll(
readIntoBuffer(
allocator,
bufferType,
vector.getMinorType(),
innerBufferValueCount,
variadicBufferIndices));
}
int nullCount;
if (type instanceof ArrowType.RunEndEncoded || type instanceof Union) {
nullCount = 0;
} else if (type instanceof ArrowType.Null) {
nullCount = valueCount;
} else {
nullCount = BitVectorHelper.getNullCount(vectorBuffers.get(0), valueCount);
}
final ArrowFieldNode fieldNode = new ArrowFieldNode(valueCount, nullCount);
vector.loadFieldBuffers(fieldNode, vectorBuffers);
/* read child vectors (if any) */
List<Field> fields = field.getChildren();
if (!fields.isEmpty()) {
List<FieldVector> vectorChildren = vector.getChildrenFromFields();
if (fields.size() != vectorChildren.size()) {
throw new IllegalArgumentException(
"fields and children are not the same size: "
+ fields.size()
+ " != "
+ vectorChildren.size());
}
nextFieldIs("children");
readToken(START_ARRAY);
for (int i = 0; i < fields.size(); i++) {
Field childField = fields.get(i);
FieldVector childVector = vectorChildren.get(i);
readFromJsonIntoVector(childField, childVector);
}
readToken(END_ARRAY);
}
}
readToken(END_OBJECT);
for (ArrowBuf buffer : vectorBuffers) {
buffer.getReferenceManager().release();
}
}