vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java [112:229]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - super(field.getName(), allocator, callBack); this.validityBuffer = allocator.getEmpty(); this.field = field; this.callBack = callBack; this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override public void initializeChildrenFromFields(List children) { checkArgument( children.size() == 1, "ListViews have one child Field. Found: %s", children.isEmpty() ? "none" : children); Field field = children.get(0); AddOrGetResult addOrGetVector = addOrGetVector(field.getFieldType()); checkArgument( addOrGetVector.isCreated(), "Child vector already existed: %s", addOrGetVector.getVector()); addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren()); this.field = new Field(this.field.getName(), this.field.getFieldType(), children); } @Override public void setInitialCapacity(int numRecords) { validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords); } /** * Specialized version of setInitialCapacity() for LargeListViewVector. This is used by some * callers when they want to explicitly control and be conservative about memory allocated for * inner data vector. This is very useful when we are working with memory constraints for a query * and have a fixed amount of memory reserved for the record batch. In such cases, we are likely * to face OOM or related problems when we reserve memory for a record batch with value count x * and do setInitialCapacity(x) such that each vector allocates only what is necessary and not the * default amount, but the multiplier forces the memory requirement to go beyond what was needed. * * @param numRecords value count * @param density density of LargeListViewVector. Density is the average size of a list per * position in the LargeListViewVector. For example, a density value of 10 implies each * position in the list vector has a list of 10 values. A density value of 0.1 implies out of * 10 positions in the list vector, 1 position has a list of size 1, and the remaining * positions are null (no lists) or empty lists. This helps in tightly controlling the memory * we provision for inner data vector. */ @Override public void setInitialCapacity(int numRecords, double density) { validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords, density); } /** * Specialized version of setInitialTotalCapacity() for LargeListViewVector. This is used by some * callers when they want to explicitly control and be conservative about memory allocated for * inner data vector. This is very useful when we are working with memory constraints for a query * and have a fixed amount of memory reserved for the record batch. In such cases, we are likely * to face OOM or related problems when we reserve memory for a record batch with value count x * and do setInitialCapacity(x) such that each vector allocates only what is necessary and not the * default amount, but the multiplier forces the memory requirement to go beyond what was needed. * * @param numRecords value count * @param totalNumberOfElements the total number of elements to allow for in this vector across * all records. */ @Override public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); super.setInitialTotalCapacity(numRecords, totalNumberOfElements); } @Override public List getChildrenFromFields() { return singletonList(getDataVector()); } /** * Load the buffers associated with this Field. * * @param fieldNode the fieldNode * @param ownBuffers the buffers for this Field (own buffers only, children not included) */ @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { if (ownBuffers.size() != 3) { throw new IllegalArgumentException( "Illegal buffer count, expected " + 3 + ", got: " + ownBuffers.size()); } ArrowBuf bitBuffer = ownBuffers.get(0); ArrowBuf offBuffer = ownBuffers.get(1); ArrowBuf szBuffer = ownBuffers.get(2); validityBuffer.getReferenceManager().release(); validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); offsetBuffer.getReferenceManager().release(); offsetBuffer = offBuffer.getReferenceManager().retain(offBuffer, allocator); sizeBuffer.getReferenceManager().release(); sizeBuffer = szBuffer.getReferenceManager().retain(szBuffer, allocator); validityAllocationSizeInBytes = checkedCastToInt(validityBuffer.capacity()); offsetAllocationSizeInBytes = offsetBuffer.capacity(); sizeAllocationSizeInBytes = sizeBuffer.capacity(); valueCount = fieldNode.getLength(); } /** Set the reader and writer indexes for the inner buffers. */ private void setReaderAndWriterIndex() { validityBuffer.readerIndex(0); offsetBuffer.readerIndex(0); sizeBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); offsetBuffer.writerIndex(0); sizeBuffer.writerIndex(0); } else { validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java [111:228]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - super(field.getName(), allocator, callBack); this.validityBuffer = allocator.getEmpty(); this.field = field; this.callBack = callBack; this.validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION); } @Override public void initializeChildrenFromFields(List children) { checkArgument( children.size() == 1, "ListViews have one child Field. Found: %s", children.isEmpty() ? "none" : children); Field field = children.get(0); AddOrGetResult addOrGetVector = addOrGetVector(field.getFieldType()); checkArgument( addOrGetVector.isCreated(), "Child vector already existed: %s", addOrGetVector.getVector()); addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren()); this.field = new Field(this.field.getName(), this.field.getFieldType(), children); } @Override public void setInitialCapacity(int numRecords) { validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords); } /** * Specialized version of setInitialCapacity() for ListViewVector. This is used by some callers * when they want to explicitly control and be conservative about memory allocated for inner data * vector. This is very useful when we are working with memory constraints for a query and have a * fixed amount of memory reserved for the record batch. In such cases, we are likely to face OOM * or related problems when we reserve memory for a record batch with value count x and do * setInitialCapacity(x) such that each vector allocates only what is necessary and not the * default amount, but the multiplier forces the memory requirement to go beyond what was needed. * * @param numRecords value count * @param density density of ListViewVector. Density is the average size of a list per position in * the ListViewVector. For example, a density value of 10 implies each position in the list * vector has a list of 10 values. A density value of 0.1 implies out of 10 positions in the * list vector, 1 position has a list of size 1, and the remaining positions are null (no * lists) or empty lists. This helps in tightly controlling the memory we provision for inner * data vector. */ @Override public void setInitialCapacity(int numRecords, double density) { validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); super.setInitialCapacity(numRecords, density); } /** * Specialized version of setInitialTotalCapacity() for ListViewVector. This is used by some * callers when they want to explicitly control and be conservative about memory allocated for * inner data vector. This is very useful when we are working with memory constraints for a query * and have a fixed amount of memory reserved for the record batch. In such cases, we are likely * to face OOM or related problems when we reserve memory for a record batch with value count x * and do setInitialCapacity(x) such that each vector allocates only what is necessary and not the * default amount, but the multiplier forces the memory requirement to go beyond what was needed. * * @param numRecords value count * @param totalNumberOfElements the total number of elements to allow for in this vector across * all records. */ @Override public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { validityAllocationSizeInBytes = getValidityBufferSizeFromCount(numRecords); super.setInitialTotalCapacity(numRecords, totalNumberOfElements); } @Override public List getChildrenFromFields() { return singletonList(getDataVector()); } /** * Load the buffers associated with this Field. * * @param fieldNode the fieldNode * @param ownBuffers the buffers for this Field (own buffers only, children not included) */ @Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers) { if (ownBuffers.size() != 3) { throw new IllegalArgumentException( "Illegal buffer count, expected " + 3 + ", got: " + ownBuffers.size()); } ArrowBuf bitBuffer = ownBuffers.get(0); ArrowBuf offBuffer = ownBuffers.get(1); ArrowBuf szBuffer = ownBuffers.get(2); validityBuffer.getReferenceManager().release(); validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); offsetBuffer.getReferenceManager().release(); offsetBuffer = offBuffer.getReferenceManager().retain(offBuffer, allocator); sizeBuffer.getReferenceManager().release(); sizeBuffer = szBuffer.getReferenceManager().retain(szBuffer, allocator); validityAllocationSizeInBytes = checkedCastToInt(validityBuffer.capacity()); offsetAllocationSizeInBytes = offsetBuffer.capacity(); sizeAllocationSizeInBytes = sizeBuffer.capacity(); valueCount = fieldNode.getLength(); } /** Set the reader and writer indexes for the inner buffers. */ private void setReaderAndWriterIndex() { validityBuffer.readerIndex(0); offsetBuffer.readerIndex(0); sizeBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); offsetBuffer.writerIndex(0); sizeBuffer.writerIndex(0); } else { validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -