Skip to content

MINOR: Preserve empty list offset buffer #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lriggs merged 16 commits into from
Jul 7, 2026
Merged

MINOR: Preserve empty list offset buffer #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
808f672
Add jar build action.
lriggs Mar 7, 2025
193c85a
Merge pull request #1 from dremio/dremio_main
lriggs Mar 7, 2025
858601a
DX-122953 Preserve empty list offset buffer
prashanthbdremio Jun 22, 2026
f5cfa00
DX-122953 Avoid NettyArrowBuf test import
prashanthbdremio Jun 22, 2026
10b259b
MINOR: Fix workflow trailing whitespace
prashanthbdremio Jun 22, 2026
e293989
MINOR: Reset list test helper capacity
prashanthbdremio Jun 23, 2026
91dafa2
Revert "MINOR: Reset list test helper capacity"
prashanthbdremio Jun 23, 2026
b9a3433
DX-122953 Preserve list offset allocation size
prashanthbdremio Jun 23, 2026
6525bbe
DX-122953 Preserve empty split transfer offset buffer
prashanthbdremio Jun 30, 2026
1a084bb
DX-122953 Complete empty list split transfer
prashanthbdremio Jul 2, 2026
56157d2
DX-122953 Complete empty large list split transfer
prashanthbdremio Jul 2, 2026
3c55ad4
DX-122953 Preserve offset bytes when expanding empty buffers
prashanthbdremio Jul 6, 2026
9d50ee1
DX-122953 Add empty split transfer coverage
prashanthbdremio Jul 6, 2026
9f1200e
Merge branch 'dremio_27.0_23_19' into prashanthbdremio/dx-122953
lriggs Jul 7, 2026
c5c4abd
Rename test method for clarity
lriggs Jul 7, 2026
b3acd70
Rename testEmptyListOffsetBuffer to testEmptyListOffsetBufferSize
lriggs Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 37 additions & 16 deletions vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,10 +305,12 @@ public void exportCDataBuffers(List<ArrowBuf> buffers, ArrowBuf buffersPtr, long

/** Set the reader and writer indexes for the inner buffers. */
private void setReaderAndWriterIndex() {
final long requiredOffsetBufferCapacity = (long) (valueCount + 1) * OFFSET_WIDTH;
validityBuffer.readerIndex(0);
offsetBuffer.readerIndex(0);
if (valueCount == 0) {
validityBuffer.writerIndex(0);
ensureEmptyOffsetBufferCapacity(requiredOffsetBufferCapacity);
} else {
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
}
Expand All @@ -319,6 +321,19 @@ private void setReaderAndWriterIndex() {
offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH);
}

private void ensureEmptyOffsetBufferCapacity(long requiredCapacity) {
if (offsetBuffer.capacity() >= requiredCapacity) {
return;
}
long previousOffsetAllocationSizeInBytes = offsetAllocationSizeInBytes;
ArrowBuf oldOffsetBuffer = offsetBuffer;
offsetBuffer = allocateOffsetBuffer(requiredCapacity);
offsetBuffer.setBytes(
0, oldOffsetBuffer, 0, Math.min(oldOffsetBuffer.capacity(), requiredCapacity));
offsetAllocationSizeInBytes = previousOffsetAllocationSizeInBytes;
oldOffsetBuffer.getReferenceManager().release();
Comment thread
prashanthbdremio marked this conversation as resolved.
}

/**
* Get the inner vectors.
*
Expand Down Expand Up @@ -674,24 +689,30 @@ public void splitAndTransfer(int startIndex, int length) {
startIndex,
length,
valueCount);
final long startPoint = offsetBuffer.getLong((long) startIndex * OFFSET_WIDTH);
final long sliceLength =
offsetBuffer.getLong((long) (startIndex + length) * OFFSET_WIDTH) - startPoint;
to.clear();
to.offsetBuffer = to.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH);
/* splitAndTransfer offset buffer */
for (int i = 0; i < length + 1; i++) {
final long relativeOffset =
offsetBuffer.getLong((long) (startIndex + i) * OFFSET_WIDTH) - startPoint;
to.offsetBuffer.setLong((long) i * OFFSET_WIDTH, relativeOffset);
if (length > 0) {
final long startPoint = offsetBuffer.getLong((long) startIndex * OFFSET_WIDTH);
final long sliceLength =
offsetBuffer.getLong((long) (startIndex + length) * OFFSET_WIDTH) - startPoint;
to.offsetBuffer = to.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH);
/* splitAndTransfer offset buffer */
for (int i = 0; i < length + 1; i++) {
final long relativeOffset =
offsetBuffer.getLong((long) (startIndex + i) * OFFSET_WIDTH) - startPoint;
to.offsetBuffer.setLong((long) i * OFFSET_WIDTH, relativeOffset);
}
/* splitAndTransfer validity buffer */
splitAndTransferValidityBuffer(startIndex, length, to);
/* splitAndTransfer data buffer */
dataTransferPair.splitAndTransfer(
checkedCastToInt(startPoint), checkedCastToInt(sliceLength));
to.lastSet = length - 1;
to.setValueCount(length);
} else {
to.ensureEmptyOffsetBufferCapacity(OFFSET_WIDTH);
dataTransferPair.splitAndTransfer(0, 0);
to.setValueCount(0);
}
/* splitAndTransfer validity buffer */
splitAndTransferValidityBuffer(startIndex, length, to);
/* splitAndTransfer data buffer */
dataTransferPair.splitAndTransfer(
checkedCastToInt(startPoint), checkedCastToInt(sliceLength));
to.lastSet = length - 1;
to.setValueCount(length);
}

@Override
Expand Down
19 changes: 19 additions & 0 deletions vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,10 +263,12 @@ public void exportCDataBuffers(List<ArrowBuf> buffers, ArrowBuf buffersPtr, long

/** Set the reader and writer indexes for the inner buffers. */
private void setReaderAndWriterIndex() {
final long requiredOffsetBufferCapacity = (long) (valueCount + 1) * OFFSET_WIDTH;
validityBuffer.readerIndex(0);
offsetBuffer.readerIndex(0);
if (valueCount == 0) {
validityBuffer.writerIndex(0);
ensureEmptyOffsetBufferCapacity(requiredOffsetBufferCapacity);
} else {
validityBuffer.writerIndex(BitVectorHelper.getValidityBufferSizeFromCount(valueCount));
}
Expand All @@ -277,6 +279,19 @@ private void setReaderAndWriterIndex() {
offsetBuffer.writerIndex((long) (valueCount + 1) * OFFSET_WIDTH);
}

private void ensureEmptyOffsetBufferCapacity(long requiredCapacity) {
if (offsetBuffer.capacity() >= requiredCapacity) {
return;
}
long previousOffsetAllocationSizeInBytes = offsetAllocationSizeInBytes;
ArrowBuf oldOffsetBuffer = offsetBuffer;
offsetBuffer = allocateOffsetBuffer(requiredCapacity);
offsetBuffer.setBytes(
0, oldOffsetBuffer, 0, Math.min(oldOffsetBuffer.capacity(), requiredCapacity));
offsetAllocationSizeInBytes = previousOffsetAllocationSizeInBytes;
oldOffsetBuffer.getReferenceManager().release();
}

/**
* Get the inner vectors.
*
Expand Down Expand Up @@ -572,6 +587,10 @@ public void splitAndTransfer(int startIndex, int length) {
dataTransferPair.splitAndTransfer(startPoint, sliceLength);
to.lastSet = length - 1;
to.setValueCount(length);
} else {
to.ensureEmptyOffsetBufferCapacity(OFFSET_WIDTH);
dataTransferPair.splitAndTransfer(0, 0);
to.setValueCount(0);
}
}

Expand Down
87 changes: 86 additions & 1 deletion vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -962,6 +962,91 @@ public void testGetBufferSizeFor() {
}
}

@Test
public void testEmptyLargeListOffsetBuffer() {
try (LargeListVector list = LargeListVector.empty("list", allocator)) {
list.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
list.allocateNew();
list.setValueCount(0);

assertEmptyLargeListOffsetBuffer(list);
}
}

@Test
public void testUnallocatedEmptyLargeListOffsetBuffer() {
try (LargeListVector list = LargeListVector.empty("list", allocator)) {
list.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
list.setValueCount(0);

assertEmptyLargeListOffsetBuffer(list);
}
}

@Test
public void testSplitAndTransferEmptyLargeListOffsetBuffer() {
try (LargeListVector source = LargeListVector.empty("source", allocator);
LargeListVector target = LargeListVector.empty("target", allocator)) {
source.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
target.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
source.allocateNew();
source.setValueCount(0);

TransferPair transferPair = source.makeTransferPair(target);
transferPair.splitAndTransfer(0, 0);

assertEmptyLargeListOffsetBuffer(target);
}
}

@Test
public void testSplitAndTransferEmptyLargeListAllocatesOffsetBuffer() {
try (LargeListVector fromVector = LargeListVector.empty("fromVector", allocator);
LargeListVector toVector = LargeListVector.empty("toVector", allocator)) {
fromVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
fromVector.allocateNew();
fromVector.setValueCount(0);

TransferPair transferPair = fromVector.makeTransferPair(toVector);
transferPair.splitAndTransfer(0, 0);

assertAllocatedEmptyLargeListOffsetBuffer(toVector);
}
}

@Test
public void testSplitAndTransferEmptyNestedLargeListAllocatesOffsetBuffers() {
try (LargeListVector fromVector = LargeListVector.empty("fromVector", allocator);
LargeListVector toVector = LargeListVector.empty("toVector", allocator)) {
fromVector.addOrGetVector(FieldType.nullable(MinorType.LARGELIST.getType()));
LargeListVector childVector = (LargeListVector) fromVector.getDataVector();
childVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
fromVector.allocateNew();
fromVector.setValueCount(0);

TransferPair transferPair = fromVector.makeTransferPair(toVector);
transferPair.splitAndTransfer(0, 0);

assertAllocatedEmptyLargeListOffsetBuffer(toVector);
assertAllocatedEmptyLargeListOffsetBuffer((LargeListVector) toVector.getDataVector());
}
}

private ArrowBuf assertEmptyLargeListOffsetBuffer(LargeListVector list) {
List<ArrowBuf> buffers = list.getFieldBuffers();
ArrowBuf offsetBuffer = buffers.get(1);
assertEquals(LargeListVector.OFFSET_WIDTH, offsetBuffer.readableBytes());
assertTrue(offsetBuffer.capacity() >= LargeListVector.OFFSET_WIDTH);
assertEquals(0L, offsetBuffer.getLong(0));
return offsetBuffer;
}

private void assertAllocatedEmptyLargeListOffsetBuffer(LargeListVector list) {
ArrowBuf offsetBuffer = list.getOffsetBuffer();
assertTrue(offsetBuffer.capacity() >= LargeListVector.OFFSET_WIDTH);
assertEquals(0L, offsetBuffer.getLong(0));
}

@Test
public void testIsEmpty() {
try (final LargeListVector vector = LargeListVector.empty("list", allocator)) {
Expand Down Expand Up @@ -1101,7 +1186,7 @@ public void testCopyValueSafeForExtensionType() throws Exception {
}

@Test
public void testEmptyLargeListOffsetBuffer() {
public void testEmptyLargeListOffsetBufferSize() {
// Test that LargeListVector has correct readableBytes after allocation.
// According to Arrow spec, offset buffer must have N+1 entries.
// Even when N=0, it should contain [0].
Expand Down
87 changes: 86 additions & 1 deletion vector/src/test/java/org/apache/arrow/vector/TestListVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1142,6 +1142,91 @@ public void testGetBufferSizeFor() {
}
}

@Test
public void testEmptyListOffsetBuffer() {
try (ListVector list = ListVector.empty("list", allocator)) {
list.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
list.allocateNew();
list.setValueCount(0);

assertEmptyListOffsetBuffer(list);
}
}

@Test
public void testUnallocatedEmptyListOffsetBuffer() {
try (ListVector list = ListVector.empty("list", allocator)) {
list.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
list.setValueCount(0);

assertEmptyListOffsetBuffer(list);
}
}

@Test
public void testSplitAndTransferEmptyListOffsetBuffer() {
try (ListVector source = ListVector.empty("source", allocator);
ListVector target = ListVector.empty("target", allocator)) {
source.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
target.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
source.allocateNew();
source.setValueCount(0);

TransferPair transferPair = source.makeTransferPair(target);
transferPair.splitAndTransfer(0, 0);

assertEmptyListOffsetBuffer(target);
}
}

@Test
public void testSplitAndTransferEmptyListAllocatesOffsetBuffer() {
try (ListVector fromVector = ListVector.empty("fromVector", allocator);
ListVector toVector = ListVector.empty("toVector", allocator)) {
fromVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
fromVector.allocateNew();
fromVector.setValueCount(0);

TransferPair transferPair = fromVector.makeTransferPair(toVector);
transferPair.splitAndTransfer(0, 0);

assertAllocatedEmptyListOffsetBuffer(toVector);
}
}

@Test
public void testSplitAndTransferEmptyNestedListAllocatesOffsetBuffers() {
try (ListVector fromVector = ListVector.empty("fromVector", allocator);
ListVector toVector = ListVector.empty("toVector", allocator)) {
fromVector.addOrGetVector(FieldType.nullable(MinorType.LIST.getType()));
ListVector childVector = (ListVector) fromVector.getDataVector();
childVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
fromVector.allocateNew();
fromVector.setValueCount(0);

TransferPair transferPair = fromVector.makeTransferPair(toVector);
transferPair.splitAndTransfer(0, 0);

assertAllocatedEmptyListOffsetBuffer(toVector);
assertAllocatedEmptyListOffsetBuffer((ListVector) toVector.getDataVector());
}
}

private ArrowBuf assertEmptyListOffsetBuffer(ListVector list) {
List<ArrowBuf> buffers = list.getFieldBuffers();
ArrowBuf offsetBuffer = buffers.get(1);
assertEquals(BaseRepeatedValueVector.OFFSET_WIDTH, offsetBuffer.readableBytes());
assertTrue(offsetBuffer.capacity() >= BaseRepeatedValueVector.OFFSET_WIDTH);
assertEquals(0, offsetBuffer.getInt(0));
return offsetBuffer;
}

private void assertAllocatedEmptyListOffsetBuffer(ListVector list) {
ArrowBuf offsetBuffer = list.getOffsetBuffer();
assertTrue(offsetBuffer.capacity() >= BaseRepeatedValueVector.OFFSET_WIDTH);
assertEquals(0, offsetBuffer.getInt(0));
}

@Test
public void testIsEmpty() {
try (final ListVector vector = ListVector.empty("list", allocator)) {
Expand Down Expand Up @@ -1380,7 +1465,7 @@ public void testCopyValueSafeForExtensionType() throws Exception {
}

@Test
public void testEmptyListOffsetBuffer() {
public void testEmptyListOffsetBufferSize() {
// Test that ListVector has correct readableBytes after allocation.
// According to Arrow spec, offset buffer must have N+1 entries.
// Even when N=0, it should contain [0].
Expand Down
Loading
Loading