From 808f672422ec2e6bf73bb258c4a2c5d382d83f52 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Fri, 7 Mar 2025 14:28:11 -0800 Subject: [PATCH 01/14] Add jar build action. --- .github/workflows/jarbuild.yml | 552 +++++++++++++++++++++++++++++++++ 1 file changed, 552 insertions(+) create mode 100644 .github/workflows/jarbuild.yml diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml new file mode 100644 index 0000000000..ae2981cd6d --- /dev/null +++ b/.github/workflows/jarbuild.yml @@ -0,0 +1,552 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: JarBuild +on: + workflow_dispatch: + inputs: + arrow_branch: + description: 'Arrow branch to build' + required: true + default: main + arrow_repo: + description: 'Arrow repository' + type: string + required: true + default: apache/arrow + release_tag_name: + description: 'release tag name' + type: string + required: true + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true +permissions: + contents: read +jobs: + source: + name: Source + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - name: Set env + run: echo "release_tag_name=$(echo $release_tag_name)" >> $GITHUB_ENV + - name: Prepare for tag + run: | + echo "${{github.event.inputs.release_tag_name}}" + ver=$(echo ${{github.event.inputs.release_tag_name}}) + version=${ver%-rc*} + version=${version#v} + rc=${ver#*-rc} + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "VERSION=${version}" + echo "RC=${rc}" >> ${GITHUB_ENV} + echo "RC=${rc}" + - name: Archive + run: | + id="apache-arrow-java-${VERSION}" + tar_gz="${id}.tar.gz" + echo "TAR_GZ=${tar_gz}" >> ${GITHUB_ENV} + git archive HEAD --prefix "${id}/" --output "${tar_gz}" + sha256sum "${tar_gz}" > "${tar_gz}.sha256" + sha512sum "${tar_gz}" > "${tar_gz}.sha512" + - name: Audit + run: | + dev/release/run_rat.sh "${TAR_GZ}" + - name: Upload source archive + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: release-source + path: | + apache-arrow-java-* + jni-linux: + name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + timeout-minutes: 120 + needs: + - source + strategy: + fail-fast: false + matrix: + platform: + - runs_on: ubuntu-latest + arch: "x86_64" + archery_arch: "amd64" + - runs_on: ubuntu-24.04-arm + arch: "aarch_64" + archery_arch: "arm64v8" + env: + # architecture name used for archery build + ARCH: ${{ matrix.platform.archery_arch }} + DOCKER_VOLUME_PREFIX: .docker/ + permissions: + contents: read + packages: write + steps: + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Extract source archive + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # run: | + # ci/scripts/download_cpp.sh + - name: Checkout Apache Arrow C++ + # if: github.event_name == 'schedule' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: ${{github.event.inputs.arrow_repo}} + ref: ${{github.event.inputs.arrow_branch}} + path: arrow + - name: Checkout apache/arrow-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-testing + path: arrow/testing + - name: Checkout apache/parquet-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/parquet-testing + path: arrow/cpp/submodules/parquet-testing + - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Cache + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: .docker + key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} + restore-keys: jni-linux-${{ matrix.platform.arch }}- + - name: Build + run: | + docker compose run vcpkg-jni + - name: Push Docker image + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main' + run: | + docker compose push vcpkg-jni + - name: Compress into single artifact to keep directory structure + run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: jni-linux-${{ matrix.platform.arch }} + path: jni-linux-${{ matrix.platform.arch }}.tar.gz + jni-macos: + name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + timeout-minutes: 45 + needs: + - source + strategy: + fail-fast: false + matrix: + platform: + - { runs_on: macos-13, arch: "x86_64"} + - { runs_on: macos-14, arch: "aarch_64" } + env: + MACOSX_DEPLOYMENT_TARGET: "14.0" + steps: + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Extract source archive + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # run: | + # ci/scripts/download_cpp.sh + - name: Checkout Apache Arrow C++ + # if: github.event_name == 'schedule' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: ${{github.event.inputs.arrow_repo}} + ref: ${{github.event.inputs.arrow_branch}} + path: arrow + - name: Checkout apache/arrow-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-testing + path: arrow/testing + - name: Checkout apache/parquet-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/parquet-testing + path: arrow/cpp/submodules/parquet-testing + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + cache: 'pip' + python-version: 3.12 + - name: Install Archery + run: pip install -e arrow/dev/archery[all] + - name: Install dependencies + run: | + # We want to use llvm@14 to avoid shared z3 + # dependency. llvm@14 doesn't depend on z3 and llvm depends + # on z3. And Homebrew's z3 provides only shared library. It + # doesn't provides static z3 because z3's CMake doesn't accept + # building both shared and static libraries at once. + # See also: Z3_BUILD_LIBZ3_SHARED in + # https://github.com/Z3Prover/z3/blob/master/README-CMake.md + # + # If llvm is installed, Apache Arrow C++ uses llvm rather than + # llvm@14 because llvm is newer than llvm@14. + brew uninstall llvm || : + + # Ensure updating python@XXX with the "--overwrite" option. + # If python@XXX is updated without "--overwrite", it causes + # a conflict error. Because Python 3 installed not by + # Homebrew exists in /usr/local on GitHub Actions. If + # Homebrew's python@XXX is updated without "--overwrite", it + # tries to replace /usr/local/bin/2to3 and so on and causes + # a conflict error. + brew update + for python_package in $(brew list | grep python@); do + brew install --overwrite ${python_package} + done + brew install --overwrite python + + if [ "$(uname -m)" = "arm64" ]; then + # pkg-config formula is deprecated but it's still installed + # in GitHub Actions runner now. We can remove this once + # pkg-config formula is removed from GitHub Actions runner. + brew uninstall pkg-config || : + brew uninstall pkg-config@0.29.2 || : + fi + + brew bundle --file=arrow/cpp/Brewfile + # We want to link aws-sdk-cpp statically but Homebrew's + # aws-sdk-cpp provides only shared library. If we have + # Homebrew's aws-sdk-cpp, our build mix Homebrew's + # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's + # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. + brew uninstall aws-sdk-cpp + # We want to use bundled RE2 for static linking. If + # Homebrew's RE2 is installed, its header file may be used. + # We uninstall Homebrew's RE2 to ensure using bundled RE2. + brew uninstall grpc || : # gRPC depends on RE2 + brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too + brew uninstall re2 + # We want to use bundled Protobuf for static linking. If + # Homebrew's Protobuf is installed, its library file may be + # used on test We uninstall Homebrew's Protobuf to ensure using + # bundled Protobuf. + brew uninstall protobuf + + brew bundle --file=Brewfile + - name: Prepare ccache + run: | + echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + - name: Cache ccache + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ccache + key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} + restore-keys: jni-macos-${{ matrix.platform.arch }}- + - name: Build + run: | + set -e + # make brew Java available to CMake + export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home + ci/scripts/jni_macos_build.sh . arrow build jni + - name: Compress into single artifact to keep directory structure + run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: jni-macos-${{ matrix.platform.arch }} + path: jni-macos-${{ matrix.platform.arch }}.tar.gz + jni-windows: + name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs_on }} + timeout-minutes: 45 + needs: + - source + strategy: + fail-fast: false + matrix: + platform: + - runs_on: windows-2019 + arch: "x86_64" + steps: + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Extract source archive + shell: bash + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + # - name: Download the latest Apache Arrow C++ + # if: github.event_name != 'schedule' + # shell: bash + # run: | + # ci/scripts/download_cpp.sh + - name: Checkout Apache Arrow C++ + # if: github.event_name == 'schedule' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: ${{github.event.inputs.arrow_repo}} + ref: ${{github.event.inputs.arrow_branch}} + path: arrow + - name: Set up Java + uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0 + with: + java-version: '11' + distribution: 'temurin' + - name: Download Timezone Database + shell: bash + run: | + arrow/ci/scripts/download_tz_database.sh + - name: Install ccache + shell: bash + run: | + env | sort + version=4.10.2 + base_name="ccache-${version}-windows-x86_64" + url="https://github.com/ccache/ccache/releases/download/v${version}/${base_name}.zip" + curl --fail --location --remote-name "${url}" + unzip "${base_name}.zip" + chmod +x "${base_name}/ccache.exe" + mv "${base_name}/ccache.exe" /usr/bin/ + rm -rf "${base_name}"{,.zip} + - name: Prepare ccache + shell: bash + run: | + echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + - name: Cache ccache + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ccache + key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }} + restore-keys: jni-windows-${{ matrix.platform.arch }}- + - name: Build + shell: cmd + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + REM For ORC + set TZDIR=/c/msys64/usr/share/zoneinfo + bash -c "ci/scripts/jni_windows_build.sh . arrow build jni" + - name: Compress into single artifact to keep directory structure + shell: bash + run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: jni-windows-${{ matrix.platform.arch }} + path: jni-windows-${{ matrix.platform.arch }}.tar.gz + binaries: + name: Binaries + runs-on: ubuntu-latest + needs: + - jni-linux + - jni-macos + - jni-windows + steps: + - name: Download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + path: artifacts + - name: Decompress artifacts + run: | + mv artifacts/*/*.tar.gz . + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + tar -xvzf jni-linux-x86_64.tar.gz + tar -xvzf jni-linux-aarch_64.tar.gz + tar -xvzf jni-macos-x86_64.tar.gz + tar -xvzf jni-macos-aarch_64.tar.gz + tar -xvzf jni-windows-x86_64.tar.gz + - name: Test that shared libraries exist + run: | + set -x + + test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so + test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so + test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.so + test -f jni/gandiva_jni/x86_64/libgandiva_jni.so + + test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so + test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so + test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.so + test -f jni/gandiva_jni/aarch_64/libgandiva_jni.so + + test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib + test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib + test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib + test -f jni/gandiva_jni/x86_64/libgandiva_jni.dylib + + test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib + test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib + test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib + test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib + + test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll + test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll + test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll + - name: Checkout apache/arrow-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: apache/arrow-testing + path: testing + - name: Cache ~/.m2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ~/.m2 + key: binaries-build-${{ hashFiles('**/*.java', '**/pom.xml') }} + restore-keys: binaries-build- + - name: Build bundled JAR and docs + run: | + ci/scripts/jni_full_build.sh . jni binaries + - name: Prepare docs + run: | + mkdir -p docs + cp -a target/site/apidocs reference + tar -cvzf reference.tar.gz reference + - name: Upload binaries + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: release-binaries + path: binaries/* + - name: Upload docs + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: reference + path: reference.tar.gz + docs: + name: Docs + needs: + - binaries + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + cache: 'pip' + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Download Javadocs + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: reference + - name: Extract source archive + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + - name: Build + run: | + cd docs + python -m venv venv + source venv/bin/activate + pip install -r requirements.txt + make html + tar -xf ../reference.tar.gz -C build/html + - name: Compress into single artifact to keep directory structure + run: tar -cvzf html.tar.gz -C docs/build html + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: release-html + path: html.tar.gz + verify: + name: Verify + needs: + - binaries + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-latest + steps: + - name: Download release artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + pattern: release-* + - name: Verify + run: | + mv release-source/* ./ + tar_gz=$(echo apache-arrow-java-*.tar.gz) + version=${tar_gz#apache-arrow-java-} + version=${version%.tar.gz} + # rc isn't used with VERIFY_DOWNLOAD=0 + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + rc="${GITHUB_REF_NAME#*-rc}" + else + rc=$(date +%Y%m%d) + fi + tar -xf ${tar_gz} + export VERIFY_DEFAULT=0 + export VERIFY_BINARY=1 + export VERIFY_SOURCE=1 + cd apache-arrow-java-${version} + mv ../${tar_gz}* ./ + mv ../release-binaries binaries + dev/release/verify_rc.sh "${version}" "${rc}" + upload: + name: Upload + needs: + - docs + - verify + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download release artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + pattern: release-* + path: artifacts + - name: Upload + run: | + # GH-499: How to create release notes? + echo "${{github.event.inputs.release_tag_name}}" + ver=$(echo ${{github.event.inputs.release_tag_name}}) + version=${ver%-rc*} + version=${version#v} + rc=${ver#*-rc} + gh release create ${{github.event.inputs.release_tag_name}} \ + --generate-notes \ + --prerelease \ + --repo ${GITHUB_REPOSITORY} \ + --title "Apache Arrow Java ${version} RC${rc}" \ + --verify-tag + # GitHub CLI does not respect their own rate limits + # https://github.com/cli/cli/issues/9586 + for artifact in artifacts/*/*; do + sleep 1 + gh release upload ${{github.event.inputs.release_tag_name}} \ + --repo ${GITHUB_REPOSITORY} \ + $artifact + done + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 858601a894a79d72f470cf455d9362f8505581f1 Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:07:31 -0400 Subject: [PATCH 02/14] DX-122953 Preserve empty list offset buffer --- .../arrow/vector/complex/LargeListVector.java | 16 +++++++-- .../arrow/vector/complex/ListVector.java | 16 +++++++-- .../arrow/vector/TestLargeListVector.java | 33 +++++++++++++++++++ .../apache/arrow/vector/TestListVector.java | 33 +++++++++++++++++++ 4 files changed, 94 insertions(+), 4 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index ed075352c9..1f44474a73 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -305,15 +305,27 @@ public void exportCDataBuffers(List buffers, ArrowBuf buffersPtr, long /** Set the reader and writer indexes for the inner buffers. */ private void setReaderAndWriterIndex() { + final long requiredOffsetBufferCapacity = (long) (valueCount + 1) * OFFSET_WIDTH; validityBuffer.readerIndex(0); offsetBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); + ensureEmptyOffsetBufferCapacity(requiredOffsetBufferCapacity); } else { validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); } + // IPC serializers use readerIndex and writerIndex to determine readable bytes. Even when the + // list is empty, the Arrow layout requires the offset buffer to contain offset[0]. + offsetBuffer.writerIndex(requiredOffsetBufferCapacity); + } + + private void ensureEmptyOffsetBufferCapacity(long requiredCapacity) { + if (offsetBuffer.capacity() >= requiredCapacity) { + return; + } + ArrowBuf oldOffsetBuffer = offsetBuffer; + offsetBuffer = allocateOffsetBuffer(requiredCapacity); + oldOffsetBuffer.getReferenceManager().release(); } /** diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 3daeb6d77b..09b8b0b92c 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -263,15 +263,27 @@ public void exportCDataBuffers(List buffers, ArrowBuf buffersPtr, long /** Set the reader and writer indexes for the inner buffers. */ private void setReaderAndWriterIndex() { + final long requiredOffsetBufferCapacity = (long) (valueCount + 1) * OFFSET_WIDTH; validityBuffer.readerIndex(0); offsetBuffer.readerIndex(0); if (valueCount == 0) { validityBuffer.writerIndex(0); - offsetBuffer.writerIndex(0); + ensureEmptyOffsetBufferCapacity(requiredOffsetBufferCapacity); } else { validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - offsetBuffer.writerIndex((valueCount + 1) * OFFSET_WIDTH); } + // IPC serializers use readerIndex and writerIndex to determine readable bytes. Even when the + // list is empty, the Arrow layout requires the offset buffer to contain offset[0]. + offsetBuffer.writerIndex(requiredOffsetBufferCapacity); + } + + private void ensureEmptyOffsetBufferCapacity(long requiredCapacity) { + if (offsetBuffer.capacity() >= requiredCapacity) { + return; + } + ArrowBuf oldOffsetBuffer = offsetBuffer; + offsetBuffer = allocateOffsetBuffer(requiredCapacity); + oldOffsetBuffer.getReferenceManager().release(); } /** diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index 101d942d2a..fdeff3da5e 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import io.netty.buffer.NettyArrowBuf; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.BaseRepeatedValueVector; @@ -955,6 +956,38 @@ public void testGetBufferSizeFor() { } } + @Test + public void testEmptyLargeListOffsetBuffer() { + try (LargeListVector list = LargeListVector.empty("list", allocator)) { + list.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + list.allocateNew(); + list.setValueCount(0); + + assertEmptyLargeListOffsetBuffer(list); + } + } + + @Test + public void testUnallocatedEmptyLargeListOffsetBufferCanBeUnwrappedAsNettyBuffer() { + try (LargeListVector list = LargeListVector.empty("list", allocator)) { + list.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + list.setValueCount(0); + + ArrowBuf offsetBuffer = assertEmptyLargeListOffsetBuffer(list); + NettyArrowBuf nettyBuffer = NettyArrowBuf.unwrapBuffer(offsetBuffer); + assertEquals(LargeListVector.OFFSET_WIDTH, nettyBuffer.readableBytes()); + } + } + + private ArrowBuf assertEmptyLargeListOffsetBuffer(LargeListVector list) { + List buffers = list.getFieldBuffers(); + ArrowBuf offsetBuffer = buffers.get(1); + assertEquals(LargeListVector.OFFSET_WIDTH, offsetBuffer.readableBytes()); + assertTrue(offsetBuffer.capacity() >= LargeListVector.OFFSET_WIDTH); + assertEquals(0L, offsetBuffer.getLong(0)); + return offsetBuffer; + } + @Test public void testIsEmpty() { try (final LargeListVector vector = LargeListVector.empty("list", allocator)) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 1d6fa39f9e..df8548d6ce 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import io.netty.buffer.NettyArrowBuf; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.AutoCloseables; @@ -1135,6 +1136,38 @@ public void testGetBufferSizeFor() { } } + @Test + public void testEmptyListOffsetBuffer() { + try (ListVector list = ListVector.empty("list", allocator)) { + list.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + list.allocateNew(); + list.setValueCount(0); + + assertEmptyListOffsetBuffer(list); + } + } + + @Test + public void testUnallocatedEmptyListOffsetBufferCanBeUnwrappedAsNettyBuffer() { + try (ListVector list = ListVector.empty("list", allocator)) { + list.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + list.setValueCount(0); + + ArrowBuf offsetBuffer = assertEmptyListOffsetBuffer(list); + NettyArrowBuf nettyBuffer = NettyArrowBuf.unwrapBuffer(offsetBuffer); + assertEquals(BaseRepeatedValueVector.OFFSET_WIDTH, nettyBuffer.readableBytes()); + } + } + + private ArrowBuf assertEmptyListOffsetBuffer(ListVector list) { + List buffers = list.getFieldBuffers(); + ArrowBuf offsetBuffer = buffers.get(1); + assertEquals(BaseRepeatedValueVector.OFFSET_WIDTH, offsetBuffer.readableBytes()); + assertTrue(offsetBuffer.capacity() >= BaseRepeatedValueVector.OFFSET_WIDTH); + assertEquals(0, offsetBuffer.getInt(0)); + return offsetBuffer; + } + @Test public void testIsEmpty() { try (final ListVector vector = ListVector.empty("list", allocator)) { From f5cfa00fea1c104220c71b4fd0696bc975229748 Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:34:35 -0400 Subject: [PATCH 03/14] DX-122953 Avoid NettyArrowBuf test import --- .../java/org/apache/arrow/vector/TestLargeListVector.java | 7 ++----- .../test/java/org/apache/arrow/vector/TestListVector.java | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index fdeff3da5e..1d089c93e3 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -25,7 +25,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import io.netty.buffer.NettyArrowBuf; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.BaseRepeatedValueVector; @@ -968,14 +967,12 @@ public void testEmptyLargeListOffsetBuffer() { } @Test - public void testUnallocatedEmptyLargeListOffsetBufferCanBeUnwrappedAsNettyBuffer() { + public void testUnallocatedEmptyLargeListOffsetBuffer() { try (LargeListVector list = LargeListVector.empty("list", allocator)) { list.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); list.setValueCount(0); - ArrowBuf offsetBuffer = assertEmptyLargeListOffsetBuffer(list); - NettyArrowBuf nettyBuffer = NettyArrowBuf.unwrapBuffer(offsetBuffer); - assertEquals(LargeListVector.OFFSET_WIDTH, nettyBuffer.readableBytes()); + assertEmptyLargeListOffsetBuffer(list); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index df8548d6ce..f128358394 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -26,7 +26,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import io.netty.buffer.NettyArrowBuf; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.AutoCloseables; @@ -1148,14 +1147,12 @@ public void testEmptyListOffsetBuffer() { } @Test - public void testUnallocatedEmptyListOffsetBufferCanBeUnwrappedAsNettyBuffer() { + public void testUnallocatedEmptyListOffsetBuffer() { try (ListVector list = ListVector.empty("list", allocator)) { list.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); list.setValueCount(0); - ArrowBuf offsetBuffer = assertEmptyListOffsetBuffer(list); - NettyArrowBuf nettyBuffer = NettyArrowBuf.unwrapBuffer(offsetBuffer); - assertEquals(BaseRepeatedValueVector.OFFSET_WIDTH, nettyBuffer.readableBytes()); + assertEmptyListOffsetBuffer(list); } } From 10b259b665204ba0a135cf015f76be277f08cdc9 Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:35:23 -0400 Subject: [PATCH 04/14] MINOR: Fix workflow trailing whitespace --- .github/workflows/jarbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index ae2981cd6d..286cc8b910 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -16,7 +16,7 @@ # under the License. name: JarBuild -on: +on: workflow_dispatch: inputs: arrow_branch: From e29398982eafb1efc128e9c625d1050ae16fc062 Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Mon, 22 Jun 2026 21:46:25 -0400 Subject: [PATCH 05/14] MINOR: Reset list test helper capacity --- .../arrow/vector/testing/ValueVectorDataPopulator.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java b/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java index 849fe6d667..d7bc987afd 100644 --- a/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java +++ b/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java @@ -633,9 +633,10 @@ public static void setVector(LargeVarCharVector vector, String... values) { /** Populate values for {@link ListVector}. */ public static void setVector(ListVector vector, List... values) { - vector.allocateNewSafe(); Types.MinorType type = Types.MinorType.INT; vector.addOrGetVector(FieldType.nullable(type.getType())); + vector.setInitialCapacity(values.length); + vector.allocateNewSafe(); IntVector dataVector = (IntVector) vector.getDataVector(); dataVector.allocateNew(); @@ -662,9 +663,10 @@ public static void setVector(ListVector vector, List... values) { /** Populate values for {@link LargeListVector}. */ public static void setVector(LargeListVector vector, List... values) { - vector.allocateNewSafe(); Types.MinorType type = Types.MinorType.INT; vector.addOrGetVector(FieldType.nullable(type.getType())); + vector.setInitialCapacity(values.length); + vector.allocateNewSafe(); IntVector dataVector = (IntVector) vector.getDataVector(); dataVector.allocateNew(); From 91dafa228356a2584cd85ba44519623add47831c Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Mon, 22 Jun 2026 21:52:34 -0400 Subject: [PATCH 06/14] Revert "MINOR: Reset list test helper capacity" This reverts commit e29398982eafb1efc128e9c625d1050ae16fc062. --- .../arrow/vector/testing/ValueVectorDataPopulator.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java b/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java index d7bc987afd..849fe6d667 100644 --- a/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java +++ b/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java @@ -633,10 +633,9 @@ public static void setVector(LargeVarCharVector vector, String... values) { /** Populate values for {@link ListVector}. */ public static void setVector(ListVector vector, List... values) { + vector.allocateNewSafe(); Types.MinorType type = Types.MinorType.INT; vector.addOrGetVector(FieldType.nullable(type.getType())); - vector.setInitialCapacity(values.length); - vector.allocateNewSafe(); IntVector dataVector = (IntVector) vector.getDataVector(); dataVector.allocateNew(); @@ -663,10 +662,9 @@ public static void setVector(ListVector vector, List... values) { /** Populate values for {@link LargeListVector}. */ public static void setVector(LargeListVector vector, List... values) { + vector.allocateNewSafe(); Types.MinorType type = Types.MinorType.INT; vector.addOrGetVector(FieldType.nullable(type.getType())); - vector.setInitialCapacity(values.length); - vector.allocateNewSafe(); IntVector dataVector = (IntVector) vector.getDataVector(); dataVector.allocateNew(); From b9a3433e60eb4552d67b1d427a0a40e7f2078125 Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Tue, 23 Jun 2026 08:14:26 -0400 Subject: [PATCH 07/14] DX-122953 Preserve list offset allocation size --- .../java/org/apache/arrow/vector/complex/LargeListVector.java | 2 ++ .../main/java/org/apache/arrow/vector/complex/ListVector.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 1f44474a73..b44f96c9d0 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -323,8 +323,10 @@ private void ensureEmptyOffsetBufferCapacity(long requiredCapacity) { if (offsetBuffer.capacity() >= requiredCapacity) { return; } + long previousOffsetAllocationSizeInBytes = offsetAllocationSizeInBytes; ArrowBuf oldOffsetBuffer = offsetBuffer; offsetBuffer = allocateOffsetBuffer(requiredCapacity); + offsetAllocationSizeInBytes = previousOffsetAllocationSizeInBytes; oldOffsetBuffer.getReferenceManager().release(); } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 09b8b0b92c..84b1e2b882 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -281,8 +281,10 @@ private void ensureEmptyOffsetBufferCapacity(long requiredCapacity) { if (offsetBuffer.capacity() >= requiredCapacity) { return; } + long previousOffsetAllocationSizeInBytes = offsetAllocationSizeInBytes; ArrowBuf oldOffsetBuffer = offsetBuffer; offsetBuffer = allocateOffsetBuffer(requiredCapacity); + offsetAllocationSizeInBytes = previousOffsetAllocationSizeInBytes; oldOffsetBuffer.getReferenceManager().release(); } From 6525bbed89f3370e108e148e1759f16fecb5d0cb Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Tue, 30 Jun 2026 17:12:23 -0400 Subject: [PATCH 08/14] DX-122953 Preserve empty split transfer offset buffer --- .../apache/arrow/vector/complex/ListVector.java | 3 +++ .../apache/arrow/vector/TestLargeListVector.java | 16 ++++++++++++++++ .../org/apache/arrow/vector/TestListVector.java | 16 ++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 84b1e2b882..2d6b819244 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -584,6 +584,9 @@ public void splitAndTransfer(int startIndex, int length) { dataTransferPair.splitAndTransfer(startPoint, sliceLength); to.lastSet = length - 1; to.setValueCount(length); + } else { + to.ensureEmptyOffsetBufferCapacity(OFFSET_WIDTH); + to.setValueCount(0); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index 1d089c93e3..b8cd4c41ce 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -976,6 +976,22 @@ public void testUnallocatedEmptyLargeListOffsetBuffer() { } } + @Test + public void testSplitAndTransferEmptyLargeListOffsetBuffer() { + try (LargeListVector source = LargeListVector.empty("source", allocator); + LargeListVector target = LargeListVector.empty("target", allocator)) { + source.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + target.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + source.allocateNew(); + source.setValueCount(0); + + TransferPair transferPair = source.makeTransferPair(target); + transferPair.splitAndTransfer(0, 0); + + assertEmptyLargeListOffsetBuffer(target); + } + } + private ArrowBuf assertEmptyLargeListOffsetBuffer(LargeListVector list) { List buffers = list.getFieldBuffers(); ArrowBuf offsetBuffer = buffers.get(1); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index f128358394..8a32959ec7 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -1156,6 +1156,22 @@ public void testUnallocatedEmptyListOffsetBuffer() { } } + @Test + public void testSplitAndTransferEmptyListOffsetBuffer() { + try (ListVector source = ListVector.empty("source", allocator); + ListVector target = ListVector.empty("target", allocator)) { + source.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + target.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + source.allocateNew(); + source.setValueCount(0); + + TransferPair transferPair = source.makeTransferPair(target); + transferPair.splitAndTransfer(0, 0); + + assertEmptyListOffsetBuffer(target); + } + } + private ArrowBuf assertEmptyListOffsetBuffer(ListVector list) { List buffers = list.getFieldBuffers(); ArrowBuf offsetBuffer = buffers.get(1); From 1a084bb9ff1bf435bc7909c1e5818acdcc6d2be0 Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:30:20 -0400 Subject: [PATCH 09/14] DX-122953 Complete empty list split transfer --- .../java/org/apache/arrow/vector/complex/ListVector.java | 1 + .../org/apache/arrow/vector/TestSplitAndTransfer.java | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 2d6b819244..86ea184994 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -586,6 +586,7 @@ public void splitAndTransfer(int startIndex, int length) { to.setValueCount(length); } else { to.ensureEmptyOffsetBufferCapacity(OFFSET_WIDTH); + dataTransferPair.splitAndTransfer(0, 0); to.setValueCount(0); } } diff --git a/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index adf4eba10c..1f5e7cea30 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -28,8 +28,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListViewVector; @@ -115,6 +117,8 @@ public void testWithEmptyVector() { TransferPair transferPair = listVector.getTransferPair(allocator); transferPair.splitAndTransfer(0, 0); assertEquals(0, transferPair.getTo().getValueCount()); + transferPair.getTo().clear(); + listVector.clear(); // BaseFixedWidthVector IntVector intVector = new IntVector("", allocator); transferPair = intVector.getTransferPair(allocator); @@ -911,6 +915,11 @@ public void testListVectorZeroStartIndexAndLength() { tp.splitAndTransfer(0, 0); assertEquals(valueCount, newListVector.getValueCount()); + List buffers = newListVector.getFieldBuffers(); + ArrowBuf offsetBuffer = buffers.get(1); + assertEquals(BaseRepeatedValueVector.OFFSET_WIDTH, offsetBuffer.readableBytes()); + assertTrue(offsetBuffer.capacity() >= BaseRepeatedValueVector.OFFSET_WIDTH); + assertEquals(0, offsetBuffer.getInt(0)); newListVector.clear(); } From 56157d214987cedfcd39124d173781aac4b622a6 Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:38:40 -0400 Subject: [PATCH 10/14] DX-122953 Complete empty large list split transfer --- .../arrow/vector/complex/LargeListVector.java | 38 +++++++++++-------- .../arrow/vector/TestSplitAndTransfer.java | 24 ++++++++++++ 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index b44f96c9d0..5455215535 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -686,24 +686,30 @@ public void splitAndTransfer(int startIndex, int length) { startIndex, length, valueCount); - final long startPoint = offsetBuffer.getLong((long) startIndex * OFFSET_WIDTH); - final long sliceLength = - offsetBuffer.getLong((long) (startIndex + length) * OFFSET_WIDTH) - startPoint; to.clear(); - to.offsetBuffer = to.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); - /* splitAndTransfer offset buffer */ - for (int i = 0; i < length + 1; i++) { - final long relativeOffset = - offsetBuffer.getLong((long) (startIndex + i) * OFFSET_WIDTH) - startPoint; - to.offsetBuffer.setLong((long) i * OFFSET_WIDTH, relativeOffset); + if (length > 0) { + final long startPoint = offsetBuffer.getLong((long) startIndex * OFFSET_WIDTH); + final long sliceLength = + offsetBuffer.getLong((long) (startIndex + length) * OFFSET_WIDTH) - startPoint; + to.offsetBuffer = to.allocateOffsetBuffer((length + 1) * OFFSET_WIDTH); + /* splitAndTransfer offset buffer */ + for (int i = 0; i < length + 1; i++) { + final long relativeOffset = + offsetBuffer.getLong((long) (startIndex + i) * OFFSET_WIDTH) - startPoint; + to.offsetBuffer.setLong((long) i * OFFSET_WIDTH, relativeOffset); + } + /* splitAndTransfer validity buffer */ + splitAndTransferValidityBuffer(startIndex, length, to); + /* splitAndTransfer data buffer */ + dataTransferPair.splitAndTransfer( + checkedCastToInt(startPoint), checkedCastToInt(sliceLength)); + to.lastSet = length - 1; + to.setValueCount(length); + } else { + to.ensureEmptyOffsetBufferCapacity(OFFSET_WIDTH); + dataTransferPair.splitAndTransfer(0, 0); + to.setValueCount(0); } - /* splitAndTransfer validity buffer */ - splitAndTransferValidityBuffer(startIndex, length, to); - /* splitAndTransfer data buffer */ - dataTransferPair.splitAndTransfer( - checkedCastToInt(startPoint), checkedCastToInt(sliceLength)); - to.lastSet = length - 1; - to.setValueCount(length); } /* diff --git a/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index 1f5e7cea30..e6823731d0 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -34,6 +34,7 @@ import org.apache.arrow.vector.complex.BaseRepeatedValueVector; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.LargeListVector; import org.apache.arrow.vector.complex.LargeListViewVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; @@ -944,6 +945,29 @@ public void testLargeListViewVectorZeroStartIndexAndLength() { } } + @Test + public void testLargeListVectorZeroStartIndexAndLength() { + try (final LargeListVector listVector = LargeListVector.empty("largelist", allocator); + final LargeListVector newListVector = LargeListVector.empty("newList", allocator)) { + + listVector.allocateNew(); + final int valueCount = 0; + listVector.setValueCount(valueCount); + + final TransferPair tp = listVector.makeTransferPair(newListVector); + + tp.splitAndTransfer(0, 0); + assertEquals(valueCount, newListVector.getValueCount()); + List buffers = newListVector.getFieldBuffers(); + ArrowBuf offsetBuffer = buffers.get(1); + assertEquals(LargeListVector.OFFSET_WIDTH, offsetBuffer.readableBytes()); + assertTrue(offsetBuffer.capacity() >= LargeListVector.OFFSET_WIDTH); + assertEquals(0L, offsetBuffer.getLong(0)); + + newListVector.clear(); + } + } + @Test public void testStructVectorZeroStartIndexAndLength() { Map metadata = new HashMap<>(); From 3c55ad4d98fa988c13097ba3f69a1801cb68eaac Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:38:53 -0400 Subject: [PATCH 11/14] DX-122953 Preserve offset bytes when expanding empty buffers --- .../java/org/apache/arrow/vector/complex/LargeListVector.java | 2 ++ .../main/java/org/apache/arrow/vector/complex/ListVector.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 5455215535..c68345f77d 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -326,6 +326,8 @@ private void ensureEmptyOffsetBufferCapacity(long requiredCapacity) { long previousOffsetAllocationSizeInBytes = offsetAllocationSizeInBytes; ArrowBuf oldOffsetBuffer = offsetBuffer; offsetBuffer = allocateOffsetBuffer(requiredCapacity); + offsetBuffer.setBytes( + 0, oldOffsetBuffer, 0, Math.min(oldOffsetBuffer.capacity(), requiredCapacity)); offsetAllocationSizeInBytes = previousOffsetAllocationSizeInBytes; oldOffsetBuffer.getReferenceManager().release(); } diff --git a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index 86ea184994..ddbbb6a562 100644 --- a/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -284,6 +284,8 @@ private void ensureEmptyOffsetBufferCapacity(long requiredCapacity) { long previousOffsetAllocationSizeInBytes = offsetAllocationSizeInBytes; ArrowBuf oldOffsetBuffer = offsetBuffer; offsetBuffer = allocateOffsetBuffer(requiredCapacity); + offsetBuffer.setBytes( + 0, oldOffsetBuffer, 0, Math.min(oldOffsetBuffer.capacity(), requiredCapacity)); offsetAllocationSizeInBytes = previousOffsetAllocationSizeInBytes; oldOffsetBuffer.getReferenceManager().release(); } From 9d50ee1a8a0996dd1d2f8e1cd8b8b2e379ff35c4 Mon Sep 17 00:00:00 2001 From: Prashanth Badari <102688956+prashanthbdremio@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:06:27 -0400 Subject: [PATCH 12/14] DX-122953 Add empty split transfer coverage --- .../arrow/vector/TestLargeListVector.java | 39 +++++++++++++++++++ .../apache/arrow/vector/TestListVector.java | 39 +++++++++++++++++++ .../arrow/vector/TestSplitAndTransfer.java | 15 +++---- 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index b8cd4c41ce..a8e5ba264a 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -992,6 +992,39 @@ public void testSplitAndTransferEmptyLargeListOffsetBuffer() { } } + @Test + public void testSplitAndTransferEmptyLargeListAllocatesOffsetBuffer() { + try (LargeListVector fromVector = LargeListVector.empty("fromVector", allocator); + LargeListVector toVector = LargeListVector.empty("toVector", allocator)) { + fromVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + fromVector.allocateNew(); + fromVector.setValueCount(0); + + TransferPair transferPair = fromVector.makeTransferPair(toVector); + transferPair.splitAndTransfer(0, 0); + + assertAllocatedEmptyLargeListOffsetBuffer(toVector); + } + } + + @Test + public void testSplitAndTransferEmptyNestedLargeListAllocatesOffsetBuffers() { + try (LargeListVector fromVector = LargeListVector.empty("fromVector", allocator); + LargeListVector toVector = LargeListVector.empty("toVector", allocator)) { + fromVector.addOrGetVector(FieldType.nullable(MinorType.LARGELIST.getType())); + LargeListVector childVector = (LargeListVector) fromVector.getDataVector(); + childVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + fromVector.allocateNew(); + fromVector.setValueCount(0); + + TransferPair transferPair = fromVector.makeTransferPair(toVector); + transferPair.splitAndTransfer(0, 0); + + assertAllocatedEmptyLargeListOffsetBuffer(toVector); + assertAllocatedEmptyLargeListOffsetBuffer((LargeListVector) toVector.getDataVector()); + } + } + private ArrowBuf assertEmptyLargeListOffsetBuffer(LargeListVector list) { List buffers = list.getFieldBuffers(); ArrowBuf offsetBuffer = buffers.get(1); @@ -1001,6 +1034,12 @@ private ArrowBuf assertEmptyLargeListOffsetBuffer(LargeListVector list) { return offsetBuffer; } + private void assertAllocatedEmptyLargeListOffsetBuffer(LargeListVector list) { + ArrowBuf offsetBuffer = list.getOffsetBuffer(); + assertTrue(offsetBuffer.capacity() >= LargeListVector.OFFSET_WIDTH); + assertEquals(0L, offsetBuffer.getLong(0)); + } + @Test public void testIsEmpty() { try (final LargeListVector vector = LargeListVector.empty("list", allocator)) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 8a32959ec7..de23694deb 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -1172,6 +1172,39 @@ public void testSplitAndTransferEmptyListOffsetBuffer() { } } + @Test + public void testSplitAndTransferEmptyListAllocatesOffsetBuffer() { + try (ListVector fromVector = ListVector.empty("fromVector", allocator); + ListVector toVector = ListVector.empty("toVector", allocator)) { + fromVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + fromVector.allocateNew(); + fromVector.setValueCount(0); + + TransferPair transferPair = fromVector.makeTransferPair(toVector); + transferPair.splitAndTransfer(0, 0); + + assertAllocatedEmptyListOffsetBuffer(toVector); + } + } + + @Test + public void testSplitAndTransferEmptyNestedListAllocatesOffsetBuffers() { + try (ListVector fromVector = ListVector.empty("fromVector", allocator); + ListVector toVector = ListVector.empty("toVector", allocator)) { + fromVector.addOrGetVector(FieldType.nullable(MinorType.LIST.getType())); + ListVector childVector = (ListVector) fromVector.getDataVector(); + childVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); + fromVector.allocateNew(); + fromVector.setValueCount(0); + + TransferPair transferPair = fromVector.makeTransferPair(toVector); + transferPair.splitAndTransfer(0, 0); + + assertAllocatedEmptyListOffsetBuffer(toVector); + assertAllocatedEmptyListOffsetBuffer((ListVector) toVector.getDataVector()); + } + } + private ArrowBuf assertEmptyListOffsetBuffer(ListVector list) { List buffers = list.getFieldBuffers(); ArrowBuf offsetBuffer = buffers.get(1); @@ -1181,6 +1214,12 @@ private ArrowBuf assertEmptyListOffsetBuffer(ListVector list) { return offsetBuffer; } + private void assertAllocatedEmptyListOffsetBuffer(ListVector list) { + ArrowBuf offsetBuffer = list.getOffsetBuffer(); + assertTrue(offsetBuffer.capacity() >= BaseRepeatedValueVector.OFFSET_WIDTH); + assertEquals(0, offsetBuffer.getInt(0)); + } + @Test public void testIsEmpty() { try (final ListVector vector = ListVector.empty("list", allocator)) { diff --git a/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index e6823731d0..a51c3e74fa 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -114,15 +114,16 @@ private void populateDenseUnionVector(final DenseUnionVector vector, int valueCo @Test public void testWithEmptyVector() { // MapVector use TransferImpl from ListVector - ListVector listVector = ListVector.empty("", allocator); - TransferPair transferPair = listVector.getTransferPair(allocator); - transferPair.splitAndTransfer(0, 0); - assertEquals(0, transferPair.getTo().getValueCount()); - transferPair.getTo().clear(); - listVector.clear(); + try (ListVector listVector = ListVector.empty("", allocator)) { + TransferPair transferPair = listVector.getTransferPair(allocator); + try (ValueVector toVector = transferPair.getTo()) { + transferPair.splitAndTransfer(0, 0); + assertEquals(0, toVector.getValueCount()); + } + } // BaseFixedWidthVector IntVector intVector = new IntVector("", allocator); - transferPair = intVector.getTransferPair(allocator); + TransferPair transferPair = intVector.getTransferPair(allocator); transferPair.splitAndTransfer(0, 0); assertEquals(0, transferPair.getTo().getValueCount()); // BaseVariableWidthVector From c5c4abdd095c7a04ce453359c6f6805db688b9e3 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 6 Jul 2026 17:18:05 -0700 Subject: [PATCH 13/14] Rename test method for clarity --- .../test/java/org/apache/arrow/vector/TestLargeListVector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index 425e82616a..e91dbabbfb 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -1186,7 +1186,7 @@ public void testCopyValueSafeForExtensionType() throws Exception { } @Test - public void testEmptyLargeListOffsetBuffer() { + public void testEmptyLargeListOffsetBufferSize() { // Test that LargeListVector has correct readableBytes after allocation. // According to Arrow spec, offset buffer must have N+1 entries. // Even when N=0, it should contain [0]. From b3acd709c6045b1dd19e7906dcf18524825e6e0c Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Mon, 6 Jul 2026 17:18:34 -0700 Subject: [PATCH 14/14] Rename testEmptyListOffsetBuffer to testEmptyListOffsetBufferSize Renamed test method to better reflect its purpose. --- .../src/test/java/org/apache/arrow/vector/TestListVector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index bce9e7cd4a..60b675a1b5 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -1465,7 +1465,7 @@ public void testCopyValueSafeForExtensionType() throws Exception { } @Test - public void testEmptyListOffsetBuffer() { + public void testEmptyListOffsetBufferSize() { // Test that ListVector has correct readableBytes after allocation. // According to Arrow spec, offset buffer must have N+1 entries. // Even when N=0, it should contain [0].