ci: resolve Arrow and Parquet from conda-forge on Unix legs

.github/actions/setup-conda-arrow/action.yml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Set up conda Arrow
+description: Install Arrow and Parquet from conda-forge for CMake builds.
+
+inputs:
+  extra-packages:
+    description: Extra conda packages to install, space separated.
+    required: false
+    default: ""
+
+runs:
+  using: composite
+  steps:
+    - name: Set up conda
+      uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
+      with:
+        miniforge-version: latest
+        channels: conda-forge
+        channel-priority: strict
+        activate-environment: arrow
+    - name: Install Arrow and Parquet
+      shell: bash -el {0}
+      env:
+        EXTRA_PACKAGES: ${{ inputs.extra-packages }}
+      run: '"${{ github.action_path }}/install-arrow.sh"'

.github/actions/setup-conda-arrow/install-arrow.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Installs Arrow and Parquet into the active conda environment and exports the
+# paths CMake and the test binaries need. Run from the setup-conda-arrow action.
+
+set -euo pipefail
+
+toolchain="${GITHUB_WORKSPACE}/cmake_modules/IcebergThirdpartyToolchain.cmake"
+
+# Keep conda Arrow in sync with the vendored build.
+arrow_version=$(sed -nE 's/^set\(ICEBERG_ARROW_BUILD_VERSION "([0-9]+\.[0-9]+\.[0-9]+)"\)$/\1/p' "${toolchain}")
+if [[ -z "${arrow_version}" ]]; then
+    echo "::error::Could not read ICEBERG_ARROW_BUILD_VERSION from ${toolchain}"
+    exit 1
+fi
+
+packages=("libarrow==${arrow_version}" "libparquet==${arrow_version}")
+if [[ -n "${EXTRA_PACKAGES:-}" ]]; then
+    read -r -a extra_packages <<< "${EXTRA_PACKAGES}"
+    packages+=("${extra_packages[@]}")
+fi
+mamba install -y "${packages[@]}"
+
+echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "${GITHUB_ENV}"
+# build_iceberg.sh reads this and fails if CMake configured vendored Arrow anyway.
+echo "ICEBERG_REQUIRE_SYSTEM_ARROW=ON" >> "${GITHUB_ENV}"
+
+if [[ "${RUNNER_OS}" == "macOS" ]]; then
+    echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:+${DYLD_FALLBACK_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "${GITHUB_ENV}"
+else
+    echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "${GITHUB_ENV}"
+fi

.github/workflows/aws_test.yml

@@ -66,7 +66,6 @@ jobs:
             s3: "ON"
             sigv4: "ON"
             bundle_awssdk: "OFF"
-            aws-sdk-features: core,config,s3,identity-management,sts,transfer
           - title: macOS 26 ARM64, S3, bundled AWS SDK
             runs-on: macos-26
             s3: "ON"
@@ -88,27 +87,6 @@ jobs:
         if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libjitterentropy3-dev
-      - name: Cache vcpkg packages
-        if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }}
-        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
-        id: vcpkg-cache
-        with:
-          path: /usr/local/share/vcpkg/installed
-          key: vcpkg-x64-linux-aws-sdk-cpp-s3-${{ matrix.s3 }}-sigv4-${{ matrix.sigv4 }}-${{ hashFiles('.github/workflows/aws_test.yml') }}
-      - name: Install AWS SDK via vcpkg
-        if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && steps.vcpkg-cache.outputs.cache-hit != 'true' }}
-        shell: bash
-        # Retry to ride out transient GitHub/mirror download failures (504s).
-        run: |
-          for attempt in 1 2 3; do
-            if vcpkg install "aws-sdk-cpp[${{ matrix.aws-sdk-features }}]:x64-linux"; then
-              exit 0
-            fi
-            echo "::warning::vcpkg install failed (attempt ${attempt}/3), retrying in 30s"
-            sleep 30
-          done
-          echo "::error::vcpkg install failed after 3 attempts"
-          exit 1
       - name: Set Ubuntu Compilers
         if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
         run: |
@@ -118,14 +96,18 @@ jobs:
         if: ${{ matrix.s3 == 'ON' }}
         shell: bash
         run: bash ci/scripts/start_minio.sh
+      - name: Set up conda Arrow and AWS SDK
+        if: ${{ matrix.bundle_awssdk == 'OFF' }}
+        uses: ./.github/actions/setup-conda-arrow
+        with:
+          # Use the same AWS SDK as Arrow to avoid incompatible symbols at runtime.
+          extra-packages: aws-sdk-cpp
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
           key-prefix: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}
       - name: Build and test Iceberg
         shell: bash
-        env:
-          CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }}
         run: ci/scripts/build_iceberg.sh "$(pwd)" OFF ON ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }}
       - name: Save sccache
         if: always()

.github/workflows/cpp-linter.yml

@@ -61,6 +61,11 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
+        with:
+          # This leg builds Hive, whose Thrift runtime needs Boost headers.
+          extra-packages: libboost-devel
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -79,7 +84,12 @@ jobs:
             -DICEBERG_SQL_SQLITE=ON \
             -DICEBERG_SQL_POSTGRESQL=ON \
             -DICEBERG_SQL_MYSQL=ON \
-            -DICEBERG_BUILD_HIVE=ON
+            -DICEBERG_BUILD_HIVE=ON \
+            -DICEBERG_BUNDLE_THRIFT=OFF
+          if [[ -d _deps/vendoredarrow-src ]]; then
+            echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+            exit 1
+          fi
           cmake --build .
       - name: Save sccache
         if: always()

.github/workflows/sanitizer_test.yml

@@ -52,6 +52,8 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -64,6 +66,10 @@ jobs:
           mkdir build && cd build
           cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON \
             -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+          if [[ -d _deps/vendoredarrow-src ]]; then
+            echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+            exit 1
+          fi
           cmake --build . --verbose
       - name: Save sccache
         if: always()

.github/workflows/sql_catalog_test.yml

@@ -97,6 +97,9 @@ jobs:
         shell: pwsh
         run: |
           vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows
+      - name: Set up conda Arrow
+        if: ${{ !startsWith(matrix.runs-on, 'windows') }}
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -115,6 +118,10 @@ jobs:
             -DCMAKE_C_COMPILER_LAUNCHER=sccache \
             -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
             ${{ matrix.cmake_extra_args }}
+          if [[ "${RUNNER_OS}" != "Windows" && -d build/_deps/vendoredarrow-src ]]; then
+            echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+            exit 1
+          fi
       - name: Build SQL catalog tests
         shell: bash
         run: cmake --build build --target sql_catalog_test

.github/workflows/test.yml

@@ -62,6 +62,8 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -94,7 +96,9 @@ jobs:
     env:
       SCCACHE_DIR: ${{ github.workspace }}/.sccache
       SCCACHE_CACHE_SIZE: "2G"
-      ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_BUILD_HIVE=ON"
+      # Prebuilt Arrow exports no Thrift target, so resolve the Thrift runtime
+      # from conda instead of Arrow's bundled build.
+      ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_BUILD_HIVE=ON -DICEBERG_BUNDLE_THRIFT=OFF"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -103,6 +107,12 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
+        with:
+          # Thrift's public C++ headers include Boost, and libboost-devel (not
+          # libboost-headers) is the package carrying BoostConfig.cmake.
+          extra-packages: libboost-devel
       - name: Restore sccache cache
         uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
@@ -140,6 +150,8 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:

ci/scripts/build_iceberg.sh

@@ -88,6 +88,11 @@ fi
 
 cmake "${CMAKE_ARGS[@]}" ${source_dir}
 
+if [[ "${ICEBERG_REQUIRE_SYSTEM_ARROW:-OFF}" == "ON" && -d _deps/vendoredarrow-src ]]; then
+    echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+    exit 1
+fi
+
 cmake --build . --target install
 if [[ "${run_tests}" == "ON" ]]; then
     ctest --output-on-failure