From 25ac397338a376924d2017e0cd5a220db9310f9a Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Tue, 6 Oct 2026 17:57:21 +0300
Subject: [PATCH 01/12] ci: build e2e Spark from the official image and add a
 Spark 4 job

The Spark e2e Dockerfile downloaded the ~300MB Spark tarball from
archive.apache.org on every run, which took 40-70 minutes. Past the
1h mark the AWS OIDC credentials expire, so send-report failed with
ExpiredToken.

Build on the official apache/spark image and fetch only the Delta and
Postgres JDBC jars from Maven Central. The spark target stays on Spark
3.3.2 + Delta 2.2.0: Spark 3.5's thrift server breaks Elementary's
DDL via PyHive (CORE-1558).

Add a spark4 target (Spark 4.2 + Delta 4.4), which also covers Spark
4's ANSI mode default.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/test-all-warehouses.yml     |  1 +
 .github/workflows/test-warehouse.yml          | 31 +++++---
 tests/e2e_dbt_project/docker-compose.yml      | 28 ++++++--
 tests/e2e_dbt_project/docker/spark/Dockerfile | 71 ++++++++++++-------
 .../docker/spark/entrypoint.sh                |  2 +-
 tests/profiles/profiles.yml.j2                |  6 +-
 6 files changed, 99 insertions(+), 40 deletions(-)

diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml
index 620f8af35..e4b56d9ea 100644
--- a/.github/workflows/test-all-warehouses.yml
+++ b/.github/workflows/test-all-warehouses.yml
@@ -114,6 +114,7 @@ jobs:
             trino,
             dremio,
             spark,
+            spark4,
             fabric,
             sqlserver,
             vertica,
diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml
index 0775a24b0..6159cb412 100644
--- a/.github/workflows/test-warehouse.yml
+++ b/.github/workflows/test-warehouse.yml
@@ -13,6 +13,7 @@ on:
           - redshift
           - databricks_catalog
           - spark
+          - spark4
           - athena
           - clickhouse
           - duckdb
@@ -127,7 +128,7 @@ jobs:
       - name: Validate workflow inputs
         run: |
           case "$WAREHOUSE_TYPE" in
-            postgres|snowflake|bigquery|redshift|databricks_catalog|spark|athena|clickhouse|duckdb|trino|dremio|fabric|sqlserver|vertica) ;;
+            postgres|snowflake|bigquery|redshift|databricks_catalog|spark|spark4|athena|clickhouse|duckdb|trino|dremio|fabric|sqlserver|vertica) ;;
             *)
               echo "Unsupported warehouse type: $WAREHOUSE_TYPE" >&2
               exit 1
@@ -233,10 +234,11 @@ jobs:
           docker compose run --rm dremio-setup
 
       - name: Start Spark
-        if: inputs.warehouse-type == 'spark'
+        if: inputs.warehouse-type == 'spark' || inputs.warehouse-type == 'spark4'
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
         run: |
-          docker compose up -d --build --wait spark-thrift
+          # spark-thrift (Spark 3) or spark4-thrift (Spark 4)
+          docker compose up -d --build --wait "${WAREHOUSE_TYPE}-thrift"
 
       - name: Start SQL Server
         if: inputs.warehouse-type == 'sqlserver'
@@ -256,7 +258,7 @@ jobs:
           python-version: "3.11"
 
       - name: Install Spark requirements
-        if: inputs.warehouse-type == 'spark'
+        if: inputs.warehouse-type == 'spark' || inputs.warehouse-type == 'spark4'
         run: sudo apt-get install -y python3-dev libsasl2-dev gcc
 
       - name: Install ODBC driver for SQL Server
@@ -298,7 +300,10 @@ jobs:
           case "$WAREHOUSE_TYPE" in
             databricks_catalog) DBT_ADAPTER="databricks" ;;
             athena) DBT_ADAPTER="athena-community" ;;
-            spark) DBT_ADAPTER_EXTRA="[PyHive]" ;;
+            spark|spark4)
+              DBT_ADAPTER="spark"
+              DBT_ADAPTER_EXTRA="[PyHive]"
+              ;;
             # dbt-sqlserver 1.12 moved pyodbc (still its default backend) to the [pyodbc] extra.
             sqlserver) DBT_ADAPTER_EXTRA="[pyodbc]" ;;
             # dbt-fabric 1.11 (requires Python >=3.11) switched pyodbc ->
@@ -343,6 +348,8 @@ jobs:
             EXTRA="$WAREHOUSE_TYPE"
             if [ "$WAREHOUSE_TYPE" = "databricks_catalog" ]; then
               EXTRA="databricks"
+            elif [ "$WAREHOUSE_TYPE" = "spark4" ]; then
+              EXTRA="spark"
             fi
             pip install ".[$EXTRA]"
           fi
@@ -368,7 +375,7 @@ jobs:
           # This enables caching the seeded database state between runs.
           IS_DOCKER=false
           case "$WAREHOUSE_TYPE" in
-            postgres|clickhouse|trino|dremio|duckdb|spark|sqlserver|vertica) IS_DOCKER=true ;;
+            postgres|clickhouse|trino|dremio|duckdb|spark|spark4|sqlserver|vertica) IS_DOCKER=true ;;
           esac
 
           if [ "$IS_DOCKER" = "true" ]; then
@@ -422,12 +429,18 @@ jobs:
 
       - name: Seed e2e dbt project (external)
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
-        if: steps.seed-cache.outputs.cache-hit != 'true' && (inputs.warehouse-type == 'dremio' || inputs.warehouse-type == 'spark')
-        run: python load_seeds_external.py "$WAREHOUSE_TYPE" "$SCHEMA_NAME" data
+        if: steps.seed-cache.outputs.cache-hit != 'true' && (inputs.warehouse-type == 'dremio' || inputs.warehouse-type == 'spark' || inputs.warehouse-type == 'spark4')
+        run: |
+          # The seeders are keyed by adapter; spark4 uses the spark adapter.
+          ADAPTER="$WAREHOUSE_TYPE"
+          if [ "$ADAPTER" = "spark4" ]; then
+            ADAPTER="spark"
+          fi
+          python load_seeds_external.py "$ADAPTER" "$SCHEMA_NAME" data
 
       - name: Seed e2e dbt project
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
-        if: steps.seed-cache.outputs.cache-hit != 'true' && inputs.warehouse-type != 'dremio' && inputs.warehouse-type != 'spark'
+        if: steps.seed-cache.outputs.cache-hit != 'true' && inputs.warehouse-type != 'dremio' && inputs.warehouse-type != 'spark' && inputs.warehouse-type != 'spark4'
         run: |
           # The seed step runs before Elementary models exist, so artifact hooks
           # would try to insert into missing Elementary relations.
diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml
index 44c616a35..76faca3e6 100644
--- a/tests/e2e_dbt_project/docker-compose.yml
+++ b/tests/e2e_dbt_project/docker-compose.yml
@@ -246,11 +246,20 @@ services:
       - dremio-lakehouse
 
   # ── Spark Thrift Server (with Delta Lake + Hive Metastore) ─────────
-  spark-thrift:
+  # spark-thrift and spark4-thrift share the port, metastore and warehouse, so
+  # run one at a time and `docker compose down -v` before switching.
+  spark-thrift: &spark-thrift
     container_name: spark-thrift
     build:
       context: ./docker/spark
       dockerfile: Dockerfile
+      # Not Spark 3.5 yet: its thrift server returns no row set for DDL/DML,
+      # which PyHive's fetchall() can't handle, and Elementary fetches the
+      # result of its CREATE TABLE AS statements.
+      args:
+        SPARK_IMAGE: apache/spark:v3.3.2
+        SCALA_BINARY_VERSION: "2.12"
+        DELTA_VERSION: 2.2.0
     ports:
       - "10000:10000"
       - "4040:4040"
@@ -261,7 +270,7 @@ services:
       --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
       --name Thrift JDBC/ODBC Server
     healthcheck:
-      test: ["CMD-SHELL", "nc -z localhost 10000"]
+      test: ["CMD", "bash", "-c", "</dev/tcp/localhost/10000"]
       interval: 10s
       timeout: 5s
       retries: 20
@@ -269,12 +278,23 @@ services:
     user: "0"
     volumes:
       - spark-warehouse:/spark-warehouse/
-      - ./docker/spark/hive-site.xml:/usr/spark/conf/hive-site.xml
-      - ./docker/spark/spark-defaults.conf:/usr/spark/conf/spark-defaults.conf
+      - ./docker/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
+      - ./docker/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
       - ./data:/seed-data:ro
     environment:
       - WAIT_FOR=spark-hive-metastore:5432
 
+  spark4-thrift:
+    <<: *spark-thrift
+    container_name: spark4-thrift
+    build:
+      context: ./docker/spark
+      dockerfile: Dockerfile
+      args:
+        SPARK_IMAGE: apache/spark:4.2.0
+        SCALA_BINARY_VERSION: "2.13"
+        DELTA_VERSION: 4.4.1
+
   spark-hive-metastore:
     image: postgres:15-alpine
     volumes:
diff --git a/tests/e2e_dbt_project/docker/spark/Dockerfile b/tests/e2e_dbt_project/docker/spark/Dockerfile
index aab2ab394..e0b34e3da 100644
--- a/tests/e2e_dbt_project/docker/spark/Dockerfile
+++ b/tests/e2e_dbt_project/docker/spark/Dockerfile
@@ -1,33 +1,54 @@
-ARG OPENJDK_VERSION=8
-FROM eclipse-temurin:${OPENJDK_VERSION}-jre
+# The official image ships the Spark distribution, so we don't download the
+# ~300MB tarball from archive.apache.org, which is heavily throttled and used
+# to take 40-70 minutes in CI.
+ARG SPARK_IMAGE=apache/spark:v3.3.2
 
-ARG SPARK_VERSION=3.3.2
-ARG HADOOP_VERSION=3
+FROM ${SPARK_IMAGE} AS spark
+
+# The Spark image has no wget/curl, so fetch the extra jars from Maven Central
+# in a separate stage.
+FROM alpine:3.22 AS jars
+ARG SCALA_BINARY_VERSION=2.12
 ARG DELTA_VERSION=2.2.0
+ARG POSTGRES_JDBC_VERSION=42.7.13
+RUN --mount=type=bind,from=spark,source=/opt/spark/jars,target=/spark-jars <<'EOF'
+set -eu
+# Delta 3.0 renamed the delta-core artifact to delta-spark.
+delta_artifact="delta-spark_${SCALA_BINARY_VERSION}"
+if [ "${DELTA_VERSION%%.*}" -lt 3 ]; then
+  delta_artifact="delta-core_${SCALA_BINARY_VERSION}"
+fi
+jars="io/delta/${delta_artifact}/${DELTA_VERSION}/${delta_artifact}-${DELTA_VERSION}.jar
+io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar
+org/postgresql/postgresql/${POSTGRES_JDBC_VERSION}/postgresql-${POSTGRES_JDBC_VERSION}.jar"
+if [ "${DELTA_VERSION%%.*}" -ge 4 ]; then
+  # Delta 4 also depends on Delta Kernel. Spark already ships Kernel's other
+  # dependencies (Hadoop, Parquet, Jackson, RoaringBitmap) except
+  # jackson-datatype-jdk8, which we match to Spark's Jackson version.
+  jackson_version=$(ls /spark-jars | sed -n 's/^jackson-databind-\(.*\)\.jar$/\1/p')
+  for artifact in delta-kernel-api delta-kernel-defaults delta-kernel-unitycatalog; do
+    jars="$jars
+io/delta/${artifact}/${DELTA_VERSION}/${artifact}-${DELTA_VERSION}.jar"
+  done
+  jars="$jars
+com/fasterxml/jackson/datatype/jackson-datatype-jdk8/${jackson_version}/jackson-datatype-jdk8-${jackson_version}.jar"
+fi
+mkdir /jars
+for jar in $jars; do
+  wget -q -P /jars "https://repo1.maven.org/maven2/${jar}"
+done
+EOF
+
+FROM spark
 
-ENV SPARK_HOME /usr/spark
-ENV PATH="/usr/spark/bin:/usr/spark/sbin:${PATH}"
+ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}"
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends wget netcat-openbsd procps libpostgresql-jdbc-java && \
-    wget -q "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
-    tar xzf "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
-    rm "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" && \
-    mv "spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" /usr/spark && \
-    ln -s /usr/share/java/postgresql-jdbc4.jar /usr/spark/jars/postgresql-jdbc4.jar && \
-    wget -q "https://repo1.maven.org/maven2/io/delta/delta-core_2.12/${DELTA_VERSION}/delta-core_2.12-${DELTA_VERSION}.jar" \
-         -P /usr/spark/jars/ && \
-    wget -q "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \
-         -P /usr/spark/jars/ && \
-    apt-get remove -y wget && \
-    apt-get autoremove -y && \
-    apt-get clean
+COPY --from=jars --chmod=644 /jars/ ${SPARK_HOME}/jars/
+COPY --chmod=755 entrypoint.sh /scripts/entrypoint.sh
 
-COPY entrypoint.sh /scripts/
-RUN chmod +x /scripts/entrypoint.sh && \
-    groupadd -r spark && useradd -r -g spark -m spark && \
-    chown -R spark:spark /usr/spark /scripts
+# Spark's default warehouse dir is relative to the working directory; keep it
+# at /spark-warehouse, where docker-compose mounts its volume.
+WORKDIR /
 
-USER spark
 ENTRYPOINT ["/scripts/entrypoint.sh"]
 CMD ["--help"]
diff --git a/tests/e2e_dbt_project/docker/spark/entrypoint.sh b/tests/e2e_dbt_project/docker/spark/entrypoint.sh
index 36b8174c9..77080f73f 100644
--- a/tests/e2e_dbt_project/docker/spark/entrypoint.sh
+++ b/tests/e2e_dbt_project/docker/spark/entrypoint.sh
@@ -10,7 +10,7 @@ if [ -n "$WAIT_FOR" ]; then
         START_TS=$(date +%s)
 
         echo Waiting for $WAIT_FOR_HOST to listen on $WAIT_FOR_PORT...
-        while ! nc -z "$WAIT_FOR_HOST" "$WAIT_FOR_PORT"; do
+        while ! (exec 3<>"/dev/tcp/$WAIT_FOR_HOST/$WAIT_FOR_PORT") 2>/dev/null; do
             if [ $(( $(date +%s) - START_TS )) -ge "$WAIT_FOR_TIMEOUT_SECONDS" ]; then
                 echo "Timed out waiting for $WAIT_FOR_HOST:$WAIT_FOR_PORT after ${WAIT_FOR_TIMEOUT_SECONDS}s"
                 exit 1
diff --git a/tests/profiles/profiles.yml.j2 b/tests/profiles/profiles.yml.j2
index 7f7c83430..171763a4d 100644
--- a/tests/profiles/profiles.yml.j2
+++ b/tests/profiles/profiles.yml.j2
@@ -62,6 +62,10 @@ elementary_tests:
       file_format: delta
       threads: 4
 
+    # Spark 4 (spark4-thrift in docker-compose) listens on the same port.
+    spark4: &spark4
+      <<: *spark
+
     sqlserver: &sqlserver
       type: sqlserver
       driver: "ODBC Driver 18 for SQL Server"
@@ -165,7 +169,7 @@ elementary_tests:
 elementary:
   target: postgres
   outputs:
-{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'spark', 'fabric', 'sqlserver', 'vertica', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %}
+{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'spark', 'spark4', 'fabric', 'sqlserver', 'vertica', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %}
 {%- for t in targets %}
     {{ t }}:
       <<: *{{ t }}

From 60bd659c2fb8b50f1004839a2fd95b5a427d762e Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Tue, 6 Oct 2026 18:28:12 +0300
Subject: [PATCH 02/12] ci: print the Spark container logs when it fails to
 start

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/test-warehouse.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml
index 6159cb412..efac6a91a 100644
--- a/.github/workflows/test-warehouse.yml
+++ b/.github/workflows/test-warehouse.yml
@@ -238,7 +238,11 @@ jobs:
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
         run: |
           # spark-thrift (Spark 3) or spark4-thrift (Spark 4)
-          docker compose up -d --build --wait "${WAREHOUSE_TYPE}-thrift"
+          SERVICE="${WAREHOUSE_TYPE}-thrift"
+          if ! docker compose up -d --build --wait "$SERVICE"; then
+            docker compose logs "$SERVICE"
+            exit 1
+          fi
 
       - name: Start SQL Server
         if: inputs.warehouse-type == 'sqlserver'

From 32bc769cb0d0df23f8512035284bc8e5a4b0498d Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Tue, 6 Oct 2026 18:39:43 +0300
Subject: [PATCH 03/12] ci: run the spark e2e target on apache/spark:3.3.3

apache/spark:v3.3.2 ships JDK 11.0.16, whose container detection fails
on the cgroup v2 GitHub runners, so Spark crashes on startup with
InstanceNotFoundException: java.nio:type=BufferPool,name=direct.
apache/spark:3.3.3 ships JDK 11.0.20.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 tests/e2e_dbt_project/docker-compose.yml      | 6 ++++--
 tests/e2e_dbt_project/docker/spark/Dockerfile | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml
index 76faca3e6..1fef23c1b 100644
--- a/tests/e2e_dbt_project/docker-compose.yml
+++ b/tests/e2e_dbt_project/docker-compose.yml
@@ -255,9 +255,11 @@ services:
       dockerfile: Dockerfile
       # Not Spark 3.5 yet: its thrift server returns no row set for DDL/DML,
       # which PyHive's fetchall() can't handle, and Elementary fetches the
-      # result of its CREATE TABLE AS statements.
+      # result of its CREATE TABLE AS statements. Not apache/spark:v3.3.2
+      # either: its JDK 11.0.16 fails container detection on cgroup v2 hosts
+      # such as the GitHub runners, which makes Spark crash on startup.
       args:
-        SPARK_IMAGE: apache/spark:v3.3.2
+        SPARK_IMAGE: apache/spark:3.3.3
         SCALA_BINARY_VERSION: "2.12"
         DELTA_VERSION: 2.2.0
     ports:
diff --git a/tests/e2e_dbt_project/docker/spark/Dockerfile b/tests/e2e_dbt_project/docker/spark/Dockerfile
index e0b34e3da..04402c91c 100644
--- a/tests/e2e_dbt_project/docker/spark/Dockerfile
+++ b/tests/e2e_dbt_project/docker/spark/Dockerfile
@@ -1,7 +1,7 @@
 # The official image ships the Spark distribution, so we don't download the
 # ~300MB tarball from archive.apache.org, which is heavily throttled and used
 # to take 40-70 minutes in CI.
-ARG SPARK_IMAGE=apache/spark:v3.3.2
+ARG SPARK_IMAGE=apache/spark:3.3.3
 
 FROM ${SPARK_IMAGE} AS spark
 

From 1384c592ce4adacf8fae5d705316059d97c9a116 Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Wed, 7 Oct 2026 01:19:38 +0300
Subject: [PATCH 04/12] ci: drop the Spark entrypoint script and its metastore
 wait loop

entrypoint.sh waited for the host:port pairs in WAIT_FOR (the Hive
metastore's Postgres) to accept connections, then ran spark-submit.
The wait is redundant: docker-compose already starts spark-thrift and
spark4-thrift only after spark-hive-metastore reports healthy
(depends_on with condition: service_healthy, backed by pg_isready),
so the loop always succeeded on its first try.

Without the loop the script was only `exec spark-submit "$@"`, which
ENTRYPOINT ["spark-submit"] does directly: the exec form passes the
container arguments through and runs spark-submit as PID 1. Also drop
the WAIT_FOR variable, which nothing reads now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6
---
 tests/e2e_dbt_project/docker-compose.yml      |  2 --
 tests/e2e_dbt_project/docker/spark/Dockerfile |  3 +--
 .../docker/spark/entrypoint.sh                | 23 -------------------
 3 files changed, 1 insertion(+), 27 deletions(-)
 delete mode 100644 tests/e2e_dbt_project/docker/spark/entrypoint.sh

diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml
index 1fef23c1b..92b367d9e 100644
--- a/tests/e2e_dbt_project/docker-compose.yml
+++ b/tests/e2e_dbt_project/docker-compose.yml
@@ -283,8 +283,6 @@ services:
       - ./docker/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
       - ./docker/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
       - ./data:/seed-data:ro
-    environment:
-      - WAIT_FOR=spark-hive-metastore:5432
 
   spark4-thrift:
     <<: *spark-thrift
diff --git a/tests/e2e_dbt_project/docker/spark/Dockerfile b/tests/e2e_dbt_project/docker/spark/Dockerfile
index 04402c91c..b75c30ac2 100644
--- a/tests/e2e_dbt_project/docker/spark/Dockerfile
+++ b/tests/e2e_dbt_project/docker/spark/Dockerfile
@@ -44,11 +44,10 @@ FROM spark
 ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}"
 
 COPY --from=jars --chmod=644 /jars/ ${SPARK_HOME}/jars/
-COPY --chmod=755 entrypoint.sh /scripts/entrypoint.sh
 
 # Spark's default warehouse dir is relative to the working directory; keep it
 # at /spark-warehouse, where docker-compose mounts its volume.
 WORKDIR /
 
-ENTRYPOINT ["/scripts/entrypoint.sh"]
+ENTRYPOINT ["spark-submit"]
 CMD ["--help"]
diff --git a/tests/e2e_dbt_project/docker/spark/entrypoint.sh b/tests/e2e_dbt_project/docker/spark/entrypoint.sh
deleted file mode 100644
index 77080f73f..000000000
--- a/tests/e2e_dbt_project/docker/spark/entrypoint.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-if [ -n "$WAIT_FOR" ]; then
-    WAIT_FOR_TIMEOUT_SECONDS="${WAIT_FOR_TIMEOUT_SECONDS:-180}"
-    IFS=';' read -a HOSTPORT_ARRAY <<< "$WAIT_FOR"
-    for HOSTPORT in "${HOSTPORT_ARRAY[@]}"
-    do
-        WAIT_FOR_HOST=${HOSTPORT%:*}
-        WAIT_FOR_PORT=${HOSTPORT#*:}
-        START_TS=$(date +%s)
-
-        echo Waiting for $WAIT_FOR_HOST to listen on $WAIT_FOR_PORT...
-        while ! (exec 3<>"/dev/tcp/$WAIT_FOR_HOST/$WAIT_FOR_PORT") 2>/dev/null; do
-            if [ $(( $(date +%s) - START_TS )) -ge "$WAIT_FOR_TIMEOUT_SECONDS" ]; then
-                echo "Timed out waiting for $WAIT_FOR_HOST:$WAIT_FOR_PORT after ${WAIT_FOR_TIMEOUT_SECONDS}s"
-                exit 1
-            fi
-            sleep 2
-        done
-    done
-fi
-
-exec spark-submit "$@"

From 9aaa2c8b5a55759938b55fd87f8c252717a4c937 Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Wed, 7 Oct 2026 01:31:18 +0300
Subject: [PATCH 05/12] ci: address review notes on the Spark e2e image

- Download the Delta and Postgres JDBC jars with ADD <url> instead of a
  separate Alpine stage with a shell script. The Dockerfile is a single
  stage again, and the Delta artifact name (delta-core_2.12 or
  delta-spark_2.13, renamed in Delta 3.0) is a build arg instead of being
  derived from the version.
- Drop the Delta Kernel and jackson-datatype-jdk8 jars from the Spark 4
  image. Plain Delta tables on a Hive metastore never load Kernel: the
  Spark 4 e2e gives the same results without them (dbt test PASS=32
  WARN=1 ERROR=60, no class loading errors).
- Use symbolic --chmod (a+r) for readability. Files ADDed from a URL
  are 600, so the jars need it.
- Health-check Spark with a beeline `select 1` instead of an open port,
  so the service is only healthy once the thrift server answers
  queries. beeline ships with the official images, takes 0-2s and exits
  non-zero when the server is unreachable.
- Replace the Spark-only container log dump with one step that prints
  the logs of all warehouse containers when a start step fails.
- Put the spark|spark4 adapter case on one line like the other entries.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6
---
 .github/workflows/test-warehouse.yml          | 18 +++----
 tests/e2e_dbt_project/docker-compose.yml      | 20 ++++++--
 tests/e2e_dbt_project/docker/spark/Dockerfile | 48 +++++--------------
 3 files changed, 38 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml
index efac6a91a..38b22f9f4 100644
--- a/.github/workflows/test-warehouse.yml
+++ b/.github/workflows/test-warehouse.yml
@@ -238,11 +238,7 @@ jobs:
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
         run: |
           # spark-thrift (Spark 3) or spark4-thrift (Spark 4)
-          SERVICE="${WAREHOUSE_TYPE}-thrift"
-          if ! docker compose up -d --build --wait "$SERVICE"; then
-            docker compose logs "$SERVICE"
-            exit 1
-          fi
+          docker compose up -d --build --wait "${WAREHOUSE_TYPE}-thrift"
 
       - name: Start SQL Server
         if: inputs.warehouse-type == 'sqlserver'
@@ -256,6 +252,13 @@ jobs:
         run: |
           docker compose up -d --wait vertica
 
+      # `docker compose up --wait` only says that a container exited or is
+      # unhealthy, so print the containers' own logs when a start step fails.
+      - name: Show warehouse service logs
+        if: failure()
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        run: docker compose logs --no-color
+
       - name: Setup Python
         uses: actions/setup-python@v6
         with:
@@ -304,10 +307,7 @@ jobs:
           case "$WAREHOUSE_TYPE" in
             databricks_catalog) DBT_ADAPTER="databricks" ;;
             athena) DBT_ADAPTER="athena-community" ;;
-            spark|spark4)
-              DBT_ADAPTER="spark"
-              DBT_ADAPTER_EXTRA="[PyHive]"
-              ;;
+            spark|spark4) DBT_ADAPTER="spark" DBT_ADAPTER_EXTRA="[PyHive]" ;;
             # dbt-sqlserver 1.12 moved pyodbc (still its default backend) to the [pyodbc] extra.
             sqlserver) DBT_ADAPTER_EXTRA="[pyodbc]" ;;
             # dbt-fabric 1.11 (requires Python >=3.11) switched pyodbc ->
diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml
index 92b367d9e..1b9590426 100644
--- a/tests/e2e_dbt_project/docker-compose.yml
+++ b/tests/e2e_dbt_project/docker-compose.yml
@@ -260,7 +260,7 @@ services:
       # such as the GitHub runners, which makes Spark crash on startup.
       args:
         SPARK_IMAGE: apache/spark:3.3.3
-        SCALA_BINARY_VERSION: "2.12"
+        DELTA_ARTIFACT: delta-core_2.12
         DELTA_VERSION: 2.2.0
     ports:
       - "10000:10000"
@@ -271,8 +271,22 @@ services:
     command: >
       --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
       --name Thrift JDBC/ODBC Server
+    # Run a query through the official image's beeline client, which checks
+    # that the thrift server opens sessions and answers queries, not only
+    # that the port is open.
     healthcheck:
-      test: ["CMD", "bash", "-c", "</dev/tcp/localhost/10000"]
+      test:
+        [
+          "CMD",
+          "beeline",
+          "-u",
+          "jdbc:hive2://localhost:10000",
+          "-n",
+          "dbt",
+          "--silent=true",
+          "-e",
+          "select 1",
+        ]
       interval: 10s
       timeout: 5s
       retries: 20
@@ -292,7 +306,7 @@ services:
       dockerfile: Dockerfile
       args:
         SPARK_IMAGE: apache/spark:4.2.0
-        SCALA_BINARY_VERSION: "2.13"
+        DELTA_ARTIFACT: delta-spark_2.13
         DELTA_VERSION: 4.4.1
 
   spark-hive-metastore:
diff --git a/tests/e2e_dbt_project/docker/spark/Dockerfile b/tests/e2e_dbt_project/docker/spark/Dockerfile
index b75c30ac2..44a2d8ac7 100644
--- a/tests/e2e_dbt_project/docker/spark/Dockerfile
+++ b/tests/e2e_dbt_project/docker/spark/Dockerfile
@@ -1,49 +1,25 @@
+# syntax=docker/dockerfile:1
+
 # The official image ships the Spark distribution, so we don't download the
 # ~300MB tarball from archive.apache.org, which is heavily throttled and used
 # to take 40-70 minutes in CI.
 ARG SPARK_IMAGE=apache/spark:3.3.3
+FROM ${SPARK_IMAGE}
 
-FROM ${SPARK_IMAGE} AS spark
-
-# The Spark image has no wget/curl, so fetch the extra jars from Maven Central
-# in a separate stage.
-FROM alpine:3.22 AS jars
-ARG SCALA_BINARY_VERSION=2.12
+# Delta 3.0 renamed delta-core to delta-spark, so the artifact name (with its
+# Scala suffix) is passed in alongside the version.
+ARG DELTA_ARTIFACT=delta-core_2.12
 ARG DELTA_VERSION=2.2.0
 ARG POSTGRES_JDBC_VERSION=42.7.13
-RUN --mount=type=bind,from=spark,source=/opt/spark/jars,target=/spark-jars <<'EOF'
-set -eu
-# Delta 3.0 renamed the delta-core artifact to delta-spark.
-delta_artifact="delta-spark_${SCALA_BINARY_VERSION}"
-if [ "${DELTA_VERSION%%.*}" -lt 3 ]; then
-  delta_artifact="delta-core_${SCALA_BINARY_VERSION}"
-fi
-jars="io/delta/${delta_artifact}/${DELTA_VERSION}/${delta_artifact}-${DELTA_VERSION}.jar
-io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar
-org/postgresql/postgresql/${POSTGRES_JDBC_VERSION}/postgresql-${POSTGRES_JDBC_VERSION}.jar"
-if [ "${DELTA_VERSION%%.*}" -ge 4 ]; then
-  # Delta 4 also depends on Delta Kernel. Spark already ships Kernel's other
-  # dependencies (Hadoop, Parquet, Jackson, RoaringBitmap) except
-  # jackson-datatype-jdk8, which we match to Spark's Jackson version.
-  jackson_version=$(ls /spark-jars | sed -n 's/^jackson-databind-\(.*\)\.jar$/\1/p')
-  for artifact in delta-kernel-api delta-kernel-defaults delta-kernel-unitycatalog; do
-    jars="$jars
-io/delta/${artifact}/${DELTA_VERSION}/${artifact}-${DELTA_VERSION}.jar"
-  done
-  jars="$jars
-com/fasterxml/jackson/datatype/jackson-datatype-jdk8/${jackson_version}/jackson-datatype-jdk8-${jackson_version}.jar"
-fi
-mkdir /jars
-for jar in $jars; do
-  wget -q -P /jars "https://repo1.maven.org/maven2/${jar}"
-done
-EOF
-
-FROM spark
 
 ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}"
 
-COPY --from=jars --chmod=644 /jars/ ${SPARK_HOME}/jars/
+# Files ADDed from a URL are only readable by root, hence a+r.
+ADD --chmod=a+r \
+    "https://repo1.maven.org/maven2/io/delta/${DELTA_ARTIFACT}/${DELTA_VERSION}/${DELTA_ARTIFACT}-${DELTA_VERSION}.jar" \
+    "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \
+    "https://repo1.maven.org/maven2/org/postgresql/postgresql/${POSTGRES_JDBC_VERSION}/postgresql-${POSTGRES_JDBC_VERSION}.jar" \
+    ${SPARK_HOME}/jars/
 
 # Spark's default warehouse dir is relative to the working directory; keep it
 # at /spark-warehouse, where docker-compose mounts its volume.

From 7d20c9051f99ffe4b7df4c58824f9689d4e8cc54 Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Wed, 7 Oct 2026 01:36:42 +0300
Subject: [PATCH 06/12] ci: reword a Dockerfile comment that the typos hook
 flagged

typos read "ADDed" as a misspelling of "added" and rewrote it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6
---
 tests/e2e_dbt_project/docker/spark/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/e2e_dbt_project/docker/spark/Dockerfile b/tests/e2e_dbt_project/docker/spark/Dockerfile
index 44a2d8ac7..6f04ed1ec 100644
--- a/tests/e2e_dbt_project/docker/spark/Dockerfile
+++ b/tests/e2e_dbt_project/docker/spark/Dockerfile
@@ -14,7 +14,7 @@ ARG POSTGRES_JDBC_VERSION=42.7.13
 
 ENV PATH="${SPARK_HOME}/bin:${SPARK_HOME}/sbin:${PATH}"
 
-# Files ADDed from a URL are only readable by root, hence a+r.
+# ADD gives files downloaded from a URL mode 600 (root only), hence a+r.
 ADD --chmod=a+r \
     "https://repo1.maven.org/maven2/io/delta/${DELTA_ARTIFACT}/${DELTA_VERSION}/${DELTA_ARTIFACT}-${DELTA_VERSION}.jar" \
     "https://repo1.maven.org/maven2/io/delta/delta-storage/${DELTA_VERSION}/delta-storage-${DELTA_VERSION}.jar" \

From 3b4675e30b69873cae09e76f086ccb3802a89aef Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Wed, 7 Oct 2026 02:01:32 +0300
Subject: [PATCH 07/12] ci: drop a Dockerfile comment that only described the
 old image

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6
---
 tests/e2e_dbt_project/docker/spark/Dockerfile | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/e2e_dbt_project/docker/spark/Dockerfile b/tests/e2e_dbt_project/docker/spark/Dockerfile
index 6f04ed1ec..4564d710a 100644
--- a/tests/e2e_dbt_project/docker/spark/Dockerfile
+++ b/tests/e2e_dbt_project/docker/spark/Dockerfile
@@ -1,8 +1,5 @@
 # syntax=docker/dockerfile:1
 
-# The official image ships the Spark distribution, so we don't download the
-# ~300MB tarball from archive.apache.org, which is heavily throttled and used
-# to take 40-70 minutes in CI.
 ARG SPARK_IMAGE=apache/spark:3.3.3
 FROM ${SPARK_IMAGE}
 

From cb53072bf53c0dc48c92576baf96b07549c26884 Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Wed, 7 Oct 2026 02:07:44 +0300
Subject: [PATCH 08/12] ci: drop the syntax directive from the Spark Dockerfile

It was there for symbolic `--chmod` (a+r), which needs Dockerfile syntax
1.14. The GitHub Ubuntu 24.04 runners have Docker 28.0.4, which ships
BuildKit 0.20 with the 1.14 frontend built in, so the directive is not
needed. Verified by building without it on BuildKit v0.20.0.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6
---
 tests/e2e_dbt_project/docker/spark/Dockerfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/e2e_dbt_project/docker/spark/Dockerfile b/tests/e2e_dbt_project/docker/spark/Dockerfile
index 4564d710a..5f67f1f59 100644
--- a/tests/e2e_dbt_project/docker/spark/Dockerfile
+++ b/tests/e2e_dbt_project/docker/spark/Dockerfile
@@ -1,5 +1,3 @@
-# syntax=docker/dockerfile:1
-
 ARG SPARK_IMAGE=apache/spark:3.3.3
 FROM ${SPARK_IMAGE}
 

From d281adf8bfd8b743f24400e0519e9f1e6653aace Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Wed, 7 Oct 2026 11:43:25 +0300
Subject: [PATCH 09/12] ci: trim the Spark e2e comments

Drop comments that restate the code next to them, and shorten the ones
that explain something non-obvious (the Spark version pin, the shared
ports and volumes, the Delta artifact rename, the ADD permissions and
the WORKDIR).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6
---
 .github/workflows/test-warehouse.yml          |  7 +------
 tests/e2e_dbt_project/docker-compose.yml      | 13 +++----------
 tests/e2e_dbt_project/docker/spark/Dockerfile |  6 ++----
 tests/profiles/profiles.yml.j2                |  1 -
 4 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml
index 38b22f9f4..150c510da 100644
--- a/.github/workflows/test-warehouse.yml
+++ b/.github/workflows/test-warehouse.yml
@@ -236,9 +236,7 @@ jobs:
       - name: Start Spark
         if: inputs.warehouse-type == 'spark' || inputs.warehouse-type == 'spark4'
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
-        run: |
-          # spark-thrift (Spark 3) or spark4-thrift (Spark 4)
-          docker compose up -d --build --wait "${WAREHOUSE_TYPE}-thrift"
+        run: docker compose up -d --build --wait "${WAREHOUSE_TYPE}-thrift"
 
       - name: Start SQL Server
         if: inputs.warehouse-type == 'sqlserver'
@@ -252,8 +250,6 @@ jobs:
         run: |
           docker compose up -d --wait vertica
 
-      # `docker compose up --wait` only says that a container exited or is
-      # unhealthy, so print the containers' own logs when a start step fails.
       - name: Show warehouse service logs
         if: failure()
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
@@ -435,7 +431,6 @@ jobs:
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
         if: steps.seed-cache.outputs.cache-hit != 'true' && (inputs.warehouse-type == 'dremio' || inputs.warehouse-type == 'spark' || inputs.warehouse-type == 'spark4')
         run: |
-          # The seeders are keyed by adapter; spark4 uses the spark adapter.
           ADAPTER="$WAREHOUSE_TYPE"
           if [ "$ADAPTER" = "spark4" ]; then
             ADAPTER="spark"
diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml
index 1b9590426..09b974ec7 100644
--- a/tests/e2e_dbt_project/docker-compose.yml
+++ b/tests/e2e_dbt_project/docker-compose.yml
@@ -246,18 +246,14 @@ services:
       - dremio-lakehouse
 
   # ── Spark Thrift Server (with Delta Lake + Hive Metastore) ─────────
-  # spark-thrift and spark4-thrift share the port, metastore and warehouse, so
-  # run one at a time and `docker compose down -v` before switching.
+  # spark-thrift and spark4-thrift share ports and volumes: run one at a time.
   spark-thrift: &spark-thrift
     container_name: spark-thrift
     build:
       context: ./docker/spark
       dockerfile: Dockerfile
-      # Not Spark 3.5 yet: its thrift server returns no row set for DDL/DML,
-      # which PyHive's fetchall() can't handle, and Elementary fetches the
-      # result of its CREATE TABLE AS statements. Not apache/spark:v3.3.2
-      # either: its JDK 11.0.16 fails container detection on cgroup v2 hosts
-      # such as the GitHub runners, which makes Spark crash on startup.
+      # Not 3.5 until CORE-1558 lands (PyHive fails on its DDL results); not
+      # v3.3.2, whose JDK crashes on the cgroup v2 GitHub runners.
       args:
         SPARK_IMAGE: apache/spark:3.3.3
         DELTA_ARTIFACT: delta-core_2.12
@@ -271,9 +267,6 @@ services:
     command: >
       --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
       --name Thrift JDBC/ODBC Server
-    # Run a query through the official image's beeline client, which checks
-    # that the thrift server opens sessions and answers queries, not only
-    # that the port is open.
     healthcheck:
       test:
         [
diff --git a/tests/e2e_dbt_project/docker/spark/Dockerfile b/tests/e2e_dbt_project/docker/spark/Dockerfile
index 5f67f1f59..8c04b870b 100644
--- a/tests/e2e_dbt_project/docker/spark/Dockerfile
+++ b/tests/e2e_dbt_project/docker/spark/Dockerfile
@@ -1,8 +1,7 @@
 ARG SPARK_IMAGE=apache/spark:3.3.3
 FROM ${SPARK_IMAGE}
 
-# Delta 3.0 renamed delta-core to delta-spark, so the artifact name (with its
-# Scala suffix) is passed in alongside the version.
+# delta-core_<scala> before Delta 3.0, delta-spark_<scala> from 3.0.
 ARG DELTA_ARTIFACT=delta-core_2.12
 ARG DELTA_VERSION=2.2.0
 ARG POSTGRES_JDBC_VERSION=42.7.13
@@ -16,8 +15,7 @@ ADD --chmod=a+r \
     "https://repo1.maven.org/maven2/org/postgresql/postgresql/${POSTGRES_JDBC_VERSION}/postgresql-${POSTGRES_JDBC_VERSION}.jar" \
     ${SPARK_HOME}/jars/
 
-# Spark's default warehouse dir is relative to the working directory; keep it
-# at /spark-warehouse, where docker-compose mounts its volume.
+# Spark's warehouse dir is relative to the workdir; /spark-warehouse is the volume.
 WORKDIR /
 
 ENTRYPOINT ["spark-submit"]
diff --git a/tests/profiles/profiles.yml.j2 b/tests/profiles/profiles.yml.j2
index 171763a4d..fc8586660 100644
--- a/tests/profiles/profiles.yml.j2
+++ b/tests/profiles/profiles.yml.j2
@@ -62,7 +62,6 @@ elementary_tests:
       file_format: delta
       threads: 4
 
-    # Spark 4 (spark4-thrift in docker-compose) listens on the same port.
     spark4: &spark4
       <<: *spark
 

From ccee9043d3ea8aecdcfb0e9af0fc82c10f5751a2 Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Wed, 7 Oct 2026 16:10:15 +0300
Subject: [PATCH 10/12] ci: remove the spark4 e2e target

Which Spark and Delta versions the e2e should run is tracked as a
separate ticket.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6
---
 .github/workflows/test-all-warehouses.yml |  1 -
 .github/workflows/test-warehouse.yml      | 27 +++++++++--------------
 tests/e2e_dbt_project/docker-compose.yml  | 14 +-----------
 tests/profiles/profiles.yml.j2            |  5 +----
 4 files changed, 12 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml
index e4b56d9ea..620f8af35 100644
--- a/.github/workflows/test-all-warehouses.yml
+++ b/.github/workflows/test-all-warehouses.yml
@@ -114,7 +114,6 @@ jobs:
             trino,
             dremio,
             spark,
-            spark4,
             fabric,
             sqlserver,
             vertica,
diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml
index 150c510da..95f086db7 100644
--- a/.github/workflows/test-warehouse.yml
+++ b/.github/workflows/test-warehouse.yml
@@ -13,7 +13,6 @@ on:
           - redshift
           - databricks_catalog
           - spark
-          - spark4
           - athena
           - clickhouse
           - duckdb
@@ -128,7 +127,7 @@ jobs:
       - name: Validate workflow inputs
         run: |
           case "$WAREHOUSE_TYPE" in
-            postgres|snowflake|bigquery|redshift|databricks_catalog|spark|spark4|athena|clickhouse|duckdb|trino|dremio|fabric|sqlserver|vertica) ;;
+            postgres|snowflake|bigquery|redshift|databricks_catalog|spark|athena|clickhouse|duckdb|trino|dremio|fabric|sqlserver|vertica) ;;
             *)
               echo "Unsupported warehouse type: $WAREHOUSE_TYPE" >&2
               exit 1
@@ -234,9 +233,10 @@ jobs:
           docker compose run --rm dremio-setup
 
       - name: Start Spark
-        if: inputs.warehouse-type == 'spark' || inputs.warehouse-type == 'spark4'
+        if: inputs.warehouse-type == 'spark'
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
-        run: docker compose up -d --build --wait "${WAREHOUSE_TYPE}-thrift"
+        run: |
+          docker compose up -d --build --wait spark-thrift
 
       - name: Start SQL Server
         if: inputs.warehouse-type == 'sqlserver'
@@ -261,7 +261,7 @@ jobs:
           python-version: "3.11"
 
       - name: Install Spark requirements
-        if: inputs.warehouse-type == 'spark' || inputs.warehouse-type == 'spark4'
+        if: inputs.warehouse-type == 'spark'
         run: sudo apt-get install -y python3-dev libsasl2-dev gcc
 
       - name: Install ODBC driver for SQL Server
@@ -303,7 +303,7 @@ jobs:
           case "$WAREHOUSE_TYPE" in
             databricks_catalog) DBT_ADAPTER="databricks" ;;
             athena) DBT_ADAPTER="athena-community" ;;
-            spark|spark4) DBT_ADAPTER="spark" DBT_ADAPTER_EXTRA="[PyHive]" ;;
+            spark) DBT_ADAPTER_EXTRA="[PyHive]" ;;
             # dbt-sqlserver 1.12 moved pyodbc (still its default backend) to the [pyodbc] extra.
             sqlserver) DBT_ADAPTER_EXTRA="[pyodbc]" ;;
             # dbt-fabric 1.11 (requires Python >=3.11) switched pyodbc ->
@@ -348,8 +348,6 @@ jobs:
             EXTRA="$WAREHOUSE_TYPE"
             if [ "$WAREHOUSE_TYPE" = "databricks_catalog" ]; then
               EXTRA="databricks"
-            elif [ "$WAREHOUSE_TYPE" = "spark4" ]; then
-              EXTRA="spark"
             fi
             pip install ".[$EXTRA]"
           fi
@@ -375,7 +373,7 @@ jobs:
           # This enables caching the seeded database state between runs.
           IS_DOCKER=false
           case "$WAREHOUSE_TYPE" in
-            postgres|clickhouse|trino|dremio|duckdb|spark|spark4|sqlserver|vertica) IS_DOCKER=true ;;
+            postgres|clickhouse|trino|dremio|duckdb|spark|sqlserver|vertica) IS_DOCKER=true ;;
           esac
 
           if [ "$IS_DOCKER" = "true" ]; then
@@ -429,17 +427,12 @@ jobs:
 
       - name: Seed e2e dbt project (external)
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
-        if: steps.seed-cache.outputs.cache-hit != 'true' && (inputs.warehouse-type == 'dremio' || inputs.warehouse-type == 'spark' || inputs.warehouse-type == 'spark4')
-        run: |
-          ADAPTER="$WAREHOUSE_TYPE"
-          if [ "$ADAPTER" = "spark4" ]; then
-            ADAPTER="spark"
-          fi
-          python load_seeds_external.py "$ADAPTER" "$SCHEMA_NAME" data
+        if: steps.seed-cache.outputs.cache-hit != 'true' && (inputs.warehouse-type == 'dremio' || inputs.warehouse-type == 'spark')
+        run: python load_seeds_external.py "$WAREHOUSE_TYPE" "$SCHEMA_NAME" data
 
       - name: Seed e2e dbt project
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
-        if: steps.seed-cache.outputs.cache-hit != 'true' && inputs.warehouse-type != 'dremio' && inputs.warehouse-type != 'spark' && inputs.warehouse-type != 'spark4'
+        if: steps.seed-cache.outputs.cache-hit != 'true' && inputs.warehouse-type != 'dremio' && inputs.warehouse-type != 'spark'
         run: |
           # The seed step runs before Elementary models exist, so artifact hooks
           # would try to insert into missing Elementary relations.
diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml
index 09b974ec7..fc3531ed4 100644
--- a/tests/e2e_dbt_project/docker-compose.yml
+++ b/tests/e2e_dbt_project/docker-compose.yml
@@ -246,8 +246,7 @@ services:
       - dremio-lakehouse
 
   # ── Spark Thrift Server (with Delta Lake + Hive Metastore) ─────────
-  # spark-thrift and spark4-thrift share ports and volumes: run one at a time.
-  spark-thrift: &spark-thrift
+  spark-thrift:
     container_name: spark-thrift
     build:
       context: ./docker/spark
@@ -291,17 +290,6 @@ services:
       - ./docker/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
       - ./data:/seed-data:ro
 
-  spark4-thrift:
-    <<: *spark-thrift
-    container_name: spark4-thrift
-    build:
-      context: ./docker/spark
-      dockerfile: Dockerfile
-      args:
-        SPARK_IMAGE: apache/spark:4.2.0
-        DELTA_ARTIFACT: delta-spark_2.13
-        DELTA_VERSION: 4.4.1
-
   spark-hive-metastore:
     image: postgres:15-alpine
     volumes:
diff --git a/tests/profiles/profiles.yml.j2 b/tests/profiles/profiles.yml.j2
index fc8586660..7f7c83430 100644
--- a/tests/profiles/profiles.yml.j2
+++ b/tests/profiles/profiles.yml.j2
@@ -62,9 +62,6 @@ elementary_tests:
       file_format: delta
       threads: 4
 
-    spark4: &spark4
-      <<: *spark
-
     sqlserver: &sqlserver
       type: sqlserver
       driver: "ODBC Driver 18 for SQL Server"
@@ -168,7 +165,7 @@ elementary_tests:
 elementary:
   target: postgres
   outputs:
-{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'spark', 'spark4', 'fabric', 'sqlserver', 'vertica', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %}
+{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'spark', 'fabric', 'sqlserver', 'vertica', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %}
 {%- for t in targets %}
     {{ t }}:
       <<: *{{ t }}

From 6b2cfca4cb8fc37606fe1aee16d0a1f15d5b1fca Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Wed, 7 Oct 2026 16:22:47 +0300
Subject: [PATCH 11/12] ci: name the JDK bug that rules out apache/spark:v3.3.2

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6
---
 tests/e2e_dbt_project/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml
index fc3531ed4..cdbffc42d 100644
--- a/tests/e2e_dbt_project/docker-compose.yml
+++ b/tests/e2e_dbt_project/docker-compose.yml
@@ -252,7 +252,7 @@ services:
       context: ./docker/spark
       dockerfile: Dockerfile
       # Not 3.5 until CORE-1558 lands (PyHive fails on its DDL results); not
-      # v3.3.2, whose JDK crashes on the cgroup v2 GitHub runners.
+      # v3.3.2, whose JDK 11.0.16 crashes on GitHub's runners (JDK-8287073).
       args:
         SPARK_IMAGE: apache/spark:3.3.3
         DELTA_ARTIFACT: delta-core_2.12

From d628084f45b8d7ea77125c6bc17a15ff98bacd51 Mon Sep 17 00:00:00 2001
From: Elad Barkay <eladb1231@gmail.com>
Date: Wed, 7 Oct 2026 17:31:38 +0300
Subject: [PATCH 12/12] ci: check the Spark metastore's readiness over TCP

On a fresh volume, the postgres entrypoint first runs a temporary,
socket-only server while it creates the database, then restarts.
pg_isready over the socket can pass during that window. The removed
entrypoint wait loop checked TCP, so Spark now relies on this
healthcheck: check 127.0.0.1 and the metastore database.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019akn5nGXcW5MA7BcVvxyo6
---
 tests/e2e_dbt_project/docker-compose.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml
index cdbffc42d..c56d26790 100644
--- a/tests/e2e_dbt_project/docker-compose.yml
+++ b/tests/e2e_dbt_project/docker-compose.yml
@@ -299,7 +299,11 @@ services:
       - POSTGRES_PASSWORD=${SPARK_METASTORE_PASSWORD:-dbt}
       - POSTGRES_DB=metastore
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U ${SPARK_METASTORE_USER:-dbt}"]
+      test:
+        [
+          "CMD-SHELL",
+          "pg_isready -h 127.0.0.1 -U ${SPARK_METASTORE_USER:-dbt} -d metastore",
+        ]
       interval: 5s
       timeout: 5s
       retries: 10