diff --git a/CHANGELOG.md b/CHANGELOG.md index 894d03a8c..6e672c94f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ All notable changes to this project will be documented in this file. - airflow, druid, hbase, nifi, opensearch-dashboards, superset, trino: Install global npm packages (cdxgen, pnpm, yarn, npm) with `--ignore-scripts`, and only resolve versions that were published at least 7 days ago (`--before`) ([#1682]). - superset: Pin npm to `10.9.9` instead of installing the latest version with `nvm install --latest-npm` ([#1682]). - opa: update golang to 1.27.1 ([#1697]). +- spark: Pin pyarrow below 26 to satisfy scikit-learn dependency on NumPy 1.x ([#1695]). ### Fixed @@ -139,6 +140,7 @@ All notable changes to this project will be documented in this file. [#1686]: https://github.com/stackabletech/docker-images/pull/1686 [#1689]: https://github.com/stackabletech/docker-images/pull/1689 [#1690]: https://github.com/stackabletech/docker-images/pull/1690 +[#1695]: https://github.com/stackabletech/docker-images/pull/1695 [#1693]: https://github.com/stackabletech/docker-images/pull/1693 [#1697]: https://github.com/stackabletech/docker-images/pull/1697 diff --git a/spark-connect-client/Dockerfile b/spark-connect-client/Dockerfile index b7a017eda..86d093812 100644 --- a/spark-connect-client/Dockerfile +++ b/spark-connect-client/Dockerfile @@ -41,8 +41,11 @@ USER ${STACKABLE_USER_UID} # Packages are intentionally installed in "user mode" to reduce the container attack surface. # - pyspark[connect] = spark connect client libs # - jupyterlab = notebook client used in demos +# - pyarrow < 26 = pyarrow 26 requires NumPy >= 2, but scikit-learn 1.3.1 requires +# NumPy 1.x. pyarrow does not declare NumPy as a dependency, thus pip cannot detect the conflict. RUN pip install --no-cache-dir --user \ "pyspark[connect]==${PRODUCT_VERSION}" \ + "pyarrow<26" \ "jupyterlab==4.4.1" \ "scikit-learn==1.3.1" \ "matplotlib==3.10.1"