From 6de755201d16251dfade53841b0bc13410451c9f Mon Sep 17 00:00:00 2001
From: Michael Weibel <mweibel@cloudscale.ch>
Date: Wed, 2 Sep 2026 16:49:00 +0200
Subject: [PATCH 1/4] sync CCM<>CSI test cluster setup, add integration tests
 on schedule

---
 .dockerignore                               |  12 +
 .github/workflows/csi-integration-tests.yml | 172 +++++++++++++
 .gitignore                                  |   2 +
 Makefile                                    |  28 ++-
 README.md                                   |  26 +-
 cmd/cloudscale-csi-plugin/Dockerfile        |  57 +++--
 helpers/bootstrap-cluster                   | 130 ----------
 helpers/clean-up                            |  14 --
 helpers/cleanup                             |  25 ++
 helpers/image-from-ref                      |  43 ++++
 helpers/run-in-test-cluster                 | 262 ++++++++++++++++++++
 helpers/test-matrix                         |  49 ++++
 test/kubernetes/integration_test.go         |  23 +-
 13 files changed, 650 insertions(+), 193 deletions(-)
 create mode 100644 .dockerignore
 create mode 100644 .github/workflows/csi-integration-tests.yml
 delete mode 100755 helpers/bootstrap-cluster
 delete mode 100755 helpers/clean-up
 create mode 100755 helpers/cleanup
 create mode 100755 helpers/image-from-ref
 create mode 100755 helpers/run-in-test-cluster
 create mode 100755 helpers/test-matrix

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000..b259346b
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,12 @@
+.git
+.gitignore
+*.md
+.github/
+/charts/
+deploy/
+examples/
+helpers/
+scripts/
+test/
+Dockerfile
+VERSION
diff --git a/.github/workflows/csi-integration-tests.yml b/.github/workflows/csi-integration-tests.yml
new file mode 100644
index 00000000..8c518b42
--- /dev/null
+++ b/.github/workflows/csi-integration-tests.yml
@@ -0,0 +1,172 @@
+name: CSI Integration Tests
+
+on:
+  push:
+    branches:
+      - master
+
+  pull_request:
+
+  # Allow to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+  # Run this regularly, to get integration tests results against new
+  # Kubernetes releases.
+  schedule:
+    - cron: '15 3 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  test-matrix:
+    name: "Get Kubernetes Releases"
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: "Generate Test Matrix"
+        id: list
+        run: 'echo "tests=$(helpers/test-matrix)" >> $GITHUB_OUTPUT'
+
+    outputs:
+      tests: ${{ steps.list.outputs.tests }}
+
+  build-image:
+    name: "Build Container Image"
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Evaluate image name
+        run: 'helpers/image-from-ref >> $GITHUB_ENV'
+
+      - name: Extract version
+        run: echo "VERSION=$(cat VERSION)" >> $GITHUB_ENV
+
+      - name: Build image
+        run: make build
+
+      - name: Export image
+        run: 'docker image save "$IMAGE" -o image.tar'
+
+      - name: Store hash
+        run: 'shasum -a 256 image.tar | tee image.tar.sha256'
+
+      - name: Store image
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: tested-image
+          path: |
+            image.tar
+            image.tar.sha256
+          retention-days: 30d
+
+  check-csi-integration:
+    # Preflight: verify the CLOUDSCALE_API_TOKEN at step-level and set an output
+    # so the integration job can be skipped.
+    #
+    # GitHub Actions limitations motivating this:
+    # - `secrets` are NOT available in `jobs.<id>.if` (job-level `if`), so you cannot
+    #   directly gate/skip a job by testing a secret there.
+    name: Check CSI Integration Configuration
+    runs-on: ubuntu-latest
+
+    outputs:
+      integration-enabled: ${{ steps.check.outputs.enabled }}
+
+    steps:
+      - id: check
+        name: Verify CLOUDSCALE_API_TOKEN is present
+        env:
+          CLOUDSCALE_API_TOKEN: ${{ secrets.CLOUDSCALE_API_TOKEN }}
+        run: |
+          if [ -n "$CLOUDSCALE_API_TOKEN" ]; then
+            echo "enabled=true" >> $GITHUB_OUTPUT
+            echo "CLOUDSCALE_API_TOKEN found — integration will run."
+          else
+            echo "enabled=false" >> $GITHUB_OUTPUT
+            echo "CLOUDSCALE_API_TOKEN not configured — skipping integration."
+          fi
+
+  integration:
+    name: "Kubernetes ${{ matrix.kubernetes }}"
+    runs-on: ubuntu-latest
+
+    needs:
+      - test-matrix
+      - build-image
+      - check-csi-integration
+    if: needs.check-csi-integration.outputs.integration-enabled == 'true'
+
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        include: "${{ fromJson(needs.test-matrix.outputs.tests) }}"
+
+    env:
+      CLOUDSCALE_API_TOKEN: ${{ secrets.CLOUDSCALE_API_TOKEN }}
+      KUBERNETES: '${{ matrix.kubernetes }}'
+      SUBNET: '${{ matrix.subnet }}'
+      CLUSTER_PREFIX: '${{ matrix.cluster_prefix }}'
+      IMAGE_SOURCE: import
+
+    # Prevent integration tests from running in parallel. Ideally this should
+    # be seuqential, but that won't work due to the following issue:
+    #
+    # https://github.com/orgs/community/discussions/5435
+    #
+    # Instead we ensure that only one integration test per supported version
+    # is run at any given time.
+    concurrency:
+      group: integration-${{ matrix.kubernetes }}
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Load image
+        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
+        with:
+          name: tested-image
+
+      - name: Validate hash
+        run: 'shasum --check image.tar.sha256'
+
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+
+      - name: Setup Helm
+        uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
+        with:
+          version: v4.2.4
+
+      - name: Evaluate image name
+        run: 'helpers/image-from-ref >> $GITHUB_ENV'
+
+      - name: Cleanup Leftovers
+        if: always()
+        run: helpers/cleanup
+
+      - name: Create Test Cluster
+        run: helpers/run-in-test-cluster
+
+      - name: Run Integration Tests
+        run: TESTARGS="-race" make test-integration
+
+      - name: Wait For Kubernetes-Internal Cleanup
+        if: always()
+        run: sleep 30
+
+      - name: Destroy Test Cluster
+        if: always()
+        run: helpers/cleanup
diff --git a/.gitignore b/.gitignore
index ad951bc7..d7af6dbb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,5 @@ charts/csi-cloudscale/charts
 cmd/cloudscale-csi-plugin/cloudscale-csi-plugin
 k8test/
 bin/
+
+__pycache__/
diff --git a/Makefile b/Makefile
index 8797c8e5..d22f27d0 100644
--- a/Makefile
+++ b/Makefile
@@ -1,11 +1,7 @@
 NAME=cloudscale-csi-plugin
 OS ?= linux
 GO_VERSION := $(shell awk '/^go/ {print $$2}' go.mod)
-ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),)
-  GIT_TREE_STATE=clean
-else
-  GIT_TREE_STATE=dirty
-endif
+GIT_TREE_STATE ?= $(shell git status --porcelain 2>/dev/null | grep -q . && echo "dirty" || echo "clean")
 COMMIT ?= $(shell git rev-parse HEAD)
 BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 LDFLAGS ?= -X github.com/cloudscale-ch/csi-cloudscale/driver.version=${VERSION} -X github.com/cloudscale-ch/csi-cloudscale/driver.commit=${COMMIT} -X github.com/cloudscale-ch/csi-cloudscale/driver.gitTreeState=${GIT_TREE_STATE}
@@ -14,6 +10,7 @@ PKG ?= github.com/cloudscale-ch/csi-cloudscale/cmd/cloudscale-csi-plugin
 VERSION ?= $(shell cat VERSION)
 CHART_VERSION ?= $(shell awk '/^version:/ {print $$2}' charts/csi-cloudscale/Chart.yaml)
 DOCKER_REPO ?= quay.io/cloudscalech/cloudscale-csi-plugin
+IMAGE ?= $(DOCKER_REPO):$(VERSION)
 
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
@@ -97,7 +94,7 @@ test: vet ## Run tests.
 .PHONY: test-integration
 test-integration: ## Run integration tests
 	@echo "==> Started integration tests"
-	@env go test -race -count 1 -v $(TESTARGS) -tags integration -parallel 4 -timeout 20m ./test/...
+	K8TEST_PATH=${PWD}/k8test go test -count 1 -v $(TESTARGS) -tags integration -parallel 4 -timeout 20m ./test/...
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter
@@ -115,13 +112,18 @@ govulncheck: govulncheck-tool ## Run govulncheck (advisory only)
 
 .PHONY: compile
 compile: ## Build the project binary
-	@echo "==> Building the project"
-	@docker run --rm -it -e GOOS=${OS} -e GOARCH=amd64 -v ${PWD}/:/app -w /app golang:${GO_VERSION}-alpine sh -c 'apk add git && go build -o cmd/cloudscale-csi-plugin/${NAME} -ldflags "$(LDFLAGS)" ${PKG}'
+	@echo "==> Building the binary"
+	CGO_ENABLED=0 go build -o cmd/cloudscale-csi-plugin/$(NAME) -ldflags "$(LDFLAGS)" $(PKG)
 
 .PHONY: build
-build: compile ## Build the docker image
+build: ## Build the docker image
 	@echo "==> Building the docker image"
-	@docker build --platform linux/amd64 -t $(DOCKER_REPO):$(VERSION) cmd/cloudscale-csi-plugin -f cmd/cloudscale-csi-plugin/Dockerfile
+	docker build --platform linux/amd64 \
+		-t $(IMAGE) \
+		--build-arg VERSION="$(VERSION)" \
+		--build-arg COMMIT="$(COMMIT)" \
+		--build-arg GIT_TREE_STATE="$(GIT_TREE_STATE)" \
+		-f cmd/cloudscale-csi-plugin/Dockerfile .
 
 .PHONY: push
 push: ## Push docker image to registry
@@ -132,9 +134,9 @@ ifeq ($(DOCKER_REPO),quay.io/cloudscalech/cloudscale-csi-plugin)
     endif
   endif
 endif
-	@echo "==> Publishing $(DOCKER_REPO):$(VERSION)"
-	@docker push $(DOCKER_REPO):$(VERSION)
-	@echo "==> Your image is now available at $(DOCKER_REPO):$(VERSION)"
+	@echo "==> Publishing $(IMAGE)"
+	@docker push $(IMAGE)
+	@echo "==> Your image is now available at $(IMAGE)"
 
 .PHONY: publish
 publish: build push clean ## Build, push, and clean
diff --git a/README.md b/README.md
index 3d85e338..9b0c7d10 100644
--- a/README.md
+++ b/README.md
@@ -418,11 +418,31 @@ This will create a binary with version `dev` and docker image pushed to
 `cloudscalech/cloudscale-csi-plugin:dev`
 
 
-To run the integration tests run the following:
+To run the integration tests locally with your local CSI build:
 
-```
-$ export KUBECONFIG=$(pwd)/kubeconfig 
+```bash
+# 1. Build and export the CSI image
+$ VERSION=dev make publish
+
+# 2. Create a test cluster with CCM and CSI
+$ export CLOUDSCALE_API_TOKEN=your-token
+$ export IMAGE=quay.io/cloudscalech/cloudscale-csi-plugin:dev
+$ helpers/run-in-test-cluster
+
+# This will:
+# - Create a Kubernetes cluster on cloudscale.ch
+# - Deploy CCM from the latest official release
+# - Deploy CSI from your local build
+
+# 3. Run integration tests
+$ export KUBECONFIG=$(pwd)/k8test/cluster/admin.conf
+$ make test-integration
+
+# Run a single test
 $ TESTARGS='-run TestPod_Single_SSD_Volume' make test-integration
+
+# 4. Clean up
+$ helpers/cleanup
 ```
 
 
diff --git a/cmd/cloudscale-csi-plugin/Dockerfile b/cmd/cloudscale-csi-plugin/Dockerfile
index 9d173048..9c2bd43a 100644
--- a/cmd/cloudscale-csi-plugin/Dockerfile
+++ b/cmd/cloudscale-csi-plugin/Dockerfile
@@ -1,30 +1,41 @@
-# Copyright 2018 DigitalOcean
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+FROM golang:1.27-alpine AS builder
+
+RUN apk add --no-cache git make bash
+
+WORKDIR /src
+
+# Copy go.mod/go.sum first for better layer caching
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy all source code
+COPY . .
+
+# Build arguments for version information
+ARG VERSION=dev
+ARG COMMIT=unknown
+ARG GIT_TREE_STATE=unknown
+
+# Build using make (ensures consistent build logic)
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make compile \
+    VERSION="${VERSION}" \
+    COMMIT="${COMMIT}" \
+    GIT_TREE_STATE="${GIT_TREE_STATE}"
+
 FROM alpine:3.23.3
 
 # e2fsprogs-extra is required for resize2fs used for the resize operation
 # blkid: block device identification tool from util-linux
 RUN apk add --no-cache ca-certificates \
-                       e2fsprogs \
-                       findmnt \
-                       xfsprogs \
-                       cryptsetup \
-                       udev \
-                       blkid \
-                       e2fsprogs-extra
-
-ADD cloudscale-csi-plugin /bin/
-ADD csi-diskinfo.sh /bin/
+    e2fsprogs \
+    findmnt \
+    xfsprogs \
+    cryptsetup \
+    udev \
+    blkid \
+    e2fsprogs-extra
+
+COPY --from=builder /src/cmd/cloudscale-csi-plugin/cloudscale-csi-plugin /bin/
+COPY --from=builder /src/cmd/cloudscale-csi-plugin/csi-diskinfo.sh /bin/
 
 ENTRYPOINT ["/bin/cloudscale-csi-plugin"]
diff --git a/helpers/bootstrap-cluster b/helpers/bootstrap-cluster
deleted file mode 100755
index 5f9a4879..00000000
--- a/helpers/bootstrap-cluster
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/env bash
-#
-# Ensures that a Kubernetes test cluster is present and updated
-#
-set -euo pipefail
-
-# Default values
-RANDOM_NUMBER=$((RANDOM % 8193))
-K8TEST_SHA="0bf850b"
-ZONE="lpg1"
-CLUSTER_PREFIX="csi-test-$RANDOM_NUMBER"
-KUBERNETES="latest"
-FLAVOR="plus-8-4"
-CONTROL_COUNT=1
-WORKER_COUNT=3
-IMAGE="ubuntu-22.04"
-VOLUME_SIZE_GB="25"
-
-# Parse command line arguments
-while [[ $# -gt 0 ]]; do
-    key="$1"
-    case $key in
-        --k8test-sha)
-            K8TEST_SHA="$2"
-            shift
-            shift
-            ;;
-        --zone)
-            ZONE="$2"
-            shift
-            shift
-            ;;
-        --cluster-prefix)
-            CLUSTER_PREFIX="$2"
-            shift
-            shift
-            ;;
-        --kubernetes)
-            KUBERNETES="$2"
-            shift
-            shift
-            ;;
-        --control-count)
-            CONTROL_COUNT="$2"
-            shift
-            shift
-            ;;
-        --worker-count)
-            WORKER_COUNT="$2"
-            shift
-            shift
-            ;;
-        --image)
-            IMAGE="$2"
-            shift
-            shift
-            ;;
-        --flavor)
-            FLAVOR="$2"
-            shift
-            shift
-            ;;
-        --volume-size-gb)
-            VOLUME_SIZE_GB="$2"
-            shift
-            shift
-            ;;
-        *)
-            echo "Unknown option: $1"
-            exit 1
-            ;;
-    esac
-done
-
-# Prepares k8test with an existing virtual env, or a newly created on
-function ensure-k8test() {
-  if ! test -d k8test; then
-    git clone git@github.com:cloudscale-ch/k8test.git
-    git -C k8test checkout "$K8TEST_SHA"
-  fi
-
-  if [[ "${VIRTUAL_ENV:-}" == "" ]]; then
-
-    if ! test -d k8test/venv; then
-      python3.12 -m venv k8test/venv
-    fi
-
-    # shellcheck source=/dev/null
-    source k8test/venv/bin/activate
-  fi
-
-  if ! command -v poetry > /dev/null; then
-    python3.12 -m pip install poetry
-  fi
-
-  if ! command -v ansible > /dev/null; then
-    poetry install --directory k8test
-  fi
-}
-
-# Launches the test cluster, if there's no inventory yet
-function ensure-inventory() {
-  if ! test -d k8test/cluster; then
-    mkdir k8test/cluster
-  fi
-
-  if ! test -f k8test/cluster/ssh.pub; then
-    ssh-keygen -t ed25519 -N '' -f k8test/cluster/ssh
-  fi
-
-  if ! test -f k8test/cluster/inventory.yml; then
-    k8test/playbooks/create-cluster.yml \
-      -e ssh_key=k8test/cluster/ssh.pub \
-      -e zone="$ZONE" \
-      -e cluster_prefix="$CLUSTER_PREFIX" \
-      -e kubernetes="$KUBERNETES" \
-      -e control_count="$CONTROL_COUNT" \
-      -e worker_count="$WORKER_COUNT" \
-      -e image="$IMAGE" \
-      -e flavor="$FLAVOR" \
-      -e volume_size_gb="$VOLUME_SIZE_GB"
-
-    # Those won't really change between runs, so update them during install
-    k8test/playbooks/update-secrets.yml \
-      -i k8test/cluster/inventory.yml
-  fi
-}
-
-ensure-k8test
-ensure-inventory
diff --git a/helpers/clean-up b/helpers/clean-up
deleted file mode 100755
index 5605e086..00000000
--- a/helpers/clean-up
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-#
-# Ensures that any Kubernetes cluster is cleaned up
-#
-set -euo pipefail
-
-if test -f k8test/cluster/inventory.yml; then
-  source k8test/venv/bin/activate
-  k8test/playbooks/destroy-cluster.yml -i k8test/cluster/inventory.yml
-fi
-
-if test -d k8test; then
-  rm -rf k8test
-fi
diff --git a/helpers/cleanup b/helpers/cleanup
new file mode 100755
index 00000000..039e6d84
--- /dev/null
+++ b/helpers/cleanup
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# Ensures that any Kubernetes cluster is cleaned up
+#
+set -euo pipefail
+
+# Make sure the k8test/helpers/release-set CLI is available
+source helpers/run-in-test-cluster
+ensure-k8test > /dev/null
+
+for attempt in 1 2 3; do
+    echo "Destroy attempt ${attempt}..."
+
+    if k8test/playbooks/destroy-cluster.yml \
+        -i k8test/cluster/inventory.yml \
+        -e cluster_prefix="$CLUSTER_PREFIX"; then
+        exit 0
+    fi
+
+    echo "Cleanup failed, retrying in 60s..."
+    sleep 60
+done
+
+echo "Cleanup failed after 3 attempts."
+exit 1
diff --git a/helpers/image-from-ref b/helpers/image-from-ref
new file mode 100755
index 00000000..b3fbaac2
--- /dev/null
+++ b/helpers/image-from-ref
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+""" Looks at the GITHUB_REF environment variable and determins the image name
+that should be used for the pipeline.
+
+For more information:
+https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
+
+The output is meant to be used with $GITHUB_ENV.
+
+"""
+
+import argparse
+import os
+import sys
+
+REPOSITORY = 'quay.io/cloudscalech/cloudscale-csi-plugin'
+
+parser = argparse.ArgumentParser(usage=__doc__)
+parser.add_argument('--ref', default=os.environ.get('GITHUB_REF'))
+
+
+def main(ref: str) -> int:
+    if not ref:
+        print("Either use --ref or set GITHUB_REF")
+        return 1
+
+    match ref.split('/'):
+        case ["refs", "heads", *branch_parts]:
+            tag = f"branch-{'-'.join(branch_parts)}"
+        case ["refs", "pull", pull_request, "merge"]:
+            tag = f"pull-request-{pull_request}"
+        case ["refs", "tags", tag_name]:
+            tag = tag_name
+        case _:
+            print(f"'{ref}' did not match any known pattern")
+            return 1
+
+    print(f"IMAGE={REPOSITORY}:{tag}")
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(**vars(parser.parse_args())))
diff --git a/helpers/run-in-test-cluster b/helpers/run-in-test-cluster
new file mode 100755
index 00000000..9b7a832c
--- /dev/null
+++ b/helpers/run-in-test-cluster
@@ -0,0 +1,262 @@
+#!/usr/bin/env bash
+#
+# Ensures that a Kubernetes test cluster is present and updated
+#
+# shellcheck source=/dev/null
+set -euo pipefail
+
+export ANSIBLE_CONFIG="$PWD"/k8test/ansible.cfg
+export KUBERNETES="${KUBERNETES-latest}"
+export CLUSTER_PREFIX="${CLUSTER_PREFIX-csi}"
+export SUBNET="${SUBNET-10.200.1.0/24}"
+export IMAGE="${IMAGE-quay.io/cloudscalech/cloudscale-csi-plugin:test}"
+export CCM_RELEASE_URL="https://github.com/cloudscale-ch/cloudscale-cloud-controller-manager/releases/latest/download/config.yml"
+
+# Prepares k8test with an existing virtual env, or a newly created on
+function ensure-k8test() {
+
+  # On GitHub actions, things are a bit different. Ansible is actually found
+  # on the runners and cloning via SSH does not work.
+  #
+  # Therefore we run a separate install block that is only meant for GitHub.
+  if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
+    if test -d k8test; then
+      source k8test/venv/bin/activate
+      return
+    fi
+
+    mkdir -p ~/.ssh/ && touch ~/.ssh/known_hosts
+    git clone https://github.com/cloudscale-ch/k8test
+    python3 -m venv k8test/venv
+    source k8test/venv/bin/activate
+    pip install poetry
+    poetry install --directory k8test
+    return
+  fi
+
+  if ! test -d k8test; then
+    git clone git@github.com:cloudscale-ch/k8test.git
+  fi
+
+  if [[ "${VIRTUAL_ENV:-}" == "" ]]; then
+
+    if ! test -d k8test/venv; then
+      python3 -m venv k8test/venv
+    fi
+
+    source k8test/venv/bin/activate
+  fi
+
+  if ! command -v poetry > /dev/null; then
+    pip install poetry
+  fi
+
+  if ! command -v ansible > /dev/null; then
+    poetry install --directory k8test
+  fi
+}
+
+# Prints a random zone
+function random-zone() {
+  arr[0]="rma1"
+  arr[1]="lpg1"
+
+  rand=$((RANDOM % 2))
+  echo "${arr[$rand]}"
+}
+
+# Launches the test cluster, if there's no admin.conf yet
+function ensure-cluster() {
+  if ! test -d k8test/cluster; then
+    mkdir k8test/cluster
+  fi
+
+  if ! test -f k8test/cluster/ssh.pub; then
+    ssh-keygen -t ed25519 -N '' -f k8test/cluster/ssh
+  fi
+
+  if ! test -f k8test/cluster/admin.conf; then
+    zone="$(random-zone)"
+
+    # First create the cluster, without configuring the network
+    k8test/playbooks/create-cluster.yml \
+      -e zone="$zone" \
+      -e ssh_key=k8test/cluster/ssh.pub \
+      -e control_count=2 \
+      -e worker_count=2 \
+      -e kubelet_extra_args='--cloud-provider=external' \
+      -e kubernetes="${KUBERNETES}" \
+      -e cluster_prefix="${CLUSTER_PREFIX}" \
+      -e subnet="${SUBNET}" \
+      --tags setup-vms,setup-controls
+
+    # Those won't really change between runs, so update them during install
+    k8test/playbooks/update-secrets.yml \
+      -i k8test/cluster/inventory.yml
+
+    # Second, get the CCM running. Without it setting the node IPs, Cilium
+    # won't work.
+    deploy-ccm
+
+    # Finally, setup the cluster to completion (includes Cilium install)
+    k8test/playbooks/create-cluster.yml \
+      -e zone="$zone" \
+      -e ssh_key=k8test/cluster/ssh.pub \
+      -e control_count=2 \
+      -e worker_count=2 \
+      -e kubelet_extra_args='--cloud-provider=external' \
+      -e kubernetes="${KUBERNETES}" \
+      -e cluster_prefix="${CLUSTER_PREFIX}" \
+      -e subnet="${SUBNET}"
+  fi
+}
+
+# Build the latest image each time
+function build-image() {
+  k8test/playbooks/build-image.yml \
+    -i k8test/cluster/inventory.yml \
+    -e dockerfile=./Dockerfile \
+    -e tag="$IMAGE" \
+    -e extra='--build-arg=VERSION=test' \
+    -l controls
+}
+
+# Import an image from the host
+function import-image() {
+  k8test/playbooks/import-image.yml \
+    -i k8test/cluster/inventory.yml \
+    -e image="$PWD/image.tar"
+}
+
+# Deploy CCM from official GitHub release
+function deploy-ccm() {
+  export KUBECONFIG=k8test/cluster/admin.conf
+  local api_url=$(echo "${CLOUDSCALE_API_URL-https://api.cloudscale.ch}" | sed 's|/v1||g; s|/$||g')
+
+  echo "Deploying CCM from ${CCM_RELEASE_URL}..."
+
+  # Download manifest, patch API URL, and apply
+  curl -sL "${CCM_RELEASE_URL}" | \
+    sed "s|https://api.cloudscale.ch/|${api_url}/|g" | \
+    kubectl apply -f -
+
+  echo "Waiting for CCM to be ready..."
+  kubectl -n kube-system rollout status daemonset/cloudscale-cloud-controller-manager --timeout=120s
+}
+
+# Deploy VolumeSnapshot CRDs and snapshot controller
+function deploy-snapshot-crds() {
+  export KUBECONFIG=k8test/cluster/admin.conf
+
+  echo "Installing VolumeSnapshot CRDs..."
+  kubectl apply -k "https://github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v8.4.0"
+
+  echo "Installing snapshot controller..."
+  kubectl apply -k "https://github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v8.4.0"
+}
+
+# Deploy CSI using Helm chart with custom image
+function deploy-csi() {
+  api_url=$(echo "${CLOUDSCALE_API_URL-https://api.cloudscale.ch}" | sed 's|/v1||g')
+
+  export KUBECONFIG=k8test/cluster/admin.conf
+
+  # Parse IMAGE to extract tag
+  local image_tag="${IMAGE##*:}"
+
+  echo "Deploying CSI with image tag: ${image_tag}..."
+
+  # Determine imagePullPolicy - force Never for local-only images
+  local image_pull_policy="IfNotPresent"
+  if [[ "${IMAGE_SOURCE-build}" == "import" ]]; then
+    image_pull_policy="Never"
+    echo "Using imagePullPolicy=Never (import mode - local images only)"
+  fi
+
+  # Generate manifest and capture for inspection
+  local manifest_file="/tmp/csi-manifest-$$.yaml"
+  helm template csi-cloudscale -n kube-system \
+    --set nameOverride=csi-cloudscale \
+    --set controller.image.tag="${image_tag}" \
+    --set node.image.tag="${image_tag}" \
+    --set controller.image.pullPolicy="${image_pull_policy}" \
+    --set node.image.pullPolicy="${image_pull_policy}" \
+    --set cloudscale.apiUrl="${api_url}/" \
+    ./charts/csi-cloudscale > "$manifest_file"
+
+  echo ""
+  echo "Deployed Image Configuration:"
+  echo "Expected image: ${IMAGE}"
+  echo "imagePullPolicy: ${image_pull_policy}"
+  echo ""
+  echo "Container images in manifest:"
+  grep -E "^\s+image:" "$manifest_file" | sed 's/^[[:space:]]*/  /' | head -10
+  echo ""
+
+  # Apply the manifest
+  kubectl apply -n kube-system -f "$manifest_file"
+
+  # Cleanup
+  rm -f "$manifest_file"
+
+  echo "Waiting for CSI controller to be ready..."
+  echo "  - Checking initial pod status..."
+  kubectl -n kube-system get pods -l app=csi-cloudscale-controller -o wide
+
+  if ! kubectl -n kube-system rollout status statefulset/csi-cloudscale-controller --timeout=180s; then
+    echo "CSI Controller rollout failed"
+    exit 1
+  fi
+
+  echo "Waiting for CSI node daemonset to be ready..."
+  echo "  - Checking node daemonset pod status..."
+  kubectl -n kube-system get pods -l app=csi-cloudscale-node -o wide
+
+  if ! kubectl -n kube-system rollout status daemonset/csi-cloudscale-node --timeout=300s; then
+    echo "CSI Node DaemonSet rollout failed"
+    exit 1
+  fi
+}
+
+# Deploy both CCM and CSI components
+function deploy-components() {
+  echo "Deploying Components"
+  echo "IMAGE_SOURCE: ${IMAGE_SOURCE-build}"
+  echo "IMAGE: ${IMAGE}"
+  echo ""
+
+  # Deploy CCM first (required for node initialization and networking)
+  deploy-ccm
+
+  # Install VolumeSnapshot CRDs and snapshot controller (required for CSI driver)
+  deploy-snapshot-crds
+
+  # Import pre-built CSI image (used in CI via IMAGE_SOURCE=import)
+  if [[ "${IMAGE_SOURCE-build}" == "import" ]]; then
+    echo "Importing CSI image from image.tar..."
+    import-image
+  fi
+
+  # Deploy CSI
+  deploy-csi
+}
+
+# Execute if not sourced
+if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then
+
+  # The image name requires a slash in it, or Podman will add `localhost/` and
+  # confuse Kubernetes.
+  if [[ "$IMAGE" != *"/"* ]]; then
+    echo "\$IMAGE has no slash: $IMAGE"
+    exit 1
+  fi
+
+  first_run=$(test ! -f k8test/cluster/inventory.yml && echo "yes" || echo "no")
+
+  ensure-k8test
+  ensure-cluster
+
+  # Deploy components on both new and existing clusters. For new clusters,
+  # this happens after the full cluster bootstrap (including Cilium) is done.
+  deploy-components
+fi
diff --git a/helpers/test-matrix b/helpers/test-matrix
new file mode 100755
index 00000000..5db5ee4f
--- /dev/null
+++ b/helpers/test-matrix
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Make sure the k8test/helpers/release-set CLI is available
+source helpers/run-in-test-cluster
+ensure-k8test > /dev/null
+
+# Returns a matrix entry for a specific Kubernetes version
+function matrixentry {
+  local k8s_version="$1"
+  local matrix_index="$2"
+
+  # Kubernetes release
+  k8test/helpers/release-set --kubernetes="$k8s_version" --limit kubernetes | jq -r '.kubernetes.version'
+
+  # The private subnet range
+  echo "10.200.$matrix_index.0/24"
+
+  # A semi-predictable cluster prefix (we want to reuse them across jobs, but
+  # not inside a job). Reuse allows us to ensure that the cleanup process
+  # at the beginning of a test will remove old runs.
+  echo "csi-matrix-$matrix_index"
+}
+
+function matrixobject {
+  local k8s_version="$1"
+  local matrix_index="$2"
+
+  matrixentry "$k8s_version" "$matrix_index" | jq --raw-input --null-input --compact-output '{
+    "kubernetes": inputs,
+    "subnet": inputs,
+    "cluster_prefix": inputs,
+  }'
+}
+
+function matrixentries {
+  # Kubernetes 1.28 (oldest supported per README compatibility matrix)
+  matrixobject "1.28" "1"
+
+  # Two newest Kubernetes releases
+  matrixobject "-1" "2"
+  matrixobject "0" "3"
+}
+
+function matrixobjects {
+  matrixentries | jq --slurp --compact-output '.'
+}
+
+matrixobjects
diff --git a/test/kubernetes/integration_test.go b/test/kubernetes/integration_test.go
index 581d732f..4ac7daca 100644
--- a/test/kubernetes/integration_test.go
+++ b/test/kubernetes/integration_test.go
@@ -14,6 +14,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"testing"
@@ -1510,19 +1511,21 @@ func TestVolumeStats(t *testing.T) {
 }
 
 func setup() error {
-	// if you want to change the loading rules (which files in which order),
-	// you can do so here
-	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
+	// Kubernetes client
+	k8test, ok := os.LookupEnv("K8TEST_PATH")
+	if !ok {
+		log.Fatalf("could not find K8TEST_PATH environment variable\n")
+	}
 
-	// if you want to change override values or bind them to flags, there are
-	// methods to help you
-	configOverrides := &clientcmd.ConfigOverrides{}
+	path := filepath.Join(k8test, "cluster", "admin.conf")
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("failed to read kubeconfig at path %q: %w", path, err)
+	}
 
-	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
-	var err error
-	config, err = kubeConfig.ClientConfig()
+	config, err = clientcmd.RESTConfigFromKubeConfig(data)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to apply kubeconfig at path %q: %w", path, err)
 	}
 
 	// create the clientset

From 6f0ce8cbb7cdd53d5854adb61ef9f5289c35b174 Mon Sep 17 00:00:00 2001
From: Michael Weibel <mweibel@cloudscale.ch>
Date: Wed, 9 Sep 2026 17:11:17 +0200
Subject: [PATCH 2/4] retry getVolumeInfo: TestPod_Single_Bulk_Luks_Volume
 sometimes takes longer until volume is usable

---
 test/kubernetes/integration_test.go | 38 ++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/test/kubernetes/integration_test.go b/test/kubernetes/integration_test.go
index 4ac7daca..21ee6a94 100644
--- a/test/kubernetes/integration_test.go
+++ b/test/kubernetes/integration_test.go
@@ -2121,7 +2121,7 @@ func getNodeName(podNamespace string, podName string) (string, error) {
 }
 
 // returns the diskinfo for the volume with the given name mounted into the given pod
-func getVolumeInfo(t *testing.T, pod *v1.Pod, volumeName string) (DiskInfo, error) {
+func getVolumeInfoOnce(t *testing.T, pod *v1.Pod, volumeName string) (DiskInfo, error) {
 	node, err := getNodeName(pod.Namespace, pod.Name)
 	if err != nil {
 		return DiskInfo{}, err
@@ -2138,6 +2138,42 @@ func getVolumeInfo(t *testing.T, pod *v1.Pod, volumeName string) (DiskInfo, erro
 	return DiskInfo{}, fmt.Errorf("cannot find volume with name %v on node %v", volumeName, node)
 }
 
+func getVolumeInfo(t *testing.T, pod *v1.Pod, volumeName string) (DiskInfo, error) {
+	start := time.Now()
+	var lastErr error
+
+	for {
+		select {
+		case <-t.Context().Done():
+			t.Logf("test context canceled while waiting for volume %s: %v", volumeName, t.Context().Err())
+			return DiskInfo{}, lastErr
+		default:
+		}
+
+		disk, err := getVolumeInfoOnce(t, pod, volumeName)
+		if err == nil {
+			return disk, nil
+		}
+
+		lastErr = err
+		elapsed := time.Since(start)
+		if elapsed >= 120*time.Second {
+			t.Logf("timeout waiting for volume %s after %v: %v", volumeName, elapsed, lastErr)
+			return DiskInfo{}, lastErr
+		}
+
+		t.Logf("waiting for volume %s to be accessible (%v elapsed): %v", volumeName, elapsed, lastErr)
+		timer := time.NewTimer(5 * time.Second)
+		select {
+		case <-t.Context().Done():
+			timer.Stop()
+			t.Logf("test context canceled while waiting for volume %s after %v: %v", volumeName, elapsed, lastErr)
+			return DiskInfo{}, lastErr
+		case <-timer.C:
+		}
+	}
+}
+
 // inspects the node and returns information about the disks from the node's perspective
 func getVolumeInfoFromNode(t *testing.T, nodeName string) ([]DiskInfo, error) {
 	diskInfo := make([]DiskInfo, 0)

From dbfb264626106a7edf1ec9611b0a28e8081e3102 Mon Sep 17 00:00:00 2001
From: Michael Weibel <mweibel@cloudscale.ch>
Date: Tue, 15 Sep 2026 11:59:44 +0200
Subject: [PATCH 3/4] address review comments

---
 .dockerignore                                 |  2 ++
 AGENTS.md                                     | 16 +++++-----
 .../Dockerfile => Dockerfile                  | 24 +++++++++++---
 Makefile                                      | 12 +++++--
 README.md                                     | 28 +----------------
 .../csi-cloudscale/templates/statefulset.yaml |  2 +-
 deploy/README.md                              | 31 +++++++++----------
 helpers/run-in-test-cluster                   | 26 ++++++++--------
 test/kubernetes/integration_test.go           | 31 +++++++------------
 9 files changed, 79 insertions(+), 93 deletions(-)
 rename cmd/cloudscale-csi-plugin/Dockerfile => Dockerfile (55%)

diff --git a/.dockerignore b/.dockerignore
index b259346b..21ece939 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -10,3 +10,5 @@ scripts/
 test/
 Dockerfile
 VERSION
+# artifact from k8test image import
+image.tar
diff --git a/AGENTS.md b/AGENTS.md
index 35e099f5..dbc4aa2c 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -33,14 +33,14 @@ The [`csi-spec.md`](csi-spec.md) is the authoritative source for this driver's b
 
 ## Project structure
 
-| Directory/File              | Purpose                                           |
-|-----------------------------|---------------------------------------------------|
-| `cmd/cloudscale-csi-plugin/` | Binary entry point and Dockerfile                |
-| `driver/`                    | CSI driver implementation (Controller, Node, etc.) |
-| `charts/csi-cloudscale/`     | Helm chart for deployment                        |
-| `deploy/kubernetes/releases/`| Pre-rendered Kubernetes manifests                |
-| `examples/kubernetes/`       | Example StorageClasses and PVCs                  |
-| `test/kubernetes/`           | Integration tests                                |
+| Directory/File                | Purpose                                            |
+|-------------------------------|----------------------------------------------------|
+| `cmd/cloudscale-csi-plugin/`  | Binary entry point                                 |
+| `driver/`                     | CSI driver implementation (Controller, Node, etc.) |
+| `charts/csi-cloudscale/`      | Helm chart for deployment                          |
+| `deploy/kubernetes/releases/` | Pre-rendered Kubernetes manifests                  |
+| `examples/kubernetes/`        | Example StorageClasses and PVCs                    |
+| `test/kubernetes/`            | Integration tests                                  |
 
 ## Key components
 
diff --git a/cmd/cloudscale-csi-plugin/Dockerfile b/Dockerfile
similarity index 55%
rename from cmd/cloudscale-csi-plugin/Dockerfile
rename to Dockerfile
index 9c2bd43a..5f876959 100644
--- a/cmd/cloudscale-csi-plugin/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,18 @@
-FROM golang:1.27-alpine AS builder
-
-RUN apk add --no-cache git make bash
+# Copyright cloudscale.ch
+# Copyright 2018 DigitalOcean
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM golang:1.27.0 AS builder
 
 WORKDIR /src
 
@@ -8,8 +20,10 @@ WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
 
-# Copy all source code
-COPY . .
+# Copy all necessary source code
+COPY Makefile ./
+COPY driver/ driver/
+COPY cmd/ cmd/
 
 # Build arguments for version information
 ARG VERSION=dev
diff --git a/Makefile b/Makefile
index d22f27d0..7a3490ab 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 NAME=cloudscale-csi-plugin
 OS ?= linux
-GO_VERSION := $(shell awk '/^go/ {print $$2}' go.mod)
+
 GIT_TREE_STATE ?= $(shell git status --porcelain 2>/dev/null | grep -q . && echo "dirty" || echo "clean")
 COMMIT ?= $(shell git rev-parse HEAD)
 BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
@@ -94,7 +94,13 @@ test: vet ## Run tests.
 .PHONY: test-integration
 test-integration: ## Run integration tests
 	@echo "==> Started integration tests"
-	K8TEST_PATH=${PWD}/k8test go test -count 1 -v $(TESTARGS) -tags integration -parallel 4 -timeout 20m ./test/...
+	@if [ -f "$(PWD)/k8test/cluster/admin.conf" ]; then \
+		echo "==> Found k8test cluster config at $(PWD)/k8test/cluster/admin.conf, using k8test kubeconfig"; \
+		KUBECONFIG=$(PWD)/k8test/cluster/admin.conf go test -count 1 -v $(TESTARGS) -tags integration -parallel 4 -timeout 20m ./test/...; \
+	else \
+		echo "==> No k8test found, using standard kubeconfig loading"; \
+		go test -count 1 -v $(TESTARGS) -tags integration -parallel 4 -timeout 20m ./test/...; \
+	fi
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter
@@ -123,7 +129,7 @@ build: ## Build the docker image
 		--build-arg VERSION="$(VERSION)" \
 		--build-arg COMMIT="$(COMMIT)" \
 		--build-arg GIT_TREE_STATE="$(GIT_TREE_STATE)" \
-		-f cmd/cloudscale-csi-plugin/Dockerfile .
+		-f Dockerfile .
 
 .PHONY: push
 push: ## Push docker image to registry
diff --git a/README.md b/README.md
index 9b0c7d10..1498383f 100644
--- a/README.md
+++ b/README.md
@@ -418,33 +418,7 @@ This will create a binary with version `dev` and docker image pushed to
 `cloudscalech/cloudscale-csi-plugin:dev`
 
 
-To run the integration tests locally with your local CSI build:
-
-```bash
-# 1. Build and export the CSI image
-$ VERSION=dev make publish
-
-# 2. Create a test cluster with CCM and CSI
-$ export CLOUDSCALE_API_TOKEN=your-token
-$ export IMAGE=quay.io/cloudscalech/cloudscale-csi-plugin:dev
-$ helpers/run-in-test-cluster
-
-# This will:
-# - Create a Kubernetes cluster on cloudscale.ch
-# - Deploy CCM from the latest official release
-# - Deploy CSI from your local build
-
-# 3. Run integration tests
-$ export KUBECONFIG=$(pwd)/k8test/cluster/admin.conf
-$ make test-integration
-
-# Run a single test
-$ TESTARGS='-run TestPod_Single_SSD_Volume' make test-integration
-
-# 4. Clean up
-$ helpers/cleanup
-```
-
+Refer to the [deploy/README.md](deploy/README.md) for instructions on how to run the integration tests.
 
 ### Release a new version
 
diff --git a/charts/csi-cloudscale/templates/statefulset.yaml b/charts/csi-cloudscale/templates/statefulset.yaml
index 69a6a770..f5fb0766 100644
--- a/charts/csi-cloudscale/templates/statefulset.yaml
+++ b/charts/csi-cloudscale/templates/statefulset.yaml
@@ -18,7 +18,7 @@ spec:
     spec:
       hostNetwork: true
       priorityClassName: system-cluster-critical
-      serviceAccount: {{ include "csi-cloudscale.controller-service-account-name" . }}
+      serviceAccountName: {{ include "csi-cloudscale.controller-service-account-name" . }}
       containers:
         - name: csi-provisioner
           image: "{{ .Values.provisioner.image.registry }}/{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}"
diff --git a/deploy/README.md b/deploy/README.md
index 91da844f..2a892c75 100644
--- a/deploy/README.md
+++ b/deploy/README.md
@@ -13,39 +13,36 @@ First bootstrap the cluster
 
     # Export your API Token obtained from http://control.cloudscale.ch
     export CLOUDSCALE_API_TOKEN="..."
+    export IMAGE=quay.io/cloudscalech/cloudscale-csi-plugin:dev
     
     # See the script for options, sensible defaults apply
-    ./helpers/bootstrap-cluster
+    ./helpers/run-in-test-cluster
+
+    # This will:
+    # - Create a Kubernetes cluster on cloudscale.ch
+    # - Deploy CCM from the latest official release
+    # - Deploy CSI from your local build
     
     # Verify cluster setup and access
     export KUBECONFIG=$PWD/k8test/cluster/admin.conf
     kubectl get nodes -o wide
 
-
-You can **either** install the driver from your working directory
-
-    # Install driver using dev image from working dir
-    # Pre-requesit: ensure the you have run `helm dependency build` as described in the main README file.
-    helm install -g -n kube-system --set controller.image.tag=dev --set node.image.tag=dev --set controller.image.pullPolicy=Always --set node.image.pullPolicy=Always ./charts/csi-cloudscale
-
-**Or** you can install a released version:
-
-    # List all released versions
-    helm search repo csi-cloudscale/csi-cloudscale  --versions
-    # Install a specific Chart version or latest if --version is omitted
-    helm install -n kube-system -g csi-cloudscale/csi-cloudscale [ --version v1.0.0 ]
-
 You can verify that the csi-driver has been installed by running the following command and checking if the csi-cloudscale pods are running:
 
+    export KUBECONFIG=$PWD/k8test/cluster/admin.conf
     kubectl get pods -n kube-system
 
 Then execute the test suite:
 
+    # integration tests automatically target the k8test-provisioned cluster
     make test-integration
 
-The get rid of the cluster:
+    # Run a single test
+    TESTARGS='-run TestPod_Single_SSD_Volume' make test-integration
+
+To get rid of the cluster:
 
-    ./helpers/clean-up
+    ./helpers/cleanup
 
 ## Debugging
 
diff --git a/helpers/run-in-test-cluster b/helpers/run-in-test-cluster
index 9b7a832c..2c19ff9a 100755
--- a/helpers/run-in-test-cluster
+++ b/helpers/run-in-test-cluster
@@ -117,8 +117,7 @@ function build-image() {
     -i k8test/cluster/inventory.yml \
     -e dockerfile=./Dockerfile \
     -e tag="$IMAGE" \
-    -e extra='--build-arg=VERSION=test' \
-    -l controls
+    -e extra='--build-arg=VERSION=test'
 }
 
 # Import an image from the host
@@ -145,7 +144,7 @@ function deploy-ccm() {
 }
 
 # Deploy VolumeSnapshot CRDs and snapshot controller
-function deploy-snapshot-crds() {
+function deploy-snapshot-components() {
   export KUBECONFIG=k8test/cluster/admin.conf
 
   echo "Installing VolumeSnapshot CRDs..."
@@ -218,23 +217,23 @@ function deploy-csi() {
   fi
 }
 
-# Deploy both CCM and CSI components
-function deploy-components() {
+# Deploy volume snapshot components, build/import CSI image and deploy CSI
+function deploy-csi-components() {
   echo "Deploying Components"
   echo "IMAGE_SOURCE: ${IMAGE_SOURCE-build}"
   echo "IMAGE: ${IMAGE}"
   echo ""
 
-  # Deploy CCM first (required for node initialization and networking)
-  deploy-ccm
-
   # Install VolumeSnapshot CRDs and snapshot controller (required for CSI driver)
-  deploy-snapshot-crds
+  deploy-snapshot-components
 
   # Import pre-built CSI image (used in CI via IMAGE_SOURCE=import)
   if [[ "${IMAGE_SOURCE-build}" == "import" ]]; then
     echo "Importing CSI image from image.tar..."
     import-image
+  else
+    echo "Building image"
+    build-image
   fi
 
   # Deploy CSI
@@ -243,7 +242,6 @@ function deploy-components() {
 
 # Execute if not sourced
 if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then
-
   # The image name requires a slash in it, or Podman will add `localhost/` and
   # confuse Kubernetes.
   if [[ "$IMAGE" != *"/"* ]]; then
@@ -256,7 +254,9 @@ if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then
   ensure-k8test
   ensure-cluster
 
-  # Deploy components on both new and existing clusters. For new clusters,
-  # this happens after the full cluster bootstrap (including Cilium) is done.
-  deploy-components
+  if [[ "$first_run" == "no" ]]; then
+    deploy-ccm
+  fi
+
+  deploy-csi-components
 fi
diff --git a/test/kubernetes/integration_test.go b/test/kubernetes/integration_test.go
index 21ee6a94..1a026498 100644
--- a/test/kubernetes/integration_test.go
+++ b/test/kubernetes/integration_test.go
@@ -14,7 +14,6 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"testing"
@@ -104,17 +103,13 @@ func TestMain(m *testing.M) {
 }
 
 func TestNode_Zone_Annotation(t *testing.T) {
-	labelSelector := "node-role.kubernetes.io/worker=true"
+	labelSelector := "csi.cloudscale.ch/zone"
 	nodes, err := client.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{
 		LabelSelector: labelSelector,
 	})
 	assert.NoError(t, err)
 
-	if !(len(nodes.Items) > 0) {
-		t.Skipf("Could not find at least one node with label %s", labelSelector)
-		return
-	}
-
+	assert.Greater(t, len(nodes.Items), 0)
 	for _, node := range nodes.Items {
 		assert.Contains(t, []string{"rma1", "lpg1"}, node.Labels["csi.cloudscale.ch/zone"])
 	}
@@ -1511,21 +1506,19 @@ func TestVolumeStats(t *testing.T) {
 }
 
 func setup() error {
-	// Kubernetes client
-	k8test, ok := os.LookupEnv("K8TEST_PATH")
-	if !ok {
-		log.Fatalf("could not find K8TEST_PATH environment variable\n")
-	}
+	// if you want to change the loading rules (which files in which order),
+	// you can do so here
+	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
 
-	path := filepath.Join(k8test, "cluster", "admin.conf")
-	data, err := os.ReadFile(path)
-	if err != nil {
-		return fmt.Errorf("failed to read kubeconfig at path %q: %w", path, err)
-	}
+	// if you want to change override values or bind them to flags, there are
+	// methods to help you
+	configOverrides := &clientcmd.ConfigOverrides{}
 
-	config, err = clientcmd.RESTConfigFromKubeConfig(data)
+	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
+	var err error
+	config, err = kubeConfig.ClientConfig()
 	if err != nil {
-		return fmt.Errorf("failed to apply kubeconfig at path %q: %w", path, err)
+		return err
 	}
 
 	// create the clientset

From 0d1bacd051b3e4a59935a8b8947d3546966f7c9d Mon Sep 17 00:00:00 2001
From: Michael Weibel <mweibel@cloudscale.ch>
Date: Wed, 16 Sep 2026 11:26:25 +0200
Subject: [PATCH 4/4] add debug info

---
 helpers/run-in-test-cluster | 47 +++++++++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/helpers/run-in-test-cluster b/helpers/run-in-test-cluster
index 2c19ff9a..125673e4 100755
--- a/helpers/run-in-test-cluster
+++ b/helpers/run-in-test-cluster
@@ -127,6 +127,49 @@ function import-image() {
     -e image="$PWD/image.tar"
 }
 
+# List CSI-related container images on all cluster nodes using ansible ad-hoc
+function list-csi-images() {
+  echo "=== CSI Images on All Nodes ==="
+#  ansible -i k8test/cluster/inventory.yml nodes -b \
+#    -m shell \
+#    -a "ctr -n k8s.io images | grep -E 'cloudscale|csi' || echo '  No CSI images found on this node'" \
+#    --timeout=30
+  ansible -i k8test/cluster/inventory.yml nodes -b \
+    -m shell \
+    -a "ctr -n k8s.io images ls" \
+    --timeout=30
+  echo ""
+}
+
+# Debug a failed CSI rollout by printing pod events, status, description and node images
+function debug-rollout-failure() {
+  local component="$1"
+  local label="$2"
+
+  echo ""
+  echo "❌ CSI ${component} rollout failed"
+  echo ""
+  echo "=== ${component} Pod Events ==="
+  kubectl -n kube-system get pods -l "${label}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | \
+    while read -r pod; do
+      echo "Pod: $pod"
+      kubectl -n kube-system describe pod "$pod" | grep -A 20 "Events:" || echo "  (No events found)"
+      echo ""
+    done
+  echo "=== ${component} Pod Status ==="
+  kubectl -n kube-system get pods -l "${label}" -o wide
+  echo ""
+  echo "=== ${component} Container Images ==="
+  kubectl -n kube-system get pods -l "${label}" \
+    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{range .status.containerStatuses[*]}  Container: {.name}, Image: {.image}, Ready: {.ready}, RestartCount: {.restartCount}{"\n"}{end}{"\n"}{end}'
+  echo ""
+  echo "=== ${component} Pod Full Description ==="
+  kubectl -n kube-system describe pods -l "${label}"
+  echo ""
+  echo "=== CSI Images on All Nodes ==="
+  list-csi-images
+}
+
 # Deploy CCM from official GitHub release
 function deploy-ccm() {
   export KUBECONFIG=k8test/cluster/admin.conf
@@ -203,7 +246,7 @@ function deploy-csi() {
   kubectl -n kube-system get pods -l app=csi-cloudscale-controller -o wide
 
   if ! kubectl -n kube-system rollout status statefulset/csi-cloudscale-controller --timeout=180s; then
-    echo "CSI Controller rollout failed"
+    debug-rollout-failure "Controller" "app=csi-cloudscale-controller"
     exit 1
   fi
 
@@ -212,7 +255,7 @@ function deploy-csi() {
   kubectl -n kube-system get pods -l app=csi-cloudscale-node -o wide
 
   if ! kubectl -n kube-system rollout status daemonset/csi-cloudscale-node --timeout=300s; then
-    echo "CSI Node DaemonSet rollout failed"
+    debug-rollout-failure "Node DaemonSet" "app=csi-cloudscale-node"
     exit 1
   fi
 }