From c3f68478e286dd6d0e8c1ae968f9f146a5b019d6 Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Sun, 11 Oct 2026 22:19:31 +0100 Subject: [PATCH 1/4] ci: Ensure HIP/CUDA_VISIBLE_DEVICES=0 is set and not all OMP_NUM_THREADS=4 PYTEST_XDIST_AUTO_NUM_WORKERS is set --- .github/workflows/docker-devito.yaml | 62 +++++++++++++++++++++------- .github/workflows/pytest-gpu.yaml | 15 +++++-- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker-devito.yaml b/.github/workflows/docker-devito.yaml index 87ab0e212c..286ec21cf8 100644 --- a/.github/workflows/docker-devito.yaml +++ b/.github/workflows/docker-devito.yaml @@ -30,7 +30,14 @@ jobs: tag_suffix: '-amd64' # Respect CUDA_VISIBLE_DEVICES set by the runner and hard-limit docker to that device. # (--gpus maps only the selected device from CUDA_VISIBLE_DEVICES) - flag: --init --cpuset-cpus="${AVAILABLE_CPUSET}" --memory="${AVAILABLE_MEM}" --memory-swap="${AVAILABLE_MEM}" --gpus "device=${CUDA_VISIBLE_DEVICES}" + flag: >- + --cpuset-cpus="${AVAILABLE_CPUSET}" + --memory="${AVAILABLE_MEM}" + --memory-swap="${AVAILABLE_MEM}" + --gpus "device=${CUDA_VISIBLE_DEVICES}" + --env CUDA_VISIBLE_DEVICES=0 + --env OMP_NUM_THREADS=4 + --env PYTEST_XDIST_AUTO_NUM_WORKERS=${PYTEST_XDIST_AUTO_NUM_WORKERS} test: 'tests/test_gpu_openacc.py tests/test_gpu_common.py' runner: ["self-hosted", "NVIDIAGPU"] @@ -40,7 +47,7 @@ jobs: platform: linux/arm64 run_tests: false tag_suffix: '-arm64' - flag: --init --gpus "device=${CUDA_VISIBLE_DEVICES:-all}" + flag: --gpus "device=${CUDA_VISIBLE_DEVICES:-all}" test: '' runner: ubuntu-24.04-arm @@ -52,7 +59,14 @@ jobs: tag_suffix: '-amd64' # Respect CUDA_VISIBLE_DEVICES set by the runner and hard-limit docker to that device. # (--gpus maps only the selected device from CUDA_VISIBLE_DEVICES) - flag: --init --cpuset-cpus="${AVAILABLE_CPUSET}" --memory="${AVAILABLE_MEM}" --memory-swap="${AVAILABLE_MEM}" --gpus "device=${CUDA_VISIBLE_DEVICES}" + flag: >- + --cpuset-cpus="${AVAILABLE_CPUSET}" + --memory="${AVAILABLE_MEM}" + --memory-swap="${AVAILABLE_MEM}" + --gpus "device=${CUDA_VISIBLE_DEVICES}" + --env CUDA_VISIBLE_DEVICES=0 + --env OMP_NUM_THREADS=4 + --env PYTEST_XDIST_AUTO_NUM_WORKERS=${PYTEST_XDIST_AUTO_NUM_WORKERS} test: 'tests/test_gpu_openacc.py tests/test_gpu_common.py' runner: ["self-hosted", "NVIDIAGPU"] @@ -62,7 +76,7 @@ jobs: platform: linux/arm64 run_tests: false tag_suffix: '-arm64' - flag: --init --gpus "device=${CUDA_VISIBLE_DEVICES:-all}" + flag: --gpus "device=${CUDA_VISIBLE_DEVICES:-all}" test: '' runner: ubuntu-24.04-arm @@ -73,7 +87,18 @@ jobs: platform: linux/amd64 run_tests: true tag_suffix: '' - flag: '--init --cpuset-cpus="${AVAILABLE_CPUSET}" --memory="${AVAILABLE_MEM}" --memory-swap="${AVAILABLE_MEM}" --network=host --device=/dev/kfd --device=/dev/dri/${HIP_DEVICE_NAME} --ipc=host --group-add video --group-add $(getent group render | cut -d: -f3) --cap-add=SYS_PTRACE --security-opt seccomp=unconfined' + flag: >- + --cpuset-cpus="${AVAILABLE_CPUSET}" + --memory="${AVAILABLE_MEM}" + --memory-swap="${AVAILABLE_MEM}" + --device=/dev/kfd --device=/dev/dri/${HIP_DEVICE_NAME} + --network=host + --ipc=host + --group-add video --group-add $(getent group render | cut -d: -f3) + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined + --env HIP_VISIBLE_DEVICES=0 + --env OMP_NUM_THREADS=4 + --env PYTEST_XDIST_AUTO_NUM_WORKERS=${PYTEST_XDIST_AUTO_NUM_WORKERS} test: 'tests/test_gpu_openmp.py' runner: ["self-hosted", "AMDGPU"] @@ -83,7 +108,6 @@ jobs: platform: linux/amd64 run_tests: false tag_suffix: '-amd64' - flag: '--init -t' test: 'tests/test_operator.py' runner: ubuntu-latest @@ -93,7 +117,6 @@ jobs: platform: linux/arm64 run_tests: false tag_suffix: '-arm64' - flag: '--init -t' test: 'tests/test_operator.py' runner: ubuntu-24.04-arm @@ -103,7 +126,6 @@ jobs: platform: linux/amd64 run_tests: true tag_suffix: '' - flag: '--init -t' test: 'tests/test_operator.py' runner: ubuntu-latest @@ -182,7 +204,7 @@ jobs: - name: Run tests if: ${{ matrix.run_tests }} run: | - docker run ${{ matrix.flag }} --rm -t --name "${CONTAINER_NAME}" \ + docker run ${{ matrix.flag }} --init -t --rm --name "${CONTAINER_NAME}" \ devitocodes/devito:${{ matrix.tag }}-dev \ pytest -n auto --dist loadscope -v ${{ matrix.test }} @@ -276,19 +298,31 @@ jobs: matrix: include: - tag: 'nvidia-nvc' - flag: --init --cpuset-cpus="${AVAILABLE_CPUSET}" --memory="${AVAILABLE_MEM}" --memory-swap="${AVAILABLE_MEM}" --gpus "device=${CUDA_VISIBLE_DEVICES:-all}" + flag: >- + --cpuset-cpus="${AVAILABLE_CPUSET}" + --memory="${AVAILABLE_MEM}" + --memory-swap="${AVAILABLE_MEM}" + --gpus "device=${CUDA_VISIBLE_DEVICES}" + --env CUDA_VISIBLE_DEVICES=0 + --env OMP_NUM_THREADS=4 + --env PYTEST_XDIST_AUTO_NUM_WORKERS=${PYTEST_XDIST_AUTO_NUM_WORKERS} test: 'tests/test_gpu_openacc.py tests/test_gpu_common.py' runner: ["self-hosted", "NVIDIAGPU"] - tag: 'nvidia-nvc12' - flag: --init --cpuset-cpus="${AVAILABLE_CPUSET}" --memory="${AVAILABLE_MEM}" --memory-swap="${AVAILABLE_MEM}" --gpus "device=${CUDA_VISIBLE_DEVICES:-all}" + flag: >- + --cpuset-cpus="${AVAILABLE_CPUSET}" + --memory="${AVAILABLE_MEM}" + --memory-swap="${AVAILABLE_MEM}" + --gpus "device=${CUDA_VISIBLE_DEVICES}" + --env CUDA_VISIBLE_DEVICES=0 + --env OMP_NUM_THREADS=4 + --env PYTEST_XDIST_AUTO_NUM_WORKERS=${PYTEST_XDIST_AUTO_NUM_WORKERS} test: 'tests/test_gpu_openacc.py tests/test_gpu_common.py' runner: ["self-hosted", "NVIDIAGPU"] - tag: 'gcc' - flag: '--init -t' test: 'tests/test_operator.py' runner: ubuntu-latest - tag: 'gcc' - flag: '--init -t' test: 'tests/test_operator.py' runner: ubuntu-24.04-arm @@ -305,6 +339,6 @@ jobs: - name: Run tests against multi-arch image run: | - docker run ${{ matrix.flag }} --rm -t --name "${CONTAINER_NAME}" \ + docker run ${{ matrix.flag }} --init -t --rm --name "${CONTAINER_NAME}" \ devitocodes/devito:${{ matrix.tag }}-dev \ pytest -n auto --dist loadscope -v ${{ matrix.test }} diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml index 3ade43ecad..40df2479e3 100644 --- a/.github/workflows/pytest-gpu.yaml +++ b/.github/workflows/pytest-gpu.yaml @@ -65,9 +65,8 @@ jobs: --cpuset-cpus="${AVAILABLE_CPUSET}" --memory="${AVAILABLE_MEM}" --memory-swap="${AVAILABLE_MEM}" - --network=host + --network=host --ipc=host --device=/dev/kfd --device=/dev/dri/${HIP_DEVICE_NAME} - --ipc=host --group-add video --group-add "$(getent group render | cut -d: -f3)" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined @@ -106,6 +105,9 @@ jobs: CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} DEVITO_LOGGING=DEBUG PYTHONFAULTHANDLER=1 + CUDA_VISIBLE_DEVICES=0 + HIP_VISIBLE_DEVICES=0 + OMP_NUM_THREADS=4 PYTEST_XDIST_AUTO_NUM_WORKERS=${PYTEST_XDIST_AUTO_NUM_WORKERS} command: | pytest \ @@ -128,7 +130,11 @@ jobs: uid: ${{ steps.build.outputs.unique }} tag: ${{ matrix.name }} args: ${{ matrix.dockerflags }} - env: PYTEST_XDIST_AUTO_NUM_WORKERS=${PYTEST_XDIST_AUTO_NUM_WORKERS} + env: | + CUDA_VISIBLE_DEVICES=0 + HIP_VISIBLE_DEVICES=0 + OMP_NUM_THREADS=4 + PYTEST_XDIST_AUTO_NUM_WORKERS=${PYTEST_XDIST_AUTO_NUM_WORKERS} command: | pytest \ -n auto --dist loadscope \ @@ -145,6 +151,9 @@ jobs: CI=true DEVITO_LOGGING=DEBUG DEVITO_MPI=1 + CUDA_VISIBLE_DEVICES=0 + HIP_VISIBLE_DEVICES=0 + OMP_NUM_THREADS=4 command: | mpiexec -n 2 \ pytest \ From 55846f5164bfe900c920d9355fc1ae28a108d939 Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Sun, 11 Oct 2026 23:29:38 +0100 Subject: [PATCH 2/4] tests: Modify decoupler marker --- conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conftest.py b/conftest.py index f33942d528..b9472f41aa 100644 --- a/conftest.py +++ b/conftest.py @@ -229,6 +229,11 @@ def decoupler(item, m): if isinstance(m, int): env_vars['DEVITO_DECOUPLER_WORKERS'] = str(m) + # Map by and bind to should be respected if set in the environment + for key in ['DEVITO_DECOUPLER_MAP_BY', 'DEVITO_DECOUPLER_BIND_TO']: + if os.environ.get(key) is not None: + env_vars[key] = os.environ[key] + testname = get_testname(item) pyversion = sys.executable From 529a01188ce7aa0ef20b3046b28d2cba1c0e634b Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Tue, 13 Oct 2026 16:57:41 +0100 Subject: [PATCH 3/4] ci: Only one OMP_THREAD with MPI --- .github/workflows/pytest-gpu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml index 40df2479e3..846b19be6b 100644 --- a/.github/workflows/pytest-gpu.yaml +++ b/.github/workflows/pytest-gpu.yaml @@ -153,7 +153,7 @@ jobs: DEVITO_MPI=1 CUDA_VISIBLE_DEVICES=0 HIP_VISIBLE_DEVICES=0 - OMP_NUM_THREADS=4 + OMP_NUM_THREADS=1 command: | mpiexec -n 2 \ pytest \ From 736a181e8627691d28fd83897bb690a9bd839f54 Mon Sep 17 00:00:00 2001 From: Jack Betteridge Date: Wed, 14 Oct 2026 14:33:27 +0100 Subject: [PATCH 4/4] ci: Stop using the host network and ipc --- .github/workflows/pytest-gpu.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml index 846b19be6b..6c1c70f9cb 100644 --- a/.github/workflows/pytest-gpu.yaml +++ b/.github/workflows/pytest-gpu.yaml @@ -65,7 +65,6 @@ jobs: --cpuset-cpus="${AVAILABLE_CPUSET}" --memory="${AVAILABLE_MEM}" --memory-swap="${AVAILABLE_MEM}" - --network=host --ipc=host --device=/dev/kfd --device=/dev/dri/${HIP_DEVICE_NAME} --group-add video --group-add "$(getent group render | cut -d: -f3)" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined @@ -153,7 +152,7 @@ jobs: DEVITO_MPI=1 CUDA_VISIBLE_DEVICES=0 HIP_VISIBLE_DEVICES=0 - OMP_NUM_THREADS=1 + OMP_NUM_THREADS=4 command: | mpiexec -n 2 \ pytest \