From 051dcf03e4cafdf3f3ab8bd22326b8a3945b0638 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 5 Oct 2026 08:25:49 -0600 Subject: [PATCH 1/5] chore(deps): refresh verified component pins for 26.11 Update monitoring, registry proxy and Slurm pins from public upstream releases. Keep the registry-cache example aligned with its role default. Leave Network Operator and MIG Manager unchanged pending verification. Signed-off-by: Doug Holt --- docs/container/nginx-docker-cache.md | 2 +- roles/alertmanager/defaults/main.yml | 2 +- roles/grafana/defaults/main.yml | 2 +- roles/nginx-docker-registry-cache/defaults/main.yml | 2 +- roles/prometheus-node-exporter/defaults/main.yml | 2 +- roles/prometheus/defaults/main.yml | 2 +- roles/slurm/defaults/main.yml | 2 +- scripts/k8s/deploy_monitoring.sh | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/container/nginx-docker-cache.md b/docs/container/nginx-docker-cache.md index 34366ad32..783cba62a 100644 --- a/docs/container/nginx-docker-cache.md +++ b/docs/container/nginx-docker-cache.md @@ -42,7 +42,7 @@ The following variables are the most common configuration you may want to adjust | Variable | Default value | Description | | ------------------------------------------ | ---------------------------------------- | ----------------------------------------------------------------------------- | -| `nginx_docker_cache_image` | `"rpardini/docker-registry-proxy:0.6.5"` | Container image used to deploy the proxy | +| `nginx_docker_cache_image` | `"rpardini/docker-registry-proxy:0.7.0"` | Container image used to deploy the proxy | | `nginx_docker_cache_registry_string` | `"registry.k8s.io quay.io k8s.gcr.io gcr.io nvcr.io"` | Space-separated list of registries to proxy; `k8s.gcr.io` is retained for older clusters while current Kubernetes images use `registry.k8s.io` | | `nginx_docker_cache_manifests` | `"false"` | Flag to determine whether to cache image manifests | | `nginx_docker_cache_manifest_default_time` | "1h" | If manifests are cached, time to cache them | diff --git a/roles/alertmanager/defaults/main.yml b/roles/alertmanager/defaults/main.yml index 7862048f9..4e32f0864 100644 --- a/roles/alertmanager/defaults/main.yml +++ b/roles/alertmanager/defaults/main.yml @@ -1,6 +1,6 @@ alertmanager_config_dir: /etc/alertmanager alertmanager_config_src: templates/alertmanager.yml.j2 -alertmanager_container: "prom/alertmanager:v0.33.0" +alertmanager_container: "prom/alertmanager:v0.34.1" alertmanager_svc_name: "docker.alertmanager.service" alertmanager_docker_volume_name: "deepops_alertmanager_metrics" alertmanager_state: started diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index 0299dae4c..b34ed35c1 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -2,7 +2,7 @@ grafana_config_dir: /etc/grafana grafana_config_template: templates/grafana.ini.j2 grafana_data_dir: /var/lib/grafana grafana_user_id: 472 -grafana_container: "grafana/grafana:13.1.0" +grafana_container: "grafana/grafana:13.2.3" grafana_svc_name: "docker.grafana.service" grafana_state: started grafana_enabled: yes diff --git a/roles/nginx-docker-registry-cache/defaults/main.yml b/roles/nginx-docker-registry-cache/defaults/main.yml index cea6bada1..2cd823805 100644 --- a/roles/nginx-docker-registry-cache/defaults/main.yml +++ b/roles/nginx-docker-registry-cache/defaults/main.yml @@ -1,6 +1,6 @@ --- nginx_docker_cache_name: "deepops-nginx-docker-cache" -nginx_docker_cache_image: "rpardini/docker-registry-proxy:0.6.5" +nginx_docker_cache_image: "rpardini/docker-registry-proxy:0.7.0" nginx_docker_cache_mirror_path: "/opt/deepops/nginx-docker-cache/mirror" nginx_docker_cache_ca_path: "/opt/deepops/nginx-docker-cache/ca" diff --git a/roles/prometheus-node-exporter/defaults/main.yml b/roles/prometheus-node-exporter/defaults/main.yml index 925f4baee..766cf626c 100644 --- a/roles/prometheus-node-exporter/defaults/main.yml +++ b/roles/prometheus-node-exporter/defaults/main.yml @@ -1,4 +1,4 @@ -node_exporter_container: "quay.io/prometheus/node-exporter:v1.11.1" +node_exporter_container: "quay.io/prometheus/node-exporter:v1.12.1" node_exporter_prom_dir: "/run/prometheus" node_exporter_svc_name: "docker.node-exporter.service" node_exporter_state: started diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml index fe354a7b8..49e2d6fcb 100644 --- a/roles/prometheus/defaults/main.yml +++ b/roles/prometheus/defaults/main.yml @@ -1,7 +1,7 @@ prometheus_config_dir: /etc/prometheus prometheus_config_src: templates/prometheus.yml.j2 prometheus_alert_rules_src: templates/alert_rules.yml.j2 -prometheus_container: "prom/prometheus:v3.13.0" +prometheus_container: "prom/prometheus:v3.14.0" prometheus_svc_name: "docker.prometheus.service" prometheus_docker_volume_name: "deepops_prometheus_metrics" prometheus_state: started diff --git a/roles/slurm/defaults/main.yml b/roles/slurm/defaults/main.yml index 5859e00f4..a8b17b48b 100644 --- a/roles/slurm/defaults/main.yml +++ b/roles/slurm/defaults/main.yml @@ -7,7 +7,7 @@ hwloc_build_dir: /opt/deepops/build/hwloc pmix_build_dir: /opt/deepops/build/pmix slurm_workflow_build: yes -slurm_version: "26.05.3" +slurm_version: "26.05.4" slurm_src_url: "https://download.schedmd.com/slurm/slurm-{{ slurm_version }}.tar.bz2" slurm_build_make_clean: no slurm_build_dir_cleanup: no diff --git a/scripts/k8s/deploy_monitoring.sh b/scripts/k8s/deploy_monitoring.sh index a11ed4ef9..e57af103a 100755 --- a/scripts/k8s/deploy_monitoring.sh +++ b/scripts/k8s/deploy_monitoring.sh @@ -24,7 +24,7 @@ if [ ! -d "${DEEPOPS_CONFIG_DIR}" ]; then fi HELM_CHARTS_REPO_PROMETHEUS="${HELM_CHARTS_REPO_PROMETHEUS:-https://prometheus-community.github.io/helm-charts}" -HELM_PROMETHEUS_CHART_VERSION="${HELM_PROMETHEUS_CHART_VERSION:-88.5.4}" +HELM_PROMETHEUS_CHART_VERSION="${HELM_PROMETHEUS_CHART_VERSION:-88.6.3}" ingress_name="ingress-nginx" PROMETHEUS_YAML_CONFIG="${PROMETHEUS_YAML_CONFIG:-${DEEPOPS_CONFIG_DIR}/helm/monitoring.yml}" From 555805d39bff02171ac299edf7e9982168394441 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 5 Oct 2026 08:46:50 -0600 Subject: [PATCH 2/5] docs(airgap): align offline mirrors with refreshed role pins Keep Slurm archives and monitoring pull, save, mirror and override examples aligned with the selected 26.11 component defaults. Signed-off-by: Doug Holt --- skills/deploy-airgapped/SKILL.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/skills/deploy-airgapped/SKILL.md b/skills/deploy-airgapped/SKILL.md index 96fd87263..adbb7f6fa 100644 --- a/skills/deploy-airgapped/SKILL.md +++ b/skills/deploy-airgapped/SKILL.md @@ -157,7 +157,7 @@ reference host; distribution repositories remain a separate prerequisite. while read -r url name; do sudo curl -fL "${url}" -o "/var/repos/downloads/${name}" done <<'EOF' - https://download.schedmd.com/slurm/slurm-26.05.1.tar.bz2 slurm-26.05.1.tar.bz2 + https://download.schedmd.com/slurm/slurm-26.05.4.tar.bz2 slurm-26.05.4.tar.bz2 https://download.open-mpi.org/release/hwloc/v2.5/hwloc-2.5.0.tar.gz hwloc-2.5.0.tar.gz https://github.com/openpmix/openpmix/releases/download/v3.2.3/pmix-3.2.3.tar.bz2 pmix-3.2.3.tar.bz2 https://github.com/mej/nhc/releases/download/1.4.3/lbnl-nhc-1.4.3.tar.xz lbnl-nhc-1.4.3.tar.xz @@ -165,7 +165,7 @@ reference host; distribution repositories remain a separate prerequisite. https://developer.download.nvidia.com/hpc-sdk/23.7/nvhpc_2023_237_Linux_x86_64_cuda_12.2.tar.gz nvhpc_2023_237_Linux_x86_64_cuda_12.2.tar.gz https://get.helm.sh/helm-v3.17.1-linux-amd64.tar.gz helm-v3.17.1-linux-amd64.tar.gz EOF - for name in slurm-26.05.1.tar.bz2 hwloc-2.5.0.tar.gz \ + for name in slurm-26.05.4.tar.bz2 hwloc-2.5.0.tar.gz \ pmix-3.2.3.tar.bz2 lbnl-nhc-1.4.3.tar.xz pyxis-0.11.1.tar.gz \ nvhpc_2023_237_Linux_x86_64_cuda_12.2.tar.gz \ helm-v3.17.1-linux-amd64.tar.gz; do @@ -217,7 +217,7 @@ reference host; distribution repositories remain a separate prerequisite. nonempty. Use these exact overrides with the published paths: ```yaml - slurm_src_url: "http://package-server/downloads/slurm-26.05.1.tar.bz2" + slurm_src_url: "http://package-server/downloads/slurm-26.05.4.tar.bz2" hwloc_src_url: "http://package-server/downloads/hwloc-2.5.0.tar.gz" pmix_src_url: "http://package-server/downloads/pmix-3.2.3.tar.bz2" nhc_src_url: "http://package-server/downloads/lbnl-nhc-1.4.3.tar.xz" @@ -405,14 +405,14 @@ reference host; distribution repositories remain a separate prerequisite. prebuilt image offline: ```bash - for image in prom/prometheus:v3.13.0 grafana/grafana:13.1.0 \ - prom/alertmanager:v0.33.0 quay.io/prometheus/node-exporter:v1.11.1 \ + for image in prom/prometheus:v3.14.0 grafana/grafana:13.2.3 \ + prom/alertmanager:v0.34.1 quay.io/prometheus/node-exporter:v1.12.1 \ nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless; do docker pull "${image}" done docker save -o /tmp/images/slurm-monitoring.tar \ - prom/prometheus:v3.13.0 grafana/grafana:13.1.0 \ - prom/alertmanager:v0.33.0 quay.io/prometheus/node-exporter:v1.11.1 \ + prom/prometheus:v3.14.0 grafana/grafana:13.2.3 \ + prom/alertmanager:v0.34.1 quay.io/prometheus/node-exporter:v1.12.1 \ nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless docker build -t deepops/prometheus-slurm-exporter:2.0.0 \ roles/prometheus-slurm-exporter/files/docker @@ -597,10 +597,10 @@ sha256sum -c SHA256SUMS docker tag "${source}" "${target}" docker push "${target}" done <<'EOF' - prom/prometheus:v3.13.0 127.0.0.1:5000/prom/prometheus:v3.13.0 - grafana/grafana:13.1.0 127.0.0.1:5000/grafana/grafana:13.1.0 - prom/alertmanager:v0.33.0 127.0.0.1:5000/prom/alertmanager:v0.33.0 - quay.io/prometheus/node-exporter:v1.11.1 127.0.0.1:5000/prometheus/node-exporter:v1.11.1 + prom/prometheus:v3.14.0 127.0.0.1:5000/prom/prometheus:v3.14.0 + grafana/grafana:13.2.3 127.0.0.1:5000/grafana/grafana:13.2.3 + prom/alertmanager:v0.34.1 127.0.0.1:5000/prom/alertmanager:v0.34.1 + quay.io/prometheus/node-exporter:v1.12.1 127.0.0.1:5000/prometheus/node-exporter:v1.12.1 nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless 127.0.0.1:5000/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless EOF ``` @@ -776,10 +776,10 @@ slurm_enable_container_registry: false standalone_container_registry_cache_enable: false slurm_exporter_build_image: false slurm_exporter_container: "registry-host:5000/deepops/prometheus-slurm-exporter:2.0.0" -prometheus_container: "registry-host:5000/prom/prometheus:v3.13.0" -grafana_container: "registry-host:5000/grafana/grafana:13.1.0" -alertmanager_container: "registry-host:5000/prom/alertmanager:v0.33.0" -node_exporter_container: "registry-host:5000/prometheus/node-exporter:v1.11.1" +prometheus_container: "registry-host:5000/prom/prometheus:v3.14.0" +grafana_container: "registry-host:5000/grafana/grafana:13.2.3" +alertmanager_container: "registry-host:5000/prom/alertmanager:v0.34.1" +node_exporter_container: "registry-host:5000/prometheus/node-exporter:v1.12.1" nvidia_dcgm_container: "registry-host:5000/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless" ``` From fdd76a04f52eb62e2c95e50d43fd38378e1d976b Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Sat, 17 Oct 2026 16:34:32 -0600 Subject: [PATCH 3/5] fix(monitoring): meet Run:ai platform chart version floor Default kube-prometheus-stack to 88.6.5 so monitoring meets the minimum version required by Run:ai platform mode. Signed-off-by: Doug Holt --- scripts/k8s/deploy_monitoring.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/k8s/deploy_monitoring.sh b/scripts/k8s/deploy_monitoring.sh index e57af103a..800ffdaa5 100755 --- a/scripts/k8s/deploy_monitoring.sh +++ b/scripts/k8s/deploy_monitoring.sh @@ -24,7 +24,7 @@ if [ ! -d "${DEEPOPS_CONFIG_DIR}" ]; then fi HELM_CHARTS_REPO_PROMETHEUS="${HELM_CHARTS_REPO_PROMETHEUS:-https://prometheus-community.github.io/helm-charts}" -HELM_PROMETHEUS_CHART_VERSION="${HELM_PROMETHEUS_CHART_VERSION:-88.6.3}" +HELM_PROMETHEUS_CHART_VERSION="${HELM_PROMETHEUS_CHART_VERSION:-88.6.5}" ingress_name="ingress-nginx" PROMETHEUS_YAML_CONFIG="${PROMETHEUS_YAML_CONFIG:-${DEEPOPS_CONFIG_DIR}/helm/monitoring.yml}" From 29562ae640eb263f3ee391cd655d9e9413ffd976 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 19 Oct 2026 08:11:25 -0600 Subject: [PATCH 4/5] chore(deps): refresh verified component versions for 26.11 Use published stable releases and align offline download examples. Select Network Operator 26.7.0, the newest chart currently published in the configured repository, rather than its newer source-only release. Retain the monitoring chart floor pending major-upgrade review. Co-Authored-By: Paperclip Signed-off-by: Doug Holt --- roles/grafana/defaults/main.yml | 2 +- roles/nvidia-gpu-operator/defaults/main.yml | 2 +- .../defaults/main.yml | 2 +- .../defaults/main.yml | 2 +- roles/nvidia-mig-manager/defaults/main.yml | 2 +- roles/nvidia-network-operator/vars/main.yaml | 2 +- roles/prometheus/defaults/main.yml | 2 +- roles/slurm/defaults/main.yml | 2 +- .../defaults/main.yml | 2 +- skills/deploy-airgapped/SKILL.md | 18 +++++++++--------- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index b34ed35c1..5b1dfb317 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -2,7 +2,7 @@ grafana_config_dir: /etc/grafana grafana_config_template: templates/grafana.ini.j2 grafana_data_dir: /var/lib/grafana grafana_user_id: 472 -grafana_container: "grafana/grafana:13.2.3" +grafana_container: "grafana/grafana:13.2.4" grafana_svc_name: "docker.grafana.service" grafana_state: started grafana_enabled: yes diff --git a/roles/nvidia-gpu-operator/defaults/main.yml b/roles/nvidia-gpu-operator/defaults/main.yml index d11ed3eb5..9d59d0e2b 100644 --- a/roles/nvidia-gpu-operator/defaults/main.yml +++ b/roles/nvidia-gpu-operator/defaults/main.yml @@ -12,7 +12,7 @@ gpu_operator_nvaie_helm_repo: "https://helm.ngc.nvidia.com/nvaie" gpu_operator_nvaie_chart_name: "nvaie/gpu-operator" # NVAIE GPU Operator may require different version, check NGC enterprise collection. -gpu_operator_chart_version: "v26.7.0" +gpu_operator_chart_version: "v26.7.2" k8s_gpu_mig_strategy: "mixed" diff --git a/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml b/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml index 4131a5a71..3120b8c61 100644 --- a/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml +++ b/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml @@ -2,6 +2,6 @@ k8s_gpu_plugin_helm_repo: "https://nvidia.github.io/k8s-device-plugin" k8s_gpu_plugin_chart_name: "nvdp/nvidia-device-plugin" k8s_gpu_plugin_release_name: "nvidia-device-plugin" -k8s_gpu_plugin_chart_version: "0.20.0" +k8s_gpu_plugin_chart_version: "0.20.1" k8s_gpu_plugin_init_error: "false" k8s_gpu_mig_strategy: "mixed" diff --git a/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml b/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml index 3effcd582..5f297e967 100644 --- a/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml +++ b/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml @@ -2,5 +2,5 @@ k8s_gpu_feature_discovery_helm_repo: "https://nvidia.github.io/k8s-device-plugin" k8s_gpu_feature_discovery_chart_name: "nvgfd/gpu-feature-discovery" k8s_gpu_feature_discovery_release_name: "gpu-feature-discovery" -k8s_gpu_feature_discovery_chart_version: "0.20.0" +k8s_gpu_feature_discovery_chart_version: "0.20.1" k8s_gpu_mig_strategy: "mixed" diff --git a/roles/nvidia-mig-manager/defaults/main.yml b/roles/nvidia-mig-manager/defaults/main.yml index 78e2e4c67..eaec7b868 100644 --- a/roles/nvidia-mig-manager/defaults/main.yml +++ b/roles/nvidia-mig-manager/defaults/main.yml @@ -4,7 +4,7 @@ # Keep the two maps separate so a single ansible_architecture value resolves # to the right token for each package type. # 'arm64' matches existing practice in roles/nvidia-dgx/vars/ubuntu-24.04.yml. -mig_manager_version: "0.14.2" +mig_manager_version: "0.15.1" mig_manager_release: "1" mig_manager_deb_arch_map: diff --git a/roles/nvidia-network-operator/vars/main.yaml b/roles/nvidia-network-operator/vars/main.yaml index 1f5e98b9c..1b34dcc62 100644 --- a/roles/nvidia-network-operator/vars/main.yaml +++ b/roles/nvidia-network-operator/vars/main.yaml @@ -6,7 +6,7 @@ # if_name must match k8s network annotation name # -nvidia_network_operator_version: "26.4.1" +nvidia_network_operator_version: "26.7.0" nvidia_network_operator_image_tag: "network-operator-v{{ nvidia_network_operator_version }}" nvidia_network_operator_namespace: "network-operator" nvidia_network_operator_ipam_type: "nv-ipam" diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml index 49e2d6fcb..3e505f414 100644 --- a/roles/prometheus/defaults/main.yml +++ b/roles/prometheus/defaults/main.yml @@ -1,7 +1,7 @@ prometheus_config_dir: /etc/prometheus prometheus_config_src: templates/prometheus.yml.j2 prometheus_alert_rules_src: templates/alert_rules.yml.j2 -prometheus_container: "prom/prometheus:v3.14.0" +prometheus_container: "prom/prometheus:v3.15.0" prometheus_svc_name: "docker.prometheus.service" prometheus_docker_volume_name: "deepops_prometheus_metrics" prometheus_state: started diff --git a/roles/slurm/defaults/main.yml b/roles/slurm/defaults/main.yml index a8b17b48b..d9707ec2a 100644 --- a/roles/slurm/defaults/main.yml +++ b/roles/slurm/defaults/main.yml @@ -7,7 +7,7 @@ hwloc_build_dir: /opt/deepops/build/hwloc pmix_build_dir: /opt/deepops/build/pmix slurm_workflow_build: yes -slurm_version: "26.05.4" +slurm_version: "26.05.5" slurm_src_url: "https://download.schedmd.com/slurm/slurm-{{ slurm_version }}.tar.bz2" slurm_build_make_clean: no slurm_build_dir_cleanup: no diff --git a/roles/standalone-container-registry/defaults/main.yml b/roles/standalone-container-registry/defaults/main.yml index 562524707..0d3e30f7d 100644 --- a/roles/standalone-container-registry/defaults/main.yml +++ b/roles/standalone-container-registry/defaults/main.yml @@ -2,7 +2,7 @@ epel_package: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" epel_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-{{ ansible_distribution_major_version }}" -standalone_container_registry_image: "registry:3.1.1" +standalone_container_registry_image: "registry:3.1.2" standalone_container_registry_port: "5000" standalone_container_registry_name: "deepops-registry" diff --git a/skills/deploy-airgapped/SKILL.md b/skills/deploy-airgapped/SKILL.md index adbb7f6fa..9db354b9c 100644 --- a/skills/deploy-airgapped/SKILL.md +++ b/skills/deploy-airgapped/SKILL.md @@ -157,7 +157,7 @@ reference host; distribution repositories remain a separate prerequisite. while read -r url name; do sudo curl -fL "${url}" -o "/var/repos/downloads/${name}" done <<'EOF' - https://download.schedmd.com/slurm/slurm-26.05.4.tar.bz2 slurm-26.05.4.tar.bz2 + https://download.schedmd.com/slurm/slurm-26.05.5.tar.bz2 slurm-26.05.5.tar.bz2 https://download.open-mpi.org/release/hwloc/v2.5/hwloc-2.5.0.tar.gz hwloc-2.5.0.tar.gz https://github.com/openpmix/openpmix/releases/download/v3.2.3/pmix-3.2.3.tar.bz2 pmix-3.2.3.tar.bz2 https://github.com/mej/nhc/releases/download/1.4.3/lbnl-nhc-1.4.3.tar.xz lbnl-nhc-1.4.3.tar.xz @@ -165,7 +165,7 @@ reference host; distribution repositories remain a separate prerequisite. https://developer.download.nvidia.com/hpc-sdk/23.7/nvhpc_2023_237_Linux_x86_64_cuda_12.2.tar.gz nvhpc_2023_237_Linux_x86_64_cuda_12.2.tar.gz https://get.helm.sh/helm-v3.17.1-linux-amd64.tar.gz helm-v3.17.1-linux-amd64.tar.gz EOF - for name in slurm-26.05.4.tar.bz2 hwloc-2.5.0.tar.gz \ + for name in slurm-26.05.5.tar.bz2 hwloc-2.5.0.tar.gz \ pmix-3.2.3.tar.bz2 lbnl-nhc-1.4.3.tar.xz pyxis-0.11.1.tar.gz \ nvhpc_2023_237_Linux_x86_64_cuda_12.2.tar.gz \ helm-v3.17.1-linux-amd64.tar.gz; do @@ -217,7 +217,7 @@ reference host; distribution repositories remain a separate prerequisite. nonempty. Use these exact overrides with the published paths: ```yaml - slurm_src_url: "http://package-server/downloads/slurm-26.05.4.tar.bz2" + slurm_src_url: "http://package-server/downloads/slurm-26.05.5.tar.bz2" hwloc_src_url: "http://package-server/downloads/hwloc-2.5.0.tar.gz" pmix_src_url: "http://package-server/downloads/pmix-3.2.3.tar.bz2" nhc_src_url: "http://package-server/downloads/lbnl-nhc-1.4.3.tar.xz" @@ -405,13 +405,13 @@ reference host; distribution repositories remain a separate prerequisite. prebuilt image offline: ```bash - for image in prom/prometheus:v3.14.0 grafana/grafana:13.2.3 \ + for image in prom/prometheus:v3.15.0 grafana/grafana:13.2.4 \ prom/alertmanager:v0.34.1 quay.io/prometheus/node-exporter:v1.12.1 \ nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless; do docker pull "${image}" done docker save -o /tmp/images/slurm-monitoring.tar \ - prom/prometheus:v3.14.0 grafana/grafana:13.2.3 \ + prom/prometheus:v3.15.0 grafana/grafana:13.2.4 \ prom/alertmanager:v0.34.1 quay.io/prometheus/node-exporter:v1.12.1 \ nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless docker build -t deepops/prometheus-slurm-exporter:2.0.0 \ @@ -597,8 +597,8 @@ sha256sum -c SHA256SUMS docker tag "${source}" "${target}" docker push "${target}" done <<'EOF' - prom/prometheus:v3.14.0 127.0.0.1:5000/prom/prometheus:v3.14.0 - grafana/grafana:13.2.3 127.0.0.1:5000/grafana/grafana:13.2.3 + prom/prometheus:v3.15.0 127.0.0.1:5000/prom/prometheus:v3.15.0 + grafana/grafana:13.2.4 127.0.0.1:5000/grafana/grafana:13.2.4 prom/alertmanager:v0.34.1 127.0.0.1:5000/prom/alertmanager:v0.34.1 quay.io/prometheus/node-exporter:v1.12.1 127.0.0.1:5000/prometheus/node-exporter:v1.12.1 nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless 127.0.0.1:5000/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless @@ -776,8 +776,8 @@ slurm_enable_container_registry: false standalone_container_registry_cache_enable: false slurm_exporter_build_image: false slurm_exporter_container: "registry-host:5000/deepops/prometheus-slurm-exporter:2.0.0" -prometheus_container: "registry-host:5000/prom/prometheus:v3.14.0" -grafana_container: "registry-host:5000/grafana/grafana:13.2.3" +prometheus_container: "registry-host:5000/prom/prometheus:v3.15.0" +grafana_container: "registry-host:5000/grafana/grafana:13.2.4" alertmanager_container: "registry-host:5000/prom/alertmanager:v0.34.1" node_exporter_container: "registry-host:5000/prometheus/node-exporter:v1.12.1" nvidia_dcgm_container: "registry-host:5000/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless" From 5eaee8db3eb5a390b7a79d8519c3e8582c856d96 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 19 Oct 2026 09:06:47 -0600 Subject: [PATCH 5/5] fix(airgap): align offline artifacts with deployment pins Mirror GPU Operator v26.7.2 and registry 3.1.2 consistently across connected-side preparation, transfer records, and offline imports. Add regression checks against role defaults, including the explicit driver override. Retain R580 to avoid an unvalidated driver-branch upgrade as part of the chart refresh. Co-Authored-By: Paperclip Signed-off-by: Doug Holt --- docs/airgap/mirror-docker-images.md | 30 +++++----- roles/nvidia-gpu-operator/defaults/main.yml | 4 ++ scripts/validation/tests/test_airgap_pins.py | 60 ++++++++++++++++++++ skills/deploy-airgapped/SKILL.md | 14 ++--- 4 files changed, 86 insertions(+), 22 deletions(-) create mode 100644 scripts/validation/tests/test_airgap_pins.py diff --git a/docs/airgap/mirror-docker-images.md b/docs/airgap/mirror-docker-images.md index e5ae73cfa..abb98f286 100644 --- a/docs/airgap/mirror-docker-images.md +++ b/docs/airgap/mirror-docker-images.md @@ -211,22 +211,22 @@ set -euo pipefail TRUSTED_RECORD_DIR="${TRUSTED_RECORD_DIR:-${HOME}/deepops-trusted-records}" mkdir -p "${TRUSTED_RECORD_DIR}" chmod 0700 "${TRUSTED_RECORD_DIR}" -REGISTRY_SOURCE="docker.io/library/registry:3.1.1" +REGISTRY_SOURCE="docker.io/library/registry:3.1.2" REGISTRY_DIGEST="$(skopeo inspect --format '{{.Digest}}' "docker://${REGISTRY_SOURCE}")" -printf '%s\n' "${REGISTRY_DIGEST}" > "${TRUSTED_RECORD_DIR}/registry-3.1.1.source-digest" +printf '%s\n' "${REGISTRY_DIGEST}" > "${TRUSTED_RECORD_DIR}/registry-3.1.2.source-digest" docker pull "registry@${REGISTRY_DIGEST}" -docker tag "registry@${REGISTRY_DIGEST}" registry:3.1.1 -docker save -o /tmp/images/registry-3.1.1.tar registry:3.1.1 +docker tag "registry@${REGISTRY_DIGEST}" registry:3.1.2 +docker save -o /tmp/images/registry-3.1.2.tar registry:3.1.2 ( cd /tmp/images - sha256sum registry-3.1.1.tar > registry-3.1.1.tar.sha256 - cp registry-3.1.1.tar.sha256 "${TRUSTED_RECORD_DIR}/registry-3.1.1.tar.sha256" + sha256sum registry-3.1.2.tar > registry-3.1.2.tar.sha256 + cp registry-3.1.2.tar.sha256 "${TRUSTED_RECORD_DIR}/registry-3.1.2.tar.sha256" ) ) ``` -Retain `${TRUSTED_RECORD_DIR}/registry-3.1.1.tar.sha256` and the recorded +Retain `${TRUSTED_RECORD_DIR}/registry-3.1.2.tar.sha256` and the recorded source digest in a separate trusted system or signed manifest. The checksum copy carried on the same removable media detects accidental corruption but does not prove authenticity if that media is replaced. @@ -255,7 +255,7 @@ umask 077 sudo apt install genisoimage TRANSFER_ROOT="/tmp/images" CUDA_NAME="nvidia-cuda-12.4.1-base-ubuntu22.04" -REGISTRY_NAME="registry-3.1.1" +REGISTRY_NAME="registry-3.1.2" ISO_STAGING_DIR="$(mktemp -d "${TMPDIR:-/tmp}/deepops-image-iso.XXXXXX")" chmod 0700 "${ISO_STAGING_DIR}" genisoimage \ @@ -302,19 +302,19 @@ host: set -euo pipefail TRUSTED_RECORD_DIR="${TRUSTED_RECORD_DIR:-${HOME}/deepops-trusted-records}" cd /tmp/images -sha256sum --check "${TRUSTED_RECORD_DIR}/registry-3.1.1.tar.sha256" -EXPECTED_REGISTRY_DIGEST="$(cat "${TRUSTED_RECORD_DIR}/registry-3.1.1.source-digest")" +sha256sum --check "${TRUSTED_RECORD_DIR}/registry-3.1.2.tar.sha256" +EXPECTED_REGISTRY_DIGEST="$(cat "${TRUSTED_RECORD_DIR}/registry-3.1.2.source-digest")" case "${EXPECTED_REGISTRY_DIGEST}" in sha256:[0-9a-f][0-9a-f]*) ;; *) exit 1 ;; esac -docker load -i registry-3.1.1.tar -LOADED_REGISTRY_ID="$(docker image inspect --format '{{.Id}}' registry:3.1.1)" +docker load -i registry-3.1.2.tar +LOADED_REGISTRY_ID="$(docker image inspect --format '{{.Id}}' registry:3.1.2)" test -n "${LOADED_REGISTRY_ID}" ) ``` -`${TRUSTED_RECORD_DIR}/registry-3.1.1.source-digest` is the separately retained +`${TRUSTED_RECORD_DIR}/registry-3.1.2.source-digest` is the separately retained or signed digest record from the connected side, delivered to this host through an independently trusted path (set `TRUSTED_RECORD_DIR` to where you placed it). The archive checksum binds the -loaded `registry:3.1.1` tag to the connected-side archive; Docker's save format +loaded `registry:3.1.2` tag to the connected-side archive; Docker's save format does not preserve repository-digest metadata, so the separately retained digest remains provenance evidence rather than an offline `RepoDigests` assertion. Do not source either trust anchor from the same untrusted transfer media as the @@ -336,7 +336,7 @@ docker run -d \ --restart=always \ --name registry \ -v registry-images:/var/lib/registry \ - registry:3.1.1 + registry:3.1.2 ``` Before binding the registry to a network-reachable address, follow the diff --git a/roles/nvidia-gpu-operator/defaults/main.yml b/roles/nvidia-gpu-operator/defaults/main.yml index 9d59d0e2b..40155f072 100644 --- a/roles/nvidia-gpu-operator/defaults/main.yml +++ b/roles/nvidia-gpu-operator/defaults/main.yml @@ -33,6 +33,10 @@ gpu_operator_grid_config_dir: "{{ deepops_dir }}/gpu_operator" # Defaults from https://github.com/NVIDIA/gpu-operator/blob/master/deployments/gpu-operator/values.yaml gpu_operator_default_runtime: "containerd" gpu_operator_driver_registry: "nvcr.io/nvidia" +# Deliberately retain the R580 pin rather than chart v26.7.2's R595 default: +# a chart refresh must not also introduce an unvalidated driver-branch upgrade. +# Used only when Operator-managed drivers are enabled; preinstalled drivers +# are not changed. Validate the target GPU/OS before overriding this version. gpu_operator_driver_version: "580.126.20" # This enables/disables NVAIE diff --git a/scripts/validation/tests/test_airgap_pins.py b/scripts/validation/tests/test_airgap_pins.py new file mode 100644 index 000000000..33e07d85e --- /dev/null +++ b/scripts/validation/tests/test_airgap_pins.py @@ -0,0 +1,60 @@ +"""Keep offline chart/image preparation aligned with the deployment roles.""" + +from pathlib import Path +import re +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[3] + + +class AirgapPinTests(unittest.TestCase): + def setUp(self): + self.skill = (ROOT / "skills/deploy-airgapped/SKILL.md").read_text() + self.gpu = yaml.safe_load( + (ROOT / "roles/nvidia-gpu-operator/defaults/main.yml").read_text() + ) + + def assert_references_match(self, text, pattern, expected): + versions = re.findall(pattern, text) + self.assertTrue(versions, f"No references matched {pattern!r}") + for version in versions: + with self.subTest(pattern=pattern, version=version): + self.assertEqual(version, expected) + + def test_gpu_operator_chart_preparation_matches_role(self): + for pattern in ( + r"Operator `(v[\d.]+)`", + r"nvidia/gpu-operator --version (\S+)", + r"gpu-operator-(v[\d.]+)\.tgz", + ): + self.assert_references_match( + self.skill, pattern, self.gpu["gpu_operator_chart_version"] + ) + + def test_registry_images_and_archives_match_role(self): + registry = yaml.safe_load( + (ROOT / "roles/standalone-container-registry/defaults/main.yml").read_text() + )["standalone_container_registry_image"] + expected = registry.removeprefix("registry:") + for path in ( + "skills/deploy-airgapped/SKILL.md", + "docs/airgap/mirror-docker-images.md", + ): + with self.subTest(path=path): + self.assert_references_match( + (ROOT / path).read_text(), r"registry[:-](\d+\.\d+\.\d+)", expected + ) + + def test_driver_image_render_matches_role_override(self): + self.assert_references_match( + self.skill, + r"--set driver.version=([\d.]+)", + self.gpu["gpu_operator_driver_version"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/deploy-airgapped/SKILL.md b/skills/deploy-airgapped/SKILL.md index 9db354b9c..651fd003b 100644 --- a/skills/deploy-airgapped/SKILL.md +++ b/skills/deploy-airgapped/SKILL.md @@ -255,7 +255,7 @@ reference host; distribution repositories remain a separate prerequisite. 2. Mirror the Helm repositories/charts required by the selected Kubernetes profile. The default tree references the stable Helm repository, GPU - Operator `v26.3.3`, and, when enabled, NFS subdir external provisioner + Operator `v26.7.2`, and, when enabled, NFS subdir external provisioner `4.0.18`. Set `gpu_operator_helm_repo` and `k8s_nfs_client_helm_repo` to internal chart repositories. @@ -265,7 +265,7 @@ reference host; distribution repositories remain a separate prerequisite. ```bash mkdir -p /tmp/charts helm repo add nvidia https://helm.ngc.nvidia.com/nvidia - helm pull nvidia/gpu-operator --version v26.3.3 --destination /tmp/charts + helm pull nvidia/gpu-operator --version v26.7.2 --destination /tmp/charts helm repo add nfs-subdir-external-provisioner \ https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner helm pull nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ @@ -310,7 +310,7 @@ reference host; distribution repositories remain a separate prerequisite. NO_HTTP_SERVER=1 ./manage-offline-files.sh helm template nvidia-gpu-operator \ - /tmp/charts/gpu-operator-v26.3.3.tgz \ + /tmp/charts/gpu-operator-v26.7.2.tgz \ --namespace gpu-operator \ --set driver.version=580.126.20 \ --set mig.strategy=mixed \ @@ -391,8 +391,8 @@ reference host; distribution repositories remain a separate prerequisite. docker pull nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 docker save -o /tmp/images/nvidia-cuda-12.4.1-base-ubuntu22.04.tar \ nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 - docker pull registry:3.1.1 - docker save -o /tmp/images/registry-3.1.1.tar registry:3.1.1 + docker pull registry:3.1.2 + docker save -o /tmp/images/registry-3.1.2.tar registry:3.1.2 ``` When Slurm monitoring stays enabled, also mirror the images selected by @@ -571,10 +571,10 @@ sha256sum -c SHA256SUMS separately and deliberately configured for the insecure endpoint: ```bash - docker load -i /tmp/images/registry-3.1.1.tar + docker load -i /tmp/images/registry-3.1.2.tar docker volume create registry-images docker run -d -p 5000:5000 --restart=always --name registry \ - -v registry-images:/var/lib/registry registry:3.1.1 + -v registry-images:/var/lib/registry registry:3.1.2 ``` 4. Import, retag, and push every archived image on that registry host. Preserve