From 2a868dc3591a615e58b6b698e5c32b1d620a0bee Mon Sep 17 00:00:00 2001
From: Doug Holt <dholt@nvidia.com>
Date: Mon, 19 Oct 2026 09:17:16 -0600
Subject: [PATCH 1/9] runai: LoadBalancer address and opt-in Gateway API
 exposure

Platform mode now waits for the ingress service's external address when the
service type is LoadBalancer and uses it for the hosts entry, the CoreDNS
entry and the API helper. runai_platform_exposure: gateway installs pinned
Gateway API CRDs (v1.4.0) and KGateway (v2.2.1), creates one Gateway and
points the Run:ai charts at it. Uninstall removes only what the role added.
Default stays ingress. Offline tests cover both paths.
---
 roles/runai/defaults/main.yml            | 38 +++++++++++-
 roles/runai/tasks/coredns.yml            |  2 +-
 roles/runai/tasks/gateway.yml            | 24 ++++++++
 roles/runai/tasks/gateway_controller.yml | 51 ++++++++++++++++
 roles/runai/tasks/gateway_remove.yml     | 44 ++++++++++++++
 roles/runai/tasks/platform.yml           | 18 +++++-
 roles/runai/tasks/platform_address.yml   | 60 ++++++++++++++++++
 roles/runai/tasks/platform_cluster.yml   |  2 +-
 roles/runai/tasks/platform_dns.yml       |  6 +-
 roles/runai/tasks/uninstall.yml          |  9 ++-
 roles/runai/vars/main.yml                | 77 +++++++++++++++++++++++-
 tests/runai/bin/kubectl                  | 29 ++++++++-
 tests/runai/gateway.yml                  | 32 ++++++++++
 tests/runai/run-tests.sh                 | 66 +++++++++++++++++++-
 14 files changed, 443 insertions(+), 15 deletions(-)
 create mode 100644 roles/runai/tasks/gateway.yml
 create mode 100644 roles/runai/tasks/gateway_controller.yml
 create mode 100644 roles/runai/tasks/gateway_remove.yml
 create mode 100644 roles/runai/tasks/platform_address.yml
 create mode 100644 tests/runai/gateway.yml

diff --git a/roles/runai/defaults/main.yml b/roles/runai/defaults/main.yml
index 04e10b092..86688a25b 100644
--- a/roles/runai/defaults/main.yml
+++ b/roles/runai/defaults/main.yml
@@ -71,7 +71,12 @@ runai_platform_ingress_chart_version: "1.54.2"
 runai_platform_ingress_release_name: "haproxy-kubernetes-ingress"
 runai_platform_ingress_namespace: "haproxy-controller"
 # NodePort is the only type live-tested in 26.11; LoadBalancer is untested in 26.11.
+# With LoadBalancer the role waits for the external address (MetalLB: see
+# scripts/k8s/deploy_loadbalancer.sh) and uses it for the hosts entry, the CoreDNS
+# entry and the validator, instead of the node address.
 runai_platform_ingress_service_type: "LoadBalancer"
+# Seconds to wait for that external address before the role stops with an explanation.
+runai_platform_address_timeout: 300
 # Single-node labs without a load balancer: serve 80/443 directly on the node.
 runai_platform_ingress_host_ports: false
 
@@ -82,9 +87,38 @@ runai_platform_tls_cert_file: ""
 runai_platform_tls_key_file: ""
 runai_platform_ca_file: ""
 
+# How the control plane is reached from outside the cluster.
+#   ingress  HAProxy ingress controller (default, unchanged)
+#   gateway  Kubernetes Gateway API: the role installs the Gateway API CRDs and the
+#            KGateway controller (unless they are already there), creates one Gateway
+#            and points the Run:ai charts at it. Nothing else changes. Pick one and
+#            uninstall in the same mode; to switch, uninstall first.
+runai_platform_exposure: "ingress"
+
+# Gateway exposure only. Versions follow the Run:ai self-hosted v3.0 Gateway API guide.
+# KGateway is used because the guide uses it, it is Apache-2.0 and conformant, and one
+# controller keeps the live test small. Any conformant controller works: set
+# runai_gateway_install_controller: false and runai_gateway_class to its class.
+runai_gateway_api_version: "v1.4.0"
+runai_gateway_api_crds_url: "https://github.com/kubernetes-sigs/gateway-api/releases/download/{{ runai_gateway_api_version }}/standard-install.yaml"
+runai_gateway_install_controller: true
+runai_gateway_class: "kgateway"
+runai_gateway_controller_version: "v2.2.1"
+runai_gateway_controller_chart: "oci://cr.kgateway.dev/kgateway-dev/charts/kgateway"
+runai_gateway_controller_crds_chart: "oci://cr.kgateway.dev/kgateway-dev/charts/kgateway-crds"
+runai_gateway_controller_namespace: "kgateway-system"
+runai_gateway_name: "runai-gateway"
+# The Gateway lives in the Run:ai cluster namespace (runai_cluster_namespace).
+# path: workloads share the control-plane domain under URL paths (any certificate).
+# host: workloads get sub-domains; needs a wildcard DNS entry and a wildcard certificate
+# (the generated lab certificate already has one).
+runai_gateway_workload_routing: "path"
+
 # Lab convenience, all off by default. Use when the domain has no DNS record.
-# Address the domain should point at (default: this host's primary address).
-runai_platform_ingress_ip: "{{ ansible_default_ipv4.address | default('') }}"
+# Address the domain should point at. Leave empty to use, in this order: the Gateway's
+# address (gateway exposure), the ingress service's external address (LoadBalancer),
+# else this host's primary address. Set it to override all of those.
+runai_platform_ingress_ip: ""
 # Add "<ingress ip> <domain>" to /etc/hosts on the control-plane host.
 runai_platform_manage_hosts_entry: false
 # Make the domain resolve inside the cluster (CoreDNS rewrite to the ingress
diff --git a/roles/runai/tasks/coredns.yml b/roles/runai/tasks/coredns.yml
index ac02afe7d..2f0c454fa 100644
--- a/roles/runai/tasks/coredns.yml
+++ b/roles/runai/tasks/coredns.yml
@@ -11,7 +11,7 @@
 - name: Plan the Corefile change
   ansible.builtin.set_fact:
     runai_corefile_old: "{{ (runai_corefile_raw.stdout | from_json).data.Corefile }}"
-    runai_corefile_block: "    hosts {\n      {{ runai_platform_ingress_ip }} {{ runai_platform_domain }}\n      fallthrough\n    }  # deepops-runai\n"
+    runai_corefile_block: "    hosts {\n      {{ runai_exposure_address | default('') }} {{ runai_platform_domain }}\n      fallthrough\n    }  # deepops-runai\n"
   no_log: true
 
 - name: Compute the new Corefile
diff --git a/roles/runai/tasks/gateway.yml b/roles/runai/tasks/gateway.yml
new file mode 100644
index 000000000..9f1b88b38
--- /dev/null
+++ b/roles/runai/tasks/gateway.yml
@@ -0,0 +1,24 @@
+---
+# One Gateway in the Run:ai cluster namespace, shaped like the Run:ai v3.0 guide. The
+# listeners accept routes from every namespace because the control plane's route lives
+# in its own namespace. Both certificate secrets come from the files used for ingress.
+- name: Create the Gateway certificate secrets
+  ansible.builtin.shell: |
+    set -euo pipefail
+    {{ runai_kubectl }} -n {{ runai_cluster_namespace }} create secret tls {{ item }} \
+      --cert {{ runai_tls_cert | quote }} --key {{ runai_tls_key | quote }} --dry-run=client -o yaml \
+      | {{ runai_kubectl }} apply -f -
+  args:
+    executable: /bin/bash
+  loop: "{{ ['runai-cluster-domain-tls-secret'] + (['runai-cluster-domain-star-tls-secret'] if runai_gateway_workload_routing == 'host' else []) }}"
+  register: runai_gw_tls
+  changed_when: "'created' in runai_gw_tls.stdout or 'configured' in runai_gw_tls.stdout"
+  no_log: true
+  become: true
+
+- name: Create the Gateway
+  ansible.builtin.command: "{{ runai_kubectl }} apply --filename -"
+  args:
+    stdin: "{{ runai_gateway_manifest | to_json }}"
+  register: runai_gw_apply
+  changed_when: "'created' in runai_gw_apply.stdout or 'configured' in runai_gw_apply.stdout"
diff --git a/roles/runai/tasks/gateway_controller.yml b/roles/runai/tasks/gateway_controller.yml
new file mode 100644
index 000000000..cd3abb2d4
--- /dev/null
+++ b/roles/runai/tasks/gateway_controller.yml
@@ -0,0 +1,51 @@
+---
+# Gateway API CRDs + KGateway controller. Installed only if missing, and marked so
+# uninstall removes exactly what this role added (never a pre-existing install).
+- name: Look for existing Gateway API CRDs
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ runai_kubectl }} get crd --output name | grep -E '\.gateway\.networking\.k8s\.io$' || true
+  args:
+    executable: /bin/bash
+  register: runai_gw_crds_before
+  changed_when: false
+  check_mode: false
+
+- name: Install the Gateway API CRDs (standard channel, pinned)
+  ansible.builtin.command: "{{ runai_kubectl }} apply --filename {{ runai_gateway_api_crds_url }}"
+  environment: "{{ proxy_env if proxy_env is defined else {} }}"
+  register: runai_gw_crds_apply
+  changed_when: true
+  when: runai_gw_crds_before.stdout_lines | length == 0
+
+- name: Mark the CRDs this role installed
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ runai_kubectl }} get crd --output name | grep -E '\.gateway\.networking\.k8s\.io$'
+    | xargs {{ runai_kubectl }} annotate --overwrite deepops.nvidia.com/runai-installed=true
+  args:
+    executable: /bin/bash
+  changed_when: true
+  when: runai_gw_crds_apply is not skipped
+
+- name: Install the KGateway CRDs
+  ansible.builtin.include_tasks: helm_release.yml
+  vars:
+    _rel:
+      release: kgateway-crds
+      namespace: "{{ runai_gateway_controller_namespace }}"
+      chart: "{{ runai_gateway_controller_crds_chart }}"
+      version: "{{ runai_gateway_controller_version }}"
+      helm_values: {}
+  when: runai_gateway_install_controller | bool
+
+- name: Install the KGateway controller
+  ansible.builtin.include_tasks: helm_release.yml
+  vars:
+    _rel:
+      release: kgateway
+      namespace: "{{ runai_gateway_controller_namespace }}"
+      chart: "{{ runai_gateway_controller_chart }}"
+      version: "{{ runai_gateway_controller_version }}"
+      helm_values: {}
+  when: runai_gateway_install_controller | bool
diff --git a/roles/runai/tasks/gateway_remove.yml b/roles/runai/tasks/gateway_remove.yml
new file mode 100644
index 000000000..43a7a0751
--- /dev/null
+++ b/roles/runai/tasks/gateway_remove.yml
@@ -0,0 +1,44 @@
+---
+# Removes only what gateway.yml and gateway_controller.yml added.
+- name: Remove the Gateway
+  ansible.builtin.command: "{{ runai_kubectl }} --namespace {{ runai_cluster_namespace }} delete gateway {{ runai_gateway_name }} --ignore-not-found"
+  register: runai_rm_gw
+  changed_when: "'deleted' in runai_rm_gw.stdout"
+  failed_when: runai_rm_gw.rc != 0 and 'the server doesn' not in runai_rm_gw.stderr
+
+- name: Remove the KGateway controller and its CRDs
+  ansible.builtin.command: "{{ runai_helm }} uninstall {{ item }} --namespace {{ runai_gateway_controller_namespace }} --wait"
+  loop: [kgateway, kgateway-crds]
+  register: runai_rm_kgw
+  changed_when: runai_rm_kgw.rc == 0
+  failed_when: runai_rm_kgw.rc != 0 and 'not found' not in runai_rm_kgw.stderr
+  when: runai_gateway_install_controller | bool
+
+- name: Note whether this role removed the controller releases
+  ansible.builtin.set_fact:
+    runai_gateway_controller_removed: "{{ runai_gateway_install_controller | bool and (runai_rm_kgw.results | selectattr('rc', 'equalto', 0) | list | length > 0) }}"
+
+# Only CRDs marked by this role, and only when no Gateway-API object is left that
+# somebody else still depends on.
+- name: Count remaining Gateways
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ runai_kubectl }} get gateways.gateway.networking.k8s.io --all-namespaces --output name 2>/dev/null | wc -l
+  args:
+    executable: /bin/bash
+  register: runai_gw_left
+  changed_when: false
+  check_mode: false
+
+- name: List the Gateway API CRDs this role installed
+  ansible.builtin.command: >-
+    {{ runai_kubectl }} get crd --output jsonpath={range .items[?(@.metadata.annotations.deepops\.nvidia\.com/runai-installed=="true")]}{.metadata.name}{"\n"}{end}
+  register: runai_gw_crds_owned
+  changed_when: false
+  check_mode: false
+
+- name: Remove the Gateway API CRDs this role installed
+  ansible.builtin.command: "{{ runai_kubectl }} delete crd {{ item }} --ignore-not-found"
+  loop: "{{ runai_gw_crds_owned.stdout_lines }}"
+  changed_when: true
+  when: runai_gw_left.stdout | trim | int == 0
diff --git a/roles/runai/tasks/platform.yml b/roles/runai/tasks/platform.yml
index bceb8fc49..16464fdf3 100644
--- a/roles/runai/tasks/platform.yml
+++ b/roles/runai/tasks/platform.yml
@@ -9,11 +9,14 @@
       - runai_platform_admin_user is match('^[^@\s]+@[^@\s]+$')
       - runai_platform_admin_password | runai_password_ok
       - (runai_platform_tls_cert_file | length > 0) == (runai_platform_tls_key_file | length > 0)
+      - runai_platform_exposure in ['ingress', 'gateway']
+      - runai_gateway_workload_routing in ['path', 'host']
     fail_msg: >-
       Platform mode needs runai_platform_domain (a DNS name), runai_platform_admin_user (an email
       address) and runai_platform_admin_password (8+ characters with a digit, lower case, upper
       case and one of -!@#$%^&*()_+=). Keep it in an Ansible vault.
-      A certificate file needs its key file too.
+      A certificate file needs its key file too. runai_platform_exposure must be ingress or gateway
+      and runai_gateway_workload_routing must be path or host.
     quiet: true
   no_log: true
 
@@ -30,11 +33,22 @@
       repo: "{{ runai_platform_ingress_repo }}"
       version: "{{ runai_platform_ingress_chart_version }}"
       helm_values: "{{ runai_platform_ingress_values }}"
-  when: runai_platform_install_ingress | bool
+  when: runai_haproxy_managed | bool
+
+- name: Install the Gateway API CRDs and the Gateway controller
+  ansible.builtin.include_tasks: gateway_controller.yml
+  when: runai_gateway_enabled | bool
 
 - name: Prepare the domain and certificates
   ansible.builtin.include_tasks: platform_tls.yml
 
+- name: Create the Gateway
+  ansible.builtin.include_tasks: gateway.yml
+  when: runai_gateway_enabled | bool
+
+- name: Find the address the domain points at
+  ansible.builtin.include_tasks: platform_address.yml
+
 - name: Make the domain resolve
   ansible.builtin.include_tasks: platform_dns.yml
 
diff --git a/roles/runai/tasks/platform_address.yml b/roles/runai/tasks/platform_address.yml
new file mode 100644
index 000000000..7e5f78ad3
--- /dev/null
+++ b/roles/runai/tasks/platform_address.yml
@@ -0,0 +1,60 @@
+---
+# Decide which address the domain points at (runai_exposure_address):
+#   1. runai_platform_ingress_ip if set
+#   2. gateway exposure: the Gateway's address (waits for the load balancer to give one)
+#   3. ingress + LoadBalancer: the controller service's external address (waits)
+#   4. otherwise this host's primary address (NodePort / host ports)
+- name: Wait for the Gateway address
+  ansible.builtin.command: >-
+    {{ runai_kubectl }} --namespace {{ runai_cluster_namespace }} get gateway {{ runai_gateway_name }}
+    --output jsonpath={.status.addresses[0].value}
+  register: runai_gw_addr
+  changed_when: false
+  check_mode: false
+  until: runai_gw_addr.rc == 0 and runai_gw_addr.stdout | length > 0
+  retries: "{{ ((runai_platform_address_timeout | int) / 10) | round(0, 'ceil') | int }}"
+  delay: 10
+  failed_when: false
+  when:
+    - runai_gateway_enabled | bool
+    - runai_platform_ingress_ip | length == 0
+    - not ansible_check_mode
+
+- name: Wait for the ingress controller's external address
+  ansible.builtin.command: >-
+    {{ runai_kubectl }} --namespace {{ runai_platform_ingress_namespace }} get service {{ runai_ingress_service_name }}
+    --output jsonpath={.status.loadBalancer.ingress[0].ip}
+  register: runai_lb_addr
+  changed_when: false
+  check_mode: false
+  until: runai_lb_addr.rc == 0 and runai_lb_addr.stdout | length > 0
+  retries: "{{ ((runai_platform_address_timeout | int) / 10) | round(0, 'ceil') | int }}"
+  delay: 10
+  failed_when: false
+  when:
+    - runai_wait_ingress_lb | bool
+    - not ansible_check_mode
+
+- name: Require an address when one was asked for
+  ansible.builtin.assert:
+    that: runai_found_address | length > 0
+    fail_msg: >-
+      No external address after {{ runai_platform_address_timeout }} seconds. A LoadBalancer service
+      needs a load balancer (DeepOps: scripts/k8s/deploy_loadbalancer.sh and an address pool in
+      config/helm/metallb.yml). Without one, use runai_platform_ingress_service_type: NodePort
+      (ingress exposure), or set runai_platform_ingress_ip yourself.
+    quiet: true
+  vars:
+    runai_found_address: "{{ (runai_gw_addr.stdout | default('')) if runai_gateway_enabled | bool else (runai_lb_addr.stdout | default('')) }}"
+  when:
+    - (runai_gateway_enabled | bool or runai_wait_ingress_lb | bool)
+    - runai_platform_ingress_ip | length == 0
+    - not ansible_check_mode
+
+- name: Set the address
+  ansible.builtin.set_fact:
+    runai_exposure_address: >-
+      {{ runai_platform_ingress_ip if runai_platform_ingress_ip | length > 0
+         else ((runai_gw_addr.stdout | default('')) if runai_gateway_enabled | bool
+         else ((runai_lb_addr.stdout | default('')) if runai_wait_ingress_lb | bool
+         else (ansible_default_ipv4.address | default('')))) }}
diff --git a/roles/runai/tasks/platform_cluster.yml b/roles/runai/tasks/platform_cluster.yml
index d75200046..91d5f61c0 100644
--- a/roles/runai/tasks/platform_cluster.yml
+++ b/roles/runai/tasks/platform_cluster.yml
@@ -27,7 +27,7 @@
           '--domain', runai_platform_domain,
           '--cluster-name', runai_cluster_name,
           '--chart-version', runai_chart_version]
-         + (['--resolve-ip', runai_platform_ingress_ip] if runai_platform_ingress_ip | length > 0 else [])
+         + (['--resolve-ip', runai_exposure_address] if runai_exposure_address | length > 0 else [])
          + (['--ca-file', runai_tls_ca] if runai_tls_ca | length > 0 else []) }}
   environment:
     RUNAI_USERNAME: "{{ runai_platform_admin_user }}"
diff --git a/roles/runai/tasks/platform_dns.yml b/roles/runai/tasks/platform_dns.yml
index a36fb4708..0665ec555 100644
--- a/roles/runai/tasks/platform_dns.yml
+++ b/roles/runai/tasks/platform_dns.yml
@@ -2,8 +2,8 @@
 # Both are off by default and meant for labs without a DNS record for the domain.
 - name: Find the address the domain should resolve to
   ansible.builtin.assert:
-    that: runai_platform_ingress_ip | length > 0
-    fail_msg: Set runai_platform_ingress_ip to the address the ingress controller answers on.
+    that: runai_exposure_address | length > 0
+    fail_msg: Set runai_platform_ingress_ip to the address the control plane answers on.
     quiet: true
   when: runai_platform_manage_hosts_entry | bool or runai_platform_coredns_rewrite | bool
 
@@ -11,7 +11,7 @@
   ansible.builtin.lineinfile:
     path: /etc/hosts
     regexp: '^\S+\s+{{ runai_platform_domain | regex_escape }}\s*# deepops-runai$'
-    line: "{{ runai_platform_ingress_ip }} {{ runai_platform_domain }} # deepops-runai"
+    line: "{{ runai_exposure_address }} {{ runai_platform_domain }} # deepops-runai"
   become: true
   when: runai_platform_manage_hosts_entry | bool
 
diff --git a/roles/runai/tasks/uninstall.yml b/roles/runai/tasks/uninstall.yml
index 2b72a86c3..42b30b4eb 100644
--- a/roles/runai/tasks/uninstall.yml
+++ b/roles/runai/tasks/uninstall.yml
@@ -23,7 +23,11 @@
   register: runai_rm_ingress
   changed_when: runai_rm_ingress.rc == 0
   failed_when: runai_rm_ingress.rc != 0 and 'not found' not in runai_rm_ingress.stderr
-  when: runai_platform_install_ingress | bool
+  when: runai_haproxy_managed | bool
+
+- name: Remove the Gateway and what was installed for it
+  ansible.builtin.include_tasks: gateway_remove.yml
+  when: runai_gateway_enabled | bool
 
 - name: Remove the CoreDNS rewrite
   ansible.builtin.include_tasks: coredns.yml
@@ -68,7 +72,8 @@
     - runai-scale-adjust
     - "{{ runai_cluster_namespace }}"
     - "{{ runai_cp_namespace }}"
-    - "{{ runai_platform_ingress_namespace if runai_platform_install_ingress | bool else omit }}"
+    - "{{ runai_platform_ingress_namespace if runai_haproxy_managed | bool else omit }}"
+    - "{{ runai_gateway_controller_namespace if (runai_gateway_enabled | bool and runai_gateway_controller_removed | default(false) | bool) else omit }}"
   register: runai_rm_ns
   changed_when: "'deleted' in runai_rm_ns.stdout"
   when: item is not none
diff --git a/roles/runai/vars/main.yml b/roles/runai/vars/main.yml
index e50f5448a..95684dded 100644
--- a/roles/runai/vars/main.yml
+++ b/roles/runai/vars/main.yml
@@ -22,6 +22,37 @@ runai_platform_ingress_values:
     daemonset:
       useHostPort: "{{ runai_platform_ingress_host_ports | bool }}"
 
+# The role installs and removes the HAProxy controller only for ingress exposure.
+runai_haproxy_managed: "{{ runai_platform_exposure == 'ingress' and runai_platform_install_ingress | bool }}"
+# True when the role waits for the ingress service's external address.
+runai_wait_ingress_lb: >-
+  {{ runai_haproxy_managed | bool and runai_platform_ingress_service_type == 'LoadBalancer'
+     and runai_platform_ingress_ip | length == 0 }}
+# The service name the HAProxy chart gives its controller (the release name contains the chart name).
+runai_ingress_service_name: "{{ runai_platform_ingress_release_name }}"
+
+# Gateway API exposure (runai_platform_exposure: gateway): Ingress off, Gateway on.
+runai_gateway_enabled: "{{ runai_platform_exposure == 'gateway' }}"
+runai_gateway_cp_values:
+  global:
+    ingress:
+      enabled: false
+    gatewayAPI:
+      enabled: true
+      parentRefs:
+        - name: "{{ runai_gateway_name }}"
+          namespace: "{{ runai_cluster_namespace }}"
+runai_gateway_cluster_values:
+  clusterConfig:
+    global:
+      ingress:
+        enabled: false
+      gatewayAPI:
+        enabled: true
+        name: "{{ runai_gateway_name }}"
+        namespace: "{{ runai_cluster_namespace }}"
+      subdomainSupport: "{{ runai_gateway_workload_routing == 'host' }}"
+
 runai_cp_base_values:
   global:
     domain: "{{ runai_platform_domain }}"
@@ -33,13 +64,13 @@ runai_cp_base_values:
     config:
       adminUsername: "{{ runai_platform_admin_user }}"
       adminPassword: "{{ runai_platform_admin_password }}"
-runai_cp_values: "{{ runai_cp_base_values | runai_deep_merge(runai_platform_cp_extra_values) }}"
+runai_cp_values: "{{ runai_cp_base_values | runai_deep_merge(runai_gateway_cp_values if runai_gateway_enabled | bool else {}) | runai_deep_merge(runai_platform_cp_extra_values) }}"
 
 # The Run:ai cluster chart turns GPU-memory fractioning on by default. That needs GPU Operator
 # >= v26.7.1 and an r615+ driver, which DeepOps does not install by default, so the fractioning
 # daemon crash-loops and Run:ai reports the cluster as "Missing prerequisites" (gpu-stack).
 # Follow the same opt-in as kai mode: off unless runai_kai_gpu_fractioning is true.
-runai_cluster_base_values:
+runai_cluster_base_values_ingress:
   kai-resource-management:
     kai-scheduler:
       global:
@@ -53,3 +84,45 @@ runai_cluster_base_values:
   global:
     customCA:
       enabled: "{{ (runai_tls_ca | default('')) | length > 0 }}"
+
+runai_cluster_base_values: "{{ runai_cluster_base_values_ingress | runai_deep_merge(runai_gateway_cluster_values if runai_gateway_enabled | bool else {}) }}"
+
+# The Gateway, as in the Run:ai v3.0 Gateway API guide. allowedRoutes lets the control plane's
+# route (in its own namespace) attach; the guide's example leaves it at the default.
+runai_gateway_listeners_base:
+  - name: https-cluster-domain
+    protocol: HTTPS
+    port: 443
+    hostname: "{{ runai_platform_domain }}"
+    tls:
+      mode: Terminate
+      certificateRefs:
+        - kind: Secret
+          name: runai-cluster-domain-tls-secret
+    allowedRoutes:
+      namespaces:
+        from: All
+runai_gateway_listeners_host:
+  - name: https-workloads
+    protocol: HTTPS
+    port: 443
+    hostname: "*.{{ runai_platform_domain }}"
+    tls:
+      mode: Terminate
+      certificateRefs:
+        - kind: Secret
+          name: runai-cluster-domain-star-tls-secret
+    allowedRoutes:
+      namespaces:
+        from: All
+runai_gateway_manifest:
+  apiVersion: gateway.networking.k8s.io/v1
+  kind: Gateway
+  metadata:
+    name: "{{ runai_gateway_name }}"
+    namespace: "{{ runai_cluster_namespace }}"
+    labels:
+      app.kubernetes.io/managed-by: deepops-runai
+  spec:
+    gatewayClassName: "{{ runai_gateway_class }}"
+    listeners: "{{ runai_gateway_listeners_base + (runai_gateway_listeners_host if runai_gateway_workload_routing == 'host' else []) }}"
diff --git a/tests/runai/bin/kubectl b/tests/runai/bin/kubectl
index 164a0dfbb..80ee7fce5 100755
--- a/tests/runai/bin/kubectl
+++ b/tests/runai/bin/kubectl
@@ -7,6 +7,9 @@ with open(os.path.join(state, "kubectl.log"), "a") as log:
     log.write(" ".join(sys.argv[1:]) + "\n")
 a = [x for x in sys.argv[1:] if x not in ("--namespace",)]
 joined = " ".join(sys.argv[1:])
+def _crds():
+    f = os.path.join(state, "gwcrds")
+    return open(f).read().split() if os.path.exists(f) else []
 gpus = os.environ.get("FAKE_GPUS", "1")
 major = os.environ.get("FAKE_DRIVER_MAJOR", "580")
 if "get nodes" in joined:
@@ -16,8 +19,32 @@ elif joined.startswith("version"):
     print(json.dumps({"clientVersion": {"gitVersion": "v1.35.4"}}))
 elif "get storageclass" in joined:
     print(json.dumps({"items": [{"metadata": {"name": "nfs", "annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}]}))
+elif "get crd" in joined and "jsonpath" in joined:
+    print("\n".join(c for c in _crds() if os.path.exists(os.path.join(state, "crd-owned"))))
 elif "get crd" in joined:
-    print("")
+    print("\n".join("customresourcedefinition.apiextensions.k8s.io/" + c for c in _crds()))
+elif joined.startswith("apply --filename http"):
+    open(os.path.join(state, "gwcrds"), "w").write("gateways.gateway.networking.k8s.io")
+    print("customresourcedefinition created")
+elif " annotate " in " " + joined + " ":
+    open(os.path.join(state, "crd-owned"), "w").write("1")
+elif "get gateway " in joined and "jsonpath" in joined:
+    if os.path.exists(os.path.join(state, "gateway")) and os.environ.get("FAKE_GW_ADDR", "192.0.2.50") != "none":
+        print(os.environ.get("FAKE_GW_ADDR", "192.0.2.50"))
+elif "get gateways" in joined:
+    if os.path.exists(os.path.join(state, "gateway")):
+        print("gateway.gateway.networking.k8s.io/runai-gateway")
+elif joined.startswith("apply --filename -") and sys.stdin.read().find('"kind": "Gateway"') >= 0:
+    open(os.path.join(state, "gateway"), "w").write("1")
+    print("gateway.gateway.networking.k8s.io/runai-gateway created")
+elif " delete gateway " in " " + joined + " ":
+    if os.path.exists(os.path.join(state, "gateway")):
+        os.remove(os.path.join(state, "gateway")); print("gateway deleted")
+elif " delete crd " in " " + joined + " ":
+    for f in ("gwcrds", "crd-owned"):
+        if os.path.exists(os.path.join(state, f)): os.remove(os.path.join(state, f))
+elif "get service" in joined and "loadBalancer" in joined:
+    print(os.environ.get("FAKE_LB_IP", "192.0.2.60") if os.environ.get("FAKE_LB_IP", "x") != "none" else "")
 elif " wait " in " " + joined + " ":
     pass
 elif "delete namespace" in joined:
diff --git a/tests/runai/gateway.yml b/tests/runai/gateway.yml
new file mode 100644
index 000000000..a81f23aa9
--- /dev/null
+++ b/tests/runai/gateway.yml
@@ -0,0 +1,32 @@
+---
+# Offline harness for the exposure tasks: runs them against fake helm/kubectl.
+- hosts: localhost
+  connection: local
+  gather_facts: false
+  vars:
+    runai_helm: "{{ lookup('env', 'RUNAI_TEST_BIN') }}/helm"
+    runai_kubectl: "{{ lookup('env', 'RUNAI_TEST_BIN') }}/kubectl"
+    runai_work_dir: "{{ lookup('env', 'FAKE_STATE') }}/work"
+    ansible_become: false   # sudo would drop the fake-state variable
+    runai_mode: platform
+    runai_platform_domain: runai.lab.test
+    runai_tls_cert: /dev/null
+    runai_tls_key: /dev/null
+    runai_platform_address_timeout: 10
+    ansible_default_ipv4: {address: 192.0.2.10}
+  tasks:
+    - name: Install exposure (gateway)
+      ansible.builtin.include_role: {name: runai, tasks_from: "{{ item }}", public: true}
+      loop: [gateway_controller, gateway, platform_address]
+      when: runai_platform_exposure == 'gateway' and runai_test_action | default('install') == 'install'
+    - name: Address only (ingress)
+      ansible.builtin.include_role: {name: runai, tasks_from: platform_address, public: true}
+      when: runai_platform_exposure == 'ingress' and runai_test_action | default('install') == 'install'
+    - name: Remove the Gateway pieces
+      ansible.builtin.include_role: {name: runai, tasks_from: gateway_remove, public: true}
+      when: runai_test_action | default('install') == 'remove'
+    - name: Show address and values
+      ansible.builtin.copy:
+        dest: "{{ lookup('env', 'FAKE_STATE') }}/result.json"
+        content: "{{ {'address': runai_exposure_address | default(''), 'cp': runai_cp_values, 'cluster': runai_cluster_base_values, 'gateway': runai_gateway_manifest} | to_json }}"
+      when: runai_test_action | default('install') == 'install'
diff --git a/tests/runai/run-tests.sh b/tests/runai/run-tests.sh
index 6c27fca76..14d9a9770 100755
--- a/tests/runai/run-tests.sh
+++ b/tests/runai/run-tests.sh
@@ -94,7 +94,71 @@ run p2 -e runai_mode=platform -e runai_platform_domain=runai.lab.test -e runai_p
 echo "$out" | grep -q "weak" && bad "login value leaked into output" || pass "login value not printed"
 unset STATE
 
-# 10. filter unit tests
+# 10. exposure: address selection and Gateway API (fake helm/kubectl)
+gw() {  # gw <name> <extra ansible args...>
+  local name="$1"; shift
+  : "${STATE:=$(mktemp -d)}"; export FAKE_STATE="$STATE"
+  out="$(ansible-playbook -i localhost, "$HERE/gateway.yml" "$@" 2>&1)"; rc=$?
+}
+jq_() { python3 -c "import json,sys; d=json.load(open('$STATE/result.json')); print(eval(sys.argv[1]))" "$1"; }
+STATE=""; gw lb -e runai_platform_exposure=ingress
+[ $rc -eq 0 ] && [ "$(jq_ "d['address']")" = "192.0.2.60" ] && pass "LoadBalancer address is used instead of the node address" || bad "LoadBalancer address ($rc)"
+[ "$(jq_ "d['cp']['global'].get('gatewayAPI')")" = "None" ] && pass "ingress exposure leaves Gateway API values out" || bad "ingress values"
+unset STATE
+STATE=""; FAKE_LB_IP=none gw lb2 -e runai_platform_exposure=ingress
+[ $rc -ne 0 ] && echo "$out" | grep -q "No external address" && pass "no LoadBalancer address stops with an explanation" || bad "LB timeout"
+unset STATE
+STATE=""; gw np -e runai_platform_exposure=ingress -e runai_platform_ingress_service_type=NodePort
+[ $rc -eq 0 ] && [ "$(jq_ "d['address']")" = "192.0.2.10" ] && pass "NodePort keeps the node address" || bad "NodePort address"
+unset STATE
+STATE=""; gw ov -e runai_platform_exposure=ingress -e runai_platform_ingress_ip=198.51.100.7
+[ $rc -eq 0 ] && [ "$(jq_ "d['address']")" = "198.51.100.7" ] && pass "explicit address wins" || bad "explicit address"
+unset STATE
+STATE=""; gw g1 -e runai_platform_exposure=gateway
+g1=$rc
+[ $g1 -eq 0 ] && [ "$(jq_ "d['address']")" = "192.0.2.50" ] && pass "gateway exposure uses the Gateway address" || bad "gateway address ($g1)"
+grep -q "standard-install.yaml" "$STATE/kubectl.log" && grep -q "v1.4.0/standard-install.yaml" "$STATE/kubectl.log" && pass "Gateway API CRDs pinned to v1.4.0" || bad "CRD pin"
+python3 - "$STATE" <<'P' && pass "KGateway chart and CRDs pinned to v2.2.1" || bad "kgateway pin"
+import json,sys
+r=json.load(open(sys.argv[1]+"/releases.json"))
+assert r["kgateway-system/kgateway"]["chart"]=="kgateway-v2.2.1" and r["kgateway-system/kgateway-crds"]["chart"]=="kgateway-crds-v2.2.1", r
+P
+python3 - "$STATE" <<'P' && pass "gateway values: ingress off, Gateway on, path routing" || bad "gateway values"
+import json,sys
+d=json.load(open(sys.argv[1]+"/result.json"))
+g=d["cp"]["global"]; assert g["ingress"]["enabled"] is False and g["gatewayAPI"]["enabled"] is True, g
+assert g["gatewayAPI"]["parentRefs"]==[{"name":"runai-gateway","namespace":"runai"}], g
+c=d["cluster"]["clusterConfig"]["global"]; assert c["ingress"]["enabled"] is False and c["gatewayAPI"]["name"]=="runai-gateway" and c["subdomainSupport"] is False, c
+assert len(d["gateway"]["spec"]["listeners"])==1
+P
+gw g2 -e runai_platform_exposure=gateway
+[ $rc -eq 0 ] && pass "gateway install converges" || bad "gateway rerun"
+gw g3 -e runai_platform_exposure=gateway -e runai_test_action=remove
+[ $rc -eq 0 ] && [ ! -e "$STATE/gateway" ] && [ ! -e "$STATE/gwcrds" ] && pass "gateway uninstall removes the Gateway and the CRDs it installed" || bad "gateway uninstall ($rc)"
+python3 - "$STATE" <<'P' && pass "gateway uninstall removes the controller releases" || bad "controller removal"
+import json,sys
+assert json.load(open(sys.argv[1]+"/releases.json"))=={}
+P
+unset STATE
+# existing Gateway API CRDs are neither replaced nor removed
+STATE="$(mktemp -d)"; echo gateways.gateway.networking.k8s.io > "$STATE/gwcrds"
+gw g4 -e runai_platform_exposure=gateway -e runai_gateway_install_controller=false
+grep -q "^apply --filename http" "$STATE/kubectl.log" && bad "pre-existing CRDs were re-applied" || pass "pre-existing Gateway API CRDs are left alone"
+gw g5 -e runai_platform_exposure=gateway -e runai_gateway_install_controller=false -e runai_test_action=remove
+[ -e "$STATE/gwcrds" ] && pass "pre-existing Gateway API CRDs survive uninstall" || bad "foreign CRDs removed"
+unset STATE
+STATE=""; gw g6 -e runai_platform_exposure=gateway -e runai_gateway_workload_routing=host
+python3 - "$STATE" <<'P' && pass "host routing adds the wildcard listener and sub-domain support" || bad "host routing"
+import json,sys
+d=json.load(open(sys.argv[1]+"/result.json"))
+assert len(d["gateway"]["spec"]["listeners"])==2 and d["cluster"]["clusterConfig"]["global"]["subdomainSupport"] is True
+P
+unset STATE
+STATE=""; run bx -e runai_mode=platform -e runai_platform_domain=runai.lab.test -e 'runai_platform_admin_password=Passw0rd!x' -e runai_platform_exposure=bogus
+[ $rc -ne 0 ] && pass "unknown exposure refused" || bad "unknown exposure"
+unset STATE
+
+# 11. filter unit tests
 python3 -m unittest discover -q -s "$HERE" -p "test_runai_helpers.py" && pass "helper unit tests" || bad "helper unit tests"
 
 exit $fail

From 03054b6abeff07621573a233ad19d2c7250f1e8b Mon Sep 17 00:00:00 2001
From: Doug Holt <dholt@nvidia.com>
Date: Mon, 19 Oct 2026 09:21:03 -0600
Subject: [PATCH 2/9] validate_runai: report and enforce the exposure path
 (ingress or gateway)

Adds exposure_reached (ingress, gateway or unknown) by matching the address
the control plane was reached on against Gateway and ingress-service
addresses, and --exposure to fail when the requested path is not the one in
use. Unit tests cover both paths and the mismatches.
---
 scripts/validation/tests/test_runai.py | 78 ++++++++++++++++++++++++++
 scripts/validation/validate_runai.py   | 77 +++++++++++++++++++++++++
 2 files changed, 155 insertions(+)

diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py
index 2f84cd2fe..99557e23f 100644
--- a/scripts/validation/tests/test_runai.py
+++ b/scripts/validation/tests/test_runai.py
@@ -328,6 +328,84 @@ def test_uninstalled_finds_leftovers(self):
         self.assertEqual(result["leftovers"], ["namespace/runai"])
 
 
+class TestExposure(unittest.TestCase):
+    GW = {"items": [{"status": {"addresses": [{"type": "IPAddress", "value": "192.0.2.50"}]}}]}
+    SVC = {"items": [{"status": {"loadBalancer": {"ingress": [{"ip": "192.0.2.60"}]}}}]}
+    NODES = {"items": [{"status": {"addresses": [{"type": "InternalIP", "address": "192.0.2.10"},
+                                                  {"type": "Hostname", "address": "n1"}]}}]}
+
+    def test_classify(self):
+        gw, ing = vr.gateway_addresses(self.GW), vr.ingress_addresses(self.SVC, self.NODES)
+        self.assertEqual(vr.classify_exposure("192.0.2.50", gw, ing), "gateway")
+        self.assertEqual(vr.classify_exposure("192.0.2.60", gw, ing), "ingress")
+        self.assertEqual(vr.classify_exposure("192.0.2.10", gw, ing), "ingress")
+        self.assertEqual(vr.classify_exposure("203.0.113.9", gw, ing), "unknown")
+        self.assertEqual(vr.classify_exposure("", gw, ing), "unknown")
+
+    def test_load_balancer_hostnames_match(self):
+        svc = {"items": [{"status": {"loadBalancer": {"ingress": [{"hostname": "lb.example"}]}}}]}
+        self.assertEqual(vr.ingress_addresses(svc, {}), {"lb.example"})
+
+    def _run(self, ip, *extra):
+        class K(FakeKube):
+            def get_json(self, a, timeout=60):
+                if "gateways.gateway.networking.k8s.io" in a:
+                    return TestExposure.GW, ""
+                if "services" in a:
+                    return TestExposure.SVC, ""
+                if a[:2] == ["get", "nodes"]:
+                    return TestExposure.NODES, ""
+                return FakeKube.get_json(self, a, timeout)
+        orig = vr.https_request
+
+        def fake(domain, rip, ca, method, path, body=None, headers=None, timeout=20):
+            if path == "/api/v2/token":
+                return 200, json.dumps({"access_token": "t"}).encode()
+            if path == "/api/v1/clusters":
+                return 200, json.dumps({"clusters": [{"name": "deepops", "status": {"platform": "Connected"}}]}).encode()
+            return 200, b""
+        vr.https_request = fake
+        try:
+            return vr.validate(args("--mode", "platform", "--platform-domain", "runai.lab.test",
+                                    "--skip-gpu-jobs", "--resolve-ip", ip, *extra),
+                               {"RUNAI_USERNAME": "u", "RUNAI_PASSWORD": "p"}, K(pods=GOOD_PODS))
+        finally:
+            vr.https_request = orig
+
+    def test_ingress_path_reported_and_accepted(self):
+        r = self._run("192.0.2.60", "--exposure", "ingress")
+        self.assertEqual(r["exposure_reached"], "ingress")
+        self.assertTrue(r["ok"], r["failures"])
+
+    def test_gateway_path_reported_and_accepted(self):
+        r = self._run("192.0.2.50", "--exposure", "gateway")
+        self.assertEqual(r["exposure_reached"], "gateway")
+        self.assertEqual(r["exposure_requested"], "gateway")
+        self.assertTrue(r["ok"], r["failures"])
+
+    def test_wrong_path_fails_both_ways(self):
+        r = self._run("192.0.2.60", "--exposure", "gateway")
+        self.assertFalse(r["ok"])
+        self.assertTrue(any("requested exposure is gateway" in f for f in r["failures"]))
+        self.assertFalse(self._run("192.0.2.50", "--exposure", "ingress")["ok"])
+
+    def test_unknown_path_fails_when_one_is_requested(self):
+        r = self._run("203.0.113.9", "--exposure", "ingress")
+        self.assertEqual(r["exposure_reached"], "unknown")
+        self.assertFalse(r["ok"])
+
+    def test_without_request_the_path_is_only_reported(self):
+        r = self._run("192.0.2.50")
+        self.assertEqual(r["exposure_reached"], "gateway")
+        self.assertTrue(r["ok"], r["failures"])
+
+    def test_exposure_needs_platform_mode_and_known_word(self):
+        with self.assertRaises(SystemExit):
+            args("--exposure", "gateway")
+        with self.assertRaises(SystemExit):
+            args("--mode", "platform", "--platform-domain", "a.test", "--skip-gpu-jobs", "--exposure", "nginx")
+
+
 class TestPlatformProject(unittest.TestCase):
     NS = "dv-abcd1234"
 
diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py
index b61db6c0d..bb5465d96 100644
--- a/scripts/validation/validate_runai.py
+++ b/scripts/validation/validate_runai.py
@@ -16,6 +16,8 @@
    and the cluster reports ``Connected``. With ``--create-project NAME`` the
    validator also creates a throw-away Run:ai project through the API, runs checks
    2 and 3 in the project's namespace with ``runai-scheduler``, then deletes the project.
+   The result also says which path the control plane was reached through (``exposure_reached``:
+   ``ingress``, ``gateway`` or ``unknown``); ``--exposure ingress|gateway`` fails on any other.
 
 ``--verify-uninstalled`` is a separate read-only mode: it passes only if no
 ``kai-scheduler``/``runai*`` namespace or Helm release is left.
@@ -779,6 +781,69 @@ def check_platform(result, args, env):
         result["platform_cluster_connected"] = True
 
 
+EXPOSURES = ("ingress", "gateway")  # fixed vocabulary; "unknown" = no match, nothing else is ever reported
+
+
+def gateway_addresses(gateways_json):
+    """Addresses of every Gateway in a ``kubectl get gateways -A -o json`` document."""
+    out = set()
+    for item in (gateways_json or {}).get("items", []):
+        for addr in (item.get("status") or {}).get("addresses") or []:
+            if isinstance(addr.get("value"), str):
+                out.add(addr["value"])
+    return out
+
+
+def ingress_addresses(services_json, nodes_json):
+    """External addresses of the ingress controller: LoadBalancer service addresses plus node
+    addresses (NodePort and host-port setups answer on a node)."""
+    out = set()
+    for item in (services_json or {}).get("items", []):
+        for lb in ((item.get("status") or {}).get("loadBalancer") or {}).get("ingress") or []:
+            for key in ("ip", "hostname"):
+                if isinstance(lb.get(key), str):
+                    out.add(lb[key])
+    for item in (nodes_json or {}).get("items", []):
+        for addr in (item.get("status") or {}).get("addresses") or []:
+            if addr.get("type") in ("InternalIP", "ExternalIP") and isinstance(addr.get("address"), str):
+                out.add(addr["address"])
+    return out
+
+
+def classify_exposure(connected_to, gw_addrs, ingress_addrs):
+    """Which path owns the address the validator connected to. Gateway wins a tie: its address is
+    the more specific one."""
+    if connected_to and connected_to in gw_addrs:
+        return "gateway"
+    if connected_to and connected_to in ingress_addrs:
+        return "ingress"
+    return "unknown"
+
+
+def resolve_target(args):
+    """The address the platform checks connect to (--resolve-ip, else what the domain resolves to)."""
+    if args.resolve_ip:
+        return args.resolve_ip
+    import socket
+    try:
+        return socket.gethostbyname(args.platform_domain)
+    except OSError:
+        return ""
+
+
+def check_exposure(kube, result, args):
+    result["exposure_requested"] = args.exposure or ""
+    gws, _ = kube.get_json(["get", "gateways.gateway.networking.k8s.io", "--all-namespaces"])
+    svcs, _ = kube.get_json(["-n", args.ingress_namespace, "get", "services"])
+    nodes, _ = kube.get_json(["get", "nodes"])
+    reached = classify_exposure(resolve_target(args), gateway_addresses(gws),
+                                ingress_addresses(svcs, nodes))
+    result["exposure_reached"] = reached
+    if args.exposure and reached != args.exposure:
+        result["failures"].append("requested exposure is %s but the control plane was reached through: %s"
+                                  % (args.exposure, reached))
+
+
 def new_result(mode):
     return {
         "schema_version": SCHEMA_VERSION, "ok": False, "mode": mode,
@@ -790,6 +855,7 @@ def new_result(mode):
         "project_created": False, "project_deleted": False,
         "uninstalled_ran": False, "uninstalled_ok": False, "leftovers": [],
         "cluster_status": {},
+        "exposure_requested": "", "exposure_reached": "",
         "failures": [],
     }
 
@@ -846,6 +912,8 @@ def validate(args, env, kube=None):
                 check_memory_enforcement(kube, result, args)
     if args.mode == "platform" and not result["platform_ran"]:
         check_platform(result, args, env)
+    if args.mode == "platform" and result["platform_api_ok"]:
+        check_exposure(kube, result, args)
     ran_ok = (result["scheduler_ready"] and
               (args.skip_gpu_jobs or (result["gpu_job_ok"] and (result["fraction_job_ok"] or result["fraction_job_skipped"]))) and
               (not args.memory_enforcement or result["memory_enforcement_ok"]) and
@@ -897,6 +965,11 @@ def parse_args(argv):
     p.add_argument("--create-project", default="",
                    help="platform mode: create this throw-away Run:ai project through the API, run the "
                         "GPU checks in its namespace, then delete it (needs RUNAI_USERNAME/RUNAI_PASSWORD)")
+    p.add_argument("--exposure", choices=EXPOSURES, default="",
+                   help="platform mode: fail unless the control plane is reached through this path "
+                        "(ingress or gateway). Without it the path is only reported (exposure_reached)")
+    p.add_argument("--ingress-namespace", default="haproxy-controller",
+                   help="namespace of the ingress controller's service (platform mode)")
     p.add_argument("--cluster-name", default="deepops", help="Run:ai cluster name to look up")
     args = p.parse_args(argv)
     for label, value in (("namespace", args.namespace), ("kai-namespace", args.kai_namespace)):
@@ -905,6 +978,10 @@ def parse_args(argv):
     if args.mode == "platform" and not args.verify_uninstalled:
         if not args.platform_domain or not FQDN_RE.match(args.platform_domain):
             p.error("--platform-domain is required in platform mode")
+    if args.exposure and args.mode != "platform":
+        p.error("--exposure needs --mode platform")
+    if not NAME_RE.match(args.ingress_namespace):
+        p.error("--ingress-namespace must be a valid Kubernetes name")
     if args.create_project and (args.mode != "platform" or not NAME_RE.match(args.create_project)):
         p.error("--create-project needs --mode platform and a valid Kubernetes name")
     if args.skip_fractions and not args.create_project:

From 7876f097fe00f76175bfd605bdea922cd6af2544 Mon Sep 17 00:00:00 2001
From: Doug Holt <dholt@nvidia.com>
Date: Mon, 19 Oct 2026 09:21:11 -0600
Subject: [PATCH 3/9] docs: describe LoadBalancer and Gateway API exposure for
 Run:ai

---
 config.example/group_vars/k8s_cluster.yml |  6 +++
 docs/k8s-cluster/runai.md                 | 47 ++++++++++++++++++++++-
 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/config.example/group_vars/k8s_cluster.yml b/config.example/group_vars/k8s_cluster.yml
index 1e377c0ec..5ab96f6a6 100644
--- a/config.example/group_vars/k8s_cluster.yml
+++ b/config.example/group_vars/k8s_cluster.yml
@@ -115,3 +115,9 @@ runai_mode: "off"
 # Lab without DNS: let the role add the hosts entry and a cluster DNS rewrite.
 #runai_platform_manage_hosts_entry: true
 #runai_platform_coredns_rewrite: true
+# How the control plane is reached: ingress (HAProxy, default) or gateway (Kubernetes
+# Gateway API with KGateway). With ingress and a LoadBalancer service the role waits for
+# the external address (deploy a load balancer first: scripts/k8s/deploy_loadbalancer.sh).
+#runai_platform_exposure: ingress
+#runai_platform_ingress_service_type: LoadBalancer   # or NodePort
+#runai_gateway_workload_routing: path               # gateway only: path or host
diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md
index 5e1f3907a..befe2f499 100644
--- a/docs/k8s-cluster/runai.md
+++ b/docs/k8s-cluster/runai.md
@@ -44,6 +44,8 @@ and any DeepOps-owned UI. The web UI in `platform` mode is the upstream one.
 | gpu-fractioning (bundled in the KAI chart) | v0.1.7 |
 | Run:ai control-plane and cluster charts | 3.0.20 |
 | HAProxy ingress controller chart (platform) | 1.54.2 |
+| Gateway API CRDs (`runai_platform_exposure: gateway`) | v1.4.0 |
+| KGateway controller and CRD charts (gateway) | v2.2.1 |
 
 All are Apache-2.0. The Run:ai images come from `nvcr.io` without an NGC key.
 Versions are variables (`runai_*_version`); pins are refreshed with each release.
@@ -163,6 +165,43 @@ The role then:
 4. installs the control plane, registers the cluster with it through its API, and
    installs the cluster chart with the short-lived install key the API returns.
 
+### Reaching the control plane: LoadBalancer or Gateway API
+
+`runai_platform_exposure` picks how traffic gets in. The default, `ingress`, is the
+HAProxy path above and is unchanged.
+
+**LoadBalancer.** With `runai_platform_ingress_service_type: LoadBalancer` (the
+default) the role waits up to `runai_platform_address_timeout` seconds (300) for the
+service to get an external address and uses it, instead of the node address, for the
+hosts entry, the CoreDNS entry and the API helper. Install a load balancer first:
+`./scripts/k8s/deploy_loadbalancer.sh` with an address pool in `config/helm/metallb.yml`
+([Load Balancer and Ingress](ingress.md)). If no address appears the role stops and says
+so; set `runai_platform_ingress_ip` to use a fixed address instead.
+
+**Gateway API** (opt in). Set `runai_platform_exposure: gateway` to follow the
+[Run:ai Gateway API guide](https://docs.nvidia.com/run-ai/self-hosted/v3.0/documentation/installation-and-configuration/config/networking/kubernetes-gateway-api).
+The role then, instead of installing HAProxy:
+
+1. installs the Gateway API CRDs (v1.4.0, standard channel) if none are present, and
+   the KGateway controller (v2.2.1) in `kgateway-system`;
+2. creates one `Gateway` (`runai-gateway`) in the Run:ai cluster namespace with a TLS
+   listener for the domain, using the same certificate as the ingress path;
+3. installs the control plane and cluster with Ingress off and Gateway API on, pointing at
+   that Gateway, and uses the Gateway's address for the lab DNS helpers.
+
+KGateway is the controller in the guide, it is Apache-2.0 and conformant. Any conformant
+controller works: set `runai_gateway_install_controller: false` and
+`runai_gateway_class` to its class; you then install it yourself. Workloads are served
+under URL paths of the one domain (`runai_gateway_workload_routing: path`, the default).
+`host` gives each workload a sub-domain: it needs wildcard DNS and a certificate that covers
+`*.<domain>` (the generated lab certificate does).
+
+Uninstall (`runai_state=absent`) removes the Gateway, the KGateway releases this role
+installed, and the Gateway API CRDs only if this role installed them and no other
+Gateway remains. It never removes a controller or CRDs that were already there. To switch
+between `ingress` and `gateway`, uninstall first and install again. Prove the path with
+`--exposure gateway` (or `ingress`) in the validator below.
+
 **Names.** `runai_platform_domain` must resolve to the ingress address from the
 nodes and from wherever you browse. Without DNS (a lab), set
 `runai_platform_manage_hosts_entry: true` (hosts-file line on the control-plane
@@ -176,9 +215,14 @@ Check it (the admin credentials go in the environment, never on the command line
 RUNAI_USERNAME=admin@example.com RUNAI_PASSWORD='...' \
   python3 scripts/validation/validate_runai.py --json --mode platform \
   --platform-domain runai.example.com --ca-file /opt/deepops/runai/ca.pem \
-  --create-project deepops-validate
+  --create-project deepops-validate --exposure ingress
 ```
 
+The result includes `exposure_reached` (`ingress`, `gateway` or `unknown`): the path that
+owns the address the validator connected to. `--exposure ingress|gateway` makes `ok` false
+if the control plane was reached any other way, so a Gateway install that quietly still
+answers through the old ingress cannot pass. For the Gateway install use `--exposure gateway`.
+
 In platform mode, pods must be submitted in a Run:ai project's namespace and use
 that project's queue. The validator does this itself: `--create-project deepops-validate`
 creates that project through the API (one GPU of quota in the first node pool), runs the
@@ -223,4 +267,5 @@ too (this deletes any queues and projects still defined).
 | Pod stays `Pending` with `schedulerName: kai-scheduler` | No queue label, or the queue does not exist: `kubectl get queues.scheduling.run.ai`. |
 | A half-GPU pod is rejected | It also asks for a whole `nvidia.com/gpu`, or the fraction is not between 0 and 1. |
 | Validator: scheduler deployments missing | The KAI operator is still starting. Wait a minute and rerun. |
+| Role stops: "No external address" | The LoadBalancer or Gateway got no address. Deploy a load balancer (`scripts/k8s/deploy_loadbalancer.sh`) or use `NodePort` / `runai_platform_ingress_ip`. |
 | Platform validator: control plane unreachable | The domain does not resolve to the ingress address, or the CA is not trusted: pass `--ca-file` and `--resolve-ip`. |

From 189ba8a7a39618cee3615aedddf4a788055802ae Mon Sep 17 00:00:00 2001
From: Doug Holt <dholt@nvidia.com>
Date: Mon, 19 Oct 2026 10:44:50 -0600
Subject: [PATCH 4/9] fix(runai): preserve Gateway ownership and verify
 exposure

Discover owned controller releases during uninstall without requiring\nthe original exposure option. Preserve foreign releases and shared\nGateway resources; link validator addresses to Run:ai routes.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 docs/k8s-cluster/runai.md                     | 22 ++++--
 roles/runai/tasks/gateway_controller.yml      | 33 +++++----
 roles/runai/tasks/gateway_release.yml         | 25 +++++++
 roles/runai/tasks/gateway_remove.yml          | 67 +++++++++++++------
 roles/runai/tasks/uninstall.yml               |  2 -
 scripts/validation/tests/test_runai.py        | 10 ++-
 .../tests/test_runai_gateway_regressions.py   | 61 +++++++++++++++++
 scripts/validation/validate_runai.py          | 52 +++++++++++---
 tests/runai/bin/kubectl                       | 29 ++++++--
 tests/runai/run-tests.sh                      |  3 +
 tests/runai/test_gateway_lifecycle.py         | 67 +++++++++++++++++++
 11 files changed, 310 insertions(+), 61 deletions(-)
 create mode 100644 roles/runai/tasks/gateway_release.yml
 create mode 100644 scripts/validation/tests/test_runai_gateway_regressions.py
 create mode 100644 tests/runai/test_gateway_lifecycle.py

diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md
index befe2f499..1fd771a4e 100644
--- a/docs/k8s-cluster/runai.md
+++ b/docs/k8s-cluster/runai.md
@@ -183,7 +183,9 @@ so; set `runai_platform_ingress_ip` to use a fixed address instead.
 The role then, instead of installing HAProxy:
 
 1. installs the Gateway API CRDs (v1.4.0, standard channel) if none are present, and
-   the KGateway controller (v2.2.1) in `kgateway-system`;
+   missing KGateway releases (v2.2.1) in `kgateway-system`, recording ownership
+   in per-release `deepops-runai-kgateway*` ConfigMaps. Existing unowned releases
+   are not upgraded or adopted;
 2. creates one `Gateway` (`runai-gateway`) in the Run:ai cluster namespace with a TLS
    listener for the domain, using the same certificate as the ingress path;
 3. installs the control plane and cluster with Ingress off and Gateway API on, pointing at
@@ -196,9 +198,14 @@ under URL paths of the one domain (`runai_gateway_workload_routing: path`, the d
 `host` gives each workload a sub-domain: it needs wildcard DNS and a certificate that covers
 `*.<domain>` (the generated lab certificate does).
 
-Uninstall (`runai_state=absent`) removes the Gateway, the KGateway releases this role
-installed, and the Gateway API CRDs only if this role installed them and no other
-Gateway remains. It never removes a controller or CRDs that were already there. To switch
+Uninstall (`runai_state=absent`, without needing to repeat the exposure option)
+removes the Gateway and discovers the ownership records. It removes owned KGateway
+releases and Gateway API CRDs only when no remaining Gateway, route or KGateway
+policy depends on them. Pre-existing releases and CRDs are preserved. The controller
+namespace is left in place because it may contain unrelated resources; an empty
+namespace alone is not an uninstall failure. `--verify-uninstalled` detects remaining
+ownership records and annotated CRDs, including resources retained for other users.
+To switch
 between `ingress` and `gateway`, uninstall first and install again. Prove the path with
 `--exposure gateway` (or `ingress`) in the validator below.
 
@@ -219,7 +226,12 @@ RUNAI_USERNAME=admin@example.com RUNAI_PASSWORD='...' \
 ```
 
 The result includes `exposure_reached` (`ingress`, `gateway` or `unknown`): the path that
-owns the address the validator connected to. `--exposure ingress|gateway` makes `ok` false
+owns the address the validator connected to, linked to an Ingress for the platform
+domain or an HTTPRoute for that domain and its referenced Gateway in the Run:ai
+namespaces. Node addresses require a NodePort ingress service; a LoadBalancer does
+not claim every node address. Ambiguous or unlinked addresses report `unknown`.
+Override `--cp-namespace` and `--cluster-namespace` if using non-default namespaces.
+`--exposure ingress|gateway` makes `ok` false
 if the control plane was reached any other way, so a Gateway install that quietly still
 answers through the old ingress cannot pass. For the Gateway install use `--exposure gateway`.
 
diff --git a/roles/runai/tasks/gateway_controller.yml b/roles/runai/tasks/gateway_controller.yml
index cd3abb2d4..8db9e23e5 100644
--- a/roles/runai/tasks/gateway_controller.yml
+++ b/roles/runai/tasks/gateway_controller.yml
@@ -28,24 +28,21 @@
   changed_when: true
   when: runai_gw_crds_apply is not skipped
 
-- name: Install the KGateway CRDs
-  ansible.builtin.include_tasks: helm_release.yml
-  vars:
-    _rel:
-      release: kgateway-crds
-      namespace: "{{ runai_gateway_controller_namespace }}"
-      chart: "{{ runai_gateway_controller_crds_chart }}"
-      version: "{{ runai_gateway_controller_version }}"
-      helm_values: {}
+- name: Look for existing KGateway releases
+  ansible.builtin.command: "{{ runai_helm }} list --all --namespace {{ runai_gateway_controller_namespace }} --output json"
+  register: runai_gw_helm_before
+  changed_when: false
+  check_mode: false
+  when: runai_gateway_install_controller | bool
+
+- name: Note existing KGateway releases
+  ansible.builtin.set_fact:
+    runai_gw_releases_before: "{{ runai_gw_helm_before.stdout | from_json | map(attribute='name') | list }}"
   when: runai_gateway_install_controller | bool
 
-- name: Install the KGateway controller
-  ansible.builtin.include_tasks: helm_release.yml
-  vars:
-    _rel:
-      release: kgateway
-      namespace: "{{ runai_gateway_controller_namespace }}"
-      chart: "{{ runai_gateway_controller_chart }}"
-      version: "{{ runai_gateway_controller_version }}"
-      helm_values: {}
+- name: Install missing or role-owned KGateway releases
+  ansible.builtin.include_tasks: gateway_release.yml
+  loop: [kgateway-crds, kgateway]
+  loop_control:
+    loop_var: runai_gw_release
   when: runai_gateway_install_controller | bool
diff --git a/roles/runai/tasks/gateway_release.yml b/roles/runai/tasks/gateway_release.yml
new file mode 100644
index 000000000..8f9548563
--- /dev/null
+++ b/roles/runai/tasks/gateway_release.yml
@@ -0,0 +1,25 @@
+---
+# A per-release marker survives reruns; never adopt an existing Helm release.
+- name: "{{ runai_gw_release }} | read ownership record"
+  ansible.builtin.command: "{{ runai_kubectl }} --namespace {{ runai_gateway_controller_namespace }} get configmap deepops-runai-{{ runai_gw_release }} --ignore-not-found --output name"
+  register: runai_gw_owner
+  changed_when: false
+  check_mode: false
+
+- name: "{{ runai_gw_release }} | install only missing or owned release"
+  ansible.builtin.include_tasks: helm_release.yml
+  vars:
+    _rel:
+      release: "{{ runai_gw_release }}"
+      namespace: "{{ runai_gateway_controller_namespace }}"
+      chart: "{{ runai_gateway_controller_crds_chart if runai_gw_release == 'kgateway-crds' else runai_gateway_controller_chart }}"
+      version: "{{ runai_gateway_controller_version }}"
+      helm_values: {}
+  when: runai_gw_release not in runai_gw_releases_before or runai_gw_owner.stdout | length > 0
+
+- name: "{{ runai_gw_release }} | record ownership of new release"
+  ansible.builtin.command:
+    argv: ["{{ runai_kubectl }}", apply, --filename, '-']
+    stdin: "{{ {'apiVersion': 'v1', 'kind': 'ConfigMap', 'metadata': {'name': 'deepops-runai-' ~ runai_gw_release, 'namespace': runai_gateway_controller_namespace}, 'data': {'release': runai_gw_release}} | to_json }}"
+  changed_when: true
+  when: runai_gw_release not in runai_gw_releases_before
diff --git a/roles/runai/tasks/gateway_remove.yml b/roles/runai/tasks/gateway_remove.yml
index 43a7a0751..68f8d586a 100644
--- a/roles/runai/tasks/gateway_remove.yml
+++ b/roles/runai/tasks/gateway_remove.yml
@@ -1,34 +1,54 @@
 ---
-# Removes only what gateway.yml and gateway_controller.yml added.
+# Discover owned resources even when absent is invoked with default exposure.
+- name: Discover Gateway resource types
+  ansible.builtin.command: "{{ runai_kubectl }} api-resources --verbs=list --output name"
+  register: runai_gw_types
+  changed_when: false
+  check_mode: false
+
 - name: Remove the Gateway
   ansible.builtin.command: "{{ runai_kubectl }} --namespace {{ runai_cluster_namespace }} delete gateway {{ runai_gateway_name }} --ignore-not-found"
   register: runai_rm_gw
   changed_when: "'deleted' in runai_rm_gw.stdout"
-  failed_when: runai_rm_gw.rc != 0 and 'the server doesn' not in runai_rm_gw.stderr
+  when: "'gateways.gateway.networking.k8s.io' in runai_gw_types.stdout_lines"
+
+# Check every route/policy as well as Gateways BEFORE removing either controller
+# release. API discovery and resource queries must fail closed on access errors.
+- name: Look for remaining Gateway API and KGateway users
+  ansible.builtin.command: "{{ runai_kubectl }} get {{ item }} --all-namespaces --output name"
+  loop: "{{ runai_gw_types.stdout_lines | select('search', '(\\.gateway\\.networking\\.k8s\\.io|\\.kgateway\\.dev)$') | reject('equalto', 'gatewayclasses.gateway.networking.k8s.io') | list }}"
+  register: runai_gw_users
+  changed_when: false
+  check_mode: false
+
+- name: Note whether Gateway resources are unused
+  ansible.builtin.set_fact:
+    runai_gw_unused: "{{ runai_gw_users.results | map(attribute='stdout') | select | list | length == 0 }}"
 
-- name: Remove the KGateway controller and its CRDs
-  ansible.builtin.command: "{{ runai_helm }} uninstall {{ item }} --namespace {{ runai_gateway_controller_namespace }} --wait"
+- name: Read controller release ownership records
+  ansible.builtin.command: "{{ runai_kubectl }} --namespace {{ runai_gateway_controller_namespace }} get configmap deepops-runai-{{ item }} --ignore-not-found --output name"
   loop: [kgateway, kgateway-crds]
+  register: runai_gw_owners
+  changed_when: false
+  check_mode: false
+
+- name: Remove unused controller releases owned by this role
+  ansible.builtin.command: "{{ runai_helm }} uninstall {{ item.item }} --namespace {{ runai_gateway_controller_namespace }} --wait"
+  loop: "{{ runai_gw_owners.results }}"
   register: runai_rm_kgw
   changed_when: runai_rm_kgw.rc == 0
   failed_when: runai_rm_kgw.rc != 0 and 'not found' not in runai_rm_kgw.stderr
-  when: runai_gateway_install_controller | bool
+  when:
+    - runai_gw_unused | bool
+    - item.stdout | length > 0
 
-- name: Note whether this role removed the controller releases
-  ansible.builtin.set_fact:
-    runai_gateway_controller_removed: "{{ runai_gateway_install_controller | bool and (runai_rm_kgw.results | selectattr('rc', 'equalto', 0) | list | length > 0) }}"
-
-# Only CRDs marked by this role, and only when no Gateway-API object is left that
-# somebody else still depends on.
-- name: Count remaining Gateways
-  ansible.builtin.shell: >-
-    set -o pipefail;
-    {{ runai_kubectl }} get gateways.gateway.networking.k8s.io --all-namespaces --output name 2>/dev/null | wc -l
-  args:
-    executable: /bin/bash
-  register: runai_gw_left
-  changed_when: false
-  check_mode: false
+- name: Remove ownership records after release cleanup
+  ansible.builtin.command: "{{ runai_kubectl }} --namespace {{ runai_gateway_controller_namespace }} delete configmap deepops-runai-{{ item.item }} --ignore-not-found"
+  loop: "{{ runai_gw_owners.results }}"
+  changed_when: true
+  when:
+    - runai_gw_unused | bool
+    - item.stdout | length > 0
 
 - name: List the Gateway API CRDs this role installed
   ansible.builtin.command: >-
@@ -37,8 +57,11 @@
   changed_when: false
   check_mode: false
 
-- name: Remove the Gateway API CRDs this role installed
+- name: Remove unused Gateway API CRDs this role installed
   ansible.builtin.command: "{{ runai_kubectl }} delete crd {{ item }} --ignore-not-found"
   loop: "{{ runai_gw_crds_owned.stdout_lines }}"
   changed_when: true
-  when: runai_gw_left.stdout | trim | int == 0
+  when: runai_gw_unused | bool
+
+# Never delete the controller namespace: it can contain unrelated releases,
+# Secrets and workloads, even when no Gateway remains.
diff --git a/roles/runai/tasks/uninstall.yml b/roles/runai/tasks/uninstall.yml
index 42b30b4eb..a1fe34122 100644
--- a/roles/runai/tasks/uninstall.yml
+++ b/roles/runai/tasks/uninstall.yml
@@ -27,7 +27,6 @@
 
 - name: Remove the Gateway and what was installed for it
   ansible.builtin.include_tasks: gateway_remove.yml
-  when: runai_gateway_enabled | bool
 
 - name: Remove the CoreDNS rewrite
   ansible.builtin.include_tasks: coredns.yml
@@ -73,7 +72,6 @@
     - "{{ runai_cluster_namespace }}"
     - "{{ runai_cp_namespace }}"
     - "{{ runai_platform_ingress_namespace if runai_haproxy_managed | bool else omit }}"
-    - "{{ runai_gateway_controller_namespace if (runai_gateway_enabled | bool and runai_gateway_controller_removed | default(false) | bool) else omit }}"
   register: runai_rm_ns
   changed_when: "'deleted' in runai_rm_ns.stdout"
   when: item is not none
diff --git a/scripts/validation/tests/test_runai.py b/scripts/validation/tests/test_runai.py
index 99557e23f..0f7402bca 100644
--- a/scripts/validation/tests/test_runai.py
+++ b/scripts/validation/tests/test_runai.py
@@ -50,6 +50,8 @@ def get_json(self, args, timeout=60):
             return self.namespaces, ""
         if "deployments" in args:
             return (self.deployments, "") if self.deployments is not None else (None, "forbidden")
+        if 'configmaps' in args:
+            return {'items': []}, ''
         if "crd" in args:
             return ({}, "") if self.crd else (None, "not found")
         if "pod" in args:
@@ -329,8 +331,8 @@ def test_uninstalled_finds_leftovers(self):
 
 
 class TestExposure(unittest.TestCase):
-    GW = {"items": [{"status": {"addresses": [{"type": "IPAddress", "value": "192.0.2.50"}]}}]}
-    SVC = {"items": [{"status": {"loadBalancer": {"ingress": [{"ip": "192.0.2.60"}]}}}]}
+    GW = {"items": [{"metadata": {"name": "runai-gateway", "namespace": "runai"}, "status": {"addresses": [{"type": "IPAddress", "value": "192.0.2.50"}]}}]}
+    SVC = {"items": [{"spec": {"type": "NodePort"}, "status": {"loadBalancer": {"ingress": [{"ip": "192.0.2.60"}]}}}]}
     NODES = {"items": [{"status": {"addresses": [{"type": "InternalIP", "address": "192.0.2.10"},
                                                   {"type": "Hostname", "address": "n1"}]}}]}
 
@@ -349,6 +351,10 @@ def test_load_balancer_hostnames_match(self):
     def _run(self, ip, *extra):
         class K(FakeKube):
             def get_json(self, a, timeout=60):
+                if 'ingresses' in a:
+                    return {'items': [{'spec': {'rules': [{'host': 'runai.lab.test'}]}}]}, ''
+                if 'httproutes.gateway.networking.k8s.io' in a:
+                    return {'items': [{'spec': {'hostnames': ['runai.lab.test'], 'parentRefs': [{'name': 'runai-gateway', 'namespace': 'runai'}]}}]}, ''
                 if "gateways.gateway.networking.k8s.io" in a:
                     return TestExposure.GW, ""
                 if "services" in a:
diff --git a/scripts/validation/tests/test_runai_gateway_regressions.py b/scripts/validation/tests/test_runai_gateway_regressions.py
new file mode 100644
index 000000000..aad6df558
--- /dev/null
+++ b/scripts/validation/tests/test_runai_gateway_regressions.py
@@ -0,0 +1,61 @@
+"""Regressions for ownership, uninstall verification and exposure identity."""
+import unittest
+from unittest.mock import patch
+from test_runai import vr, args, FakeKube
+
+NODES = {'items': [{'status': {'addresses': [{'type': 'InternalIP', 'address': '192.0.2.10'}]}}]}
+
+
+class Regressions(unittest.TestCase):
+    def test_no_ingress_service_does_not_claim_node(self):
+        self.assertEqual(vr.ingress_addresses({'items': []}, NODES), set())
+
+    def test_loadbalancer_does_not_claim_node(self):
+        svc = {'items': [{'spec': {'type': 'LoadBalancer'}, 'status': {'loadBalancer': {'ingress': [{'ip': '192.0.2.60'}]}}}]}
+        self.assertEqual(vr.ingress_addresses(svc, NODES), {'192.0.2.60'})
+
+    def exposure(self, routes):
+        class K(FakeKube):
+            def get_json(self, cmd, timeout=60):
+                if 'gateways.gateway.networking.k8s.io' in cmd:
+                    return {'items': [{'metadata': {'name': 'unrelated', 'namespace': 'other'}, 'status': {'addresses': [{'value': '192.0.2.10'}]}}]}, ''
+                if 'ingresses' in cmd:
+                    return {'items': routes}, ''
+                if 'httproutes.gateway.networking.k8s.io' in cmd:
+                    return {'items': []}, ''
+                if 'services' in cmd:
+                    return {'items': [{'spec': {'type': 'NodePort'}}]}, ''
+                if cmd == ['get', 'nodes']:
+                    return NODES, ''
+                return super().get_json(cmd, timeout)
+        result = vr.new_result('platform')
+        vr.check_exposure(K(), result, args('--mode', 'platform', '--platform-domain', 'runai.example', '--skip-gpu-jobs', '--resolve-ip', '192.0.2.10', '--exposure', 'ingress'))
+        return result
+
+    def test_unrelated_gateway_cannot_override_runai_ingress(self):
+        r = self.exposure([{'spec': {'rules': [{'host': 'runai.example'}]}}])
+        self.assertEqual(r['exposure_reached'], 'ingress')
+        self.assertEqual(r['failures'], [])
+
+    def test_missing_runai_routes_cannot_prove_ingress(self):
+        r = self.exposure([])
+        self.assertEqual(r['exposure_reached'], 'unknown')
+        self.assertTrue(r['failures'])
+
+    def test_uninstall_detects_owned_gateway_artifacts(self):
+        class K(FakeKube):
+            def get_json(self, cmd, timeout=60):
+                if cmd == ['get', 'crd']:
+                    return {'items': [{'metadata': {'name': 'gateways.gateway.networking.k8s.io', 'annotations': {'deepops.nvidia.com/runai-installed': 'true'}}}]}, ''
+                if 'configmaps' in cmd:
+                    return {'items': [{'metadata': {'name': 'deepops-runai-kgateway', 'namespace': 'kgateway-system'}}]}, ''
+                return super().get_json(cmd, timeout)
+        with patch.object(vr, 'run', return_value=(0, '[]', '')):
+            r = vr.validate(args('--verify-uninstalled'), {}, K())
+        self.assertFalse(r['ok'])
+        self.assertIn('crd/gateways.gateway.networking.k8s.io', r['leftovers'])
+        self.assertIn('release-owner/kgateway-system/deepops-runai-kgateway', r['leftovers'])
+
+    def test_uninstall_helm_failure_is_not_clean(self):
+        with patch.object(vr, 'run', return_value=(1, '', 'denied')):
+            self.assertFalse(vr.validate(args('--verify-uninstalled'), {}, FakeKube())['ok'])
diff --git a/scripts/validation/validate_runai.py b/scripts/validation/validate_runai.py
index bb5465d96..1aab42a5e 100644
--- a/scripts/validation/validate_runai.py
+++ b/scripts/validation/validate_runai.py
@@ -795,15 +795,16 @@ def gateway_addresses(gateways_json):
 
 
 def ingress_addresses(services_json, nodes_json):
-    """External addresses of the ingress controller: LoadBalancer service addresses plus node
-    addresses (NodePort and host-port setups answer on a node)."""
+    """LoadBalancer addresses, or node addresses when a NodePort service exists."""
     out = set()
     for item in (services_json or {}).get("items", []):
         for lb in ((item.get("status") or {}).get("loadBalancer") or {}).get("ingress") or []:
             for key in ("ip", "hostname"):
                 if isinstance(lb.get(key), str):
                     out.add(lb[key])
-    for item in (nodes_json or {}).get("items", []):
+    node_services = [s for s in (services_json or {}).get('items', [])
+                     if s.get('spec', {}).get('type') == 'NodePort']
+    for item in (nodes_json or {}).get("items", []) if node_services else []:
         for addr in (item.get("status") or {}).get("addresses") or []:
             if addr.get("type") in ("InternalIP", "ExternalIP") and isinstance(addr.get("address"), str):
                 out.add(addr["address"])
@@ -811,8 +812,9 @@ def ingress_addresses(services_json, nodes_json):
 
 
 def classify_exposure(connected_to, gw_addrs, ingress_addrs):
-    """Which path owns the address the validator connected to. Gateway wins a tie: its address is
-    the more specific one."""
+    """Ambiguous addresses cannot prove either exposure path."""
+    if connected_to in gw_addrs and connected_to in ingress_addrs:
+        return "unknown"
     if connected_to and connected_to in gw_addrs:
         return "gateway"
     if connected_to and connected_to in ingress_addrs:
@@ -836,8 +838,24 @@ def check_exposure(kube, result, args):
     gws, _ = kube.get_json(["get", "gateways.gateway.networking.k8s.io", "--all-namespaces"])
     svcs, _ = kube.get_json(["-n", args.ingress_namespace, "get", "services"])
     nodes, _ = kube.get_json(["get", "nodes"])
-    reached = classify_exposure(resolve_target(args), gateway_addresses(gws),
-                                ingress_addresses(svcs, nodes))
+    ing_addrs, gw_addrs = set(), set()
+    for namespace in (args.cp_namespace, args.cluster_namespace):
+        ing, _ = kube.get_json(['-n', namespace, 'get', 'ingresses'])
+        if any(args.platform_domain in [r.get('host') for r in i.get('spec', {}).get('rules', [])]
+               for i in (ing or {}).get('items', [])):
+            ing_addrs.update(ingress_addresses(svcs, nodes))
+        routes, _ = kube.get_json(['-n', namespace, 'get', 'httproutes.gateway.networking.k8s.io'])
+        for route in (routes or {}).get('items', []):
+            if args.platform_domain not in route.get('spec', {}).get('hostnames', []):
+                continue
+            for ref in route.get('spec', {}).get('parentRefs', []):
+                if ref.get('kind', 'Gateway') != 'Gateway' or ref.get('group', 'gateway.networking.k8s.io') != 'gateway.networking.k8s.io':
+                    continue
+                matches = {'items': [g for g in (gws or {}).get('items', [])
+                                    if g.get('metadata', {}).get('name') == ref.get('name')
+                                    and g.get('metadata', {}).get('namespace') == ref.get('namespace', namespace)]}
+                gw_addrs.update(gateway_addresses(matches))
+    reached = classify_exposure(resolve_target(args), gw_addrs, ing_addrs)
     result["exposure_reached"] = reached
     if args.exposure and reached != args.exposure:
         result["failures"].append("requested exposure is %s but the control plane was reached through: %s"
@@ -866,7 +884,10 @@ def verify_uninstalled(kube, result):
     if ns is None:
         result["failures"].append("could not list namespaces: %s" % err)
         return
-    rc, out, _ = run(["helm", "list", "--all-namespaces", "-o", "json"], timeout=60)
+    rc, out, err = run(["helm", "list", "--all", "--all-namespaces", "-o", "json"], timeout=60)
+    if rc != 0:
+        result['failures'].append('could not list Helm releases: %s' % err)
+        return
     releases = []
     if rc == 0 and out:
         try:
@@ -875,6 +896,19 @@ def verify_uninstalled(kube, result):
             result["failures"].append("could not parse helm list output")
             return
     result["leftovers"] = leftovers(ns, releases)
+    crds, err = kube.get_json(['get', 'crd'])
+    owners, owner_err = kube.get_json(['get', 'configmaps', '--all-namespaces'])
+    if crds is None or owners is None:
+        result['failures'].append('could not inspect Gateway ownership: %s %s' % (err, owner_err))
+        return
+    for crd in crds.get('items', []):
+        meta = crd.get('metadata', {})
+        if meta.get('annotations', {}).get('deepops.nvidia.com/runai-installed') == 'true':
+            result['leftovers'].append('crd/' + meta.get('name', ''))
+    for owner in owners.get('items', []):
+        meta = owner.get('metadata', {})
+        if meta.get('name') in ('deepops-runai-kgateway', 'deepops-runai-kgateway-crds'):
+            result['leftovers'].append('release-owner/%s/%s' % (meta.get('namespace', ''), meta['name']))
     if result["leftovers"]:
         result["failures"].append("leftover objects: %s" % ", ".join(result["leftovers"]))
     else:
@@ -970,6 +1004,8 @@ def parse_args(argv):
                         "(ingress or gateway). Without it the path is only reported (exposure_reached)")
     p.add_argument("--ingress-namespace", default="haproxy-controller",
                    help="namespace of the ingress controller's service (platform mode)")
+    p.add_argument('--cp-namespace', default='runai-backend', help='Run:ai control-plane route namespace')
+    p.add_argument('--cluster-namespace', default='runai', help='Run:ai cluster route namespace')
     p.add_argument("--cluster-name", default="deepops", help="Run:ai cluster name to look up")
     args = p.parse_args(argv)
     for label, value in (("namespace", args.namespace), ("kai-namespace", args.kai_namespace)):
diff --git a/tests/runai/bin/kubectl b/tests/runai/bin/kubectl
index 80ee7fce5..abc34820e 100755
--- a/tests/runai/bin/kubectl
+++ b/tests/runai/bin/kubectl
@@ -12,7 +12,19 @@ def _crds():
     return open(f).read().split() if os.path.exists(f) else []
 gpus = os.environ.get("FAKE_GPUS", "1")
 major = os.environ.get("FAKE_DRIVER_MAJOR", "580")
-if "get nodes" in joined:
+if joined.startswith('api-resources'):
+    if os.environ.get('FAKE_DISCOVERY_FAIL'):
+        sys.stderr.write('forbidden'); sys.exit(1)
+    print('gateways.gateway.networking.k8s.io\nhttproutes.gateway.networking.k8s.io')
+elif 'get configmap deepops-runai-' in joined:
+    name = sys.argv[sys.argv.index('configmap') + 1]
+    if os.path.exists(os.path.join(state, name)):
+        print('configmap/' + name)
+elif 'delete configmap deepops-runai-' in joined:
+    name = sys.argv[sys.argv.index('configmap') + 1]
+    if os.path.exists(os.path.join(state, name)):
+        os.remove(os.path.join(state, name))
+elif "get nodes" in joined:
     print(json.dumps({"items": [{"metadata": {"name": "n1", "labels": {"nvidia.com/gpu.present": "true", "nvidia.com/cuda.driver-version.major": major}},
                                  "status": {"allocatable": {"nvidia.com/gpu": gpus}}}]}))
 elif joined.startswith("version"):
@@ -31,12 +43,21 @@ elif " annotate " in " " + joined + " ":
 elif "get gateway " in joined and "jsonpath" in joined:
     if os.path.exists(os.path.join(state, "gateway")) and os.environ.get("FAKE_GW_ADDR", "192.0.2.50") != "none":
         print(os.environ.get("FAKE_GW_ADDR", "192.0.2.50"))
+elif 'get httproutes' in joined:
+    if os.path.exists(os.path.join(state, 'foreign-route')):
+        print('httproute.gateway.networking.k8s.io/foreign')
 elif "get gateways" in joined:
     if os.path.exists(os.path.join(state, "gateway")):
         print("gateway.gateway.networking.k8s.io/runai-gateway")
-elif joined.startswith("apply --filename -") and sys.stdin.read().find('"kind": "Gateway"') >= 0:
-    open(os.path.join(state, "gateway"), "w").write("1")
-    print("gateway.gateway.networking.k8s.io/runai-gateway created")
+    if os.path.exists(os.path.join(state, "foreign-gateway")):
+        print("gateway.gateway.networking.k8s.io/foreign")
+elif joined.startswith("apply --filename -"):
+    manifest = json.load(sys.stdin)
+    if manifest['kind'] == 'ConfigMap':
+        open(os.path.join(state, manifest['metadata']['name']), 'w').write('1')
+    elif manifest['kind'] == 'Gateway':
+        open(os.path.join(state, "gateway"), "w").write("1")
+        print("gateway.gateway.networking.k8s.io/runai-gateway created")
 elif " delete gateway " in " " + joined + " ":
     if os.path.exists(os.path.join(state, "gateway")):
         os.remove(os.path.join(state, "gateway")); print("gateway deleted")
diff --git a/tests/runai/run-tests.sh b/tests/runai/run-tests.sh
index 14d9a9770..897f09f2f 100755
--- a/tests/runai/run-tests.sh
+++ b/tests/runai/run-tests.sh
@@ -158,6 +158,9 @@ STATE=""; run bx -e runai_mode=platform -e runai_platform_domain=runai.lab.test
 [ $rc -ne 0 ] && pass "unknown exposure refused" || bad "unknown exposure"
 unset STATE
 
+# Ownership, foreign users and default absent exercise the real task files.
+python3 -m unittest discover -q -s "$HERE" -p "test_gateway_lifecycle.py" && pass "Gateway lifecycle regressions" || bad "Gateway lifecycle regressions"
+
 # 11. filter unit tests
 python3 -m unittest discover -q -s "$HERE" -p "test_runai_helpers.py" && pass "helper unit tests" || bad "helper unit tests"
 
diff --git a/tests/runai/test_gateway_lifecycle.py b/tests/runai/test_gateway_lifecycle.py
new file mode 100644
index 000000000..595e89680
--- /dev/null
+++ b/tests/runai/test_gateway_lifecycle.py
@@ -0,0 +1,67 @@
+"""Execute real Ansible tasks against the bounded offline Kubernetes/Helm doubles."""
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parent.parent
+
+
+class Lifecycle(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.state = Path(self.tmp.name)
+        self.addCleanup(self.tmp.cleanup)
+        self.env = dict(os.environ, FAKE_STATE=str(self.state), RUNAI_TEST_BIN=str(HERE / 'bin'),
+                        ANSIBLE_ROLES_PATH=str(ROOT / 'roles'), ANSIBLE_CONFIG=str(ROOT / 'ansible.cfg'))
+
+    def play(self, remove=False, standard=False):
+        play = 'site.yml' if standard else 'gateway.yml'
+        opts = ['runai_state=absent'] if standard else ['runai_platform_exposure=gateway'] + (['runai_test_action=remove'] if remove else [])
+        p = subprocess.run(['ansible-playbook', '-i', 'localhost,', str(HERE / play)] +
+                           [a for opt in opts for a in ['-e', opt]], env=self.env, capture_output=True, text=True)
+        self.assertEqual(p.returncode, 0, p.stdout + p.stderr)
+
+    def releases(self):
+        return json.loads((self.state / 'releases.json').read_text())
+
+    def test_foreign_controller_is_not_upgraded_or_removed(self):
+        foreign = {'kgateway-system/' + n: {'ns': 'kgateway-system', 'chart': n + '-v1.0.0', 'values': {'foreign': True}}
+                   for n in ('kgateway', 'kgateway-crds')}
+        (self.state / 'releases.json').write_text(json.dumps(foreign))
+        self.play()
+        self.assertEqual(self.releases(), foreign)
+        self.play(remove=True)
+        self.assertEqual(self.releases(), foreign)
+
+    def test_other_gateway_preserves_owned_controller(self):
+        self.play()
+        (self.state / 'foreign-gateway').write_text('1')
+        self.play(remove=True)
+        self.assertEqual(set(self.releases()), {'kgateway-system/kgateway', 'kgateway-system/kgateway-crds'})
+        self.assertTrue((self.state / 'gwcrds').exists())
+
+    def test_route_without_gateway_preserves_controller(self):
+        self.play()
+        (self.state / 'foreign-route').write_text('1')
+        self.play(remove=True)
+        self.assertEqual(set(self.releases()), {'kgateway-system/kgateway', 'kgateway-system/kgateway-crds'})
+        self.assertTrue((self.state / 'gwcrds').exists())
+
+    def test_failed_discovery_does_not_uninstall_controller(self):
+        self.play()
+        self.env['FAKE_DISCOVERY_FAIL'] = '1'
+        p = subprocess.run(['ansible-playbook', '-i', 'localhost,', str(HERE / 'gateway.yml'),
+                            '-e', 'runai_platform_exposure=gateway', '-e', 'runai_test_action=remove'],
+                           env=self.env, capture_output=True, text=True)
+        self.assertNotEqual(p.returncode, 0)
+        self.assertEqual(set(self.releases()), {'kgateway-system/kgateway', 'kgateway-system/kgateway-crds'})
+
+    def test_standard_absent_discovers_owned_gateway_install(self):
+        self.play()
+        self.play(standard=True)
+        self.assertEqual(self.releases(), {})
+        self.assertFalse((self.state / 'gwcrds').exists())

From dae40e30a2757591bf0c79e4ce667ab65197d36c Mon Sep 17 00:00:00 2001
From: Doug Holt <dholt@nvidia.com>
Date: Mon, 19 Oct 2026 20:15:28 -0600
Subject: [PATCH 5/9] docs(runai): explain MetalLB strict ARP for IPVS users

Document new and existing cluster configuration and the iptables-only
release validation scope.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 docs/k8s-cluster/ingress.md |  2 ++
 docs/k8s-cluster/runai.md   | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/docs/k8s-cluster/ingress.md b/docs/k8s-cluster/ingress.md
index 02882b5aa..e7d682dfa 100644
--- a/docs/k8s-cluster/ingress.md
+++ b/docs/k8s-cluster/ingress.md
@@ -21,6 +21,8 @@ DeepOps provides scripts you can run to configure a simple Load Balancer and/or
 
 Modify `config/helm/metallb.yml` to configure the IP range that the load balancer will hand out.
 
+With IPVS kube-proxy (Kubespray's default), MetalLB L2 requires `kube_proxy_strict_arp: true` before deployment; see [strict ARP setup and existing-cluster steps](runai.md#reaching-the-control-plane-loadbalancer-or-gateway-api).
+
 Run the script to deploy the load balancer:
 
 ```bash
diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md
index 1fd771a4e..3757c35e2 100644
--- a/docs/k8s-cluster/runai.md
+++ b/docs/k8s-cluster/runai.md
@@ -178,6 +178,26 @@ hosts entry, the CoreDNS entry and the API helper. Install a load balancer first
 ([Load Balancer and Ingress](ingress.md)). If no address appears the role stops and says
 so; set `runai_platform_ingress_ip` to use a fixed address instead.
 
+**MetalLB L2 with IPVS kube-proxy.** Kubespray defaults to IPVS mode. Before
+deploying the cluster, set the following in `config/group_vars/k8s_cluster.yml`:
+
+```yaml
+kube_proxy_strict_arp: true
+```
+
+On an existing cluster, edit the `kube-proxy` ConfigMap in `kube-system` and set
+`ipvs.strictARP: true` inside its `config.conf`, then restart kube-proxy so every
+instance loads the change. Keep the group variable above set as well so a later
+Kubespray run preserves it. Without strict ARP, nodes can answer ARP for the
+LoadBalancer address even when they are not the MetalLB L2 announcer, sending
+traffic to the wrong node. This applies to both the ingress LoadBalancer and a
+Gateway backed by a LoadBalancer service; see the
+[upstream MetalLB requirement](https://metallb.io/installation/#preparation).
+
+**26.11 validation scope.** The release validation uses kube-proxy in iptables
+mode, not IPVS. IPVS operation relies on the strict ARP setting above; it is an
+upstream MetalLB requirement, not a separately live-tested path for this release.
+
 **Gateway API** (opt in). Set `runai_platform_exposure: gateway` to follow the
 [Run:ai Gateway API guide](https://docs.nvidia.com/run-ai/self-hosted/v3.0/documentation/installation-and-configuration/config/networking/kubernetes-gateway-api).
 The role then, instead of installing HAProxy:

From 2c533c23a9f17c32b55dd410ad01d41527016957 Mon Sep 17 00:00:00 2001
From: Doug Holt <dholt@nvidia.com>
Date: Tue, 20 Oct 2026 04:32:38 -0600
Subject: [PATCH 6/9] fix(runai): preserve Gateway CRD ownership query
 arguments

Pass the JSONPath through command argv so its range space, annotation
key escapes, string comparison and newline quotes reach kubectl intact.
Record argument boundaries in the offline kubectl double and verify the
exact ownership selector through the real Ansible removal tasks.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 roles/runai/tasks/gateway_remove.yml  |  9 +++++++--
 tests/runai/bin/kubectl               |  3 +++
 tests/runai/test_gateway_lifecycle.py | 14 ++++++++++++++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/roles/runai/tasks/gateway_remove.yml b/roles/runai/tasks/gateway_remove.yml
index 68f8d586a..29f8f19b6 100644
--- a/roles/runai/tasks/gateway_remove.yml
+++ b/roles/runai/tasks/gateway_remove.yml
@@ -51,8 +51,13 @@
     - item.stdout | length > 0
 
 - name: List the Gateway API CRDs this role installed
-  ansible.builtin.command: >-
-    {{ runai_kubectl }} get crd --output jsonpath={range .items[?(@.metadata.annotations.deepops\.nvidia\.com/runai-installed=="true")]}{.metadata.name}{"\n"}{end}
+  ansible.builtin.command:
+    argv:
+      - "{{ runai_kubectl }}"
+      - get
+      - crd
+      - --output
+      - 'jsonpath={range .items[?(@.metadata.annotations.deepops\.nvidia\.com/runai-installed=="true")]}{.metadata.name}{"\n"}{end}'
   register: runai_gw_crds_owned
   changed_when: false
   check_mode: false
diff --git a/tests/runai/bin/kubectl b/tests/runai/bin/kubectl
index abc34820e..c361341e9 100755
--- a/tests/runai/bin/kubectl
+++ b/tests/runai/bin/kubectl
@@ -5,6 +5,9 @@ state = os.environ["FAKE_STATE"]
 os.makedirs(state, exist_ok=True)
 with open(os.path.join(state, "kubectl.log"), "a") as log:
     log.write(" ".join(sys.argv[1:]) + "\n")
+# Preserve argument boundaries so lifecycle tests catch command parsing bugs.
+with open(os.path.join(state, "kubectl-argv.jsonl"), "a") as log:
+    log.write(json.dumps(sys.argv[1:]) + "\n")
 a = [x for x in sys.argv[1:] if x not in ("--namespace",)]
 joined = " ".join(sys.argv[1:])
 def _crds():
diff --git a/tests/runai/test_gateway_lifecycle.py b/tests/runai/test_gateway_lifecycle.py
index 595e89680..1763fa1a4 100644
--- a/tests/runai/test_gateway_lifecycle.py
+++ b/tests/runai/test_gateway_lifecycle.py
@@ -60,6 +60,20 @@ def test_failed_discovery_does_not_uninstall_controller(self):
         self.assertNotEqual(p.returncode, 0)
         self.assertEqual(set(self.releases()), {'kgateway-system/kgateway', 'kgateway-system/kgateway-crds'})
 
+    def test_owned_crd_query_preserves_jsonpath_argument(self):
+        self.play()
+        self.play(remove=True)
+        calls = [json.loads(line) for line in
+                 (self.state / 'kubectl-argv.jsonl').read_text().splitlines()]
+        queries = [args for args in calls if args[:3] == ['get', 'crd', '--output']
+                   and any(arg.startswith('jsonpath=') for arg in args)]
+        # One argument must retain the range space, escaped annotation key,
+        # quoted string comparison and newline escape, exactly as kubectl needs.
+        self.assertEqual(queries, [[
+            'get', 'crd', '--output',
+            r'jsonpath={range .items[?(@.metadata.annotations.deepops\.nvidia\.com/runai-installed=="true")]}{.metadata.name}{"\n"}{end}',
+        ]])
+
     def test_standard_absent_discovers_owned_gateway_install(self):
         self.play()
         self.play(standard=True)

From fe51fb788c6f6486f0724128864bff20086ec1fd Mon Sep 17 00:00:00 2001
From: Doug Holt <dholt@nvidia.com>
Date: Tue, 20 Oct 2026 07:22:27 -0600
Subject: [PATCH 7/9] fix(runai): validate Gateway lifecycle command arguments

Place CRD resources before annotation assignments and skip xargs when no
resources match. Build the uninstall namespace list without an omit
placeholder, which otherwise reaches kubectl on Ansible 2.17.

Make the offline kubectl and Helm doubles reject malformed arguments and
unknown commands. Exercise the complete Gateway platform install, rerun
and removal path with real Ansible tasks, local TLS generation and an
offline API transport, covering both path and host routing.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
---
 roles/runai/tasks/gateway_controller.yml |   2 +-
 roles/runai/tasks/uninstall.yml          |  13 +-
 tests/runai/bin/fake_cli.py              |  37 +++
 tests/runai/bin/helm                     |  99 +++++---
 tests/runai/bin/kubectl                  | 308 +++++++++++++++++------
 tests/runai/bin/python3                  |  41 +++
 tests/runai/run-tests.sh                 |   3 +
 tests/runai/test_gateway_lifecycle.py    |  24 ++
 tests/runai/test_gateway_platform.py     | 132 ++++++++++
 9 files changed, 539 insertions(+), 120 deletions(-)
 create mode 100644 tests/runai/bin/fake_cli.py
 create mode 100755 tests/runai/bin/python3
 create mode 100644 tests/runai/test_gateway_platform.py

diff --git a/roles/runai/tasks/gateway_controller.yml b/roles/runai/tasks/gateway_controller.yml
index 8db9e23e5..c5b9ef13d 100644
--- a/roles/runai/tasks/gateway_controller.yml
+++ b/roles/runai/tasks/gateway_controller.yml
@@ -22,7 +22,7 @@
   ansible.builtin.shell: >-
     set -o pipefail;
     {{ runai_kubectl }} get crd --output name | grep -E '\.gateway\.networking\.k8s\.io$'
-    | xargs {{ runai_kubectl }} annotate --overwrite deepops.nvidia.com/runai-installed=true
+    | xargs -r -I{} {{ runai_kubectl }} annotate --overwrite {} deepops.nvidia.com/runai-installed=true
   args:
     executable: /bin/bash
   changed_when: true
diff --git a/roles/runai/tasks/uninstall.yml b/roles/runai/tasks/uninstall.yml
index a1fe34122..044a873da 100644
--- a/roles/runai/tasks/uninstall.yml
+++ b/roles/runai/tasks/uninstall.yml
@@ -64,17 +64,12 @@
 # them behind is not clean, so remove them here.
 - name: Remove namespaces left behind
   ansible.builtin.command: "{{ runai_kubectl }} delete namespace {{ item }} --ignore-not-found --wait --timeout=300s"
-  loop:
-    - "{{ runai_kai_namespace }}"
-    - kai-resource-reservation
-    - runai-reservation
-    - runai-scale-adjust
-    - "{{ runai_cluster_namespace }}"
-    - "{{ runai_cp_namespace }}"
-    - "{{ runai_platform_ingress_namespace if runai_haproxy_managed | bool else omit }}"
+  loop: >-
+    {{ [runai_kai_namespace, 'kai-resource-reservation', 'runai-reservation',
+        'runai-scale-adjust', runai_cluster_namespace, runai_cp_namespace]
+       + ([runai_platform_ingress_namespace] if runai_haproxy_managed | bool else []) }}
   register: runai_rm_ns
   changed_when: "'deleted' in runai_rm_ns.stdout"
-  when: item is not none
 
 - name: List CRDs from these charts
   ansible.builtin.shell: >-
diff --git a/tests/runai/bin/fake_cli.py b/tests/runai/bin/fake_cli.py
new file mode 100644
index 000000000..3f4f65bb5
--- /dev/null
+++ b/tests/runai/bin/fake_cli.py
@@ -0,0 +1,37 @@
+"""Strict, deliberately bounded CLI grammar for the offline Run:ai tests."""
+import sys
+
+
+def fail(message):
+    sys.stderr.write(message + '\n')
+    raise SystemExit(1)
+
+
+def parse(argv, valued, flags):
+    """Keep positional order; reject unknown flags and missing option values."""
+    positional, options = [], {}
+    index = 0
+    while index < len(argv):
+        arg = argv[index]
+        if not arg.startswith('-') or arg == '-':
+            positional.append(arg)
+        else:
+            key, sep, value = arg.partition('=')
+            if key in valued:
+                if not sep:
+                    index += 1
+                    if index == len(argv) or (argv[index].startswith('--')):
+                        fail('missing value for ' + key)
+                    value = argv[index]
+                options[key] = value
+            elif key in flags and not sep:
+                options[key] = True
+            else:
+                fail('unsupported option: ' + arg)
+        index += 1
+    return positional, options
+
+
+def count(args, expected):
+    if len(args) != expected:
+        fail('expected %d positional arguments, got %r' % (expected, args))
diff --git a/tests/runai/bin/helm b/tests/runai/bin/helm
index 1dc7c3d02..7e2ad53f7 100755
--- a/tests/runai/bin/helm
+++ b/tests/runai/bin/helm
@@ -1,39 +1,66 @@
 #!/usr/bin/env python3
-"""Fake helm for offline role tests. State lives in $FAKE_STATE (a directory)."""
-import json, os, sys
-state = os.environ["FAKE_STATE"]
-os.makedirs(state, exist_ok=True)
-db = os.path.join(state, "releases.json")
-rels = {}
-if os.path.exists(db):
-    with open(db) as fh:
-        rels = json.load(fh)
-with open(os.path.join(state, "calls.log"), "a") as log:
-    log.write(" ".join(a for a in sys.argv[1:] if a != "-") + "\n")
+"""Strict offline Helm double. State lives only in $FAKE_STATE."""
+import json
+import os
+from pathlib import Path
+import sys
+
+from fake_cli import count, fail, parse
+
+state = Path(os.environ['FAKE_STATE'])
+state.mkdir(exist_ok=True)
+db = state / 'releases.json'
+rels = json.loads(db.read_text()) if db.exists() else {}
 a = sys.argv[1:]
-def opt(name, default=""):
-    return a[a.index(name) + 1] if name in a else default
-def save():
-    with open(db, "w") as fh:
-        json.dump(rels, fh)
-if a[0] == "list":
-    ns = opt("--namespace")
-    out = [dict(name=k.split("/", 1)[1], namespace=v["ns"], chart=v["chart"], status="deployed")
-           for k, v in rels.items() if "--all-namespaces" in a or v["ns"] == ns]
-    print(json.dumps(out))
-elif a[0] == "get" and a[1] == "values":
-    key = opt("--namespace") + "/" + a[2]
-    print(json.dumps(rels[key]["values"]) if key in rels else "null")
-elif a[0] == "upgrade":
-    rel, chart = a[2], a[3]
-    values = json.loads(sys.stdin.read() or "{}")
-    name = chart.rstrip("/").split("/")[-1]
-    rels[opt("--namespace") + "/" + rel] = dict(ns=opt("--namespace"), chart=name + "-" + opt("--version"), values=values)
-    save()
-elif a[0] == "uninstall":
-    key = opt("--namespace") + "/" + a[1]
+with (state / 'calls.log').open('a') as log:
+    log.write(' '.join(a) + '\n')
+with (state / 'helm-argv.jsonl').open('a') as log:
+    log.write(json.dumps(a) + '\n')
+if not a:
+    fail('helm command required')
+verb = a[0]
+valued = {
+    'list': ['--namespace', '--output'],
+    'get': ['--namespace', '--output'],
+    'upgrade': ['--namespace', '--version', '--repo', '--timeout', '--values'],
+    'uninstall': ['--namespace', '--timeout'],
+}
+flags = {
+    'list': ['--all', '--all-namespaces'], 'get': [],
+    'upgrade': ['--install', '--create-namespace', '--wait'], 'uninstall': ['--wait'],
+}
+if verb not in valued:
+    fail('unsupported helm command')
+p, opts = parse(a[1:], valued[verb], flags[verb])
+ns = opts.get('--namespace', '')
+if verb == 'list':
+    count(p, 0)
+    if opts.get('--output') != 'json':
+        fail('unsupported list output')
+    print(json.dumps([dict(name=k.split('/', 1)[1], namespace=v['ns'], chart=v['chart'], status=v.get('status', 'deployed'))
+                      for k, v in rels.items() if '--all-namespaces' in opts or v['ns'] == ns]))
+elif verb == 'get':
+    count(p, 2)
+    if p[0] != 'values' or opts.get('--output') != 'json':
+        fail('unsupported get')
+    key = ns + '/' + p[1]
     if key not in rels:
-        sys.stderr.write("Error: uninstall: Release not loaded: %s: release: not found\n" % a[1]); sys.exit(1)
-    del rels[key]; save()
-else:
-    sys.stderr.write("fake helm: unsupported %s\n" % a); sys.exit(3)
+        fail('release: not found')
+    print(json.dumps(rels[key]['values']))
+elif verb == 'upgrade':
+    count(p, 2)
+    if '--install' not in opts or not ns or not opts.get('--version') or opts.get('--values') != '-':
+        fail('upgrade requires install, namespace, version and stdin values')
+    values = json.load(sys.stdin)
+    if not isinstance(values, dict):
+        fail('Helm values must be a mapping')
+    rel, chart = p
+    rels[ns + '/' + rel] = dict(ns=ns, chart=chart.rstrip('/').split('/')[-1] + '-' + opts['--version'], values=values)
+    db.write_text(json.dumps(rels))
+elif verb == 'uninstall':
+    count(p, 1)
+    key = ns + '/' + p[0]
+    if key not in rels:
+        fail('Error: uninstall: Release not loaded: %s: release: not found' % p[0])
+    del rels[key]
+    db.write_text(json.dumps(rels))
diff --git a/tests/runai/bin/kubectl b/tests/runai/bin/kubectl
index c361341e9..c72f0028e 100755
--- a/tests/runai/bin/kubectl
+++ b/tests/runai/bin/kubectl
@@ -1,77 +1,237 @@
 #!/usr/bin/env python3
-"""Fake kubectl for offline role tests."""
-import json, os, sys
-state = os.environ["FAKE_STATE"]
-os.makedirs(state, exist_ok=True)
-with open(os.path.join(state, "kubectl.log"), "a") as log:
-    log.write(" ".join(sys.argv[1:]) + "\n")
-# Preserve argument boundaries so lifecycle tests catch command parsing bugs.
-with open(os.path.join(state, "kubectl-argv.jsonl"), "a") as log:
-    log.write(json.dumps(sys.argv[1:]) + "\n")
-a = [x for x in sys.argv[1:] if x not in ("--namespace",)]
-joined = " ".join(sys.argv[1:])
-def _crds():
-    f = os.path.join(state, "gwcrds")
-    return open(f).read().split() if os.path.exists(f) else []
-gpus = os.environ.get("FAKE_GPUS", "1")
-major = os.environ.get("FAKE_DRIVER_MAJOR", "580")
-if joined.startswith('api-resources'):
+"""Bounded kubectl double: strict argv, JSON stdin, persistent offline resources.
+
+Unknown invocations fail rather than silently succeeding. This checks the CLI
+boundary, not Kubernetes schema admission, controllers or networking.
+"""
+import json
+import os
+from pathlib import Path
+import re
+import sys
+
+from fake_cli import count, fail, parse
+
+state = Path(os.environ['FAKE_STATE'])
+state.mkdir(exist_ok=True)
+a = sys.argv[1:]
+with (state / 'kubectl.log').open('a') as log:
+    log.write(' '.join(a) + '\n')
+with (state / 'kubectl-argv.jsonl').open('a') as log:
+    log.write(json.dumps(a) + '\n')
+
+# Only the command/option combinations exercised by the role are implemented.
+ns = ''
+if a and a[0] in ('-n', '--namespace'):
+    ns, a = a[1], a[2:]
+if not a:
+    fail('command required')
+verb, rest = a[0], a[1:]
+valued = {
+    'version': ['--output'], 'api-resources': ['--verbs', '--output'],
+    'get': ['--output', '-o'], 'apply': ['--filename', '-f'],
+    'annotate': [], 'label': [], 'delete': ['--timeout'],
+    'create': ['--dry-run', '-o', '--cert', '--key', '--from-file'],
+    'patch': ['--type', '--patch'], 'rollout': [],
+    'wait': ['--for', '--timeout'],
+}
+flags = {
+    'get': ['--all-namespaces', '--ignore-not-found'],
+    'annotate': ['--overwrite'], 'label': ['--overwrite'],
+    'delete': ['--ignore-not-found', '--wait'],
+}
+if verb not in valued:
+    fail('unsupported kubectl command: ' + verb)
+p, opts = parse(rest, valued[verb] + ['-n', '--namespace'], flags.get(verb, []))
+ns = opts.get('--namespace', opts.get('-n', ns))
+output = opts.get('--output', opts.get('-o', ''))
+owned_query = r'jsonpath={range .items[?(@.metadata.annotations.deepops\.nvidia\.com/runai-installed=="true")]}{.metadata.name}{"\n"}{end}'
+if output.startswith('jsonpath=') and output not in (
+        owned_query, 'jsonpath={.status.addresses[0].value}',
+        'jsonpath={.status.loadBalancer.ingress[0].ip}'):
+    fail('unsupported or malformed jsonpath argument: ' + output)
+
+
+def lines(name):
+    path = state / name
+    return path.read_text().splitlines() if path.exists() else []
+
+
+def save_lines(name, values):
+    path = state / name
+    if values:
+        path.write_text('\n'.join(values))
+    else:
+        path.unlink(missing_ok=True)
+
+
+def manifests():
+    path = state / 'manifests.json'
+    return json.loads(path.read_text()) if path.exists() else {}
+
+
+def store(doc):
+    db = manifests()
+    key = '/'.join([doc['kind'], doc['metadata'].get('namespace', ''), doc['metadata']['name']])
+    db[key] = doc
+    (state / 'manifests.json').write_text(json.dumps(db))
+
+
+if verb == 'version':
+    count(p, 0)
+    print(json.dumps({'clientVersion': {'gitVersion': 'v1.35.4'}}))
+elif verb == 'api-resources':
+    count(p, 0)
     if os.environ.get('FAKE_DISCOVERY_FAIL'):
-        sys.stderr.write('forbidden'); sys.exit(1)
+        fail('forbidden')
     print('gateways.gateway.networking.k8s.io\nhttproutes.gateway.networking.k8s.io')
-elif 'get configmap deepops-runai-' in joined:
-    name = sys.argv[sys.argv.index('configmap') + 1]
-    if os.path.exists(os.path.join(state, name)):
-        print('configmap/' + name)
-elif 'delete configmap deepops-runai-' in joined:
-    name = sys.argv[sys.argv.index('configmap') + 1]
-    if os.path.exists(os.path.join(state, name)):
-        os.remove(os.path.join(state, name))
-elif "get nodes" in joined:
-    print(json.dumps({"items": [{"metadata": {"name": "n1", "labels": {"nvidia.com/gpu.present": "true", "nvidia.com/cuda.driver-version.major": major}},
-                                 "status": {"allocatable": {"nvidia.com/gpu": gpus}}}]}))
-elif joined.startswith("version"):
-    print(json.dumps({"clientVersion": {"gitVersion": "v1.35.4"}}))
-elif "get storageclass" in joined:
-    print(json.dumps({"items": [{"metadata": {"name": "nfs", "annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}]}))
-elif "get crd" in joined and "jsonpath" in joined:
-    print("\n".join(c for c in _crds() if os.path.exists(os.path.join(state, "crd-owned"))))
-elif "get crd" in joined:
-    print("\n".join("customresourcedefinition.apiextensions.k8s.io/" + c for c in _crds()))
-elif joined.startswith("apply --filename http"):
-    open(os.path.join(state, "gwcrds"), "w").write("gateways.gateway.networking.k8s.io")
-    print("customresourcedefinition created")
-elif " annotate " in " " + joined + " ":
-    open(os.path.join(state, "crd-owned"), "w").write("1")
-elif "get gateway " in joined and "jsonpath" in joined:
-    if os.path.exists(os.path.join(state, "gateway")) and os.environ.get("FAKE_GW_ADDR", "192.0.2.50") != "none":
-        print(os.environ.get("FAKE_GW_ADDR", "192.0.2.50"))
-elif 'get httproutes' in joined:
-    if os.path.exists(os.path.join(state, 'foreign-route')):
-        print('httproute.gateway.networking.k8s.io/foreign')
-elif "get gateways" in joined:
-    if os.path.exists(os.path.join(state, "gateway")):
-        print("gateway.gateway.networking.k8s.io/runai-gateway")
-    if os.path.exists(os.path.join(state, "foreign-gateway")):
-        print("gateway.gateway.networking.k8s.io/foreign")
-elif joined.startswith("apply --filename -"):
-    manifest = json.load(sys.stdin)
-    if manifest['kind'] == 'ConfigMap':
-        open(os.path.join(state, manifest['metadata']['name']), 'w').write('1')
-    elif manifest['kind'] == 'Gateway':
-        open(os.path.join(state, "gateway"), "w").write("1")
-        print("gateway.gateway.networking.k8s.io/runai-gateway created")
-elif " delete gateway " in " " + joined + " ":
-    if os.path.exists(os.path.join(state, "gateway")):
-        os.remove(os.path.join(state, "gateway")); print("gateway deleted")
-elif " delete crd " in " " + joined + " ":
-    for f in ("gwcrds", "crd-owned"):
-        if os.path.exists(os.path.join(state, f)): os.remove(os.path.join(state, f))
-elif "get service" in joined and "loadBalancer" in joined:
-    print(os.environ.get("FAKE_LB_IP", "192.0.2.60") if os.environ.get("FAKE_LB_IP", "x") != "none" else "")
-elif " wait " in " " + joined + " ":
-    pass
-elif "delete namespace" in joined:
-    print("namespace \"x\" deleted")
-else:
-    pass
+elif verb == 'get':
+    if not p:
+        fail('resource required')
+    kind = p[0]
+    if kind == 'nodes':
+        count(p, 1)
+        print(json.dumps({'items': [{'metadata': {'name': 'n1', 'labels': {
+            'nvidia.com/gpu.present': 'true', 'nvidia.com/cuda.driver-version.major': os.environ.get('FAKE_DRIVER_MAJOR', '580')}},
+            'status': {'allocatable': {'nvidia.com/gpu': os.environ.get('FAKE_GPUS', '1')}}}]}))
+    elif kind == 'storageclass':
+        count(p, 1)
+        print(json.dumps({'items': [{'metadata': {'name': 'nfs', 'annotations': {'storageclass.kubernetes.io/is-default-class': 'true'}}}]}))
+    elif kind == 'crd':
+        count(p, 1)
+        if output == owned_query:
+            print('\n'.join(c for c in lines('gwcrds') if c in lines('crd-owned')))
+        elif output == 'name':
+            print('\n'.join('customresourcedefinition.apiextensions.k8s.io/' + c for c in lines('gwcrds')))
+        else:
+            fail('unsupported CRD output')
+    elif kind == 'configmap':
+        count(p, 2)
+        name = p[1]
+        if name.startswith('deepops-runai-'):
+            if (state / name).exists():
+                print('configmap/' + name)
+        elif name in ('coredns', 'nodelocaldns'):
+            if output == 'name':
+                print('configmap/' + name)
+            elif output == 'json':
+                core = (state / name).read_text() if (state / name).exists() else '.:53 {\n    forward . /etc/resolv.conf\n}\n'
+                print(json.dumps({'data': {'Corefile': core}}))
+            else:
+                fail('unsupported ConfigMap output')
+        else:
+            fail('unsupported ConfigMap')
+    elif kind == 'gateway':
+        count(p, 2)
+        if output != 'jsonpath={.status.addresses[0].value}':
+            fail('unsupported Gateway output')
+        addr = os.environ.get('FAKE_GW_ADDR', '192.0.2.50')
+        if (state / 'gateway').exists() and addr != 'none':
+            print(addr)
+    elif kind == 'service':
+        count(p, 2)
+        if output != 'jsonpath={.status.loadBalancer.ingress[0].ip}':
+            fail('unsupported service output')
+        addr = os.environ.get('FAKE_LB_IP', '192.0.2.60')
+        if addr != 'none':
+            print(addr)
+    elif kind in ('gateways.gateway.networking.k8s.io', 'httproutes.gateway.networking.k8s.io'):
+        count(p, 1)
+        if kind.startswith('gateways'):
+            if (state / 'gateway').exists():
+                print('gateway.gateway.networking.k8s.io/runai-gateway')
+            if (state / 'foreign-gateway').exists():
+                print('gateway.gateway.networking.k8s.io/foreign')
+        elif (state / 'foreign-route').exists():
+            print('httproute.gateway.networking.k8s.io/foreign')
+    else:
+        fail('unsupported get resource: ' + kind)
+elif verb in ('annotate', 'label'):
+    changed, resources = False, []
+    for operand in p:
+        if '=' in operand:
+            changed = True
+        elif changed:
+            fail('all resources must be specified before annotation changes')
+        else:
+            resources.append(operand)
+    if not resources or not changed:
+        fail('resource and assignment required')
+    if verb == 'annotate':
+        names = [r.split('/', 1)[-1] for r in resources]
+        if not set(names) <= set(lines('gwcrds')):
+            fail('CRD not found')
+        save_lines('crd-owned', sorted(set(lines('crd-owned') + names)))
+    else:
+        count(resources, 2)
+        if 'Secret/' + ns + '/' + resources[1] not in manifests():
+            fail('secret not found')
+elif verb == 'create':
+    if opts.get('--dry-run') != 'client' or output != 'yaml':
+        fail('only dry-run manifest generation is supported')
+    if p[:1] == ['namespace']:
+        count(p, 2)
+        doc = {'apiVersion': 'v1', 'kind': 'Namespace', 'metadata': {'name': p[1]}}
+    elif p[:1] == ['secret']:
+        count(p, 3)
+        if p[1] == 'tls':
+            for key in ('--cert', '--key'):
+                if key not in opts or not Path(opts[key]).is_file() and opts[key] != '/dev/null':
+                    fail('missing TLS input')
+        elif p[1] == 'generic':
+            if not Path(opts.get('--from-file', '=').split('=', 1)[-1]).is_file():
+                fail('missing secret input')
+        else:
+            fail('unsupported secret type')
+        doc = {'apiVersion': 'v1', 'kind': 'Secret', 'metadata': {'name': p[2], 'namespace': ns}}
+    else:
+        fail('unsupported create')
+    print(json.dumps(doc))  # JSON is a YAML subset; no secret bytes in logs/state.
+elif verb == 'apply':
+    count(p, 0)
+    filename = opts.get('--filename', opts.get('-f'))
+    if filename and filename.startswith('https://'):
+        save_lines('gwcrds', ['gateways.gateway.networking.k8s.io', 'httproutes.gateway.networking.k8s.io'])
+        print('customresourcedefinition created')
+    elif filename == '-':
+        doc = json.load(sys.stdin)
+        store(doc)
+        if doc['kind'] == 'ConfigMap':
+            (state / doc['metadata']['name']).write_text('1')
+        elif doc['kind'] == 'Gateway':
+            (state / 'gateway').write_text('1')
+        elif doc['kind'] not in ('Namespace', 'Secret'):
+            fail('unsupported manifest kind')
+        print(doc['kind'].lower() + '/' + doc['metadata']['name'] + ' created')
+    else:
+        fail('unsupported apply filename')
+elif verb == 'delete':
+    count(p, 2)
+    kind, name = p
+    if kind == 'gateway':
+        (state / 'gateway').unlink(missing_ok=True)
+    elif kind == 'configmap' and name.startswith('deepops-runai-'):
+        (state / name).unlink(missing_ok=True)
+    elif kind == 'crd':
+        save_lines('gwcrds', [c for c in lines('gwcrds') if c != name])
+        save_lines('crd-owned', [c for c in lines('crd-owned') if c != name])
+    elif kind == 'namespace':
+        if not re.fullmatch(r'[a-z0-9]([-a-z0-9]*[a-z0-9])?', name):
+            fail('invalid namespace name: ' + name)
+        db = {k: v for k, v in manifests().items() if v['metadata'].get('namespace') != name}
+        (state / 'manifests.json').write_text(json.dumps(db))
+    else:
+        fail('unsupported delete')
+    print(kind + '/' + name + ' deleted')
+elif verb == 'patch':
+    count(p, 2)
+    if p[0] != 'configmap' or p[1] not in ('coredns', 'nodelocaldns') or opts.get('--type') != 'merge':
+        fail('unsupported patch')
+    (state / p[1]).write_text(json.loads(opts['--patch'])['data']['Corefile'])
+elif verb == 'rollout':
+    count(p, 2)
+    if p not in (['restart', 'deployment/coredns'], ['restart', 'daemonset/nodelocaldns']):
+        fail('unsupported rollout')
+elif verb == 'wait':
+    if not p or '--for' not in opts:
+        fail('wait needs a resource and condition')
diff --git a/tests/runai/bin/python3 b/tests/runai/bin/python3
new file mode 100755
index 000000000..b65ee7060
--- /dev/null
+++ b/tests/runai/bin/python3
@@ -0,0 +1,41 @@
+#!/usr/bin/python3
+"""Run the real API helper with an offline HTTP transport; delegate other Python.
+
+Only used in the full-path test PATH. Credentials and response fixtures are
+synthetic. No network or TLS-handshake claim is made by this test.
+"""
+import importlib.util
+import json
+import os
+from pathlib import Path
+import sys
+
+if len(sys.argv) < 2 or Path(sys.argv[1]).name != 'runai_api.py':
+    os.execv('/usr/bin/python3', ['/usr/bin/python3'] + sys.argv[1:])
+
+spec = importlib.util.spec_from_file_location('runai_api', sys.argv[1])
+api = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(api)
+state = Path(os.environ['FAKE_STATE'])
+
+
+def request(self, method, path, body=None, form=None):
+    with (state / 'api-calls.jsonl').open('a') as log:
+        log.write(json.dumps([method, path]) + '\n')
+    if (method, path) == ('POST', '/api/v2/token'):
+        assert form['grant_type'] == 'password' and form['username'] and form['password']
+        return {'access_token': 'offline-token'}
+    assert self.bearer == 'offline-token'
+    if (method, path) == ('GET', '/api/v1/clusters'):
+        return {'clusters': []}
+    if (method, path) == ('POST', '/api/v1/clusters'):
+        assert body['name'] == 'deepops'
+        return {'uuid': 'offline-cluster'}
+    if (method, path) == ('GET', '/api/v1/clusters/offline-cluster/cluster-install-info?version=3.0.20'):
+        return {'installationStr': '--set controlPlane.url=https://runai.lab.test --set cluster.uid=offline-cluster',
+                'bootstrapToken': 'offline-bootstrap'}
+    raise AssertionError('unsupported HTTP request: %s %s' % (method, path))
+
+
+api.Client.request = request
+sys.exit(api.main(sys.argv[2:]))
diff --git a/tests/runai/run-tests.sh b/tests/runai/run-tests.sh
index 897f09f2f..f5c998499 100755
--- a/tests/runai/run-tests.sh
+++ b/tests/runai/run-tests.sh
@@ -161,6 +161,9 @@ unset STATE
 # Ownership, foreign users and default absent exercise the real task files.
 python3 -m unittest discover -q -s "$HERE" -p "test_gateway_lifecycle.py" && pass "Gateway lifecycle regressions" || bad "Gateway lifecycle regressions"
 
+# Full platform path, including TLS, DNS, registration, rerun and absent.
+python3 -m unittest discover -v -s "$HERE" -p "test_gateway_platform.py" && pass "Gateway full platform path" || bad "Gateway full platform path"
+
 # 11. filter unit tests
 python3 -m unittest discover -q -s "$HERE" -p "test_runai_helpers.py" && pass "helper unit tests" || bad "helper unit tests"
 
diff --git a/tests/runai/test_gateway_lifecycle.py b/tests/runai/test_gateway_lifecycle.py
index 1763fa1a4..67890c933 100644
--- a/tests/runai/test_gateway_lifecycle.py
+++ b/tests/runai/test_gateway_lifecycle.py
@@ -6,6 +6,8 @@
 import tempfile
 import unittest
 
+import yaml
+
 HERE = Path(__file__).resolve().parent
 ROOT = HERE.parent.parent
 
@@ -25,6 +27,28 @@ def play(self, remove=False, standard=False):
                            [a for opt in opts for a in ['-e', opt]], env=self.env, capture_output=True, text=True)
         self.assertEqual(p.returncode, 0, p.stdout + p.stderr)
 
+    def mark_crds(self, names):
+        (self.state / 'gwcrds').write_text('\n'.join(names))
+        tasks = yaml.safe_load((ROOT / 'roles/runai/tasks/gateway_controller.yml').read_text())
+        task = next(t for t in tasks if t['name'] == 'Mark the CRDs this role installed')
+        command = task['ansible.builtin.shell'].replace('{{ runai_kubectl }}', str(HERE / 'bin/kubectl'))
+        return subprocess.run(['/bin/bash', '-c', command], env=self.env, capture_output=True, text=True)
+
+    def test_annotation_resources_precede_assignment(self):
+        result = self.mark_crds(['gateways.gateway.networking.k8s.io', 'httproutes.gateway.networking.k8s.io'])
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertTrue((self.state / 'crd-owned').exists())
+        calls = [json.loads(line) for line in (self.state / 'kubectl-argv.jsonl').read_text().splitlines()]
+        annotations = [a for a in calls if a[0] == 'annotate']
+        for name in ['gateways', 'httproutes']:
+            resource = 'customresourcedefinition.apiextensions.k8s.io/' + name + '.gateway.networking.k8s.io'
+            self.assertTrue(any(resource in a and a.index(resource) < a.index('deepops.nvidia.com/runai-installed=true') for a in annotations))
+
+    def test_empty_crd_list_does_not_annotate(self):
+        self.mark_crds([])
+        calls = [json.loads(line) for line in (self.state / 'kubectl-argv.jsonl').read_text().splitlines()]
+        self.assertFalse(any(a[0] == 'annotate' for a in calls), calls)
+
     def releases(self):
         return json.loads((self.state / 'releases.json').read_text())
 
diff --git a/tests/runai/test_gateway_platform.py b/tests/runai/test_gateway_platform.py
new file mode 100644
index 000000000..056f50aaf
--- /dev/null
+++ b/tests/runai/test_gateway_platform.py
@@ -0,0 +1,132 @@
+"""Whole-role offline Gateway install/rerun/remove, not Ansible --check.
+
+Execute a temporary role copy with ONLY root ownership mapped to the invoking
+user and /etc/hosts redirected into the fixture. All task logic, shell parsing,
+filters, TLS generation and the API helper remain real. External Kubernetes,
+Helm and HTTP operations are bounded doubles; nothing contacts a cluster.
+"""
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+import yaml
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parent.parent
+
+
+class PlatformPath(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory(prefix='runai-platform-')
+        self.addCleanup(self.tmp.cleanup)
+        self.state = Path(self.tmp.name)
+        role = self.state / 'roles/runai'
+        shutil.copytree(ROOT / 'roles/runai', role)
+        for path in (role / 'tasks').glob('*.yml'):
+            text = path.read_text().replace('owner: root', 'owner: "%s"' % os.getuid())
+            text = text.replace('group: root', 'group: "%s"' % os.getgid())
+            text = text.replace('path: /etc/hosts', 'path: "' + str(self.state / 'hosts') + '"')
+            path.write_text(text)
+        (self.state / 'hosts').write_text('127.0.0.1 localhost\n')
+        self.prerequisite = {'monitoring/kube-prometheus-stack': {
+            'ns': 'monitoring', 'chart': 'kube-prometheus-stack-88.6.5', 'values': {}}}
+        (self.state / 'releases.json').write_text(json.dumps(self.prerequisite))
+        self.playbook = self.state / 'site.yml'
+        self.playbook.write_text(yaml.safe_dump([{
+            'hosts': 'localhost', 'connection': 'local', 'gather_facts': False,
+            'vars': {
+                'ansible_become': False, 'ansible_python_interpreter': sys.executable,
+                'runai_kubectl': str(HERE / 'bin/kubectl'), 'runai_helm': str(HERE / 'bin/helm'),
+                'runai_work_dir': str(self.state / 'work'),
+                'runai_mode': 'platform', 'runai_platform_exposure': 'gateway',
+                'runai_platform_domain': 'runai.lab.test',
+                'runai_platform_admin_password': 'Offline-test1!',
+                'runai_platform_manage_hosts_entry': True,
+                'runai_platform_coredns_rewrite': True,
+                'runai_platform_address_timeout': 1,
+            }, 'roles': ['runai'],
+        }]))
+        self.env = dict(os.environ, FAKE_STATE=str(self.state),
+                        PATH=str(HERE / 'bin') + os.pathsep + os.environ['PATH'],
+                        ANSIBLE_CONFIG=str(ROOT / 'ansible.cfg'),
+                        ANSIBLE_ROLES_PATH=str(self.state / 'roles'))
+
+    def play(self, *extra):
+        result = subprocess.run(['ansible-playbook', '-i', 'localhost,', str(self.playbook), *extra],
+                                env=self.env, capture_output=True, text=True, timeout=180)
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+    def read_json(self, name):
+        return json.loads((self.state / name).read_text())
+
+    def calls(self, name):
+        return [json.loads(line) for line in (self.state / name).read_text().splitlines()]
+
+    def check_round_trip(self, routing):
+        extra = ['-e', 'runai_gateway_workload_routing=' + routing]
+        self.play(*extra)
+        releases = self.read_json('releases.json')
+        self.assertEqual(set(releases) - set(self.prerequisite), {
+            'kgateway-system/kgateway', 'kgateway-system/kgateway-crds',
+            'runai-backend/runai-backend', 'runai/runai-cluster'})
+        cp = releases['runai-backend/runai-backend']['values']['global']
+        cluster = releases['runai/runai-cluster']['values']
+        self.assertFalse(cp['ingress']['enabled'])
+        self.assertTrue(cp['gatewayAPI']['enabled'])
+        self.assertEqual(cluster['controlPlane']['url'], 'https://runai.lab.test')
+        self.assertEqual(cluster['cluster']['uid'], 'offline-cluster')
+        self.assertEqual(cluster['clusterConfig']['global']['subdomainSupport'], routing == 'host')
+        manifests = self.read_json('manifests.json')
+        gateway = manifests['Gateway/runai/runai-gateway']
+        self.assertEqual(len(gateway['spec']['listeners']), 2 if routing == 'host' else 1)
+        for listener in gateway['spec']['listeners']:
+            for cert in listener['tls']['certificateRefs']:
+                self.assertIn('Secret/runai/' + cert['name'], manifests)
+        for cm in ('coredns', 'nodelocaldns'):
+            self.assertIn('192.0.2.50 runai.lab.test', (self.state / cm).read_text())
+        self.assertIn('192.0.2.50 runai.lab.test', (self.state / 'hosts').read_text())
+        upgrades = [a for a in self.calls('helm-argv.jsonl') if a[0] == 'upgrade']
+        api_calls = self.calls('api-calls.jsonl')
+        self.assertEqual(len(upgrades), 4)
+        self.assertEqual(len(api_calls), 4)
+        # Exercise supplied-certificate selection on the converged rerun too.
+        self.play(*extra, '-e', 'runai_platform_tls_cert_file=' + str(self.state / 'work/tls.crt'),
+                  '-e', 'runai_platform_tls_key_file=' + str(self.state / 'work/tls.key'),
+                  '-e', 'runai_platform_ca_file=' + str(self.state / 'work/ca.pem'))
+        self.assertEqual([a for a in self.calls('helm-argv.jsonl') if a[0] == 'upgrade'], upgrades)
+        self.assertEqual(self.calls('api-calls.jsonl'), api_calls)
+        self.play('-e', 'runai_state=absent')
+        self.assertEqual(self.read_json('releases.json'), self.prerequisite)
+        self.assertFalse((self.state / 'gateway').exists())
+        self.assertFalse((self.state / 'gwcrds').exists())
+        for cm in ('coredns', 'nodelocaldns', 'hosts'):
+            self.assertNotIn('deepops-runai', (self.state / cm).read_text())
+        self.play('-e', 'runai_state=absent')
+        self.assertEqual(self.read_json('releases.json'), self.prerequisite)
+
+    def test_gateway_absent_does_not_delete_ingress_namespace(self):
+        self.play('-e', 'runai_state=absent')
+        deletions = [a[2] for a in self.calls('kubectl-argv.jsonl') if a[:2] == ['delete', 'namespace']]
+        self.assertEqual(deletions, ['kai-scheduler', 'kai-resource-reservation',
+                                    'runai-reservation', 'runai-scale-adjust', 'runai', 'runai-backend'])
+
+    def test_ingress_absent_still_deletes_managed_ingress_namespace(self):
+        self.play('-e', 'runai_state=absent', '-e', 'runai_platform_exposure=ingress')
+        deletions = [a[2] for a in self.calls('kubectl-argv.jsonl') if a[:2] == ['delete', 'namespace']]
+        self.assertEqual(deletions[-1], 'haproxy-controller')
+        self.assertEqual(len(deletions), 7)
+
+    def test_path_routing_full_install_rerun_remove(self):
+        self.check_round_trip('path')
+
+    def test_host_routing_full_install_rerun_remove(self):
+        self.check_round_trip('host')
+
+
+if __name__ == '__main__':
+    unittest.main()

From d373e70fba8c68ddd7d8af8cf15c31bf175cfe1a Mon Sep 17 00:00:00 2001
From: Doug Holt <dholt@nvidia.com>
Date: Tue, 20 Oct 2026 13:07:40 -0600
Subject: [PATCH 8/9] fix(runai): remove owned Gateway controller runtime
 secret

Clean up the runtime HMAC Secret after uninstalling an unused controller
owned by the role, before removing its ownership record. Helm does not
manage this object. Preserve foreign controllers and unrelated Secrets.

Exercise scoped cleanup, repeated removal, default absent, foreign users,
and CRD-only ownership through the offline Ansible lifecycle tests.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
Signed-off-by: Doug Holt <dholt@nvidia.com>
---
 roles/runai/tasks/gateway_remove.yml  | 10 ++++++
 tests/runai/bin/kubectl               |  9 +++++
 tests/runai/test_gateway_lifecycle.py | 48 +++++++++++++++++++++++++++
 3 files changed, 67 insertions(+)

diff --git a/roles/runai/tasks/gateway_remove.yml b/roles/runai/tasks/gateway_remove.yml
index 29f8f19b6..bd9ee900e 100644
--- a/roles/runai/tasks/gateway_remove.yml
+++ b/roles/runai/tasks/gateway_remove.yml
@@ -42,6 +42,16 @@
     - runai_gw_unused | bool
     - item.stdout | length > 0
 
+# KGateway creates this Secret at runtime, outside the Helm release. Remove it
+# only after the owned controller is stopped, before dropping its ownership record.
+- name: Remove the owned controller runtime HMAC Secret
+  ansible.builtin.command: "{{ runai_kubectl }} --namespace {{ runai_gateway_controller_namespace }} delete secret oauth2-hmac-secret --ignore-not-found"
+  register: runai_rm_gw_hmac
+  changed_when: "'deleted' in runai_rm_gw_hmac.stdout"
+  when:
+    - runai_gw_unused | bool
+    - runai_gw_owners.results | selectattr('item', 'equalto', 'kgateway') | map(attribute='stdout') | select | list | length > 0
+
 - name: Remove ownership records after release cleanup
   ansible.builtin.command: "{{ runai_kubectl }} --namespace {{ runai_gateway_controller_namespace }} delete configmap deepops-runai-{{ item.item }} --ignore-not-found"
   loop: "{{ runai_gw_owners.results }}"
diff --git a/tests/runai/bin/kubectl b/tests/runai/bin/kubectl
index c72f0028e..396e3b49a 100755
--- a/tests/runai/bin/kubectl
+++ b/tests/runai/bin/kubectl
@@ -215,6 +215,15 @@ elif verb == 'delete':
     elif kind == 'crd':
         save_lines('gwcrds', [c for c in lines('gwcrds') if c != name])
         save_lines('crd-owned', [c for c in lines('crd-owned') if c != name])
+    elif kind == 'secret':
+        db = manifests()
+        key = 'Secret/' + ns + '/' + name
+        if key not in db:
+            if '--ignore-not-found' in opts:
+                sys.exit(0)
+            fail('secret not found')
+        del db[key]
+        (state / 'manifests.json').write_text(json.dumps(db))
     elif kind == 'namespace':
         if not re.fullmatch(r'[a-z0-9]([-a-z0-9]*[a-z0-9])?', name):
             fail('invalid namespace name: ' + name)
diff --git a/tests/runai/test_gateway_lifecycle.py b/tests/runai/test_gateway_lifecycle.py
index 67890c933..5d29237a2 100644
--- a/tests/runai/test_gateway_lifecycle.py
+++ b/tests/runai/test_gateway_lifecycle.py
@@ -52,26 +52,69 @@ def test_empty_crd_list_does_not_annotate(self):
     def releases(self):
         return json.loads((self.state / 'releases.json').read_text())
 
+    def seed_controller_secrets(self):
+        # Runtime-created objects are not part of the Helm release. No values
+        # are needed: the role must delete only this name in this namespace.
+        path = self.state / 'manifests.json'
+        db = json.loads(path.read_text()) if path.exists() else {}
+        for namespace, name in [('kgateway-system', 'oauth2-hmac-secret'),
+                                ('kgateway-system', 'unrelated-secret'),
+                                ('other-controller', 'oauth2-hmac-secret')]:
+            db[f'Secret/{namespace}/{name}'] = {
+                'apiVersion': 'v1', 'kind': 'Secret',
+                'metadata': {'namespace': namespace, 'name': name},
+            }
+        path.write_text(json.dumps(db))
+        return self.secrets()
+
+    def secrets(self):
+        db = json.loads((self.state / 'manifests.json').read_text())
+        return {k: v for k, v in db.items() if v['kind'] == 'Secret'}
+
+    def test_owned_controller_runtime_secret_cleanup_is_scoped_and_repeatable(self):
+        self.play()
+        expected = self.seed_controller_secrets()
+        del expected['Secret/kgateway-system/oauth2-hmac-secret']
+        self.play(remove=True)
+        self.assertEqual(self.secrets(), expected)
+        self.assertEqual(self.releases(), {})
+        self.play(remove=True)
+        self.assertEqual(self.secrets(), expected)
+
+    def test_crds_ownership_alone_does_not_authorize_secret_cleanup(self):
+        self.play()
+        expected = self.seed_controller_secrets()
+        (self.state / 'deepops-runai-kgateway').unlink()
+        self.play(remove=True)
+        self.assertEqual(self.secrets(), expected)
+        self.assertEqual(set(self.releases()), {'kgateway-system/kgateway'})
+
     def test_foreign_controller_is_not_upgraded_or_removed(self):
         foreign = {'kgateway-system/' + n: {'ns': 'kgateway-system', 'chart': n + '-v1.0.0', 'values': {'foreign': True}}
                    for n in ('kgateway', 'kgateway-crds')}
         (self.state / 'releases.json').write_text(json.dumps(foreign))
         self.play()
         self.assertEqual(self.releases(), foreign)
+        expected = self.seed_controller_secrets()
         self.play(remove=True)
         self.assertEqual(self.releases(), foreign)
+        self.assertEqual(self.secrets(), expected)
 
     def test_other_gateway_preserves_owned_controller(self):
         self.play()
         (self.state / 'foreign-gateway').write_text('1')
+        expected = self.seed_controller_secrets()
         self.play(remove=True)
+        self.assertEqual(self.secrets(), expected)
         self.assertEqual(set(self.releases()), {'kgateway-system/kgateway', 'kgateway-system/kgateway-crds'})
         self.assertTrue((self.state / 'gwcrds').exists())
 
     def test_route_without_gateway_preserves_controller(self):
         self.play()
         (self.state / 'foreign-route').write_text('1')
+        expected = self.seed_controller_secrets()
         self.play(remove=True)
+        self.assertEqual(self.secrets(), expected)
         self.assertEqual(set(self.releases()), {'kgateway-system/kgateway', 'kgateway-system/kgateway-crds'})
         self.assertTrue((self.state / 'gwcrds').exists())
 
@@ -100,6 +143,11 @@ def test_owned_crd_query_preserves_jsonpath_argument(self):
 
     def test_standard_absent_discovers_owned_gateway_install(self):
         self.play()
+        expected = self.seed_controller_secrets()
+        # Standard absent also removes the Run:ai namespace and its TLS secret.
+        del expected['Secret/runai/runai-cluster-domain-tls-secret']
+        del expected['Secret/kgateway-system/oauth2-hmac-secret']
         self.play(standard=True)
+        self.assertEqual(self.secrets(), expected)
         self.assertEqual(self.releases(), {})
         self.assertFalse((self.state / 'gwcrds').exists())

From 009ea2e89453c72c591be4334d216435fd5f9ff5 Mon Sep 17 00:00:00 2001
From: Doug Holt <dholt@nvidia.com>
Date: Tue, 20 Oct 2026 16:05:59 -0600
Subject: [PATCH 9/9] docs(runai): record 26.11 live test of LoadBalancer and
 Gateway API exposure

Signed-off-by: Doug Holt <dholt@nvidia.com>
---
 docs/k8s-cluster/runai.md     | 6 ++++--
 roles/runai/defaults/main.yml | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/k8s-cluster/runai.md b/docs/k8s-cluster/runai.md
index 3757c35e2..eb56786ec 100644
--- a/docs/k8s-cluster/runai.md
+++ b/docs/k8s-cluster/runai.md
@@ -26,8 +26,10 @@ and any DeepOps-owned UI. The web UI in `platform` mode is the upstream one.
 - `platform`, which you must provide before running the role:
   - a default StorageClass (`playbooks/k8s-cluster/nfs-client-provisioner.yml`)
   - a load balancer for a `LoadBalancer` service ([Load Balancer and Ingress](ingress.md)),
-    or set `runai_platform_ingress_service_type: NodePort` (only NodePort was
-    live-tested in 26.11; `LoadBalancer` is untested in 26.11)
+    or set `runai_platform_ingress_service_type: NodePort` (26.11 live-tested
+    NodePort, and `LoadBalancer` with MetalLB on a single node for both the
+    ingress and the Gateway API paths; reaching that address from another
+    machine was not part of the test)
   - `kube-prometheus-stack` 88.6.5 or newer (`scripts/k8s/deploy_monitoring.sh`
     with `HELM_PROMETHEUS_CHART_VERSION` set)
   - a DNS name for the control plane, and a TLS certificate for it
diff --git a/roles/runai/defaults/main.yml b/roles/runai/defaults/main.yml
index 86688a25b..1eb4cf38f 100644
--- a/roles/runai/defaults/main.yml
+++ b/roles/runai/defaults/main.yml
@@ -70,7 +70,7 @@ runai_platform_ingress_repo: "https://haproxytech.github.io/helm-charts"
 runai_platform_ingress_chart_version: "1.54.2"
 runai_platform_ingress_release_name: "haproxy-kubernetes-ingress"
 runai_platform_ingress_namespace: "haproxy-controller"
-# NodePort is the only type live-tested in 26.11; LoadBalancer is untested in 26.11.
+# 26.11 live-tested NodePort and LoadBalancer (MetalLB, single node, ingress and Gateway API).
 # With LoadBalancer the role waits for the external address (MetalLB: see
 # scripts/k8s/deploy_loadbalancer.sh) and uses it for the hosts entry, the CoreDNS
 # entry and the validator, instead of the node address.