From a64a699f7086ff8225661147673393ce25667c1b Mon Sep 17 00:00:00 2001 From: Jesper Terkelsen Date: Tue, 13 Oct 2026 16:23:22 +0200 Subject: [PATCH] fix(actions): read arc-job-report metrics from mimir-internal at 15 s ARC cAdvisor and kube-state-metrics stopped landing in grafanacloud-prom around 2026-10-01, so every job in the report came back as "no metrics" and no runner suggestions were made. Query mimir-internal instead. mimir-internal scrapes every 15 s, so query at a 15 s step with a 1 min rate window instead of 60 s / 2 min, which caught only 2-minute averages and hid CPU bursts. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/actions/arc-job-report/README.md | 9 ++++----- .github/actions/arc-job-report/action.yml | 2 +- .github/actions/arc-job-report/arc_job_report.py | 2 +- .github/actions/arc-job-report/job_resources.py | 11 +++++------ 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/actions/arc-job-report/README.md b/.github/actions/arc-job-report/README.md index bec75f3..c3d7d6d 100644 --- a/.github/actions/arc-job-report/README.md +++ b/.github/actions/arc-job-report/README.md @@ -62,21 +62,20 @@ an ARC runner (`arc-*` / `self-hosted-*`) are skipped. | Input | Required | Description | |---|---|---| | `github-token` | yes | Reads the run's jobs and writes the comment. Needs `actions: read` and `pull-requests: write`. | -| `grafana-token` | yes | Grafana service-account token that can read the `grafanacloud-prom` datasource. | +| `grafana-token` | yes | Grafana service-account token that can read the `mimir-internal` Prometheus datasource. | | `run-url` | no | Run to report on. Defaults to the run that triggered the `workflow_run` event. | ## How to read the numbers -Metrics come from cAdvisor and kube-state-metrics, scraped every **60 s**. +Metrics come from cAdvisor and kube-state-metrics in `mimir-internal`, scraped every **15 s**. - **Peak memory is reliable.** It is the highest working-set sample over the job. -- **CPU is not, for short jobs.** It is the highest 60 s average, so short bursts +- **CPU is not, for short jobs.** It is the highest 1 min average, so short bursts are averaged away. In service-ocpi, the CPU-based rule suggested 4 vCPU for the integration shards, but a trial run made every shard 17–97 s slower. Trial a smaller runner before moving to it. - **Jobs under 2 minutes are left out of the suggestions.** They get too few - samples. Jobs under a minute have none at all (CPU needs two samples), so they - show `–`. There is no point waiting for them: a runner pod's samples stop once + samples. There is no point waiting for them: a runner pod's samples stop once its job ends and the pod is deleted. The suggested size is the smallest catalog runner with memory ≥ 1.3 × peak and diff --git a/.github/actions/arc-job-report/action.yml b/.github/actions/arc-job-report/action.yml index f6edd1a..f772014 100644 --- a/.github/actions/arc-job-report/action.yml +++ b/.github/actions/arc-job-report/action.yml @@ -10,7 +10,7 @@ inputs: required: true grafana-token: description: >- - Grafana service-account token with read access to the `grafanacloud-prom` datasource, + Grafana service-account token with read access to the `mimir-internal` Prometheus datasource, for example `secrets.GRAFANA_CLOUD_TOKEN`. required: true run-url: diff --git a/.github/actions/arc-job-report/arc_job_report.py b/.github/actions/arc-job-report/arc_job_report.py index ba53ff6..8f812bf 100644 --- a/.github/actions/arc-job-report/arc_job_report.py +++ b/.github/actions/arc-job-report/arc_job_report.py @@ -127,7 +127,7 @@ def render(run, results, previous_body): "|---|---|", *job_rows(results), "", - *(["💡 **Suggested runner changes.** Memory is the exact peak; CPU is the max of 60 s averages, so " + *(["💡 **Suggested runner changes.** Memory is the exact peak; CPU is the max of 1 min averages, so " "trial before downsizing on it. Jobs under 2 min have too few samples and are left out.", "", "| Job | Change | CPU | Memory |", diff --git a/.github/actions/arc-job-report/job_resources.py b/.github/actions/arc-job-report/job_resources.py index dc45af0..8423170 100644 --- a/.github/actions/arc-job-report/job_resources.py +++ b/.github/actions/arc-job-report/job_resources.py @@ -6,8 +6,7 @@ Needs `gh` (authenticated) and either GRAFANA_TOKEN (a Grafana service-account token with read access to the Prometheus datasource) or `gcx` (a context with that datasource). -A job's runner name (GitHub API `runner_name`) is its ARC pod name; metrics are scraped every 60 s, -so jobs shorter than ~2 minutes get only a sample or two. +A job's runner name (GitHub API `runner_name`) is its ARC pod name; metrics are scraped every 15 s. Copied from monta-app/kube-manifests infra/arc/scripts/job-resources.py @ dd43b0172; the only change is the GRAFANA_TOKEN query path in prom(). @@ -22,7 +21,7 @@ import urllib.request from datetime import datetime -DATASOURCE = "grafanacloud-prom" +DATASOURCE = "ffzoywkuhm5tsd" # mimir-internal GRAFANA = "https://montaapp.grafana.net" ARC_RUNNER = re.compile(r"^(self-hosted|arc)-") # (name, arch, vCPU, memory GiB) -- keep in sync with infra/arc/README.md @@ -40,7 +39,7 @@ def sh(*args): return subprocess.run(args, capture_output=True, text=True, check=True).stdout -def prom(expr, start, end, step=60): +def prom(expr, start, end, step=15): token = os.environ.get("GRAFANA_TOKEN") if token: query = urllib.parse.urlencode({"query": expr, "start": start, "end": end, "step": step}) @@ -88,7 +87,7 @@ def analyse(job): pod, start, end = job["runner_name"], ts(job["started_at"]), ts(job["completed_at"]) sel = f'cluster="internal", namespace="arc", pod="{pod}"' lo, hi = start - 60, end + 60 - cpu = values(prom(f'sum(rate(container_cpu_usage_seconds_total{{{sel}, container=~"runner|dind"}}[2m]))', lo, hi)) + cpu = values(prom(f'sum(rate(container_cpu_usage_seconds_total{{{sel}, container=~"runner|dind"}}[1m]))', lo, hi)) mem = values(prom(f'sum(container_memory_working_set_bytes{{{sel}, container=~"runner|dind"}})', lo, hi)) req = values(prom(f'max(kube_pod_container_resource_requests{{{sel}, container="runner", resource="cpu"}})', lo, hi)) lim = values(prom(f'max(kube_pod_container_resource_limits{{{sel}, container="runner", resource="memory"}})', lo, hi)) @@ -120,7 +119,7 @@ def analyse(job): }) notes = [] if duration < 120: - notes.append("short job: only a few 60 s samples, low confidence") + notes.append("short job: few samples, low confidence") if res["node"]["vcpu"] and p95 >= 0.85 * res["node"]["vcpu"]: notes.append("CPU-bound on this node: a larger size may make it faster") if res["oom_kills"]: