From ddc0ffdca8f4047e07fa688138444194f20e6b44 Mon Sep 17 00:00:00 2001 From: Christian Weinberger Date: Wed, 7 Oct 2026 09:32:43 +0200 Subject: [PATCH] fix(deploy): reach the changelog identity service over cluster DNS [CPONETOPS-1176] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tailnet steps could never have worked where they ran. `create-changelog` is on `arc-arm64-1cpu-2gb-nodocker`, whose pods get neither `/dev/net/tun` nor `NET_ADMIN`, so `tailscaled` never came up — service-ecotap-gateway run 37486823410 shows `failed to connect to local tailscaled` eight times, the step green only because of `continue-on-error`, and the wait step skipped on `outcome == 'failure'`. Of the ARC pools only the legacy `self-hosted-arm64-2xl` has Tailscale. So every release notification since the feature landed has been on the GitHub-profile-email fallback it was meant to replace. The runners are in the same EKS cluster (`internal`) as the identity service, so there was never a perimeter to cross: with a `*.svc.cluster.local` value in `CHANGELOG_IDENTITY_RESOLVE_URL` the lookup resolves directly. Measured from an `arc-arm64-1cpu-2gb-nodocker` runner: 200 in ~80 ms, and ~27 s of dead Tailscale install and retry gone from the job. `TAILSCALE_AUTHKEY` stays declared on both workflows even though nothing reads it. Passing a secret a reusable workflow does not declare is a startup error, and service-ecotap-gateway passes it explicitly in both of its deploy workflows; the input goes once those callers drop it. Co-Authored-By: Claude Opus 5 --- .github/workflows/deploy-kotlin-v2.yml | 44 +++----------------------- .github/workflows/deploy-kotlin.yml | 44 +++----------------------- CLAUDE.md | 11 ++++--- docs/workflow-guide.md | 8 ++--- 4 files changed, 21 insertions(+), 86 deletions(-) diff --git a/.github/workflows/deploy-kotlin-v2.yml b/.github/workflows/deploy-kotlin-v2.yml index cb4ede9..5bb27dc 100644 --- a/.github/workflows/deploy-kotlin-v2.yml +++ b/.github/workflows/deploy-kotlin-v2.yml @@ -207,7 +207,7 @@ on: description: 'GitHub token for changelog generation' TAILSCALE_AUTHKEY: required: false - description: 'Tailscale auth key. Lets the changelog job join the tailnet to reach the internal identity service (repo/org variable CHANGELOG_IDENTITY_RESOLVE_URL) used to tag release notification contributors; leave unset to skip' + description: 'Unused. The changelog job reaches the identity service over cluster DNS now, so nothing here joins the tailnet. Still declared because a caller that passes it explicitly fails to start if the input disappears; drop it from your secrets block and this can go' ARGOCD_TOKEN: required: false description: 'ArgoCD authentication token for API access' @@ -333,45 +333,7 @@ jobs: if: always() && inputs.enable-changelog && needs.deploy.result == 'success' runs-on: ${{ github.event.repository.private && 'arc-arm64-1cpu-2gb-nodocker' || 'ubuntu-24.04-arm' }} timeout-minutes: 5 - env: - TAILSCALE_AUTHKEY: ${{ secrets.TAILSCALE_AUTHKEY }} steps: - # The identity service that maps GitHub logins to Slack users is VPN-only, so the - # runner joins the tailnet first. Only needed when a release notification is enabled - # and an identity URL is configured. Best-effort: a failure here must never fail the - # changelog job, the CLI falls back to less reliable email lookups instead. - - name: Tailscale - id: tailscale - if: ${{ inputs.changelog-release-notify-channel != '' && vars.CHANGELOG_IDENTITY_RESOLVE_URL != '' && env.TAILSCALE_AUTHKEY != '' }} - continue-on-error: true - uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3 - with: - authkey: ${{ env.TAILSCALE_AUTHKEY }} - hostname: "github-changelog-${{ github.run_id }}" - # v4 passes --accept-routes itself; repeating it in args fails `tailscale up` - args: "--login-server https://headscale.monta.com" - # `tailscale up` returns before routes are usable, so the first request can hit a - # half-open path. Poll the identity service until it answers, otherwise the CLI's - # single best-effort lookup could silently fall back to GitHub profile emails. - # Output stays quiet on purpose: the URL is an internal hostname. - - name: Wait for the identity service to be reachable over the tailnet - if: ${{ steps.tailscale.outcome == 'success' }} - continue-on-error: true - shell: bash - env: - IDENTITY_RESOLVE_URL: ${{ vars.CHANGELOG_IDENTITY_RESOLVE_URL }} - run: | - MAX_ATTEMPTS=8 - for attempt in $(seq 1 "${MAX_ATTEMPTS}"); do - if curl --silent --fail --connect-timeout 3 --max-time 5 \ - --output /dev/null "${IDENTITY_RESOLVE_URL}?github=probe" 2>/dev/null; then - echo "✅ Identity service reachable (attempt ${attempt}/${MAX_ATTEMPTS})" - exit 0 - fi - echo "… attempt ${attempt}/${MAX_ATTEMPTS} missed; retrying in 3s" - sleep 3 - done - echo "⚠️ Identity service not reachable; contributors may fall back to GitHub profile emails" - name: Run changelog cli action uses: monta-app/changelog-cli-action@main with: @@ -399,4 +361,8 @@ jobs: jira-project-blacklist: ${{ vars.CHANGELOG_JIRA_PROJECT_BLACKLIST }} monitoring-urls: ${{ inputs.changelog-monitoring-urls }} release-notify-channel: ${{ inputs.changelog-release-notify-channel }} + # Cluster-local URL: this job lands on an ARC runner inside the same EKS cluster as + # the identity service, so it resolves over cluster DNS with no VPN hop. Public repos + # fall back to `ubuntu-24.04-arm` (see runs-on), where the name does not resolve — the + # CLI then degrades to GitHub-profile-email lookups instead of failing. identity-resolve-url: ${{ vars.CHANGELOG_IDENTITY_RESOLVE_URL }} diff --git a/.github/workflows/deploy-kotlin.yml b/.github/workflows/deploy-kotlin.yml index 9375829..5b2c3aa 100644 --- a/.github/workflows/deploy-kotlin.yml +++ b/.github/workflows/deploy-kotlin.yml @@ -195,7 +195,7 @@ on: description: 'GitHub token for changelog generation' TAILSCALE_AUTHKEY: required: false - description: 'Tailscale auth key. Lets the changelog job join the tailnet to reach the internal identity service (repo/org variable CHANGELOG_IDENTITY_RESOLVE_URL) used to tag release notification contributors; leave unset to skip' + description: 'Unused. The changelog job reaches the identity service over cluster DNS now, so nothing here joins the tailnet. Still declared because a caller that passes it explicitly fails to start if the input disappears; drop it from your secrets block and this can go' ARGOCD_TOKEN: required: false description: 'ArgoCD authentication token for API access' @@ -318,45 +318,7 @@ jobs: if: always() && inputs.enable-changelog && needs.deploy.result == 'success' runs-on: ${{ github.event.repository.private && 'arc-arm64-1cpu-2gb-nodocker' || 'ubuntu-24.04-arm' }} timeout-minutes: 5 - env: - TAILSCALE_AUTHKEY: ${{ secrets.TAILSCALE_AUTHKEY }} steps: - # The identity service that maps GitHub logins to Slack users is VPN-only, so the - # runner joins the tailnet first. Only needed when a release notification is enabled - # and an identity URL is configured. Best-effort: a failure here must never fail the - # changelog job, the CLI falls back to less reliable email lookups instead. - - name: Tailscale - id: tailscale - if: ${{ inputs.changelog-release-notify-channel != '' && vars.CHANGELOG_IDENTITY_RESOLVE_URL != '' && env.TAILSCALE_AUTHKEY != '' }} - continue-on-error: true - uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3 - with: - authkey: ${{ env.TAILSCALE_AUTHKEY }} - hostname: "github-changelog-${{ github.run_id }}" - # v4 passes --accept-routes itself; repeating it in args fails `tailscale up` - args: "--login-server https://headscale.monta.com" - # `tailscale up` returns before routes are usable, so the first request can hit a - # half-open path. Poll the identity service until it answers, otherwise the CLI's - # single best-effort lookup could silently fall back to GitHub profile emails. - # Output stays quiet on purpose: the URL is an internal hostname. - - name: Wait for the identity service to be reachable over the tailnet - if: ${{ steps.tailscale.outcome == 'success' }} - continue-on-error: true - shell: bash - env: - IDENTITY_RESOLVE_URL: ${{ vars.CHANGELOG_IDENTITY_RESOLVE_URL }} - run: | - MAX_ATTEMPTS=8 - for attempt in $(seq 1 "${MAX_ATTEMPTS}"); do - if curl --silent --fail --connect-timeout 3 --max-time 5 \ - --output /dev/null "${IDENTITY_RESOLVE_URL}?github=probe" 2>/dev/null; then - echo "✅ Identity service reachable (attempt ${attempt}/${MAX_ATTEMPTS})" - exit 0 - fi - echo "… attempt ${attempt}/${MAX_ATTEMPTS} missed; retrying in 3s" - sleep 3 - done - echo "⚠️ Identity service not reachable; contributors may fall back to GitHub profile emails" - name: Run changelog cli action uses: monta-app/changelog-cli-action@main with: @@ -384,4 +346,8 @@ jobs: jira-project-blacklist: ${{ vars.CHANGELOG_JIRA_PROJECT_BLACKLIST }} monitoring-urls: ${{ inputs.changelog-monitoring-urls }} release-notify-channel: ${{ inputs.changelog-release-notify-channel }} + # Cluster-local URL: this job lands on an ARC runner inside the same EKS cluster as + # the identity service, so it resolves over cluster DNS with no VPN hop. Public repos + # fall back to `ubuntu-24.04-arm` (see runs-on), where the name does not resolve — the + # CLI then degrades to GitHub-profile-email lookups instead of failing. identity-resolve-url: ${{ vars.CHANGELOG_IDENTITY_RESOLVE_URL }} diff --git a/CLAUDE.md b/CLAUDE.md index 4dd983d..d29f556 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,12 +1,15 @@ # Claude Code Notes ## Last Documentation Update -- **Date**: 2026-10-06 -- **Latest SHA**: 340af8f (check for newer commits) -- **Changes**: Small shared-workflow jobs run on the Docker-less ARC runner +- **Date**: 2026-10-07 +- **Latest SHA**: f252b1f (check for newer commits) +- **Changes**: The changelog job reaches the identity service over cluster DNS; its Tailscale steps are gone + +## Recent Changes (2026-10-07) +1. **Changelog identity lookup goes over cluster DNS, not the tailnet**: the `Tailscale` and `Wait for the identity service...` steps are gone from `deploy-kotlin.yml` and `deploy-kotlin-v2.yml`; `create-changelog` now just passes `vars.CHANGELOG_IDENTITY_RESOLVE_URL` to `changelog-cli-action`. The tailnet step never worked where it ran: `create-changelog` is on `arc-arm64-1cpu-2gb-nodocker`, which gives the pod no `/dev/net/tun` or `NET_ADMIN`, so `tailscaled` could not start (`failed to connect to local tailscaled` × 8, service-ecotap-gateway run 37486823410). `continue-on-error` hid that as a green step, and the wait step skipped on `outcome == 'failure'` — so every release notification since the feature landed used the GitHub-profile-email fallback. Of the ARC pools only the legacy `self-hosted-arm64-2xl` has Tailscale. The runners live in the same EKS cluster (`internal`) as the identity service, so the variable now holds a `*.svc.cluster.local` URL and resolves directly; measured from an ARC runner, 200 in ~80 ms. Public-repo callers run on `ubuntu-24.04-arm`, where that name does not resolve and the CLI falls back as before. `TAILSCALE_AUTHKEY` stays declared on both workflows although nothing reads it: an undeclared secret that a caller passes explicitly is a startup error, and service-ecotap-gateway passes it in both its deploy workflows. ## Recent Changes (2026-10-06) -1. **Changelog job can join the tailnet to resolve Slack users**: `deploy-kotlin.yml` and `deploy-kotlin-v2.yml` `create-changelog` jobs pass the repo/org variable `CHANGELOG_IDENTITY_RESOLVE_URL` to `changelog-cli-action` (`identity-resolve-url`), so the CLI can map GitHub logins to Slack user IDs for the release notification. The endpoint is VPN-only, so when `changelog-release-notify-channel`, that variable and the new optional `TAILSCALE_AUTHKEY` secret are all set, the job first joins the tailnet (`tailscale/github-action`, same login server and pinned version as `pull-request-kotlin`) and polls the service until it answers (`tailscale up` returns before routes are usable). Both steps are `continue-on-error` and best-effort: any failure just leaves the CLI on its GitHub-profile-email fallback, never failing the changelog job. The URL is deliberately a variable, not in the workflow, because this repo is public. Needs a `changelog-cli` release that includes the identity resolver (pinned in `changelog-cli-action`); the passthrough is inert until then. +1. **Changelog job resolves Slack users via the identity API** (tailnet part superseded 2026-10-07, above): `deploy-kotlin.yml` and `deploy-kotlin-v2.yml` `create-changelog` jobs pass the repo/org variable `CHANGELOG_IDENTITY_RESOLVE_URL` to `changelog-cli-action` (`identity-resolve-url`), so the CLI can map GitHub logins to Slack user IDs for the release notification. The URL is deliberately a variable, not in the workflow, because this repo is public. Needs a `changelog-cli` release that includes the identity resolver (pinned in `changelog-cli-action`); the passthrough is inert until then. 1. **Small jobs on `arc-arm64-1cpu-2gb-nodocker`**: the shared workflows' small jobs that need no Docker moved there from `arc-arm64-2cpu-4gb` (kube-manifests#7784 added the set: 0.5 CPU guaranteed, 2 GiB, no dind, ~14 per node): the Setup/converter jobs (`component-build`, `component-test-kotlin`, `component-test-python`, `pull-request-{bun,kotlin,react}`, `sonar-cloud`), `check-migration-order`, the `component-deploy(-v2)` deploy job, `create-release-tag`, the changelog and Slack-token deprecation jobs, `track-pending-release`, `pr-digest`, `allow-/block-deploys` and rollback's `push-manifest`. Jobs that need Docker stay on `arc-arm64-2cpu-4gb`: rollback's `slackNotification` (`rtCamp/action-slack-notify` runs a `docker://` image), this repo's `lint-workflows` (`reviewdog/action-actionlint` is a Docker action), and `publish-tech-docs` (a docs build). Anything added to these jobs must not use `container:`, `services:` or a Docker-based action. 1. **The shared workflows' own jobs run on ARC**: the 17 jobs that named a GitHub-hosted label (`component-deploy(-v2)` deploy, `create-release-tag`, the deploy workflows' changelog and Slack-token deprecation jobs, `rollback`, `track-pending-release`, `pr-digest`, `publish-tech-docs`, `allow-/block-deploys`, `semgrep-security-scan`, and this repo's own `pull-request.yml`) use `${{ github.event.repository.private && 'arc-arm64-2cpu-4gb' || 'ubuntu-24.04-arm' }}`; semgrep gets `arc-arm64-4cpu-12gb`. Public callers, and events without a repository in the payload (`schedule`), get GitHub's standard arm64 runner. The deploy and rollback jobs rely on `yq` v4 and `jq` from the runner, both in the ARC image; `argocd` is downloaded by `argocd-wait-sync`, and nothing calls the `helm` CLI. diff --git a/docs/workflow-guide.md b/docs/workflow-guide.md index 0180979..85f8ce5 100644 --- a/docs/workflow-guide.md +++ b/docs/workflow-guide.md @@ -393,11 +393,11 @@ This conditional logic ensures the workflow continues properly even when optiona | `SLACK_APP_TOKEN` | Yes | Slack token for notifications | | `MANIFEST_REPO_PAT` | Yes | GitHub PAT for updating kube-manifests | | `SENTRY_AUTH_TOKEN` | No | Sentry authentication token | -| `TAILSCALE_AUTHKEY` | No | Tailscale auth key. With `changelog-release-notify-channel` and the `CHANGELOG_IDENTITY_RESOLVE_URL` variable set, the changelog job joins the tailnet to resolve contributors' Slack accounts (see below) | +| `TAILSCALE_AUTHKEY` | No | Unused, and safe to drop from your `secrets:` block. The changelog job reaches the identity service over cluster DNS now (see below); the input is only still declared so callers that pass it explicitly don't fail at startup | | `AWS_CDN_ACCESS_KEY_ID` | No | CDN access key for S3 access | | `AWS_CDN_SECRET_ACCESS_KEY` | No | CDN secret key for S3 access | -**Tagging contributors in the release notification:** the notification tags PR authors, co-authors and approvers with real Slack mentions. Resolving a GitHub login to a Slack user goes through an internal, VPN-only identity service, so the URL is *not* in the workflow - set it as a repo/org **variable** `CHANGELOG_IDENTITY_RESOLVE_URL` (full endpoint URL) and pass the `TAILSCALE_AUTHKEY` secret (`secrets: inherit` covers it). When all three of `changelog-release-notify-channel`, the variable and the secret are set, the changelog job joins the tailnet and waits for the service to answer before running the CLI. Without them, or if joining fails, the job still succeeds and the CLI falls back to less reliable GitHub-profile-email lookups. +**Tagging contributors in the release notification:** the notification tags PR authors, co-authors and approvers with real Slack mentions. Resolving a GitHub login to a Slack user goes through an internal identity service, so the URL is *not* in the workflow - set it as a repo/org **variable** `CHANGELOG_IDENTITY_RESOLVE_URL` (full endpoint URL) and set `changelog-release-notify-channel`; nothing else is needed. In a private repo the changelog job runs on an ARC runner inside the same cluster as the service, so it reaches it over cluster DNS. A public repo runs on a GitHub-hosted runner, where that name doesn't resolve - the job still succeeds and the CLI falls back to less reliable GitHub-profile-email lookups. ### Example Usage: @@ -594,7 +594,7 @@ jobs: | `SLACK_APP_TOKEN` | Yes | Slack token for notifications | | `MANIFEST_REPO_PAT` | Yes | GitHub PAT for updating service repository | | `SENTRY_AUTH_TOKEN` | No | Sentry authentication token | -| `TAILSCALE_AUTHKEY` | No | Tailscale auth key. With `changelog-release-notify-channel` and the `CHANGELOG_IDENTITY_RESOLVE_URL` variable set, the changelog job joins the tailnet to resolve contributors' Slack accounts (see below) | +| `TAILSCALE_AUTHKEY` | No | Unused, and safe to drop from your `secrets:` block. The changelog job reaches the identity service over cluster DNS now (see below); the input is only still declared so callers that pass it explicitly don't fail at startup | | `AWS_CDN_ACCESS_KEY_ID` | No | CDN access key for S3 access | | `AWS_CDN_SECRET_ACCESS_KEY` | No | CDN secret key for S3 access | | `LOKALISE_TOKEN` | No | Lokalise API token | @@ -604,7 +604,7 @@ jobs: | `CHANGELOG_GITHUB_TOKEN` | No | GitHub token for changelog generation | | `ARGOCD_TOKEN` | No | ArgoCD authentication token | -**Tagging contributors in the release notification:** the notification tags PR authors, co-authors and approvers with real Slack mentions. Resolving a GitHub login to a Slack user goes through an internal, VPN-only identity service, so the URL is *not* in the workflow - set it as a repo/org **variable** `CHANGELOG_IDENTITY_RESOLVE_URL` (full endpoint URL) and pass the `TAILSCALE_AUTHKEY` secret (`secrets: inherit` covers it). When all three of `changelog-release-notify-channel`, the variable and the secret are set, the changelog job joins the tailnet and waits for the service to answer before running the CLI. Without them, or if joining fails, the job still succeeds and the CLI falls back to less reliable GitHub-profile-email lookups. +**Tagging contributors in the release notification:** the notification tags PR authors, co-authors and approvers with real Slack mentions. Resolving a GitHub login to a Slack user goes through an internal identity service, so the URL is *not* in the workflow - set it as a repo/org **variable** `CHANGELOG_IDENTITY_RESOLVE_URL` (full endpoint URL) and set `changelog-release-notify-channel`; nothing else is needed. In a private repo the changelog job runs on an ARC runner inside the same cluster as the service, so it reaches it over cluster DNS. A public repo runs on a GitHub-hosted runner, where that name doesn't resolve - the job still succeeds and the CLI falls back to less reliable GitHub-profile-email lookups. ### Outputs: - `slack-message-id`: Slack message ID for notifications