fix(terraform): store saved plans beside the state in S3, not as artifacts

.github/workflows/terraform-stack.yml

@@ -4,7 +4,9 @@ name: Terraform Stack
 # emitted by terraform-discover.yml. See docs/workflow-guide.md.
 #
 # The caller's job must grant: contents: read, pull-requests: write,
-# issues: write, actions: write, and id-token: write when using an OIDC role.
+# issues: write, and id-token: write when using an OIDC role. The stack's
+# credentials must be able to read, write and delete objects under
+# tfplans/<state key>/ in the state bucket (see below).
 
 on:
   workflow_call:
@@ -32,9 +34,6 @@ on:
         description: Overrides .terraform-version in the stack directory or repository root.
         type: string
         default: ""
-      plan-retention-days:
-        type: number
-        default: 7
       runner:
         type: string
         default: linux-arm64
@@ -65,11 +64,6 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
-      - name: Slug the stack path
-        id: slug
-        working-directory: .
-        run: echo "value=$(printf '%s' "$STACK" | tr '/' '.')" >> "$GITHUB_OUTPUT"
-
       - name: Resolve Terraform version
         id: tfver
         working-directory: .
@@ -120,6 +114,25 @@ jobs:
       - name: Terraform init
         run: terraform init -input=false -lock-timeout=5m
 
+      # The saved plan embeds the full prior state and every variable value, so it is
+      # as secret as the state. It is kept in the state bucket, where only the state's
+      # readers can get it - never as an Actions artifact, which anyone with read access
+      # to the repository can download. One shared tfplans/ prefix lets a single bucket
+      # lifecycle rule expire plans that are never applied.
+      - name: Locate the plan store
+        id: store
+        if: inputs.command == 'plan' || inputs.command == 'apply'
+        run: |
+          set -eu
+          backend=$(jq -c '.backend // empty' .terraform/terraform.tfstate)
+          if [ "$(printf '%s' "$backend" | jq -r '.type // empty')" != "s3" ]; then
+            echo "::error title=Unsupported backend::$STACK must use the s3 backend; the saved plan is stored beside the state."
+            exit 1
+          fi
+          printf '%s' "$backend" | jq -r '
+            "prefix=s3://" + .config.bucket + "/tfplans/" + .config.key,
+            "region=" + (.config.region // "")' >> "$GITHUB_OUTPUT"
+
       - name: Terraform fmt
         if: inputs.command == 'plan'
         run: terraform fmt -check -recursive
@@ -144,12 +157,10 @@ jobs:
 
       - name: Upload plan
         if: inputs.command == 'plan' && steps.plan.outputs.failed != 'true'
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
-        with:
-          name: tfplan-${{ steps.slug.outputs.value }}-${{ github.event.pull_request.head.sha || github.sha }}
-          path: ${{ inputs.stack }}/tfplan
-          retention-days: ${{ inputs.plan-retention-days }}
-          if-no-files-found: error
+        env:
+          URI: ${{ steps.store.outputs.prefix }}/${{ github.event.pull_request.head.sha || github.sha }}.tfplan
+          AWS_REGION: ${{ steps.store.outputs.region || inputs.aws-region }}
+        run: aws s3 cp --only-show-errors tfplan "$URI"
 
       - name: Comment plan on the pull request
         if: inputs.command == 'plan' && github.event_name == 'pull_request'
@@ -222,35 +233,21 @@ jobs:
       - name: Download the reviewed plan
         id: fetch
         if: inputs.command == 'apply'
-        working-directory: .
+        env:
+          PREFIX: ${{ steps.store.outputs.prefix }}
+          AWS_REGION: ${{ steps.store.outputs.region || inputs.aws-region }}
         run: |
           set -eu
-          slug='${{ steps.slug.outputs.value }}'
-          find_artifact() {
-            gh api "/repos/$REPO/actions/artifacts?name=tfplan-$slug-$1&per_page=100" \
-              --jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | last // empty'
-          }
-          art=$(find_artifact "${{ github.sha }}")
-          if [ -z "$art" ]; then
+          uri="$PREFIX/${{ github.sha }}.tfplan"
+          if ! aws s3 cp --only-show-errors "$uri" tfplan 2>/dev/null; then
             head=$(gh api "/repos/$REPO/commits/${{ github.sha }}/pulls" --jq '.[0].head.sha // empty')
-            [ -n "$head" ] && art=$(find_artifact "$head")
+            uri="$PREFIX/$head.tfplan"
+            if [ -z "$head" ] || ! aws s3 cp --only-show-errors "$uri" tfplan; then
+              echo "::error title=No reviewed plan::No saved plan for $STACK at ${{ github.sha }}. Dispatch the caller with command recover for this stack."
+              exit 1
+            fi
           fi
-          if [ -z "$art" ]; then
-            echo "::error title=No reviewed plan::No plan artifact for $STACK at ${{ github.sha }}. Dispatch the caller with command recover for this stack."
-            exit 1
-          fi
-          printf '%s' "$art" | jq -r '
-            "name=" + .name,
-            "id="   + (.id | tostring),
-            "run="  + (.workflow_run.id | tostring)' >> "$GITHUB_OUTPUT"
-
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
-        if: inputs.command == 'apply'
-        with:
-          name: ${{ steps.fetch.outputs.name }}
-          run-id: ${{ steps.fetch.outputs.run }}
-          github-token: ${{ github.token }}
-          path: ${{ inputs.stack }}
+          echo "uri=$uri" >> "$GITHUB_OUTPUT"
 
       - name: Terraform apply
         id: apply
@@ -326,11 +323,16 @@ jobs:
               })
             }
 
-      - name: Close the failure issue and drop the plan
+      - name: Drop the applied plan
+        if: inputs.command == 'apply' && steps.apply.outcome == 'success'
+        env:
+          URI: ${{ steps.fetch.outputs.uri }}
+          AWS_REGION: ${{ steps.store.outputs.region || inputs.aws-region }}
+        run: aws s3 rm --only-show-errors "$URI" || echo "::warning::Could not delete $URI"
+
+      - name: Close the failure issue
         if: inputs.command != 'plan' && steps.apply.outcome == 'success'
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
-        env:
-          ARTIFACT_ID: ${{ steps.fetch.outputs.id }}
         with:
           script: |
             const { owner, repo } = context.repo
@@ -346,13 +348,6 @@ jobs:
               })
               await github.rest.issues.update({ owner, repo, issue_number: issue.number, state: 'closed' })
             }
-            if (process.env.ARTIFACT_ID) try {
-              await github.rest.actions.deleteArtifact({
-                owner, repo, artifact_id: Number(process.env.ARTIFACT_ID),
-              })
-            } catch (e) {
-              core.warning(`Could not delete the plan artifact: ${e.message}`)
-            }
 
       - name: Fail on apply error
         if: steps.apply.outcome == 'failure'

CLAUDE.md

@@ -2,10 +2,11 @@
 
 ## Last Documentation Update
 - **Date**: 2026-10-07
-- **Latest SHA**: f252b1f (check for newer commits)
-- **Changes**: The changelog job reaches the identity service over cluster DNS; its Tailscale steps are gone
+- **Latest SHA**: 17d99ba (check for newer commits)
+- **Changes**: Terraform plans are stored beside the state in S3 instead of as Actions artifacts
 
 ## Recent Changes (2026-10-07)
+1. **Terraform plans are stored beside the state, not as Actions artifacts**: `terraform-stack.yml` writes the saved plan to `s3://<bucket>/tfplans/<state key>/<pr head sha>.tfplan`, read from the initialised backend in `.terraform/terraform.tfstate`, and the apply downloads and then deletes it. A saved plan is a zip of the full prior state (`tfstate`, `tfstate-prev`) plus every variable value, `sensitive` or not, and Actions artifacts are downloadable by anyone with read access to the repository — the whole org. Verified locally: a `sensitive` data-source value was absent from the text plan and present in `terraform show -json tfplan`. The shared `tfplans/` prefix is what lets one bucket lifecycle rule expire unapplied plans and the noncurrent versions left by the post-apply delete (S3 lifecycle filters only match a literal key prefix); the cost is that roles scoped to their own state prefix need `tfplans/<prefix>/*` added. Stacks must use the `s3` backend; anything else fails with `Unsupported backend`. `plan-retention-days` and the `Slug the stack path` step are gone, and callers no longer need `actions: write`. The runner needs the AWS CLI. Callers pinned to an older SHA keep uploading artifacts until they bump, and a PR planned under the old pin finds no plan after the bump, so it needs a fresh plan or a `recover`.
 1. **Changelog identity lookup goes over cluster DNS, not the tailnet**: the `Tailscale` and `Wait for the identity service...` steps are gone from `deploy-kotlin.yml` and `deploy-kotlin-v2.yml`; `create-changelog` now just passes `vars.CHANGELOG_IDENTITY_RESOLVE_URL` to `changelog-cli-action`. The tailnet step never worked where it ran: `create-changelog` is on `arc-arm64-1cpu-2gb-nodocker`, which gives the pod no `/dev/net/tun` or `NET_ADMIN`, so `tailscaled` could not start (`failed to connect to local tailscaled` × 8, service-ecotap-gateway run 37486823410). `continue-on-error` hid that as a green step, and the wait step skipped on `outcome == 'failure'` — so every release notification since the feature landed used the GitHub-profile-email fallback. Of the ARC pools only the legacy `self-hosted-arm64-2xl` has Tailscale. The runners live in the same EKS cluster (`internal`) as the identity service, so the variable now holds a `*.svc.cluster.local` URL and resolves directly; measured from an ARC runner, 200 in ~80 ms. Public-repo callers run on `ubuntu-24.04-arm`, where that name does not resolve and the CLI falls back as before. `TAILSCALE_AUTHKEY` stays declared on both workflows although nothing reads it: an undeclared secret that a caller passes explicitly is a startup error, and service-ecotap-gateway passes it in both its deploy workflows.
 
 ## Recent Changes (2026-10-06)

docs/workflow-guide.md

@@ -1326,7 +1326,9 @@ terraform providers lock -platform=linux_arm64 -platform=darwin_arm64
 
 ### Apply runs the reviewed plan
 
-Plan uploads `tfplan` as an artifact named `tfplan-<stack.path>-<sha>`, where `<sha>` is the **pull request head** commit — not `github.sha`, which on a `pull_request` event is the ephemeral merge commit and never matches anything that lands on `main`. Apply resolves the head SHA back from the commit it is running on, downloads that artifact and runs `terraform apply tfplan` — it never re-plans, so what merges is what was reviewed. Apply fails loudly when no artifact matches, which is the intended behaviour for a direct push to `main`.
+Plan stores `tfplan` in the stack's own state bucket, at `s3://<bucket>/tfplans/<state key>/<sha>.tfplan`, where `<sha>` is the **pull request head** commit — not `github.sha`, which on a `pull_request` event is the ephemeral merge commit and never matches anything that lands on `main`. Apply resolves the head SHA back from the commit it is running on, downloads that plan, runs `terraform apply tfplan` and deletes it — it never re-plans, so what merges is what was reviewed. Apply fails loudly when no plan matches, which is the intended behaviour for a direct push to `main`.
+
+The plan is not an Actions artifact because a saved plan is a zip of the full prior state plus every variable value, `sensitive` or not (`terraform show -json tfplan` prints them), and anyone with read access to the repository can download artifacts. Beside the state it is readable by exactly the principals that can already read the state. This needs the `s3` backend, and the stack's credentials must be allowed `s3:GetObject`, `s3:PutObject` and `s3:DeleteObject` on `tfplans/<state key>/*` — a role scoped to its own key prefix needs that path added. Plans of pull requests that never merge, and the old versions an apply's delete leaves in a versioned bucket, stay until a lifecycle rule on `tfplans/` expires them; add one to every state bucket.
 
 Combine with a **merge queue** and `merge_group` in the caller: the queue tests each PR against the queue head, so the plan attached to the landing commit already accounts for everything merging ahead of it.
 
@@ -1336,7 +1338,7 @@ The stack is left partially applied. The workflow re-plans in place, then opens
 
 Recovery is always a forward apply — never a hand-rolled rollback. Create the `terraform-failed` label in the repository before first use.
 
-Re-running the failed job does not recover: a partial apply moved the state on, so the saved plan is stale and Terraform rejects it again. Recover by dispatching the caller with `stack` set; that runs `command: recover`, which plans and applies that one stack in the same job under its environment. The plan is not reviewed — read the leftover diff in the issue first. A merged PR whose apply found no plan artifact (plan still running, failed, or expired) is recovered the same way.
+Re-running the failed job does not recover: a partial apply moved the state on, so the saved plan is stale and Terraform rejects it again. Recover by dispatching the caller with `stack` set; that runs `command: recover`, which plans and applies that one stack in the same job under its environment. The plan is not reviewed — read the leftover diff in the issue first. A merged PR whose apply found no saved plan (plan still running, failed, or deleted) is recovered the same way.
 
 ### Caller
 
@@ -1385,7 +1387,6 @@ jobs:
       contents: read
       pull-requests: write
       issues: write
-      actions: write
       id-token: write
     uses: monta-app/github-workflows/.github/workflows/terraform-stack.yml@v1
     with: