diff --git a/.github/workflows/terraform-discover.yml b/.github/workflows/terraform-discover.yml index 6682801..8967484 100644 --- a/.github/workflows/terraform-discover.yml +++ b/.github/workflows/terraform-discover.yml @@ -21,6 +21,10 @@ on: description: Select every stack regardless of the diff. Use for scheduled drift runs. type: boolean default: false + ignore: + description: Extended regex of changed paths that select nothing, on top of *.md. Example '^scripts/' + type: string + default: "" runner: type: string default: linux-arm64 @@ -55,7 +59,8 @@ jobs: A stack is a directory holding backend.tf. A changed file inside one belongs to that stack. A changed file inside a local module belongs to every stack that calls that module, directly or through other modules - so a shared - module change plans its callers and nothing else. A file that is neither, + module change plans its callers and nothing else. A .tf symlinked into a + stack counts as that stack calling the directory the link points into. A file that is neither, such as a repository root file or CI config, could affect anything and selects every stack. @@ -87,6 +92,11 @@ jobs: referenced = collections.defaultdict(set) for path in tf: here = os.path.dirname(path) + full = os.path.join(root, path) + if os.path.islink(full): + target = os.path.relpath(os.path.dirname(os.path.realpath(full)), os.path.realpath(root)) + if not target.startswith(".."): + referenced[target].add(here) try: body = open(os.path.join(root, path), encoding="utf-8").read() except OSError: @@ -166,6 +176,7 @@ jobs: ACCOUNTS: ${{ inputs.accounts }} FORCE_ALL: ${{ inputs.all }} ONLY: ${{ inputs.only }} + IGNORE: ${{ inputs.ignore }} EVENT: ${{ github.event_name }} BASE_PR: ${{ github.event.pull_request.base.sha }} BASE_MERGE_GROUP: ${{ github.event.merge_group.base_sha }} @@ -194,7 +205,7 @@ jobs: selected=$all_stacks echo "Selecting every stack (event=$EVENT force-all=$FORCE_ALL base='$base')." else - git diff --name-only "$base" HEAD -- | grep -Ev '\.md$' > "$RUNNER_TEMP/changed.txt" || true + git diff --name-only "$base" HEAD -- | grep -Ev "\.md\$${IGNORE:+|$IGNORE}" > "$RUNNER_TEMP/changed.txt" || true selected=$(python3 "$RUNNER_TEMP/resolve_stacks.py" < "$RUNNER_TEMP/changed.txt") if [ "$selected" = "__ALL__" ]; then selected=$all_stacks diff --git a/.github/workflows/terraform-stack.yml b/.github/workflows/terraform-stack.yml index db027d0..062eab8 100644 --- a/.github/workflows/terraform-stack.yml +++ b/.github/workflows/terraform-stack.yml @@ -30,6 +30,18 @@ on: aws-region: type: string default: eu-west-1 + aws-session-duration: + description: Seconds the OIDC session lasts. Raise it for applies that run past an hour; the role's MaxSessionDuration must allow it. + type: number + default: 3600 + parallelism: + description: Terraform -parallelism for plan and apply. + type: number + default: 10 + tailscale: + description: Join the tailnet before init, for providers that reach private endpoints. + type: boolean + default: false terraform-version: description: Overrides .terraform-version in the stack directory or repository root. type: string @@ -45,6 +57,9 @@ on: tf-vars-json: description: JSON object of Terraform variables, written to the stack as ci.auto.tfvars.json. required: false + tailscale-authkey: + description: Required when tailscale is true. + required: false jobs: terraform: @@ -92,6 +107,7 @@ jobs: with: role-to-assume: ${{ inputs.aws-role }} aws-region: ${{ inputs.aws-region }} + role-duration-seconds: ${{ inputs.aws-session-duration }} - name: Configure AWS credentials via static keys if: inputs.aws-role == '' @@ -101,6 +117,14 @@ jobs: aws-secret-access-key: ${{ secrets.aws-secret-access-key }} aws-region: ${{ inputs.aws-region }} + - name: Join the tailnet + if: inputs.tailscale + uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3 + with: + authkey: ${{ secrets.tailscale-authkey }} + hostname: github-${{ github.run_id }} + args: "--login-server https://headscale.monta.com" + - name: Write CI variables env: TF_VARS_JSON: ${{ secrets.tf-vars-json }} @@ -145,7 +169,7 @@ jobs: if: inputs.command == 'plan' || inputs.command == 'recover' run: | set -eu - terraform plan -no-color -input=false -lock-timeout=5m \ + terraform plan -no-color -input=false -lock-timeout=5m -parallelism=${{ inputs.parallelism }} \ -out=tfplan -detailed-exitcode > plan.txt 2>&1 && code=0 || code=$? tail -c 400000 plan.txt case "$code" in @@ -261,11 +285,11 @@ jobs: continue-on-error: true run: | set -o pipefail - terraform apply -input=false -lock-timeout=5m -no-color tfplan 2>&1 | tee apply.txt + terraform apply -input=false -lock-timeout=5m -parallelism=${{ inputs.parallelism }} -no-color tfplan 2>&1 | tee apply.txt - name: Re-plan after a failed apply if: steps.apply.outcome == 'failure' - run: terraform plan -no-color -input=false -lock-timeout=5m > replan.txt 2>&1 || true + run: terraform plan -no-color -input=false -lock-timeout=5m -parallelism=${{ inputs.parallelism }} > replan.txt 2>&1 || true - name: Report the failed apply if: steps.apply.outcome == 'failure' diff --git a/docs/workflow-guide.md b/docs/workflow-guide.md index 3f6a450..477ab03 100644 --- a/docs/workflow-guide.md +++ b/docs/workflow-guide.md @@ -1316,7 +1316,9 @@ Two reusable workflows replace the per-repo Terraform pipelines: A **stack** is any directory containing `backend.tf`. That directory is simultaneously the unit of state, the CI job, the state lock and the blast radius. Discovery walks the diff and maps each changed file to the nearest ancestor directory holding a `backend.tf`, so adding a stack requires no workflow edits. -A change inside a local module selects every stack that calls that module, directly or through other modules — discovery reads the `source = "../../modules/x"` paths out of the configuration, so nothing has to be declared or maintained. A module no stack calls selects nothing. Only a path that is neither a stack nor Terraform code — a repository root file, `.github/**` — fans out to **every** stack, because it could affect any of them. Changes to `*.md` are ignored. +A change inside a local module selects every stack that calls that module, directly or through other modules — discovery reads the `source = "../../modules/x"` paths out of the configuration, so nothing has to be declared or maintained. A `.tf` file symlinked into stacks (e.g. one shared `users.tf` linked into every stack beside it) works the same way: changing the target selects every stack holding a link to it. A module no stack calls selects nothing. Only a path that is neither a stack nor Terraform code — a repository root file, `.github/**` — fans out to **every** stack, because it could affect any of them. Changes to `*.md` are ignored, and so are paths matching the caller's `ignore` regex (e.g. `^scripts/` for helper scripts that never touch state). + +Optional per-stack inputs on `terraform-stack.yml`: `tailscale: true` (with the `tailscale-authkey` secret) joins the tailnet before init, for providers that talk to private endpoints such as a database; `parallelism` sets Terraform's `-parallelism` for plan and apply; `aws-session-duration` lengthens the OIDC session for applies that run past an hour (the role's `MaxSessionDuration` must allow it). Discovery passes any extra key on an `accounts[]` entry through to the matrix, so these can be set per path prefix. Each stack needs `.terraform-version` (its own, or one at the repository root) and a committed `.terraform.lock.hcl`. Generate the lock file for the runner architecture: