From 626f0fc555d06655a4d37c195a53db8cecc6dce6 Mon Sep 17 00:00:00 2001 From: Astrid Gealer Date: Mon, 19 Oct 2026 23:38:57 +0100 Subject: [PATCH 1/2] sec(ci): delete the GCP staging VPC peering before removing it from state (#330) The destroy-gcp job removed module.networking.google_service_networking_connection.private_vpc_connection from state before deleting the peering, deleted it on a hardcoded `cudly-staging-vpc`, and swallowed the delete's failure with `2>/dev/null || echo`. Unlike the root-level Cloud SQL addresses fixed in #1971, this address is module-qualified, so the state rm was real: a failed or misdirected delete left a live peering no state tracked. Move the peering cleanup into scripts/delete-owned-vpc-peering.sh, run as its own step between the Cloud SQL delete and terraform destroy. It deletes only while the state still tracks the address (exact match via select-owned-name.sh), reads the network from the state's network_name output, swallows nothing, and runs `terraform state rm` only after the delete returned 0. Add scripts/test-vpc-peering-delete-scope.sh as the fourth sibling guard suite (stubbed terraform/gcloud behavior, workflow wiring and step order, and a sweep of workflows and scripts/ for inline peering deletes), wired into ci.yml and ci-success. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 30 ++ .github/workflows/cleanup-staging.yml | 24 +- scripts/delete-owned-vpc-peering.sh | 146 +++++++++ scripts/lib/code-scan-awk.sh | 12 +- scripts/test-vpc-peering-delete-scope.sh | 388 +++++++++++++++++++++++ 5 files changed, 587 insertions(+), 13 deletions(-) create mode 100755 scripts/delete-owned-vpc-peering.sh create mode 100755 scripts/test-vpc-peering-delete-scope.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b885e287e..d0f0681c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1124,6 +1124,35 @@ jobs: - name: Run Cloud SQL scope self-tests run: bash scripts/test-cloud-sql-delete-scope.sh + # Assert that cleanup-staging.yml's destroy-gcp job deletes the Service + # Networking peering on the network its state names (the `network_name` + # output, not a hardcoded `cudly-staging-vpc`), and removes the peering from + # state only after that delete succeeded, with nothing swallowed (#330). The + # step it replaces removed the module-qualified address from state first and + # swallowed the delete's failure, which left a live peering no state + # tracked. The suite runs scripts/delete-owned-vpc-peering.sh end to end + # against stubbed terraform and gcloud, asserts the step order inside + # destroy-gcp (Cloud SQL, then peering, then destroy), and sweeps every + # workflow plus a recursive glob of scripts/ so the script stays the only + # place that runs `gcloud services vpc-peerings delete`. + # Fast (shell only), so it always runs. + vpc-peering-delete-scope: + name: VPC peering delete scope + runs-on: ubuntu-latest + # Same shape as ecr-delete-selection above: this job checks out the tree and + # runs a shell script against it, so `contents: read` is all it needs. + permissions: + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + - name: Run VPC peering scope self-tests + run: bash scripts/test-vpc-peering-delete-scope.sh + # Assert that every job applying a `compute_platform` writes the Terraform # state namespace that platform owns: `compute_platform=lambda` into # github-/, `compute_platform=fargate` into github-fargate-/. The @@ -1204,6 +1233,7 @@ jobs: - ecr-delete-selection - rds-deletion-protection-scope - cloud-sql-delete-scope + - vpc-peering-delete-scope - aws-tfstate-platform-key - azure-kv-access-policy if: always() diff --git a/.github/workflows/cleanup-staging.yml b/.github/workflows/cleanup-staging.yml index cabff1944..c57a041d5 100644 --- a/.github/workflows/cleanup-staging.yml +++ b/.github/workflows/cleanup-staging.yml @@ -405,14 +405,22 @@ jobs: # per the zero-expression-interpolation-in-run-blocks rule from #1641. PROJECT="$GCP_PROJECT_ID" ./scripts/delete-owned-cloud-sql-instance.sh terraform/environments/gcp "$PROJECT" - # Remove the Service Networking Connection from state and delete directly - # GCP needs extra time after Cloud SQL deletion to release VPC peering - cd terraform/environments/gcp - terraform state rm module.networking.google_service_networking_connection.private_vpc_connection 2>/dev/null || true - gcloud services vpc-peerings delete \ - --service=servicenetworking.googleapis.com \ - --network=cudly-staging-vpc \ - --project="$PROJECT" --quiet 2>/dev/null || echo "VPC peering already gone or not found" + + # After the Cloud SQL step (the peering cannot go while the instance still + # uses it) and before `terraform destroy`. Deletes the peering on the + # network named by this state's `network_name` output, and removes it + # from state only after that delete succeeds. The Cloud SQL step used to do + # this inline: state rm first, delete on a hardcoded `cudly-staging-vpc`, + # and the delete's failure swallowed, which left a live peering untracked + # (#330). Rationale: the script's header. + - name: Delete the Service Networking peering this state owns + env: + GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} + run: | + # Routed via env: rather than interpolated into this script's source, + # per the zero-expression-interpolation-in-run-blocks rule from #1641. + PROJECT="$GCP_PROJECT_ID" + ./scripts/delete-owned-vpc-peering.sh terraform/environments/gcp "$PROJECT" - name: Terraform Destroy env: diff --git a/scripts/delete-owned-vpc-peering.sh b/scripts/delete-owned-vpc-peering.sh new file mode 100755 index 000000000..a4fe4b14d --- /dev/null +++ b/scripts/delete-owned-vpc-peering.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# delete-owned-vpc-peering.sh +# +# Deletes the Service Networking peering a Terraform state owns, then removes +# it from that state. Nothing is removed from state unless the delete +# succeeded. +# +# Usage: delete-owned-vpc-peering.sh TERRAFORM_STATE_DIR GCP_PROJECT_ID +# +# Runs before `terraform destroy` on the GCP staging state, after +# delete-owned-cloud-sql-instance.sh: the peering cannot be removed while the +# Cloud SQL instance still uses it, and Terraform's own destroy of +# google_service_networking_connection fails on that "producer services still +# using this connection" ordering, so the workflow deletes it directly. +# +# The step this replaces (#330) ran +# +# terraform state rm module.networking.google_service_networking_connection.private_vpc_connection 2>/dev/null || true +# gcloud services vpc-peerings delete ... --network=cudly-staging-vpc ... 2>/dev/null || echo "..." +# +# which had two defects: +# +# 1. State was removed BEFORE the delete, and the delete's failure was +# swallowed. Unlike the root-level Cloud SQL addresses #1971 corrected, +# this address is module-qualified, so the state rm was real: a failed +# delete left a live peering that no state tracked any more, and the +# following `terraform destroy` then failed on google_compute_network +# with nothing left to reconcile against. Here the delete runs first, its +# exit status is not masked, and `terraform state rm` runs only after it +# returned 0. +# 2. The network was the literal "cudly-staging-vpc" rather than the name the +# state publishes. A renamed network would have turned the delete into a +# "not found" that the `|| echo` reported as "already gone", while the +# state rm still ran. The network is now read from the state's +# `network_name` output (terraform/environments/gcp/outputs.tf -> +# module.networking.network_name -> google_compute_network.main.name), +# which is the network google_service_networking_connection +# .private_vpc_connection is attached to +# (terraform/modules/networking/gcp/main.tf). +# +# Whether there is anything to delete is decided by the state, not by gcloud: +# the peering is deleted only while this state still tracks it. That makes a +# re-run after a completed cleanup a no-op (the address is already gone from +# state), and a re-run after a FAILED delete a retry (the address is still +# there). The address is compared by exact equality against an unfiltered +# `terraform state list`, by scripts/select-owned-name.sh, rather than with +# `terraform state list ADDRESS`: that form exits 1 both when the address is +# absent and when the state cannot be read (measured on Terraform 1.15.8), so +# the two cannot be told apart without swallowing an error. +# +# `gcloud services vpc-peerings delete` waits for the operation unless `--async` +# is passed, so its exit status IS the deletion confirmation, the same way +# `gcloud sql instances delete` is in the Cloud SQL script. No `--async` here. +# +# Nothing is swallowed: no `2>/dev/null`, no `|| true`, no `|| echo`. +# +# Exit codes: +# 0 completed, including the "peering no longer in state" case, which is +# what a re-run after a completed cleanup looks like +# 1 the peering is in state but the network it belongs to cannot be read +# from a usable `network_name` output (state is left untouched) +# 2 usage error (wrong arity, a state directory that does not exist, or a +# malformed project id) +# * anything terraform, jq or the selector fails with, unmasked; a failed +# delete exits with gcloud's own status and leaves state untouched + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +PEERING_ADDR="module.networking.google_service_networking_connection.private_vpc_connection" + +if [[ $# -ne 2 ]]; then + echo "usage: $(basename "$0") TERRAFORM_STATE_DIR GCP_PROJECT_ID" >&2 + exit 2 +fi + +STATE_DIR="$1" +PROJECT="$2" + +if [[ ! -d "$STATE_DIR" ]]; then + echo "error: terraform state directory '${STATE_DIR}' does not exist" >&2 + exit 2 +fi + +# Same project id shape check as delete-owned-cloud-sql-instance.sh: a +# whitespace-only or malformed value would otherwise reach `gcloud --project=`. +if ! printf '%s' "$PROJECT" | grep -Eq '^[a-z][a-z0-9-]{5,29}$'; then + echo "error: GCP project id is unset or malformed ('${PROJECT}'); refusing to run VPC peering cleanup" >&2 + exit 2 +fi + +# Unfiltered on purpose: the equality test is the selector's. A failed +# `terraform state list` fails the pipeline under pipefail, and so the step. +IN_STATE="$(terraform -chdir="$STATE_DIR" state list \ + | "${SCRIPT_DIR}/select-owned-name.sh" "$PEERING_ADDR")" + +if [[ -z "$IN_STATE" ]]; then + echo "State no longer tracks ${PEERING_ADDR}; nothing to delete, state left untouched." + exit 0 +fi + +OUTPUTS_JSON="$(terraform -chdir="$STATE_DIR" output -json)" +# On its own line so a non-JSON `terraform output` fails here under `set -e`, +# loudly, instead of reading as an absent output below. +OUTPUTS_TYPE="$(jq -r 'type' <<<"$OUTPUTS_JSON")" +if [[ "$OUTPUTS_TYPE" != "object" ]]; then + echo "error: 'terraform output -json' on '${STATE_DIR}' returned a ${OUTPUTS_TYPE}, not an object" >&2 + exit 1 +fi + +if ! NETWORK="$(jq -er '.network_name.value' <<<"$OUTPUTS_JSON")"; then + echo "error: state '${STATE_DIR}' tracks ${PEERING_ADDR}, but its" >&2 + echo " 'network_name' output is absent or null, so the network the peering" >&2 + echo " belongs to cannot be identified. Refusing to fall back to a hardcoded" >&2 + echo " network name. State left untouched." >&2 + exit 1 +fi + +# RFC 1035, which is what google_compute_network accepts. An empty or +# whitespace-bearing value is a broken output, not a network. +if ! printf '%s' "$NETWORK" | grep -Eq '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'; then + echo "error: state '${STATE_DIR}' publishes 'network_name' as '${NETWORK}', which" >&2 + echo " is not a VPC network name. Inspect" >&2 + echo " 'terraform -chdir=${STATE_DIR} output -json' before destroying anything." >&2 + echo " State left untouched." >&2 + exit 1 +fi + +echo "Deleting the Service Networking peering on network '${NETWORK}'..." +if gcloud services vpc-peerings delete \ + --service=servicenetworking.googleapis.com \ + --network="$NETWORK" \ + --project="$PROJECT" --quiet; then + echo "Service Networking peering on network '${NETWORK}' deleted." +else + rc=$? + echo "error: deleting the Service Networking peering on network '${NETWORK}' failed" >&2 + echo " (exit ${rc}); ${PEERING_ADDR} is left in state so a re-run retries it." >&2 + echo " If the peering is confirmed gone already, remove the address by hand:" >&2 + echo " terraform -chdir=${STATE_DIR} state rm ${PEERING_ADDR}" >&2 + exit "$rc" +fi + +# Only after the delete above returned 0. +terraform -chdir="$STATE_DIR" state rm "$PEERING_ADDR" diff --git a/scripts/lib/code-scan-awk.sh b/scripts/lib/code-scan-awk.sh index eedc142c1..153a730f7 100644 --- a/scripts/lib/code-scan-awk.sh +++ b/scripts/lib/code-scan-awk.sh @@ -3,9 +3,10 @@ # # Shared awk helper functions for the guard suites that scan workflow and shell # sources for what a step actually runs: test-ecr-delete-selection.sh, -# test-rds-deletion-protection-scope.sh, test-aws-tfstate-platform-key.sh and -# test-cloud-sql-delete-scope.sh. Sourced, not executed; it defines one -# variable, AWK_CODE_FUNCS, to be prepended to an awk program. +# test-rds-deletion-protection-scope.sh, test-aws-tfstate-platform-key.sh, +# test-cloud-sql-delete-scope.sh and test-vpc-peering-delete-scope.sh. Sourced, +# not executed; it defines one variable, AWK_CODE_FUNCS, to be prepended to an +# awk program. # # Shared rather than copied because these functions encode the rule that # separates code that RUNS a command from prose that only mentions it, and every @@ -54,7 +55,7 @@ # build_swept_scripts SCRIPTS_DIR # # Sets SWEPT_SCRIPTS to every `*.sh` anywhere under SCRIPTS_DIR, at any depth, -# excluding the four guard suites themselves. +# excluding the guard suites themselves. # # Discovered rather than named file by file, in every suite, because naming the # scripts already known to be guarded is the same defect the suites exist to @@ -96,7 +97,8 @@ build_swept_scripts() { while IFS= read -r -d '' candidate; do case "$(basename "$candidate")" in test-rds-deletion-protection-scope.sh | test-ecr-delete-selection.sh | \ - test-aws-tfstate-platform-key.sh | test-cloud-sql-delete-scope.sh) continue ;; + test-aws-tfstate-platform-key.sh | test-cloud-sql-delete-scope.sh | \ + test-vpc-peering-delete-scope.sh) continue ;; esac SWEPT_SCRIPTS+=("$candidate") done < <(find "$dir" -type f -name '*.sh' -print0 2>/dev/null | sort -z) diff --git a/scripts/test-vpc-peering-delete-scope.sh b/scripts/test-vpc-peering-delete-scope.sh new file mode 100755 index 000000000..0dba44dee --- /dev/null +++ b/scripts/test-vpc-peering-delete-scope.sh @@ -0,0 +1,388 @@ +#!/usr/bin/env bash +# test-vpc-peering-delete-scope.sh +# +# Asserts that the GCP staging destroy deletes the Service Networking peering +# its Terraform state owns, on the network that state names, and removes it +# from state only after that delete succeeded (#330). +# +# The step this guards used to run `terraform state rm` on the module-qualified +# peering address FIRST, then `gcloud services vpc-peerings delete` on the +# literal network `cudly-staging-vpc` with `2>/dev/null || echo ...`. A failed +# or misdirected delete therefore reported success and left a live peering no +# state tracked. Fourth sibling of test-ecr-delete-selection.sh, +# test-rds-deletion-protection-scope.sh and test-cloud-sql-delete-scope.sh. +# +# Exits 0 when all cases pass; exits 1 on any failure. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +WORKFLOW_DIR="${REPO_ROOT}/.github/workflows" + +# code_of() / invokes() / build_swept_scripts, shared with the sibling suites. +# shellcheck source=scripts/lib/code-scan-awk.sh +. "${SCRIPT_DIR}/lib/code-scan-awk.sh" + +DELETE_SCRIPT="${SCRIPT_DIR}/delete-owned-vpc-peering.sh" +DELETE_CMD_RE='gcloud[[:space:]]+services[[:space:]]+vpc-peerings[[:space:]]+delete' +PEERING_ADDR="module.networking.google_service_networking_connection.private_vpc_connection" +# Read by the awk programs through ENVIRON, so a step name never has to be +# spliced into awk source. +export STEP="Delete the Service Networking peering this state owns" +export SQL_STEP="Delete the Cloud SQL instance this state owns" + +pass=0 +fail=0 + +# check LABEL CONDITION_RESULT DETAIL -- CONDITION_RESULT is "0" for a pass. +check() { + if [[ "$2" == "0" ]]; then + echo "PASS: $1" + ((pass++)) || true + else + echo "FAIL: $1" + echo " $3" + ((fail++)) || true + fi +} + +# awk_check LABEL FILE DETAIL AWK_PROGRAM -- the program's exit status decides. +awk_check() { + local label="$1" file="$2" detail="$3" program="$4" rc=0 + if [[ ! -f "$file" ]]; then + check "$label" 1 "file not found at ${file}" + return + fi + awk -v SQ="'" -v cmdre="$DELETE_CMD_RE" "${AWK_CODE_FUNCS}${program}" "$file" || rc=$? + check "$label" "$rc" "$detail" +} + +# --- Terraform: the output names the network the peering is attached to ----- +# +# The script trusts `network_name` to be the peering's network. Break any link +# of that chain and every behavior case below stays green while the real +# delete targets the wrong network (or none). + +# block_has HEADER_RE LINE_RE -> an awk program passing when exactly one line +# inside the block opened by HEADER_RE matches LINE_RE. `[{]`/`[}]` rather than +# bare braces, which start an interval expression in ERE under mawk (CI's awk). +block_has() { + printf '%s' ' + code_of($0) ~ /^'"$1"'[[:space:]]*[{]/ { in_block = 1; next } + in_block && code_of($0) ~ /^[}]/ { in_block = 0 } + in_block && code_of($0) ~ /'"$2"'/ { hits++ } + END { exit !(hits == 1) }' +} + +awk_check "the gcp environment publishes network_name from the networking module" \ + "${REPO_ROOT}/terraform/environments/gcp/outputs.tf" "expected exactly one match" \ + "$(block_has 'output[[:space:]]+"network_name"' 'value[[:space:]]*=[[:space:]]*module[.]networking[.]network_name[[:space:]]*$')" + +awk_check "the networking module publishes the real google_compute_network name" \ + "${REPO_ROOT}/terraform/modules/networking/gcp/outputs.tf" "expected exactly one match" \ + "$(block_has 'output[[:space:]]+"network_name"' 'value[[:space:]]*=[[:space:]]*google_compute_network[.]main[.]name[[:space:]]*$')" + +awk_check "the peering resource is attached to that same network" \ + "${REPO_ROOT}/terraform/modules/networking/gcp/main.tf" "expected exactly one match" \ + "$(block_has 'resource[[:space:]]+"google_service_networking_connection"[[:space:]]+"private_vpc_connection"' 'network[[:space:]]*=[[:space:]]*google_compute_network[.]main[.]id[[:space:]]*$')" + +awk_check "the gcp environment instantiates that module as module.networking" \ + "${REPO_ROOT}/terraform/environments/gcp/networking.tf" "expected exactly one match" \ + "$(block_has 'module[[:space:]]+"networking"' 'source[[:space:]]*=[[:space:]]*"[.][.]\/[.][.]\/modules\/networking\/gcp"')" + +# --- Wiring: the destroy job routes through the script, in order ------------ +# +# Exactly one step with this name, running the script with the pinned state +# directory and project argument. The regex ends at end-of-line, so a trailing +# `|| true` fails it. +CLEANUP="${WORKFLOW_DIR}/cleanup-staging.yml" +awk_check "exactly one '${STEP}' step, and it calls the script" "$CLEANUP" \ + "expected one step running './scripts/delete-owned-vpc-peering.sh terraform/environments/gcp \"\$PROJECT\"' (step renamed, duplicated, call changed, or '|| true' appended)" ' + function finish() { if (in_step) { steps++; if (!has_call) unwired++ } in_step = 0; has_call = 0 } + $0 ~ ("^[[:space:]]*-[[:space:]]+name:[[:space:]]*" ENVIRON["STEP"] "[[:space:]]*$") { finish(); in_step = 1; next } + /^[[:space:]]*-[[:space:]]+name:/ { finish() } + in_step && code_of($0) ~ /^[[:space:]]*[.]\/scripts\/delete-owned-vpc-peering[.]sh[[:space:]]+terraform\/environments\/gcp[[:space:]]+"[$]PROJECT"[[:space:]]*$/ { has_call = 1 } + END { finish(); exit !(steps == 1 && unwired == 0) }' + +# Inside the destroy-gcp job, the peering step sits after the Cloud SQL step +# (the peering cannot go while the instance uses it) and before the destroy +# (which fails on the network while the peering exists). Job keys are the +# two-space-indented `name:` lines under `jobs:`. +awk_check "destroy-gcp runs Cloud SQL delete, then the peering delete, then terraform destroy" "$CLEANUP" \ + "expected the three steps in that order inside the destroy-gcp job" ' + /^ [A-Za-z0-9_-]+:[[:space:]]*$/ { in_job = ($0 ~ /^ destroy-gcp:/) } + in_job && /^[[:space:]]*-[[:space:]]+name:/ { + name = $0; sub(/^[[:space:]]*-[[:space:]]+name:[[:space:]]*/, "", name); sub(/[[:space:]]*$/, "", name) + n++ + if (name == ENVIRON["SQL_STEP"]) sql = n + if (name == ENVIRON["STEP"]) peer = n + if (name == "Terraform Destroy") destroy = n + } + END { exit !(sql > 0 && peer == sql + 1 && destroy == peer + 1) }' + +# --- The script, statically ------------------------------------------------- +# +# Behavior below shows what the script does against stubs; these pin the +# text a stub cannot see, such as a literal network name reintroduced on a +# path the stubs never take. +awk_check "$(basename "$DELETE_SCRIPT") deletes once, on the output-named network, and removes state after" \ + "$DELETE_SCRIPT" \ + "expected: one network_name read via jq -er, one delete with --network=\"\$NETWORK\", no --async, no literal network, one state rm of \"\$PEERING_ADDR\" after the delete" ' + code_of($0) ~ /NETWORK="[$][(]jq[[:space:]]+-er[[:space:]]+.\.network_name\.value./ { reads++ } + invokes($0, cmdre) { deletes++; if (!delete_line) delete_line = FNR } + code_of($0) ~ /--network="[$]NETWORK"/ { by_var++ } + code_of($0) ~ /--network=[^"]/ { literal++ } + code_of($0) ~ /--async/ { async++ } + code_of($0) ~ /cudly-/ { literal++ } + code_of($0) ~ /terraform[[:space:]]+-chdir=[^[:space:]]+[[:space:]]+state[[:space:]]+rm[[:space:]]+"[$]PEERING_ADDR"[[:space:]]*$/ { rms++; rm_line = FNR } + invokes($0, "state[[:space:]]+rm") { any_rm++ } + END { exit !(reads == 1 && deletes == 1 && by_var == 1 && literal == 0 && async == 0 && \ + rms == 1 && any_rm == 1 && rm_line > delete_line) }' + +awk_check "$(basename "$DELETE_SCRIPT") pins the module-qualified peering address" \ + "$DELETE_SCRIPT" "expected PEERING_ADDR=\"${PEERING_ADDR}\" exactly once" ' + code_of($0) ~ /^PEERING_ADDR="module[.]networking[.]google_service_networking_connection[.]private_vpc_connection"$/ { hits++ } + END { exit !(hits == 1) }' + +awk_check "$(basename "$DELETE_SCRIPT") swallows nothing" "$DELETE_SCRIPT" \ + "found 2>/dev/null, || true or || echo in code (#330: the old step swallowed the delete)" ' + code_of($0) ~ /2>[[:space:]]*\/dev\/null/ { bad++ } + code_of($0) ~ /[|][|][[:space:]]*(true|echo|:)/ { bad++ } + END { exit !(bad == 0) }' + +# --- Behavior: the script end to end against stubbed terraform and gcloud -- +# +# Both stubs log to ONE file, so "state rm after delete" is read off a single +# call log. The gcloud stub's delete branch is an ALLOWLIST of the exact +# argument vector: `--async` (which would make the exit status meaningless) or +# any extra flag fails as behavior, not only as text. +STUB_DIR="$(mktemp -d)" +STUB_STATE="$(mktemp -d)" +CALLS="$(mktemp)" +FIXTURE_DIR="$(mktemp -d)" +trap 'rm -rf "$STUB_DIR" "$STUB_STATE" "$FIXTURE_DIR"; rm -f "$CALLS"' EXIT +export CALLS + +cat >"${STUB_DIR}/terraform" <<'EOF' +#!/usr/bin/env bash +echo "terraform $*" >>"$CALLS" +case "$2 $3" in + "state list") + [[ "${STATE_LIST_FAILS:-0}" == "1" ]] && { echo "state list failed" >&2; exit 1; } + printf '%s' "$STATE_LIST" + ;; + "state rm") + [[ "${STATE_RM_FAILS:-0}" == "1" ]] && exit 3 + ;; + output*) + printf '%s' "$TF_OUTPUT_JSON" + ;; +esac +exit 0 +EOF + +cat >"${STUB_DIR}/gcloud" <<'EOF' +#!/usr/bin/env bash +echo "gcloud $*" >>"$CALLS" +if [[ "$#" -ne 7 || "$1 $2 $3" != "services vpc-peerings delete" || \ + "$4" != "--service=servicenetworking.googleapis.com" || "$5" != --network=* || \ + "$6" != --project=* || "$7" != "--quiet" ]]; then + echo "stub: unexpected gcloud invocation: $*" >&2 + exit 99 +fi +[[ "${DELETE_FAILS:-0}" == "1" ]] && { echo "delete failed" >&2; exit 254; } +exit 0 +EOF +chmod +x "${STUB_DIR}/terraform" "${STUB_DIR}/gcloud" + +PROJECT_OK="cudly-staging" +# Not the old literal, so a hardcoded `cudly-staging-vpc` cannot pass. +NETWORK_OK="cudly-renamed-vpc" +OUTPUTS_OK="{\"network_name\":{\"value\":\"${NETWORK_OK}\"},\"database_instance_name\":{\"value\":\"cudly-staging-postgres\"}}" +# The real address among its near-misses, as `terraform state list` prints them. +STATE_WITH_PEERING="module.networking.google_compute_network.main +module.networking.google_compute_global_address.private_ip_address +${PEERING_ADDR} +module.networking.google_vpc_access_connector.main +" + +# run_script ARGS... -> STUB_EXIT, STUB_ERR, and the call log in $CALLS +run_script() { + : >"$CALLS" + local errfile + errfile="$(mktemp)" + STUB_EXIT=0 + PATH="${STUB_DIR}:${PATH}" "$DELETE_SCRIPT" "$@" /dev/null 2>"$errfile" || STUB_EXIT=$? + STUB_ERR="$(cat "$errfile")" + rm -f "$errfile" +} + +count_calls() { grep -c -- "$1" "$CALLS" || true; } +log() { printf 'exit %s, calls: %s' "$STUB_EXIT" "$(tr '\n' ';' <"$CALLS")"; } + +export STATE_LIST="$STATE_WITH_PEERING" TF_OUTPUT_JSON="$OUTPUTS_OK" + +# The golden path, FIRST: every refusal case below is also satisfied by a +# script that never deletes anything. +run_script "$STUB_STATE" "$PROJECT_OK" +check "behavior: golden path exits 0" "$([[ $STUB_EXIT -eq 0 ]] && echo 0 || echo 1)" "$(log)" +check "behavior: exactly one peering delete, on the output-named network and the given project" \ + "$([[ $(count_calls "vpc-peerings delete") -eq 1 ]] && grep -qx "gcloud services vpc-peerings delete --service=servicenetworking.googleapis.com --network=${NETWORK_OK} --project=${PROJECT_OK} --quiet" "$CALLS" && echo 0 || echo 1)" "$(log)" +check "behavior: no call names the old hardcoded network" \ + "$(! grep -q "cudly-staging-vpc" "$CALLS" && echo 0 || echo 1)" "$(log)" +check "behavior: exactly one state rm, of the exact peering address" \ + "$([[ $(count_calls "state rm") -eq 1 ]] && grep -qx "terraform -chdir=${STUB_STATE} state rm ${PEERING_ADDR}" "$CALLS" && echo 0 || echo 1)" "$(log)" +check "behavior: the state rm comes after the delete" \ + "$([[ $(grep -n "vpc-peerings delete" "$CALLS" | cut -d: -f1) -lt $(grep -n "state rm" "$CALLS" | cut -d: -f1) ]] && echo 0 || echo 1)" "$(log)" + +# The #330 defect as behavior: a failed delete fails the step with gcloud's +# own status AND leaves the address in state, so a re-run retries it. +export DELETE_FAILS=1 +run_script "$STUB_STATE" "$PROJECT_OK" +check "behavior: a failed delete exits with gcloud's status and removes no state" \ + "$([[ $STUB_EXIT -eq 254 && $(count_calls "state rm") -eq 0 ]] && echo 0 || echo 1)" "$(log)" +check "behavior: the failed-delete error names the address left in state" \ + "$([[ "$STUB_ERR" == *"left in state"* && "$STUB_ERR" == *"$PEERING_ADDR"* ]] && echo 0 || echo 1)" "stderr: ${STUB_ERR}" +unset DELETE_FAILS + +export STATE_RM_FAILS=1 +run_script "$STUB_STATE" "$PROJECT_OK" +check "behavior: a failed state rm fails the step after exactly one delete" \ + "$([[ $STUB_EXIT -ne 0 && $(count_calls "vpc-peerings delete") -eq 1 ]] && echo 0 || echo 1)" "$(log)" +unset STATE_RM_FAILS + +# Not in state: a re-run after a completed cleanup. Near-misses of the address +# (prefix, suffix, other module, leading space) must not count as present. +export STATE_LIST="module.networking.google_compute_network.main +${PEERING_ADDR}_old +module.networking_old.google_service_networking_connection.private_vpc_connection + ${PEERING_ADDR} +module.networking.google_service_networking_connection.private_vpc_connection[0] +" +run_script "$STUB_STATE" "$PROJECT_OK" +check "behavior: peering not in state (only near-miss addresses) exits 0, deletes nothing, removes nothing" \ + "$([[ $STUB_EXIT -eq 0 && $(count_calls "^gcloud") -eq 0 && $(count_calls "state rm") -eq 0 ]] && echo 0 || echo 1)" "$(log)" + +export STATE_LIST="" +run_script "$STUB_STATE" "$PROJECT_OK" +check "behavior: an empty state exits 0 without calling gcloud" \ + "$([[ $STUB_EXIT -eq 0 && $(count_calls "^gcloud") -eq 0 ]] && echo 0 || echo 1)" "$(log)" + +# The address on an unterminated final line is still found. +export STATE_LIST="module.networking.google_compute_network.main +${PEERING_ADDR}" +run_script "$STUB_STATE" "$PROJECT_OK" +check "behavior: the address on an unterminated last line is still deleted" \ + "$([[ $STUB_EXIT -eq 0 && $(count_calls "vpc-peerings delete") -eq 1 ]] && echo 0 || echo 1)" "$(log)" + +export STATE_LIST="$STATE_WITH_PEERING" STATE_LIST_FAILS=1 +run_script "$STUB_STATE" "$PROJECT_OK" +check "behavior: an unreadable state fails the step without calling gcloud" \ + "$([[ $STUB_EXIT -ne 0 && $(count_calls "^gcloud") -eq 0 && $(count_calls "state rm") -eq 0 ]] && echo 0 || echo 1)" "$(log)" +unset STATE_LIST_FAILS + +# In state, but the network cannot be named: refuse, and leave state alone. +# Each value is its own failure mode; the last one is flag-shaped. +while IFS='|' read -r label outputs; do + [[ -n "$label" ]] || continue + export TF_OUTPUT_JSON="$outputs" + run_script "$STUB_STATE" "$PROJECT_OK" + check "behavior: ${label} fails without calling gcloud or state rm" \ + "$([[ $STUB_EXIT -ne 0 && $(count_calls "^gcloud") -eq 0 && $(count_calls "state rm") -eq 0 ]] && echo 0 || echo 1)" "$(log)" +done <<'EOF' +non-JSON terraform output|Warning: something +a JSON array as terraform output|[] +no outputs at all|{} +network_name absent|{"database_instance_name":{"value":"cudly-staging-postgres"}} +network_name null|{"network_name":{"value":null}} +network_name empty|{"network_name":{"value":""}} +network_name with whitespace|{"network_name":{"value":"cudly staging vpc"}} +network_name upper case|{"network_name":{"value":"Cudly-Staging-VPC"}} +network_name flag-shaped|{"network_name":{"value":"--project=other"}} +EOF +export TF_OUTPUT_JSON="$OUTPUTS_OK" + +for bad_project in "" " " "Cudly-Staging" "cudly"; do + run_script "$STUB_STATE" "$bad_project" + check "behavior: malformed project '${bad_project}' exits 2 without calling terraform or gcloud" \ + "$([[ $STUB_EXIT -eq 2 && ! -s "$CALLS" ]] && echo 0 || echo 1)" "$(log)" +done + +run_script +check "behavior: no arguments exits 2" "$([[ $STUB_EXIT -eq 2 ]] && echo 0 || echo 1)" "$(log)" +run_script "$STUB_STATE" "$PROJECT_OK" extra +check "behavior: three arguments exits 2" "$([[ $STUB_EXIT -eq 2 ]] && echo 0 || echo 1)" "$(log)" +run_script "${STUB_STATE}/does-not-exist" "$PROJECT_OK" +check "behavior: a missing state directory exits 2" "$([[ $STUB_EXIT -eq 2 ]] && echo 0 || echo 1)" "$(log)" + +# --- Sweep: the script is the only place that deletes a peering ------------- +# +# Keyed on the dangerous command, not on a file name, over every workflow and +# a recursive glob of scripts/, so a new inline delete (the #330 shape) is +# caught wherever it is added. + +# sweep_sites FILE... -> one "FILE:LINE" per invocation, then "total N" +sweep_sites() { + awk -v SQ="'" -v cmdre="$DELETE_CMD_RE" "$AWK_CODE_FUNCS"' + invokes($0, cmdre) { print FILENAME ":" FNR; total++ } + END { print "total " total + 0 }' "$@" +} + +# assert_sweep LABEL EXPECTED_SITES_EXCEPT_SCRIPT EXPECTED_TOTAL FILE... +assert_sweep() { + local label="$1" expected_other="$2" expected_total="$3" report other + shift 3 + report="$(sweep_sites "$@")" + # `|| true` only on grep, whose exit 1 means "no other site": the clean case. + other="$({ grep -v -e "^total " -e "^${DELETE_SCRIPT}:" <<<"$report" || true; } | sed 's/:[0-9]*$//' | sort -u | tr '\n' ' ')" + check "$label" \ + "$([[ "$other" == "$expected_other" && "$report" == *"total ${expected_total}" ]] && echo 0 || echo 1)" \ + "expected other sites '${expected_other}' and total ${expected_total}, got: $(tr '\n' ' ' <<<"$report")" +} + +build_swept_scripts "$SCRIPT_DIR" +shopt -s nullglob +WORKFLOWS=("${WORKFLOW_DIR}"/*.yml "${WORKFLOW_DIR}"/*.yaml) +shopt -u nullglob + +swept_has_script=1 +# `${arr[@]+...}`: bash 3.2 (macOS) treats an empty "${arr[@]}" as unbound under set -u. +for f in ${SWEPT_SCRIPTS[@]+"${SWEPT_SCRIPTS[@]}"}; do [[ "$f" == "$DELETE_SCRIPT" ]] && swept_has_script=0; done +check "the swept set holds workflows and includes $(basename "$DELETE_SCRIPT")" \ + "$([[ ${#WORKFLOWS[@]} -gt 0 && $swept_has_script -eq 0 ]] && echo 0 || echo 1)" \ + "${#WORKFLOWS[@]} workflow(s); an empty swept set has no violations either" + +# Exactly one site in the whole tree, and it is the script: zero would mean the +# sweep stopped recognizing the command, which passes vacuously. +assert_sweep "every 'gcloud services vpc-peerings delete' in .github/workflows and scripts/ is the script's" \ + "" 1 ${WORKFLOWS[@]+"${WORKFLOWS[@]}"} ${SWEPT_SCRIPTS[@]+"${SWEPT_SCRIPTS[@]}"} + +# The sweep itself, over fixtures. The unwired one is the pre-fix step from +# origin/main with its two comment lines dropped. +cat >"${FIXTURE_DIR}/unwired.yml" <<'EOF' + - name: Delete the Cloud SQL instance this state owns + run: | + ./scripts/delete-owned-cloud-sql-instance.sh terraform/environments/gcp "$PROJECT" + cd terraform/environments/gcp + terraform state rm module.networking.google_service_networking_connection.private_vpc_connection 2>/dev/null || true + gcloud services vpc-peerings delete \ + --service=servicenetworking.googleapis.com \ + --network=cudly-staging-vpc \ + --project="$PROJECT" --quiet 2>/dev/null || echo "VPC peering already gone or not found" +EOF +cat >"${FIXTURE_DIR}/prose.yml" <<'EOF' + - name: Mentions the command without running it + run: | + # every `gcloud services vpc-peerings delete` goes through the script + echo "gcloud services vpc-peerings delete is not run here" +EOF + +assert_sweep "an inline peering delete (the pre-fix step) is flagged" \ + "${FIXTURE_DIR}/unwired.yml " 1 "${FIXTURE_DIR}/unwired.yml" +assert_sweep "a step that only mentions the command is not a delete site" \ + "" 0 "${FIXTURE_DIR}/prose.yml" + +echo +echo "passed: ${pass}, failed: ${fail}" +[[ "$fail" -eq 0 ]] From 46df186295a3c47214aa80fe3fa5edcc2ae8fe04 Mon Sep 17 00:00:00 2001 From: Astrid Gealer Date: Sun, 25 Oct 2026 23:58:45 +0000 Subject: [PATCH 2/2] fix(ci): run terraform destroy when the GCP peering delete fails (#330) The peering step added for #330 sat before `Terraform Destroy` with no `if:`, so a failed peering delete (most likely GCP still releasing the peering right after the Cloud SQL delete) skipped the destroy and left the whole staging stack running and billing. Before #1144 the destroy still removed everything except the network. The peering step is now continue-on-error with id delete-peering, so the destroy runs after it either way. A new last step, gated on `!cancelled() && steps.delete-peering.outcome == 'failure'`, fails the job so the failure stays visible and is not swallowed. The script is unchanged: it still removes the address from state only after a successful delete, so the destroy keeps tracking a peering whose delete failed. test-vpc-peering-delete-scope.sh gains a small model of the runner's step semantics for destroy-gcp. It pins that a failed peering delete still runs the destroy and fails the job, that a failed destroy does not hide the peering failure, and that a failed Cloud SQL delete still skips the destroy. Five mutants of the real workflow check that the model is not vacuous. The existing failed-delete behavior case still pins that no state is removed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/cleanup-staging.yml | 20 ++++ scripts/test-vpc-peering-delete-scope.sh | 143 ++++++++++++++++++++++- 2 files changed, 161 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cleanup-staging.yml b/.github/workflows/cleanup-staging.yml index c57a041d5..64ece9153 100644 --- a/.github/workflows/cleanup-staging.yml +++ b/.github/workflows/cleanup-staging.yml @@ -413,7 +413,17 @@ jobs: # this inline: state rm first, delete on a hardcoded `cudly-staging-vpc`, # and the delete's failure swallowed, which left a live peering untracked # (#330). Rationale: the script's header. + # + # continue-on-error so a failed delete does not skip `Terraform Destroy`: + # the likely failure is GCP still releasing the peering just after the + # Cloud SQL delete, and skipping destroy would leave the whole staging + # stack running (and billing) instead of only the network. The script + # leaves the address in state when the delete fails, so the destroy below + # still tracks the peering. The failure is NOT swallowed: the last step + # fails the job when this step's outcome is failure. - name: Delete the Service Networking peering this state owns + id: delete-peering + continue-on-error: true env: GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} run: | @@ -431,3 +441,13 @@ jobs: terraform destroy \ -var-file="github-staging.tfvars" \ -auto-approve + + # Surfaces the peering step's failure, which continue-on-error turned + # into a passing step so that `Terraform Destroy` above still ran. + # !cancelled() rather than the implicit success(), so it still reports + # when the destroy failed too. + - name: Fail if the peering delete failed + if: ${{ !cancelled() && steps.delete-peering.outcome == 'failure' }} + run: | + echo "::error::Deleting the Service Networking peering failed; its address is still in Terraform state. Terraform Destroy ran anyway. See the 'Delete the Service Networking peering this state owns' step log, then re-run this job." + exit 1 diff --git a/scripts/test-vpc-peering-delete-scope.sh b/scripts/test-vpc-peering-delete-scope.sh index 0dba44dee..fc9a769d2 100755 --- a/scripts/test-vpc-peering-delete-scope.sh +++ b/scripts/test-vpc-peering-delete-scope.sh @@ -9,8 +9,10 @@ # peering address FIRST, then `gcloud services vpc-peerings delete` on the # literal network `cudly-staging-vpc` with `2>/dev/null || echo ...`. A failed # or misdirected delete therefore reported success and left a live peering no -# state tracked. Fourth sibling of test-ecr-delete-selection.sh, -# test-rds-deletion-protection-scope.sh and test-cloud-sql-delete-scope.sh. +# state tracked. It also asserts that a failed peering delete still lets +# `terraform destroy` run and still fails the job. Fourth sibling of +# test-ecr-delete-selection.sh, test-rds-deletion-protection-scope.sh and +# test-cloud-sql-delete-scope.sh. # # Exits 0 when all cases pass; exits 1 on any failure. @@ -121,6 +123,143 @@ awk_check "destroy-gcp runs Cloud SQL delete, then the peering delete, then terr } END { exit !(sql > 0 && peer == sql + 1 && destroy == peer + 1) }' +# --- Wiring: a failed peering delete still destroys, and still fails the job -- +# +# Before the peering step existed, a failed peering delete only cost the +# network: `terraform destroy` had already removed everything else. A plain +# failing step would skip the destroy and leave the whole stack up (and +# billing). So the peering step is continue-on-error, and a later step fails +# the job on its outcome, so the failure is not swallowed. The script itself +# leaves the address in state on a failed delete (the behavior cases below). +# +# simulate_destroy_gcp FILE FAILING_STEP_NAMES -> one "ran NAME" / "skipped +# NAME" per destroy-gcp step, then "job success|failure". A small model of the +# runner's step semantics, for the subset of `if:` this job may use: +# success(), failure(), always(), cancelled(), !cancelled() and +# steps.ID.outcome|conclusion ==/!= 'VALUE', joined by `&&`. An `if:` with no +# status function gets the runner's implicit success(). A step fails when it +# is named in FAILING_STEP_NAMES (`|`-separated) or when the last code line of +# its run block is `exit 1`. continue-on-error makes a failed step's +# conclusion success while its outcome stays failure. Anything outside that +# subset (an `||`, a multi-line `if:`, an unknown function) is an error, so +# the model cannot silently misread a changed condition. +simulate_destroy_gcp() { + FAILING="$2" awk -v SQ="'" "${AWK_CODE_FUNCS}"' + function trim(s) { sub(/^[[:space:]]+/, "", s); sub(/[[:space:]]+$/, "", s); return s } + function unsupported(what) { print "unsupported: " what; bad = 1; exit 2 } + function value_of(line) { sub(/^[[:space:]]*(-[[:space:]]+)?[A-Za-z_-]+:[[:space:]]*/, "", line); return trim(code_of(line)) } + /^ [A-Za-z0-9_-]+:[[:space:]]*$/ { in_job = ($0 ~ /^ destroy-gcp:/); in_run = 0; next } + !in_job { next } + /^ -[[:space:]]/ { n++; in_run = 0 } + n == 0 { next } + in_run && /^[[:space:]]*$/ { next } + in_run && /^ / { c = trim(code_of($0)); if (c != "") last[n] = c; next } + { in_run = 0 } + /^ -[[:space:]]+name:/ || /^ name:/ { name[n] = value_of($0) } + /^ id:/ { id[n] = value_of($0) } + /^ if:/ { cond[n] = value_of($0); if (cond[n] ~ /^[|>]/) unsupported("multi-line if: in step " n) } + /^ continue-on-error:/ { + v = value_of($0) + if (v != "true" && v != "false") unsupported("continue-on-error: " v) + coe[n] = (v == "true") + } + /^ run:[[:space:]]*[|]/ { in_run = 1; next } + /^ run:/ { last[n] = value_of($0) } + END { + if (bad) exit 2 + nf = split(ENVIRON["FAILING"], f, "|") + for (i = 1; i <= nf; i++) failing[f[i]] = 1 + job_ok = 1 + for (i = 1; i <= n; i++) { + e = cond[i] + sub(/^[$][{][{]/, "", e); sub(/[}][}]$/, "", e); e = trim(e) + run = 1; has_status = 0 + if (e != "") { + nt = split(e, t, /&&/) + for (k = 1; k <= nt; k++) { + term = trim(t[k]) + if (term == "success()") { has_status = 1; r = job_ok } + else if (term == "failure()") { has_status = 1; r = !job_ok } + else if (term == "always()") { has_status = 1; r = 1 } + else if (term == "!cancelled()") { has_status = 1; r = 1 } + else if (term == "cancelled()") { has_status = 1; r = 0 } + else if (term ~ ("^steps[.][A-Za-z0-9_-]+[.](outcome|conclusion)[[:space:]]*[!=]=[[:space:]]*" SQ "(success|failure|skipped|cancelled)" SQ "$")) { + split(term, ref, /[.]/); sid = ref[2] + field = (term ~ /[.]outcome/) ? "outcome" : "conclusion" + want = term; sub("^[^" SQ "]*" SQ, "", want); sub(SQ "$", "", want) + got = (field == "outcome") ? out_by_id[sid] : concl_by_id[sid] + r = (term ~ /!=/) ? (got != want) : (got == want) + } + else unsupported("if term: " term) + run = run && r + } + } + if (!has_status) run = run && job_ok + if (run) { + fails = (name[i] in failing) || (last[i] ~ /^exit[[:space:]]+1$/) + o = fails ? "failure" : "success" + cl = (fails && !coe[i]) ? "failure" : "success" + if (cl == "failure") job_ok = 0 + print "ran " name[i] + } else { + o = "skipped"; cl = "skipped" + print "skipped " name[i] + } + if (id[i] != "") { out_by_id[id[i]] = o; concl_by_id[id[i]] = cl } + } + print "job " (job_ok ? "success" : "failure") + }' "$1" +} + +# destroy_gcp_failure_handling_rc FILE -> 0 when every scenario below holds. +destroy_gcp_failure_handling_rc() { + local file="$1" out + # Nothing fails: the job passes and destroy runs. + out="$(simulate_destroy_gcp "$file" "")" || return 1 + [[ "$out" == *$'\nran Terraform Destroy\n'* && "$out" == *$'\njob success' ]] || return 1 + # The peering delete fails: destroy still runs, and the job fails. + out="$(simulate_destroy_gcp "$file" "$STEP")" || return 1 + [[ "$out" == *$'\nran Terraform Destroy\n'* && "$out" == *$'\njob failure' ]] || return 1 + # Both fail: the job still fails (the reporting step is not skipped by the + # destroy failure, so the peering failure stays named in the log). + out="$(simulate_destroy_gcp "$file" "${STEP}|Terraform Destroy")" || return 1 + [[ "$out" == *$'\njob failure' && "$(grep -c '^ran ' <<<"$out")" -eq "$(grep -c '^ran \|^skipped ' <<<"$out")" ]] || return 1 + # The Cloud SQL delete fails: destroy stays skipped (it deadlocks while the + # instance exists), so the fix above did not become an `always()` on destroy. + out="$(simulate_destroy_gcp "$file" "$SQL_STEP")" || return 1 + [[ "$out" == *$'\nskipped Terraform Destroy\n'* && "$out" == *$'\njob failure' ]] || return 1 +} + +check "destroy-gcp: a failed peering delete still runs Terraform Destroy and still fails the job" \ + "$(destroy_gcp_failure_handling_rc "$CLEANUP" && echo 0 || echo 1)" \ + "simulated: none -> $(simulate_destroy_gcp "$CLEANUP" "" | tr '\n' ';') peering -> $(simulate_destroy_gcp "$CLEANUP" "$STEP" | tr '\n' ';')" + +# The model, against mutants of the real workflow, so the check above cannot +# pass vacuously. Each mutant is one plausible regression. +MUTANT_DIR="$(mktemp -d)" +mutant_flagged() { + local label="$1" program="$2" m="${MUTANT_DIR}/cleanup-staging.yml" + awk "$program" "$CLEANUP" >"$m" + if cmp -s "$m" "$CLEANUP"; then + check "mutant flagged: ${label}" 1 "the mutation did not change the workflow; update the mutation" + return + fi + check "mutant flagged: ${label}" "$(destroy_gcp_failure_handling_rc "$m" && echo 1 || echo 0)" \ + "the mutant passed: $(simulate_destroy_gcp "$m" "$STEP" | tr '\n' ';')" +} +# awk rather than sed: `a\` and in-place forms differ between GNU and BSD sed. +mutant_flagged "peering step without continue-on-error (destroy skipped)" \ + '$0 != " continue-on-error: true"' +mutant_flagged "failure-reporting step removed (failure swallowed)" \ + '$0 == " - name: Fail if the peering delete failed" { stop = 1 } !stop' +mutant_flagged "peering step id renamed (reporting step never fires)" \ + '{ if ($0 == " id: delete-peering") $0 = " id: peering"; print }' +mutant_flagged "reporting step on the implicit success() (silent when destroy also fails)" \ + '{ sub(/[!]cancelled[(][)] && /, ""); print }' +mutant_flagged "Terraform Destroy on always() (runs over a live Cloud SQL instance)" \ + '{ print } $0 == " - name: Terraform Destroy" { print " if: always()" }' +rm -rf "$MUTANT_DIR" + # --- The script, statically ------------------------------------------------- # # Behavior below shows what the script does against stubs; these pin the