From 96fac4b208de373e31c3048d98f4f293d54c1637 Mon Sep 17 00:00:00 2001 From: Astrid Gealer Date: Mon, 19 Oct 2026 04:48:39 +0100 Subject: [PATCH 1/5] feat(iac/aws): add opt-in multi_az to the RDS database module The AWS database module had no way to express a Multi-AZ standby, so the AWS environment could only ever run a single-AZ instance while GCP (REGIONAL) and Azure (ZoneRedundant) prod are already HA. Add a multi_az module variable (default false) passed to aws_db_instance.main, and a database_multi_az environment variable (default false) wired through database.tf and reported in deployment_info. The default keeps dev/staging unchanged: an explicit multi_az = false matches the provider's existing value, so it plans no RDS modification there. A terraform test (mock providers, plan only) pins the default and that true reaches the instance; CI now runs it, and tests-only changes to the module no longer trigger the Lambda deploy. A Go guard pins the environment wiring. Refs #719. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 9 ++++ .github/workflows/deploy-aws-lambda.yml | 1 + terraform/environments/aws/database.tf | 3 ++ .../aws/database_ha_guard_test.go | 46 +++++++++++++++++++ terraform/environments/aws/outputs.tf | 1 + terraform/environments/aws/variables.tf | 8 +++- terraform/modules/database/aws/main.tf | 4 ++ .../database/aws/tests/multi_az.tftest.hcl | 37 +++++++++++++++ terraform/modules/database/aws/variables.tf | 8 +++- 9 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 terraform/environments/aws/database_ha_guard_test.go create mode 100644 terraform/modules/database/aws/tests/multi_az.tftest.hcl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b885e287e..3d797ebb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -613,6 +613,15 @@ jobs: terraform -chdir="$d" test done + # multi_az must default off and reach aws_db_instance.main (#719). + - name: Test AWS database module + if: matrix.cloud == 'aws' + run: | + d=terraform/modules/database/aws + terraform -chdir="$d" init -backend=false -lockfile=readonly -input=false + terraform -chdir="$d" fmt -check -recursive + terraform -chdir="$d" test + # The post-deploy /health gate and the GCP uptime check must read the # body's top-level "status", not just the 200 (#649). - name: Test /health gate modules diff --git a/.github/workflows/deploy-aws-lambda.yml b/.github/workflows/deploy-aws-lambda.yml index 388a41d98..7cc846fba 100644 --- a/.github/workflows/deploy-aws-lambda.yml +++ b/.github/workflows/deploy-aws-lambda.yml @@ -52,6 +52,7 @@ on: - 'terraform/modules/compute/aws/lambda/**' - '!terraform/modules/compute/aws/lambda/tests/**' - 'terraform/modules/database/aws/**' + - '!terraform/modules/database/aws/tests/**' - 'terraform/modules/secrets/aws/**' - 'terraform/modules/networking/aws/**' - 'terraform/environments/aws/**' diff --git a/terraform/environments/aws/database.tf b/terraform/environments/aws/database.tf index d57c935ce..383c38787 100644 --- a/terraform/environments/aws/database.tf +++ b/terraform/environments/aws/database.tf @@ -20,6 +20,9 @@ module "database" { allocated_storage = var.database_allocated_storage max_allocated_storage = var.database_max_allocated_storage + # High availability + multi_az = var.database_multi_az + # Networking vpc_id = module.networking.vpc_id vpc_cidr = var.vpc_cidr diff --git a/terraform/environments/aws/database_ha_guard_test.go b/terraform/environments/aws/database_ha_guard_test.go new file mode 100644 index 000000000..33e633506 --- /dev/null +++ b/terraform/environments/aws/database_ha_guard_test.go @@ -0,0 +1,46 @@ +// Regression test for issue #719: the AWS prod database was a single-AZ +// burstable db.t4g.micro and the database module could not express Multi-AZ, +// while GCP (REGIONAL) and Azure (ZoneRedundant) prod were already HA. The +// module half is covered by terraform/modules/database/aws/tests; this pins +// the environment wiring. +package aws_test + +import ( + "os" + "path/filepath" + "regexp" + "testing" +) + +func TestAWSDatabaseMultiAZIsWired(t *testing.T) { + cases := []struct { + file string + re *regexp.Regexp + what string + }{ + { + file: "variables.tf", + re: regexp.MustCompile(`variable\s+"database_multi_az"\s*\{[^}]*type\s*=\s*bool[^}]*default\s*=\s*false`), + what: `variable "database_multi_az" (bool, default false)`, + }, + { + file: "database.tf", + re: regexp.MustCompile(`(?m)^\s*multi_az\s*=\s*var\.database_multi_az\s*$`), + what: "multi_az = var.database_multi_az on module \"database\"", + }, + { + file: filepath.Join("..", "..", "modules", "database", "aws", "main.tf"), + re: regexp.MustCompile(`resource\s+"aws_db_instance"\s+"main"\s*\{[\s\S]*?\n\s*multi_az\s*=\s*var\.multi_az\s*\n`), + what: `multi_az = var.multi_az on aws_db_instance.main`, + }, + } + for _, c := range cases { + data, err := os.ReadFile(c.file) + if err != nil { + t.Fatalf("reading %s: %v", c.file, err) + } + if !c.re.Match(data) { + t.Errorf("%s: missing %s", c.file, c.what) + } + } +} diff --git a/terraform/environments/aws/outputs.tf b/terraform/environments/aws/outputs.tf index b2b1d22b9..aa4de9d33 100644 --- a/terraform/environments/aws/outputs.tf +++ b/terraform/environments/aws/outputs.tf @@ -207,6 +207,7 @@ output "deployment_info" { nat_enabled = false # Using IPv6 dual-stack, no NAT Gateway needed vpc_endpoints = false # Not needed with IPv6 db_instance_class = var.database_instance_class + db_multi_az = var.database_multi_az db_storage_gb = var.database_allocated_storage } } diff --git a/terraform/environments/aws/variables.tf b/terraform/environments/aws/variables.tf index ce8c7d12d..b374108da 100644 --- a/terraform/environments/aws/variables.tf +++ b/terraform/environments/aws/variables.tf @@ -128,11 +128,17 @@ variable "database_engine_version" { } variable "database_instance_class" { - description = "RDS instance class" + description = "RDS instance class. Burstable db.t* classes throttle once their CPU credits run out under sustained load, so use a non-burstable class for production." type = string default = "db.t4g.micro" } +variable "database_multi_az" { + description = "Provision the RDS instance as Multi-AZ (standby in a second AZ, automatic failover). Roughly doubles the instance cost, so it is off by default; enable it for production." + type = bool + default = false +} + variable "database_allocated_storage" { description = "Initial allocated storage in GB" type = number diff --git a/terraform/modules/database/aws/main.tf b/terraform/modules/database/aws/main.tf index 97ae7b31d..2a234e34e 100644 --- a/terraform/modules/database/aws/main.tf +++ b/terraform/modules/database/aws/main.tf @@ -129,6 +129,10 @@ resource "aws_db_instance" "main" { auto_minor_version_upgrade = true instance_class = var.instance_class + # Multi-AZ standby in a second AZ with synchronous replication and automatic + # failover (#719). Opt-in: it roughly doubles the instance cost. + multi_az = var.multi_az + db_name = var.database_name username = var.master_username password = local.db_password diff --git a/terraform/modules/database/aws/tests/multi_az.tftest.hcl b/terraform/modules/database/aws/tests/multi_az.tftest.hcl new file mode 100644 index 000000000..8a1fd0237 --- /dev/null +++ b/terraform/modules/database/aws/tests/multi_az.tftest.hcl @@ -0,0 +1,37 @@ +# Multi-AZ support (#719). The module could not express a standby replica, so +# prod on AWS had no HA while GCP (REGIONAL) and Azure (ZoneRedundant) did. +# The environment wiring is pinned by TestAWSDatabaseMultiAZIsWired in +# terraform/environments/aws. +mock_provider "aws" {} +mock_provider "random" {} + +variables { + stack_name = "cudly-test" + permissions_boundary_arn = "arn:aws:iam::123456789012:policy/cudly-boundary" + vpc_id = "vpc-0123456789abcdef0" + vpc_cidr = "10.0.0.0/16" + private_subnet_ids = ["subnet-0aaaaaaaaaaaaaaaa", "subnet-0bbbbbbbbbbbbbbbb"] + admin_email = "admin@example.com" +} + +run "single_az_by_default" { + command = plan + + assert { + condition = aws_db_instance.main.multi_az == false + error_message = "multi_az must default to false so dev/staging keep their single-AZ cost profile." + } +} + +run "multi_az_reaches_the_instance" { + command = plan + + variables { + multi_az = true + } + + assert { + condition = aws_db_instance.main.multi_az == true + error_message = "multi_az = true must provision a Multi-AZ standby on aws_db_instance.main." + } +} diff --git a/terraform/modules/database/aws/variables.tf b/terraform/modules/database/aws/variables.tf index 9f5f6ad1d..9f0ab686a 100644 --- a/terraform/modules/database/aws/variables.tf +++ b/terraform/modules/database/aws/variables.tf @@ -61,11 +61,17 @@ variable "engine_version" { } variable "instance_class" { - description = "RDS instance class (e.g., db.t4g.micro, db.t4g.small)" + description = "RDS instance class (e.g., db.t4g.micro, db.m7g.large). Burstable db.t* classes run on CPU credits and throttle once those are spent under sustained load, so use a non-burstable class for production." type = string default = "db.t4g.micro" } +variable "multi_az" { + description = "Provision a Multi-AZ standby with automatic failover. Roughly doubles the instance cost; enable for production. The DB subnet group must span at least two AZs." + type = bool + default = false +} + variable "allocated_storage" { description = "Initial allocated storage in GB" type = number From 2346682cf493448d9a7395d06beffc3b7db5da58 Mon Sep 17 00:00:00 2001 From: Astrid Gealer Date: Mon, 19 Oct 2026 04:49:21 +0100 Subject: [PATCH 2/5] chore(iac/aws): run prod RDS as Multi-AZ on a non-burstable class Set database_multi_az = true and database_instance_class = "db.m7g.large" (2 vCPU / 8 GiB, matching GCP prod db-custom-2-7680 REGIONAL and Azure prod GP_Standard_D2s_v3 ZoneRedundant) in github-prod.tfvars and the prod profile example, and pin both with a Go guard so prod cannot fall back to the single-AZ burstable db.t4g.micro. This raises the AWS prod database cost and is an owner decision: AWS prod is not provisioned yet, so it only takes effect on prod's first apply. Kept as its own commit so it can be dropped without touching the opt-in support. Closes #719. Co-Authored-By: Claude Opus 5.5 --- .../aws/database_ha_guard_test.go | 40 ++++++++++++++++++- terraform/environments/aws/github-prod.tfvars | 5 +++ terraform/profiles/aws/prod.tfvars.example | 2 + 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/terraform/environments/aws/database_ha_guard_test.go b/terraform/environments/aws/database_ha_guard_test.go index 33e633506..d39292345 100644 --- a/terraform/environments/aws/database_ha_guard_test.go +++ b/terraform/environments/aws/database_ha_guard_test.go @@ -2,16 +2,54 @@ // burstable db.t4g.micro and the database module could not express Multi-AZ, // while GCP (REGIONAL) and Azure (ZoneRedundant) prod were already HA. The // module half is covered by terraform/modules/database/aws/tests; this pins -// the environment wiring. +// the environment wiring and the prod values. package aws_test import ( "os" "path/filepath" "regexp" + "strings" "testing" ) +// tfvarsValue returns the right-hand side of a top-level `name = value` +// assignment in a tfvars file, ignoring comment lines and trailing comments. +func tfvarsValue(t *testing.T, path, name string) (string, bool) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + re := regexp.MustCompile(`^` + regexp.QuoteMeta(name) + `\s*=\s*(.*)$`) + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if i := strings.Index(line, "#"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + if m := re.FindStringSubmatch(line); m != nil { + return strings.Trim(strings.TrimSpace(m[1]), `"`), true + } + } + return "", false +} + +func TestAWSProdDatabaseIsHighlyAvailable(t *testing.T) { + const prod = "github-prod.tfvars" + + if v, ok := tfvarsValue(t, prod, "database_multi_az"); !ok || v != "true" { + t.Errorf("%s: database_multi_az = %q (set: %v), want true: prod RDS needs a standby in a second AZ", prod, v, ok) + } + + class, ok := tfvarsValue(t, prod, "database_instance_class") + if !ok { + t.Fatalf("%s: database_instance_class not set, so prod falls back to the burstable db.t4g.micro default", prod) + } + if regexp.MustCompile(`^db\.t\d`).MatchString(class) { + t.Errorf("%s: database_instance_class = %q is burstable (CPU credits); prod needs a non-burstable class", prod, class) + } +} + func TestAWSDatabaseMultiAZIsWired(t *testing.T) { cases := []struct { file string diff --git a/terraform/environments/aws/github-prod.tfvars b/terraform/environments/aws/github-prod.tfvars index d19ff7d2f..db7c30aa3 100644 --- a/terraform/environments/aws/github-prod.tfvars +++ b/terraform/environments/aws/github-prod.tfvars @@ -41,10 +41,15 @@ fargate_max_capacity = 20 # ============================================== # Database (RDS PostgreSQL) # ============================================== +# Non-burstable and Multi-AZ (#719): 2 vCPU / 8 GiB with a standby in a second +# AZ, matching GCP prod (db-custom-2-7680, REGIONAL) and Azure prod +# (GP_Standard_D2s_v3, ZoneRedundant). database_name = "cudly" database_username = "cudly" database_engine_version = "16" +database_instance_class = "db.m7g.large" +database_multi_az = true database_backup_retention_days = 30 database_deletion_protection = true database_skip_final_snapshot = false diff --git a/terraform/profiles/aws/prod.tfvars.example b/terraform/profiles/aws/prod.tfvars.example index 659716676..8ff836abf 100644 --- a/terraform/profiles/aws/prod.tfvars.example +++ b/terraform/profiles/aws/prod.tfvars.example @@ -31,6 +31,8 @@ admin_email = "admin@example.com" database_name = "cudly" database_username = "cudly" database_engine_version = "16.6" +database_instance_class = "db.m7g.large" # Non-burstable (#719) +database_multi_az = true # Standby in a second AZ (#719) database_backup_retention_days = 30 # Longer retention for prod database_deletion_protection = true # Protect against deletion database_skip_final_snapshot = false From f546a7a9a7b84a73bee77d937900ae2eb8e41ba0 Mon Sep 17 00:00:00 2001 From: Astrid Gealer Date: Mon, 19 Oct 2026 11:25:47 +0100 Subject: [PATCH 3/5] build(terraform): record linux provider hashes in the database module lock file The AWS database module's lock file held only the darwin h1 hashes for hashicorp/aws 5.100.0 and hashicorp/random 3.8.1. On the Linux CI runner `init -lockfile=readonly` therefore only warned, and the new `terraform test` step then rejected the installed providers as not matching the lock file. Recorded with `terraform providers lock` for linux_amd64, darwin_arm64 and darwin_amd64, the same set the compute module lock files use (#949). Versions are unchanged, and the aws hashes match those lock files. Co-Authored-By: Claude Opus 5.5 --- terraform/modules/database/aws/.terraform.lock.hcl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terraform/modules/database/aws/.terraform.lock.hcl b/terraform/modules/database/aws/.terraform.lock.hcl index 24a92392c..94afa1cf4 100644 --- a/terraform/modules/database/aws/.terraform.lock.hcl +++ b/terraform/modules/database/aws/.terraform.lock.hcl @@ -6,6 +6,8 @@ provider "registry.terraform.io/hashicorp/aws" { constraints = "~> 5.0" hashes = [ "h1:Ijt7pOlB7Tr7maGQIqtsLFbl7pSMIj06TVdkoSBcYOw=", + "h1:edXOJWE4ORX8Fm+dpVpICzMZJat4AX0VRCAy/xkcOc0=", + "h1:hd45qFU5cFuJMpFGdUniU9mVIr5LYVWP1uMeunBpYYs=", "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", @@ -28,6 +30,8 @@ provider "registry.terraform.io/hashicorp/random" { version = "3.8.1" constraints = "~> 3.0" hashes = [ + "h1:Eexl06+6J+s75uD46+WnZtpJZYRVUMB0AiuPBifK6Jc=", + "h1:fdfOl1HabDT42XLH8qjmfTbVZpgQZ5lyOyOa+GQhm0w=", "h1:u8AKlWVDTH5r9YLSeswoVEjiY72Rt4/ch7U+61ZDkiQ=", "zh:08dd03b918c7b55713026037c5400c48af5b9f468f483463321bd18e17b907b4", "zh:0eee654a5542dc1d41920bbf2419032d6f0d5625b03bd81339e5b33394a3e0ae", From 3941be7fddaf4aa00f639fe0ccb3e040377fde53 Mon Sep 17 00:00:00 2001 From: Astrid Gealer Date: Fri, 23 Oct 2026 01:30:04 +0000 Subject: [PATCH 4/5] test(iac/aws): scope the multi_az wiring guard to aws_db_instance.main The module-case regex matched the first `multi_az = var.multi_az` anywhere after the aws_db_instance.main header, so the guard still passed if the line moved into a later resource. Match it only inside that top-level block. Co-Authored-By: Claude Opus 5.5 --- .../aws/database_ha_guard_test.go | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/terraform/environments/aws/database_ha_guard_test.go b/terraform/environments/aws/database_ha_guard_test.go index d39292345..4134bbe40 100644 --- a/terraform/environments/aws/database_ha_guard_test.go +++ b/terraform/environments/aws/database_ha_guard_test.go @@ -50,11 +50,27 @@ func TestAWSProdDatabaseIsHighlyAvailable(t *testing.T) { } } +// topLevelBlock returns the text of the top-level block that starts with +// header, up to its closing brace in column 0 (terraform fmt layout), or "" +// when the header is absent. +func topLevelBlock(data []byte, header *regexp.Regexp) string { + loc := header.FindIndex(data) + if loc == nil { + return "" + } + rest := string(data[loc[0]:]) + if end := strings.Index(rest, "\n}"); end >= 0 { + return rest[:end+2] + } + return rest +} + func TestAWSDatabaseMultiAZIsWired(t *testing.T) { cases := []struct { - file string - re *regexp.Regexp - what string + file string + block *regexp.Regexp // when set, re must match inside this top-level block + re *regexp.Regexp + what string }{ { file: "variables.tf", @@ -67,9 +83,10 @@ func TestAWSDatabaseMultiAZIsWired(t *testing.T) { what: "multi_az = var.database_multi_az on module \"database\"", }, { - file: filepath.Join("..", "..", "modules", "database", "aws", "main.tf"), - re: regexp.MustCompile(`resource\s+"aws_db_instance"\s+"main"\s*\{[\s\S]*?\n\s*multi_az\s*=\s*var\.multi_az\s*\n`), - what: `multi_az = var.multi_az on aws_db_instance.main`, + file: filepath.Join("..", "..", "modules", "database", "aws", "main.tf"), + block: regexp.MustCompile(`(?m)^resource\s+"aws_db_instance"\s+"main"\s*\{`), + re: regexp.MustCompile(`(?m)^\s*multi_az\s*=\s*var\.multi_az\s*$`), + what: `multi_az = var.multi_az on aws_db_instance.main`, }, } for _, c := range cases { @@ -77,6 +94,9 @@ func TestAWSDatabaseMultiAZIsWired(t *testing.T) { if err != nil { t.Fatalf("reading %s: %v", c.file, err) } + if c.block != nil { + data = []byte(topLevelBlock(data, c.block)) + } if !c.re.Match(data) { t.Errorf("%s: missing %s", c.file, c.what) } From 8ae58fd7370457d6d51c37e6bc947a069aa811c5 Mon Sep 17 00:00:00 2001 From: Astrid Gealer Date: Wed, 28 Oct 2026 00:19:39 +0000 Subject: [PATCH 5/5] fix(deps): bump golang.org/x/net to v0.62.0 (GO-2026-6860, GO-2026-6866) govulncheck in the Docker image build now fails on these two advisories, both fixed in x/net v0.62.0. Co-Authored-By: Claude Opus 5.5 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5fb3a2f61..abcf4fb6c 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( go.opentelemetry.io/otel/metric v1.45.0 // indirect go.opentelemetry.io/otel/trace v1.45.0 // indirect golang.org/x/crypto v0.58.0 - golang.org/x/net v0.61.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.37.0 golang.org/x/sync v0.24.0 golang.org/x/sys v0.49.0 // indirect diff --git a/go.sum b/go.sum index 06deac0b9..83df52e66 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= -golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= -golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=