diff --git a/.badges/operations.svg b/.badges/operations.svg index f29f7c0811..3a5c6a5059 100644 --- a/.badges/operations.svg +++ b/.badges/operations.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ PARITY entries PARITY entries - 9318 - 9318 + 9762 + 9762 diff --git a/.badges/parity.svg b/.badges/parity.svg index ebf24c67a5..c3f218c91e 100644 --- a/.badges/parity.svg +++ b/.badges/parity.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ parity parity - 169 A · 87 B · 3 C - 169 A · 87 B · 3 C + 169 A · 92 B · 3 C + 169 A · 92 B · 3 C diff --git a/.badges/services.svg b/.badges/services.svg index 69b929ea37..9a8662f4f8 100644 --- a/.badges/services.svg +++ b/.badges/services.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ AWS services AWS services - 261 - 261 + 266 + 266 diff --git a/.github/actions/lint-setup/action.yml b/.github/actions/lint-setup/action.yml new file mode 100644 index 0000000000..9b357a847d --- /dev/null +++ b/.github/actions/lint-setup/action.yml @@ -0,0 +1,67 @@ +name: lint-setup +description: Disk, swap, Go and golangci-lint setup shared by the lint jobs +runs: + using: composite + steps: + # Full-repo lint (go.mod/.golangci.yml changed) exhausts the default runner disk; lint needs no docker images or apt packages. + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + - name: Keep Go temp files and build cache off the /tmp tmpfs + shell: bash + run: | + # /tmp is a small tmpfs on hosted runners; use /mnt when mounted, else the root disk. + if mountpoint -q /mnt; then base=/mnt; else base="$RUNNER_TEMP"; fi + sudo mkdir -p "$base/gotmp" "$base/gocache" + sudo chown -R "$(id -u):$(id -g)" "$base/gotmp" "$base/gocache" + { + echo "GOTMPDIR=$base/gotmp" + echo "TMPDIR=$base/gotmp" + echo "GOCACHE=$base/gocache" + } >> "$GITHUB_ENV" + df -h / /tmp "$base" || true + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: go.mod + check-latest: true + cache: true + + - name: Download Go modules (retry proxy stream errors) + shell: bash + run: | + for i in 1 2 3 4 5; do + go mod download && exit 0 + sleep $((i * 15)) + done + exit 1 + + # Full-repo lint (go.mod changed) exceeds runner RAM; the runner is killed without swap. + - name: Add swap + shell: bash + run: | + sudo swapoff -a || true + sudo fallocate -l 16G /mnt/lint.swap + sudo chmod 600 /mnt/lint.swap + sudo mkswap /mnt/lint.swap + sudo swapon /mnt/lint.swap + + - name: Read golangci-lint version + id: golangci-lint-version + shell: bash + run: | + echo "version=$(sed -n 's/^ GOLANGCI_LINT_VERSION: \(.*\)$/\1/p' Taskfile.yml)" >> "$GITHUB_OUTPUT" + + - name: Install golangci-lint + shell: bash + run: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b "$(go env GOPATH)/bin" "v${{ steps.golangci-lint-version.outputs.version }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a24239cd7..1acb4358a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,86 +52,81 @@ jobs: env: NODE_OPTIONS: --max-old-space-size=8192 - lint: + # Required status check "lint" is the aggregator at the bottom of this group. + # Config change (or no base): sharded full-repo lint. Otherwise: changed + # packages plus importers of any go.mod-bumped module (lint-changed.sh). + lint-scope: runs-on: ubuntu-latest + outputs: + mode: ${{ steps.mode.outputs.mode }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: token: ${{ env.GH_CI_TOKEN }} fetch-depth: 0 - # Full-repo lint (go.mod/.golangci.yml changed) exhausts the default runner disk; lint needs no docker images or apt packages. - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main - with: - tool-cache: false - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true - swap-storage: true - - - name: Keep Go temp files and build cache off the /tmp tmpfs + - name: Decide lint scope + id: mode run: | - # /tmp is a small tmpfs on hosted runners; use /mnt when mounted, else the root disk. - if mountpoint -q /mnt; then base=/mnt; else base="$RUNNER_TEMP"; fi - sudo mkdir -p "$base/gotmp" "$base/gocache" - sudo chown -R "$(id -u):$(id -g)" "$base/gotmp" "$base/gocache" - { - echo "GOTMPDIR=$base/gotmp" - echo "TMPDIR=$base/gotmp" - echo "GOCACHE=$base/gocache" - } >> "$GITHUB_ENV" - df -h / /tmp "$base" || true + set -euo pipefail + mode=full + if git rev-parse --verify -q origin/main >/dev/null 2>&1; then + merge_base=$(git merge-base HEAD origin/main) + if git diff --quiet "$merge_base" HEAD -- .golangci.yml; then + mode=changed + fi + fi + echo "lint: scope=$mode" + echo "mode=$mode" >> "$GITHUB_OUTPUT" - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + lint-changed: + needs: [lint-scope] + if: needs.lint-scope.outputs.mode == 'changed' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - go-version-file: go.mod - check-latest: true - cache: true + token: ${{ env.GH_CI_TOKEN }} + fetch-depth: 0 - - name: Download Go modules (retry proxy stream errors) - run: | - for i in 1 2 3 4 5; do - go mod download && exit 0 - sleep $((i * 15)) - done - exit 1 + - uses: ./.github/actions/lint-setup - # Full-repo lint (go.mod changed) exceeds runner RAM; the runner is killed without swap. - - name: Add swap - run: | - sudo swapoff -a || true - sudo fallocate -l 16G /mnt/lint.swap - sudo chmod 600 /mnt/lint.swap - sudo mkswap /mnt/lint.swap - sudo swapon /mnt/lint.swap + - name: Run golangci-lint on changed packages + run: bash scripts/lint-changed.sh - - name: Read golangci-lint version - id: golangci-lint-version - run: | - echo "version=$(sed -n 's/^ GOLANGCI_LINT_VERSION: \(.*\)$/\1/p' Taskfile.yml)" >> "$GITHUB_OUTPUT" + lint-shard: + needs: [lint-scope] + if: needs.lint-scope.outputs.mode == 'full' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [svc-a, svc-b-c, svc-d-e, svc-f-h, svc-i-m, svc-n-o, svc-p-r, svc-s, svc-t-z, pkgs, rest] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + token: ${{ env.GH_CI_TOKEN }} + fetch-depth: 0 - - name: Install golangci-lint - run: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b "$(go env GOPATH)/bin" "v${{ steps.golangci-lint-version.outputs.version }}" + - uses: ./.github/actions/lint-setup + + - name: Run golangci-lint shard + run: bash scripts/lint-shard.sh "${{ matrix.shard }}" - - name: Run golangci-lint (changed packages or full repo on config change) + lint: + needs: [lint-scope, lint-changed, lint-shard] + if: always() + runs-on: ubuntu-latest + steps: + - name: Require every lint job to pass or be skipped by scope + env: + RESULTS: ${{ needs.lint-scope.result }} ${{ needs.lint-changed.result }} ${{ needs.lint-shard.result }} run: | - set -euo pipefail - if git rev-parse --verify -q origin/main >/dev/null 2>&1; then - merge_base=$(git merge-base HEAD origin/main) - if git diff --name-only "$merge_base" HEAD -- .golangci.yml go.mod | grep -q .; then - echo "lint: .golangci.yml or go.mod changed vs origin/main — running full-repo lint" - golangci-lint run --timeout 55m ./... - else - bash scripts/lint-changed.sh - fi - else - golangci-lint run --timeout 55m ./... - fi + echo "lint: results: $RESULTS" + case " $RESULTS " in + *" failure "*|*" cancelled "*) exit 1 ;; + esac + [[ "${{ needs.lint-scope.result }}" == success ]] # Fails fast on tag-gated compile breaks (e2e, integration) before the slow # e2e/integration/terraform jobs run any tests -- see gopherstack-0bpp. @@ -862,6 +857,34 @@ jobs: GOPHERSTACK_TEST_IMAGE: gopherstack:tc-local run: go test -count=1 -timeout 10m ./... + helm-tests: + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + token: ${{ env.GH_CI_TOKEN }} + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: go.mod + check-latest: true + cache: true + + - name: Download binary + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: gopherstack-binary + path: bin/ + + - name: Lint, render and install the Helm chart on k3s + env: + GOPHERSTACK_TEST_DOCKER: "1" + run: | + chmod +x bin/gopherstack-linux + go test -count=1 -timeout 15m -v ./test/helm/ + integration: runs-on: ubuntu-latest needs: [integration-tests] diff --git a/Dockerfile b/Dockerfile index 05ebb2a088..dd996a1267 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,13 +32,22 @@ RUN CGO_ENABLED=0 go build \ -ldflags="-w -s -extldflags '-static -fno-PIC'" \ -o gopherstack -# Final stage -FROM scratch +# Writable state dir for the non-root runtime user (65532:65532) +FROM --platform=$BUILDPLATFORM alpine:3.22@sha256:2e0b81d60597f449baa56dc25d355367869d2dca4cd57e866f1e2b52af3eb55f AS state +RUN mkdir /data -WORKDIR /root/ +# Final stage: distroless static (nonroot) has a world-searchable / (containerd would unpack a bare +# scratch / as 0700 root), plus ca-certificates, tzdata and a 1777 /tmp. +FROM gcr.io/distroless/static:nonroot@sha256:25ac8f0c3b2334a9eb05452266bd4697864a8c8ef0ef71e5e6480d7f92868329 + +COPY --from=state --chown=65532:65532 /data /data + +WORKDIR /usr/local/bin + +COPY --from=builder --chown=0:0 --chmod=755 /app/gopherstack . + +USER 65532:65532 -# Copy the Pre-built binary from the previous stage -COPY --from=builder /app/gopherstack . # Expose port 8000 to the outside world EXPOSE 8000 diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index e9789d37eb..6f28a4b006 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,12 +1,21 @@ # Dockerfile used by GoReleaser for multi-arch builds -FROM scratch +FROM --platform=$BUILDPLATFORM alpine:3.22@sha256:2e0b81d60597f449baa56dc25d355367869d2dca4cd57e866f1e2b52af3eb55f AS state +RUN mkdir /data + +# distroless static (nonroot): world-searchable /, ca-certificates, tzdata, 1777 /tmp, UID 65532. +FROM gcr.io/distroless/static:nonroot@sha256:25ac8f0c3b2334a9eb05452266bd4697864a8c8ef0ef71e5e6480d7f92868329 ARG TARGETPLATFORM -WORKDIR /root/ +COPY --from=state --chown=65532:65532 /data /data + +WORKDIR /usr/local/bin # Copy the binary from GoReleaser context -COPY $TARGETPLATFORM/gopherstack . +COPY --chown=0:0 --chmod=755 $TARGETPLATFORM/gopherstack . + +USER 65532:65532 + # Expose port 8000 EXPOSE 8000 diff --git a/Dockerfile.test b/Dockerfile.test index 2e28f8490a..e1a041e359 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,8 +1,14 @@ -# Lightweight test image — expects a pre-built static binary at bin/gopherstack-linux -# (from `task build-linux`; this is a separate path from `task build`'s bin/gopherstack -# so the two targets can't clobber each other — see gopherstack-gooq). -FROM scratch -COPY bin/gopherstack-linux /root/gopherstack +# Lightweight test image: expects a pre-built static binary at bin/gopherstack-linux +# (from `task build-linux`; separate from `task build`'s bin/gopherstack — see gopherstack-gooq). +# Same non-root layout as the release image. Must build with the classic (non-BuildKit) +# builder testcontainers uses: no $BUILDPLATFORM, no COPY --chmod (the binary is already 0755). +FROM alpine:3.22@sha256:2e0b81d60597f449baa56dc25d355367869d2dca4cd57e866f1e2b52af3eb55f AS state +RUN mkdir /data + +FROM gcr.io/distroless/static:nonroot@sha256:25ac8f0c3b2334a9eb05452266bd4697864a8c8ef0ef71e5e6480d7f92868329 +COPY --from=state --chown=65532:65532 /data /data +COPY --chown=0:0 bin/gopherstack-linux /usr/local/bin/gopherstack +USER 65532:65532 EXPOSE 8000 EXPOSE 1883 -CMD ["/root/gopherstack"] +CMD ["/usr/local/bin/gopherstack"] diff --git a/README.md b/README.md index 0714e3ef8d..2baa36698c 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,14 @@ docker compose up -d For Lambda support the container needs access to a container runtime — see [docs/docker.md](docs/docker.md). +### Kubernetes (Helm) + +```bash +helm install gopherstack ./charts/gopherstack +``` + +See [docs/kubernetes.md](docs/kubernetes.md). + ### Testcontainers (Go) Gopherstack ships a reusable [Testcontainers for Go](https://golang.testcontainers.org/) @@ -339,6 +347,7 @@ A single switch picks the default for every container-backed engine (RDS/Aurora, | `--transparent-ca-dir` | `TRANSPARENT_CA_DIR` | ephemeral | Directory to persist the CA so it is trusted once. | | `--transparent-container-ca-path` | `TRANSPARENT_CONTAINER_CA_PATH` | temp file (skipped when containerised) | CA PEM path visible to the Docker daemon; required to inject the CA when gopherstack itself runs in Docker. | | `--rds-engine` | `RDS_ENGINE` | `stub` | RDS engine mode: `stub` (metadata only) or `docker` (real `postgres`/`mysql`/`mariadb` container per DB instance, one per Aurora cluster; Data API stays on SQLite; `RDS_DB_HOST` overrides the advertised host, default `127.0.0.1`; ports bind loopback only unless it is non-loopback, then all interfaces; Postgres runs `postgis/postgis` + pgvector with RDS-style `aws_s3`/`aws_lambda` extensions unless `RDS_POSTGRES_EXTENSIONS=false`; `RDS_SQLSERVER_ENGINE=docker` opts `sqlserver-*` into a real SQL Server container (amd64 only; accepts the Microsoft EULA); `RDS_CALLOUT_HOST`/`RDS_CALLOUT_BIND` set the address containers dial / the callout listener binds (default: container bridge gateway, never a wildcard)). | +| `--dsql-engine` | `DSQL_ENGINE` | `stub` | Aurora DSQL engine mode: `stub` (control plane only) or `docker` (one `postgres` container per cluster behind a TLS proxy that accepts `GenerateDbConnectAuthToken` / `GenerateDBConnectAdminAuthToken` tokens as the password for user `admin`; `CREATE INDEX ASYNC`, `sys.jobs`, `sys.wait_for_job`; connect address at `/_gopherstack/dsql/clusters/{id}`, CA at `/_gopherstack/dsql/ca.pem`; `DSQL_ENGINE_IMAGE` picks the image, default `postgres:17`). | | `--docdb-engine` | `DOCDB_ENGINE` | `stub` | DocumentDB engine mode: `stub` (metadata only) or `docker` (one plaintext `mongo` container per cluster, `mongo:4.4` for 3.6/4.0 and `mongo:5.0` for 5.0; `DOCDB_HOST` overrides the advertised host, default `127.0.0.1`; ports bind loopback only unless it is non-loopback, then all interfaces). | | `--redshift-engine` | `REDSHIFT_ENGINE` | `stub` | Redshift engine mode: `stub` (metadata only) or `docker` (one `postgres:17` container per provisioned cluster with MasterUsername/MasterUserPassword/DBName; Endpoint reports the published `127.0.0.1` port and the Redshift Data API runs SQL against it; `REDSHIFT_HOST` overrides the advertised host; Serverless workgroups stay metadata-only). | | `--neptune-engine` | `NEPTUNE_ENGINE` | `stub` | Neptune engine mode: `stub` (metadata only), `embedded` (in-process Gremlin, openCypher and Bolt over one persisted property graph; no container runtime) or `docker` (the same in-process graph by default; `NEPTUNE_GREMLIN=container` runs one `tinkerpop/gremlin-server` container per cluster instead, TinkerPop 3.4.13 for engine 1.0/1.1, 3.5.7 for 1.2, 3.6.8 for 1.3, 3.7.3 later). Endpoint, ReaderEndpoint, Port and member instances report the published `127.0.0.1` port, Gremlin over WebSocket/HTTP at `/gremlin`; `NEPTUNE_HOST` overrides the advertised host; no IAM auth. | @@ -730,16 +739,16 @@ Every service links to its own page with a coverage breakdown — audited operat | [DocumentDB](services/docdb/README.md) | A | 55 | 2 gaps; 5 structural gaps; 1 deferred | | [DynamoDB](services/dynamodb/README.md) | A | — | 15 families; 3 gaps; 3 structural gaps; 2 deferred | | [DynamoDB Streams](services/dynamodbstreams/README.md) | A | 4 | clean | -| [ElastiCache](services/elasticache/README.md) | A | 75 | 2 gaps; 4 structural gaps; 1 deferred | -| [MemoryDB](services/memorydb/README.md) | A | 45 | 3 gaps; 2 structural gaps; 3 deferred | -| [Neptune](services/neptune/README.md) | A | — | 13 families; 6 gaps; 2 structural gaps; 2 deferred | +| [ElastiCache](services/elasticache/README.md) | A | 75 | 3 gaps; 4 structural gaps; 1 deferred | +| [MemoryDB](services/memorydb/README.md) | A | 45 | 4 gaps; 2 structural gaps; 3 deferred | +| [Neptune](services/neptune/README.md) | A | — | 13 families; 9 gaps; 2 structural gaps; 2 deferred | | [QLDB](services/qldb/README.md) | Removed | — | removed service | | [QLDB Session](services/qldbsession/README.md) | Removed | — | removed service | | [RDS](services/rds/README.md) | A | 52 | 7 gaps; 7 structural gaps | | [RDS Data](services/rdsdata/README.md) | A | 7 | 4 structural gaps | | [Redshift](services/redshift/README.md) | A | 9 | 5 gaps; 5 structural gaps | | [Redshift Data](services/redshiftdata/README.md) | A | 12 | 4 structural gaps; 1 deferred | -| [Timestream Query](services/timestreamquery/README.md) | A | 12 | 6 gaps; 1 structural gap | +| [Timestream Query](services/timestreamquery/README.md) | A | 12 | 7 gaps; 1 structural gap | | [Timestream Write](services/timestreamwrite/README.md) | A | 19 | 1 structural gap | ### Networking & Content Delivery @@ -799,7 +808,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [Lake Formation](services/lakeformation/README.md) | A | 61 | 1 gap; 2 structural gaps | | [Managed Streaming for Kafka](services/kafka/README.md) | A | 64 | 3 gaps; 1 structural gap | | [Managed Workflows for Apache Airflow](services/mwaa/README.md) | A | 12 | 4 gaps; 1 structural gap; 1 deferred | -| [OpenSearch](services/opensearch/README.md) | A | 19 | 1 gap; 1 structural gap | +| [OpenSearch](services/opensearch/README.md) | A | 19 | 4 gaps; 1 structural gap | | [QuickSight](services/quicksight/README.md) | A | 81 | 3 gaps; 7 structural gaps | ### Security @@ -827,7 +836,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [Cognito Identity](services/cognitoidentity/README.md) | A | 23 | 1 gap; 4 deferred | | [Cognito Identity Provider](services/cognitoidp/README.md) | A | 71 | 6 gaps; 3 structural gaps | | [Directory Service](services/directoryservice/README.md) | A | 80 | 1 gap; 2 structural gaps; 1 deferred | -| [IAM](services/iam/README.md) | A | 38 | 4 gaps; 4 structural gaps | +| [IAM](services/iam/README.md) | A | 38 | 5 gaps; 3 structural gaps | | [IAM Access Analyzer](services/accessanalyzer/README.md) | A | 39 | 1 gap; 5 structural gaps; 1 deferred | | [IAM Identity Center (SSO)](services/ssoadmin/README.md) | A | 56 | 1 gap | | [IAM Identity Center OIDC](services/ssooidc/README.md) | B | 4 | 1 gap; 1 structural gap | @@ -854,7 +863,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [Cost Explorer](services/ce/README.md) | A | 37 | 1 gap; 5 structural gaps; 2 deferred | | [Fault Injection Simulator](services/fis/README.md) | A | 26 | 2 structural gaps; 1 deferred | | [OpsWorks](services/opsworks/README.md) | A | 32 | 1 gap; 2 structural gaps; 1 deferred | -| [Organizations](services/organizations/README.md) | A | 63 | 2 structural gaps | +| [Organizations](services/organizations/README.md) | A | 63 | 1 gap; 2 structural gaps | | [Resource Access Manager](services/ram/README.md) | A | 36 | 2 gaps; 1 structural gap; 2 deferred | | [Resource Groups](services/resourcegroups/README.md) | A | 23 | 2 structural gaps | | [Resource Groups Tagging API](services/resourcegroupstaggingapi/README.md) | A | 9 | 2 structural gaps; 1 deferred | @@ -890,7 +899,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [SageMaker](services/sagemaker/README.md) | A | 70 | 1 gap; 19 structural gaps | | [SageMaker Runtime](services/sagemakerruntime/README.md) | A | 3 | 1 gap; 1 structural gap | | [Textract](services/textract/README.md) | A | 25 | 2 structural gaps; 1 deferred | -| [Transcribe](services/transcribe/README.md) | A | 44 | 2 structural gaps | +| [Transcribe](services/transcribe/README.md) | A | 44 | 6 gaps; 2 structural gaps | | [Translate](services/translate/README.md) | A | 19 | 1 gap; 2 structural gaps | ### Media @@ -925,51 +934,52 @@ Every service links to its own page with a coverage breakdown — audited operat | Service | Parity | PARITY Entries | Notes | |---|---|---|---| -| [Azure App Configuration](services/azureappconfig/README.md) | B | 14 | 13 gaps | +| [Azure AI Search](services/azuresearch/README.md) | B | 19 | 8 gaps | +| [Azure App Configuration](services/azureappconfig/README.md) | B | 14 | 12 gaps | | [Azure Blob Storage](services/azureblob/README.md) | C | 29 | 5 gaps; 3 deferred | -| [Azure Cache for Redis](services/azureredis/README.md) | B | 11 | 13 gaps | -| [Azure Container Registry](services/azurecontainerregistry/README.md) | B | 24 | 10 gaps | +| [Azure Cache for Redis](services/azureredis/README.md) | B | 11 | 12 gaps | +| [Azure Container Registry](services/azurecontainerregistry/README.md) | B | 24 | 9 gaps | | [Azure Cosmos DB](services/cosmosdb/README.md) | A | 35 | clean | -| [Azure Event Grid](services/azureeventgrid/README.md) | B | 9 | 15 gaps | -| [Azure Event Hubs](services/azureeventhubs/README.md) | B | 15 | 9 gaps | -| [Azure Files](services/azurefile/README.md) | B | 38 | 11 gaps | -| [Azure Functions](services/azurefunctions/README.md) | B | 6 | 12 gaps | +| [Azure Event Grid](services/azureeventgrid/README.md) | B | 9 | 14 gaps | +| [Azure Event Hubs](services/azureeventhubs/README.md) | B | 15 | 8 gaps | +| [Azure Files](services/azurefile/README.md) | B | 41 | 10 gaps | +| [Azure Functions](services/azurefunctions/README.md) | B | 10 | 12 gaps | | [Azure Key Vault](services/azurekeyvault/README.md) | B | 15 | 4 gaps | | [Azure Monitor Logs](services/azuremonitor/README.md) | B | 20 | 5 gaps | | [Azure Queue Storage](services/azurequeue/README.md) | C | 13 | 6 gaps; 1 deferred | -| [Azure SQL Database](services/azuresql/README.md) | B | 14 | 11 gaps | +| [Azure SQL Database](services/azuresql/README.md) | B | 14 | 10 gaps | | [Azure Service Bus](services/azureservicebus/README.md) | B | 25 | 3 gaps; 4 deferred | | [Azure Table Storage](services/azuretable/README.md) | C | 14 | 4 gaps; 3 deferred | -| [Microsoft Graph](services/azuregraph/README.md) | B | 28 | 10 gaps | +| [Microsoft Graph](services/azuregraph/README.md) | B | 33 | 10 gaps | ### GCP | Service | Parity | PARITY Entries | Notes | |---|---|---|---| -| [API Gateway](services/gcpapigateway/README.md) | B | 25 | 7 gaps | -| [Artifact Registry](services/gcpartifactregistry/README.md) | B | 40 | 13 gaps | +| [API Gateway](services/gcpapigateway/README.md) | B | 25 | 6 gaps | +| [Artifact Registry](services/gcpartifactregistry/README.md) | B | 50 | 15 gaps | | [BigQuery](services/gcpbigquery/README.md) | B | 23 | 8 gaps | | [Cloud Bigtable](services/gcpbigtable/README.md) | B | 43 | 8 gaps | -| [Cloud Build](services/gcpcloudbuild/README.md) | B | 20 | 11 gaps | +| [Cloud Build](services/gcpcloudbuild/README.md) | B | 20 | 9 gaps | | [Cloud DNS](services/gcpdns/README.md) | B | — | 8 families; 5 gaps | | [Cloud Datastore](services/gcpdatastore/README.md) | B | 13 | 5 gaps | | [Cloud Functions](services/gcpfunctions/README.md) | B | 8 | 8 gaps | | [Cloud IAM](services/gcpiam/README.md) | B | 31 | 5 gaps | | [Cloud Logging](services/gcplogging/README.md) | B | 40 | 7 gaps | -| [Cloud Monitoring](services/gcpmonitoring/README.md) | B | 53 | 11 gaps | +| [Cloud Monitoring](services/gcpmonitoring/README.md) | B | 53 | 8 gaps | | [Cloud Run](services/gcprun/README.md) | B | 20 | 5 gaps | | [Cloud SQL](services/gcpsql/README.md) | B | 42 | 7 gaps | | [Cloud Scheduler](services/gcpscheduler/README.md) | B | 10 | 4 gaps | | [Cloud Spanner](services/gcpspanner/README.md) | B | 18 | 6 gaps | | [Cloud Tasks](services/gcpcloudtasks/README.md) | B | 20 | 3 gaps | -| [Compute Engine](services/gcpcompute/README.md) | B | — | 8 families; 10 gaps | -| [Eventarc](services/gcpeventarc/README.md) | B | 24 | 10 gaps | +| [Compute Engine](services/gcpcompute/README.md) | B | — | 8 families; 9 gaps | +| [Eventarc](services/gcpeventarc/README.md) | B | 47 | 9 gaps | | [Google Cloud Batch](services/gcpbatch/README.md) | B | 14 | 9 gaps | | [Google Cloud Firestore](services/gcpfirestore/README.md) | B | 15 | 4 gaps | | [Google Cloud KMS](services/gcpkms/README.md) | B | 35 | 4 gaps | | [Google Cloud Pub/Sub](services/gcppubsub/README.md) | B | 38 | 6 gaps | | [Google Cloud Resource Manager and Service Usage](services/gcpresourcemanager/README.md) | B | 39 | 9 gaps | -| [Google Cloud Storage](services/gcpstorage/README.md) | B | 28 | 8 gaps | +| [Google Cloud Storage](services/gcpstorage/README.md) | B | 37 | 9 gaps | | [Google Secret Manager](services/gcpsecretmanager/README.md) | B | 18 | 5 gaps | | [Identity Platform](services/gcpidentitytoolkit/README.md) | B | 1 | 6 gaps | | [Kubernetes Engine](services/gcpcontainer/README.md) | B | — | 5 families; 5 gaps | @@ -980,12 +990,15 @@ Every service links to its own page with a coverage breakdown — audited operat | Service | Parity | PARITY Entries | Notes | |---|---|---|---| -| [OCI API Gateway](services/ociapigateway/README.md) | B | 22 | 10 gaps | +| [OCI API Gateway](services/ociapigateway/README.md) | B | 52 | 13 gaps | | [OCI Artifacts and Container Registry](services/ociartifacts/README.md) | B | 37 | 7 gaps | +| [OCI Autonomous Database](services/ocidatabase/README.md) | B | 21 | 7 gaps | | [OCI Container Engine for Kubernetes](services/ocicontainerengine/README.md) | B | 20 | 9 gaps | | [OCI Container Instances](services/ocicontainerinstances/README.md) | B | 18 | 7 gaps | -| [OCI Core Services](services/ocicore/README.md) | B | 135 | 11 gaps | +| [OCI Core Services](services/ocicore/README.md) | B | 227 | 11 gaps | +| [OCI DNS](services/ocidns/README.md) | B | 52 | 7 gaps | | [OCI Events](services/ocievents/README.md) | B | 6 | 6 gaps | +| [OCI File Storage](services/ocifilestorage/README.md) | B | 74 | 7 gaps | | [OCI Functions](services/ocifunctions/README.md) | B | 16 | 9 gaps | | [OCI Identity](services/ociidentity/README.md) | B | 70 | 9 gaps | | [OCI Load Balancer](services/ociloadbalancer/README.md) | B | 61 | 9 gaps | @@ -994,10 +1007,10 @@ Every service links to its own page with a coverage breakdown — audited operat | [OCI MySQL HeatWave Database Service](services/ocimysql/README.md) | B | 32 | 9 gaps | | [OCI NoSQL Database Cloud Service](services/ocinosql/README.md) | B | 21 | 9 gaps | | [OCI Notifications](services/ocinotifications/README.md) | B | 16 | 7 gaps | -| [OCI Object Storage](services/ociobjectstorage/README.md) | B | 45 | 11 gaps | +| [OCI Object Storage](services/ociobjectstorage/README.md) | B | 52 | 10 gaps | | [OCI Queue](services/ociqueue/README.md) | B | 19 | 9 gaps | | [OCI Streaming](services/ocistreaming/README.md) | B | 30 | 9 gaps | -| [OCI Vault](services/ocivault/README.md) | B | 45 | 11 gaps | +| [OCI Vault](services/ocivault/README.md) | B | 57 | 12 gaps | ### Other @@ -1006,25 +1019,26 @@ Every service links to its own page with a coverage breakdown — audited operat | [Amp](services/amp/README.md) | B | 44 | 9 gaps; 2 structural gaps | | [AppStream 2.0](services/appstream/README.md) | A | 44 | 5 structural gaps | | [Appintegrations](services/appintegrations/README.md) | B | 23 | 3 gaps; 1 structural gap | -| [Azurearm](services/azurearm/README.md) | B | 20 | 69 gaps; 3 deferred | +| [Azurearm](services/azurearm/README.md) | B | 21 | 74 gaps; 3 deferred | | [Azurestoragevhost](services/azurestoragevhost/README.md) | B | 2 | 2 gaps; 1 deferred | | [Bcmdataexports](services/bcmdataexports/README.md) | B | 12 | 6 gaps | | [Bedrockagentcore](services/bedrockagentcore/README.md) | B | 30 | 11 gaps; 2 structural gaps | -| [Bedrockagentcorecontrol](services/bedrockagentcorecontrol/README.md) | B | 171 | 12 gaps; 2 structural gaps | +| [Bedrockagentcorecontrol](services/bedrockagentcorecontrol/README.md) | B | 171 | 13 gaps; 2 structural gaps | | [Budgets](services/budgets/README.md) | B | 26 | 8 gaps | | [Cloudfrontkeyvaluestore](services/cloudfrontkeyvaluestore/README.md) | A | 6 | 2 structural gaps | | [Cloudhsmv2](services/cloudhsmv2/README.md) | B | 18 | 7 gaps; 3 structural gaps | | [Codegurureviewer](services/codegurureviewer/README.md) | B | 14 | 5 gaps; 2 structural gaps | -| [Connect](services/connect/README.md) | B | 312 | 21 gaps; 2 structural gaps | +| [Connect](services/connect/README.md) | B | 374 | 20 gaps; 2 structural gaps | | [Cur](services/cur/README.md) | B | 7 | 6 gaps | | [Directconnect](services/directconnect/README.md) | A | 73 | 2 gaps; 8 structural gaps; 1 deferred | -| [Dsql](services/dsql/README.md) | A | 16 | 3 structural gaps | +| [Dsql](services/dsql/README.md) | A | 16 | 8 gaps; 2 structural gaps | | [Ecrpublic](services/ecrpublic/README.md) | A | 23 | 3 structural gaps | | [Globalaccelerator](services/globalaccelerator/README.md) | B | 56 | 4 gaps; 2 structural gaps | | [Grafana](services/grafana/README.md) | A | 25 | 1 structural gap | | [HealthOmics](services/omics/README.md) | A | — | 25 families; 5 structural gaps; 1 deferred | | [Kafkaconnect](services/kafkaconnect/README.md) | A | 19 | 3 gaps; 3 structural gaps | | [Kinesisvideo](services/kinesisvideo/README.md) | A | 32 | 3 structural gaps | +| [Lambdaweb](services/lambdaweb/README.md) | B | 20 | 11 gaps; 1 structural gap | | [Lightsail](services/lightsail/README.md) | A | — | 29 families; 1 gap; 6 structural gaps; 2 deferred | | [Managed Blockchain](services/managedblockchain/README.md) | A | 27 | 3 structural gaps | | [Marketplaceentitlementservice](services/marketplaceentitlementservice/README.md) | B | 1 | 3 gaps; 1 structural gap | @@ -1148,6 +1162,7 @@ cdk deploy - [Quickstart](docs/quickstart.md) - [Running with Docker](docs/docker.md) +- [Running on Kubernetes (Helm)](docs/kubernetes.md) - [Security group enforcement](docs/security-groups.md) - [Migrating from LocalStack](docs/migration.md) - [Architecture](docs/architecture/README.md) diff --git a/Taskfile.yml b/Taskfile.yml index eecd1b307a..1f7b6441b0 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -165,6 +165,21 @@ tasks: -ldflags "-w -s -X {{.VERSION_PKG}}.Build={{.BUILD_VERSION}}" \ -o bin/{{.BINARY_NAME}}-linux . + helm-test: + desc: Lint and golden-render charts/gopherstack with the pinned helm image (needs docker) + env: + GOPHERSTACK_TEST_DOCKER: "1" + cmds: + - go test -count=1 -run 'TestChart_(Lint|GoldenTemplate|SchemaRejects)' ./test/helm/ + + helm-install-test: + desc: Install charts/gopherstack into a k3s cluster and call it (needs docker) + deps: [build-linux] + env: + GOPHERSTACK_TEST_DOCKER: "1" + cmds: + - go test -count=1 -timeout 15m -run TestChart_InstallK3s -v ./test/helm/ + install-deps: desc: Install pinned golangci-lint, fieldalignment and dev MCP CLIs run: once diff --git a/charts/gopherstack/.helmignore b/charts/gopherstack/.helmignore new file mode 100644 index 0000000000..483618d6ec --- /dev/null +++ b/charts/gopherstack/.helmignore @@ -0,0 +1,4 @@ +.git +*.swp +*.bak +*.tmp diff --git a/charts/gopherstack/Chart.yaml b/charts/gopherstack/Chart.yaml new file mode 100644 index 0000000000..805c6466d9 --- /dev/null +++ b/charts/gopherstack/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: gopherstack +description: Gopherstack, a local multi-cloud (AWS, Azure, GCP, OCI) service emulator +type: application +version: 0.1.0 +appVersion: "1.9.1" +kubeVersion: ">=1.25.0-0" +home: https://github.com/blackbirdworks/gopherstack +sources: + - https://github.com/blackbirdworks/gopherstack +keywords: + - aws + - emulator + - localstack + - testing diff --git a/charts/gopherstack/templates/NOTES.txt b/charts/gopherstack/templates/NOTES.txt new file mode 100644 index 0000000000..1b3b2829d7 --- /dev/null +++ b/charts/gopherstack/templates/NOTES.txt @@ -0,0 +1,29 @@ +Gopherstack {{ .Chart.AppVersion }} is deploying as {{ include "gopherstack.fullname" . }}. + +Point AWS SDKs and the CLI at the service: +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.hosts }} + export AWS_ENDPOINT_URL=http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }} +{{- end }} +{{- else if eq .Values.service.type "NodePort" }} + export NODE_PORT=$(kubectl get svc {{ include "gopherstack.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.ports[0].nodePort}') + export AWS_ENDPOINT_URL=http://:$NODE_PORT +{{- else if eq .Values.service.type "LoadBalancer" }} + export AWS_ENDPOINT_URL=http://$(kubectl get svc {{ include "gopherstack.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):{{ .Values.service.port }} +{{- else }} + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "gopherstack.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + export AWS_ENDPOINT_URL=http://localhost:{{ .Values.service.port }} +{{- end }} + export AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_DEFAULT_REGION={{ .Values.config.region }} + +In-cluster endpoint: http://{{ include "gopherstack.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} +Health: {{ .Values.probes.path }} +{{- if not .Values.persistence.enabled }} + +State is in memory and is lost when the pod restarts. Set persistence.enabled=true to keep it. +{{- end }} +{{- if not .Values.dind.enabled }} + +Docker-backed engines (RDS, EKS, Lambda containers, ...) need a container runtime and run +as stubs in this pod. Set dind.enabled=true and config.engines=docker to use a DinD sidecar. +{{- end }} diff --git a/charts/gopherstack/templates/_helpers.tpl b/charts/gopherstack/templates/_helpers.tpl new file mode 100644 index 0000000000..3d25b70bf8 --- /dev/null +++ b/charts/gopherstack/templates/_helpers.tpl @@ -0,0 +1,48 @@ +{{- define "gopherstack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "gopherstack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "gopherstack.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "gopherstack.labels" -}} +helm.sh/chart: {{ include "gopherstack.chart" . }} +{{ include "gopherstack.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "gopherstack.selectorLabels" -}} +app.kubernetes.io/name: {{ include "gopherstack.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "gopherstack.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} +{{- default (include "gopherstack.fullname" .) .Values.serviceAccount.name -}} +{{- else -}} +{{- default "default" .Values.serviceAccount.name -}} +{{- end -}} +{{- end -}} + +{{- define "gopherstack.image" -}} +{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) -}} +{{- end -}} + +{{- define "gopherstack.pvcName" -}} +{{- default (include "gopherstack.fullname" .) .Values.persistence.existingClaim -}} +{{- end -}} diff --git a/charts/gopherstack/templates/configmap.yaml b/charts/gopherstack/templates/configmap.yaml new file mode 100644 index 0000000000..c4c1f4cc45 --- /dev/null +++ b/charts/gopherstack/templates/configmap.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "gopherstack.fullname" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} +data: + PORT: {{ .Values.service.port | quote }} + REGION: {{ .Values.config.region | quote }} + ACCOUNT_ID: {{ .Values.config.accountId | quote }} + LOG_LEVEL: {{ .Values.config.logLevel | quote }} + GOPHERSTACK_ENGINES: {{ .Values.config.engines | quote }} + GOPHERSTACK_ENFORCE_IAM: {{ .Values.config.enforceIam | quote }} + GOPHERSTACK_LOCALSTACK_COMPAT: {{ .Values.config.localstackCompat | quote }} + {{- with .Values.config.services }} + GOPHERSTACK_SERVICES: {{ join "," . | quote }} + {{- end }} + {{- with .Values.config.disableServices }} + GOPHERSTACK_DISABLE_SERVICES: {{ join "," . | quote }} + {{- end }} + {{- if .Values.persistence.enabled }} + PERSIST: "true" + GOPHERSTACK_DATA_DIR: {{ .Values.persistence.mountPath | quote }} + {{- end }} + {{- if .Values.dind.enabled }} + DOCKER_HOST: "tcp://127.0.0.1:2375" + {{- end }} + {{- if eq .Values.container.runtime "kubernetes" }} + CONTAINER_RUNTIME: "kubernetes" + CONTAINER_K8S_NAMESPACE: {{ default .Release.Namespace .Values.container.kubernetes.namespace | quote }} + CONTAINER_K8S_FETCH_IMAGE: {{ .Values.container.kubernetes.fetchImage | quote }} + CONTAINER_K8S_CODE_PORT: {{ .Values.container.kubernetes.codePort | quote }} + {{- end }} + {{- range $k, $v := .Values.config.extra }} + {{ $k }}: {{ $v | quote }} + {{- end }} diff --git a/charts/gopherstack/templates/deployment.yaml b/charts/gopherstack/templates/deployment.yaml new file mode 100644 index 0000000000..1e58087f02 --- /dev/null +++ b/charts/gopherstack/templates/deployment.yaml @@ -0,0 +1,142 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "gopherstack.fullname" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "gopherstack.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "gopherstack.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "gopherstack.serviceAccountName" . }} + automountServiceAccountToken: {{ eq .Values.container.runtime "kubernetes" }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: gopherstack + image: {{ include "gopherstack.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "gopherstack.fullname" . }} + {{- with .Values.envFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if or .Values.env (eq .Values.container.runtime "kubernetes") }} + env: + {{- if eq .Values.container.runtime "kubernetes" }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LAMBDA_DOCKER_HOST + value: $(POD_IP) + - name: CONTAINER_K8S_ADVERTISE_HOST + value: $(POD_IP) + {{- end }} + {{- with .Values.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- if eq .Values.container.runtime "kubernetes" }} + - name: fn-code + containerPort: {{ .Values.container.kubernetes.codePort }} + protocol: TCP + {{- end }} + {{- range .Values.service.extraPorts }} + - name: {{ .name }} + containerPort: {{ .port }} + protocol: {{ default "TCP" .protocol }} + {{- end }} + livenessProbe: + httpGet: + path: {{ .Values.probes.path }} + port: http + {{- toYaml .Values.probes.liveness | nindent 12 }} + readinessProbe: + httpGet: + path: {{ .Values.probes.path }} + port: http + {{- toYaml .Values.probes.readiness | nindent 12 }} + startupProbe: + httpGet: + path: {{ .Values.probes.path }} + port: http + {{- toYaml .Values.probes.startup | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: tmp + mountPath: /tmp + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + {{- end }} + {{- if .Values.dind.enabled }} + - name: dind + image: {{ printf "%s:%s" .Values.dind.image.repository .Values.dind.image.tag | quote }} + imagePullPolicy: {{ .Values.dind.image.pullPolicy }} + args: ["--host=tcp://127.0.0.1:2375"] + env: + - name: DOCKER_TLS_CERTDIR + value: "" + securityContext: + privileged: true + runAsNonRoot: false + runAsUser: 0 + resources: + {{- toYaml .Values.dind.resources | nindent 12 }} + volumeMounts: + - name: dind-storage + mountPath: /var/lib/docker + {{- end }} + volumes: + - name: tmp + emptyDir: {} + {{- if .Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ include "gopherstack.pvcName" . }} + {{- end }} + {{- if .Values.dind.enabled }} + - name: dind-storage + emptyDir: {} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/gopherstack/templates/ingress.yaml b/charts/gopherstack/templates/ingress.yaml new file mode 100644 index 0000000000..85c3425799 --- /dev/null +++ b/charts/gopherstack/templates/ingress.yaml @@ -0,0 +1,35 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "gopherstack.fullname" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ default "Prefix" .pathType }} + backend: + service: + name: {{ include "gopherstack.fullname" $ }} + port: + name: http + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/gopherstack/templates/pvc.yaml b/charts/gopherstack/templates/pvc.yaml new file mode 100644 index 0000000000..b03f5251af --- /dev/null +++ b/charts/gopherstack/templates/pvc.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "gopherstack.fullname" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 4 }} + {{- with .Values.persistence.storageClass }} + storageClassName: {{ if eq . "-" }}""{{ else }}{{ . | quote }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} +{{- end }} diff --git a/charts/gopherstack/templates/rbac.yaml b/charts/gopherstack/templates/rbac.yaml new file mode 100644 index 0000000000..0645402e4a --- /dev/null +++ b/charts/gopherstack/templates/rbac.yaml @@ -0,0 +1,43 @@ +{{- if eq .Values.container.runtime "kubernetes" }} +{{- $ns := default .Release.Namespace .Values.container.kubernetes.namespace }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "gopherstack.fullname" . }}-pods + namespace: {{ $ns }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + {{- if .Values.container.kubernetes.ecs.enabled }} + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "patch", "delete"] + {{- end }} + {{- if and .Values.container.kubernetes.ecs.enabled .Values.container.kubernetes.ecs.exec }} + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "gopherstack.fullname" . }}-pods + namespace: {{ $ns }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "gopherstack.fullname" . }}-pods +subjects: + - kind: ServiceAccount + name: {{ include "gopherstack.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/gopherstack/templates/service.yaml b/charts/gopherstack/templates/service.yaml new file mode 100644 index 0000000000..fc6b33c6d0 --- /dev/null +++ b/charts/gopherstack/templates/service.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "gopherstack.fullname" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + {{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- range .Values.service.extraPorts }} + - name: {{ .name }} + port: {{ .port }} + targetPort: {{ .name }} + protocol: {{ default "TCP" .protocol }} + {{- if and (eq $.Values.service.type "NodePort") .nodePort }} + nodePort: {{ .nodePort }} + {{- end }} + {{- end }} + selector: + {{- include "gopherstack.selectorLabels" . | nindent 4 }} diff --git a/charts/gopherstack/templates/serviceaccount.yaml b/charts/gopherstack/templates/serviceaccount.yaml new file mode 100644 index 0000000000..76b5c9be2d --- /dev/null +++ b/charts/gopherstack/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "gopherstack.serviceAccountName" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ eq .Values.container.runtime "kubernetes" }} +{{- end }} diff --git a/charts/gopherstack/values.schema.json b/charts/gopherstack/values.schema.json new file mode 100644 index 0000000000..e68c59cb16 --- /dev/null +++ b/charts/gopherstack/values.schema.json @@ -0,0 +1,159 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "image": { + "type": "object", + "required": ["repository"], + "properties": { + "repository": {"type": "string", "minLength": 1}, + "tag": {"type": "string"}, + "pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]} + } + }, + "imagePullSecrets": {"type": "array"}, + "nameOverride": {"type": "string"}, + "fullnameOverride": {"type": "string"}, + "config": { + "type": "object", + "properties": { + "region": {"type": "string", "minLength": 1}, + "accountId": {"type": "string", "pattern": "^[0-9]{12}$"}, + "logLevel": {"enum": ["debug", "info", "warn", "error"]}, + "engines": {"enum": ["stub", "auto", "docker"]}, + "enforceIam": {"type": "boolean"}, + "localstackCompat": {"type": "boolean"}, + "services": {"type": "array", "items": {"type": "string"}}, + "disableServices": {"type": "array", "items": {"type": "string"}}, + "extra": {"type": "object", "additionalProperties": {"type": ["string", "number", "boolean"]}} + } + }, + "container": { + "type": "object", + "properties": { + "runtime": {"enum": ["docker", "kubernetes"]}, + "kubernetes": { + "type": "object", + "properties": { + "namespace": {"type": "string"}, + "fetchImage": {"type": "string", "minLength": 1}, + "codePort": {"type": "integer", "minimum": 1, "maximum": 65535}, + "ecs": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "exec": {"type": "boolean"} + } + } + } + } + } + }, + "env": {"type": "array"}, + "envFrom": {"type": "array"}, + "service": { + "type": "object", + "properties": { + "type": {"enum": ["ClusterIP", "NodePort", "LoadBalancer"]}, + "port": {"type": "integer", "minimum": 1, "maximum": 65535}, + "nodePort": {"type": ["integer", "null"], "minimum": 30000, "maximum": 32767}, + "annotations": {"type": "object"}, + "extraPorts": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "port"], + "properties": { + "name": {"type": "string", "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", "maxLength": 15}, + "port": {"type": "integer", "minimum": 1, "maximum": 65535}, + "protocol": {"enum": ["TCP", "UDP"]}, + "nodePort": {"type": "integer", "minimum": 30000, "maximum": 32767} + } + } + } + } + }, + "ingress": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "className": {"type": "string"}, + "annotations": {"type": "object"}, + "hosts": { + "type": "array", + "items": { + "type": "object", + "required": ["host", "paths"], + "properties": { + "host": {"type": "string"}, + "paths": { + "type": "array", + "items": { + "type": "object", + "required": ["path"], + "properties": { + "path": {"type": "string"}, + "pathType": {"enum": ["Prefix", "Exact", "ImplementationSpecific"]} + } + } + } + } + } + }, + "tls": {"type": "array"} + } + }, + "persistence": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "size": {"type": "string"}, + "storageClass": {"type": "string"}, + "accessModes": {"type": "array", "items": {"type": "string"}}, + "existingClaim": {"type": "string"}, + "mountPath": {"type": "string", "pattern": "^/"}, + "annotations": {"type": "object"} + } + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": {"type": "boolean"}, + "name": {"type": "string"}, + "annotations": {"type": "object"} + } + }, + "dind": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "image": { + "type": "object", + "properties": { + "repository": {"type": "string"}, + "tag": {"type": "string"}, + "pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]} + } + }, + "resources": {"type": "object"} + } + }, + "probes": { + "type": "object", + "properties": { + "path": {"type": "string", "pattern": "^/"}, + "liveness": {"type": "object"}, + "readiness": {"type": "object"}, + "startup": {"type": "object"} + } + }, + "podAnnotations": {"type": "object"}, + "podLabels": {"type": "object"}, + "podSecurityContext": {"type": "object"}, + "securityContext": {"type": "object"}, + "resources": {"type": "object"}, + "nodeSelector": {"type": "object"}, + "tolerations": {"type": "array"}, + "affinity": {"type": "object"} + } +} diff --git a/charts/gopherstack/values.yaml b/charts/gopherstack/values.yaml new file mode 100644 index 0000000000..bfc76f4c7e --- /dev/null +++ b/charts/gopherstack/values.yaml @@ -0,0 +1,133 @@ +image: + repository: ghcr.io/blackbirdworks/gopherstack + # Defaults to the chart appVersion. + tag: "" + pullPolicy: IfNotPresent +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +# Gopherstack keeps state in one process; the Deployment always runs one replica. +config: + region: us-east-1 + accountId: "000000000000" + logLevel: info + # Engine default for container-backed services: stub | auto | docker. + engines: stub + enforceIam: false + localstackCompat: false + # Only enable / disable these services (empty services = all). + services: [] + disableServices: [] + # Any other environment variable, as KEY: value. + extra: {} + +env: [] +envFrom: [] + +service: + type: ClusterIP + port: 8000 + nodePort: null + annotations: {} + # Dedicated per-cloud listeners, e.g. Azure Blob 10000, Service Bus 10003, Cosmos 8081, + # GCP Pub/Sub 8085, Storage 4443. Each entry needs the matching *_PORT env var if it is not the default. + extraPorts: [] + # - name: azure-blob + # port: 10000 + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: gopherstack.local + paths: + - path: / + pathType: Prefix + tls: [] + +persistence: + enabled: false + size: 1Gi + storageClass: "" + accessModes: + - ReadWriteOnce + existingClaim: "" + mountPath: /data + annotations: {} + +serviceAccount: + create: true + name: "" + annotations: {} + +# How container-backed engines (Lambda today) run their containers. +# docker: the docker daemon (see dind below, or DOCKER_HOST via config.extra). +# kubernetes: one Pod per container in the release namespace, through the Kubernetes API. Needs no privileged +# sidecar; the chart adds a namespace-scoped Role (pods, pods/log) and mounts the ServiceAccount token. +# ECS tasks run as one Pod each; see container.kubernetes.ecs for the extra grants they need. +container: + runtime: docker + kubernetes: + # Namespace for function Pods; empty means the release namespace. + namespace: "" + # Pod-side initContainer image that downloads and unpacks function code (needs sh, wget, unzip). + fetchImage: busybox:1.37.0 + # Port Gopherstack serves function code on, reachable from function Pods by Pod IP. + codePort: 4599 + # ECS tasks as Pods (set config.engines=docker or GOPHERSTACK_ECS_RUNTIME=docker; one Pod per task). + ecs: + # Grants secrets create/patch/delete: container secrets are injected from a Secret the task's Pod owns. + enabled: false + # Grants pods/exec create so ECS Exec (ExecuteCommand) can open sessions. Needs ecs.enabled. + exec: false + +# Docker-in-Docker sidecar so docker-backed engines work in a cluster. Runs privileged. +dind: + enabled: false + image: + repository: docker + tag: 27-dind + pullPolicy: IfNotPresent + resources: {} + +probes: + path: /_gopherstack/health + liveness: + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readiness: + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + startup: + periodSeconds: 2 + timeoutSeconds: 3 + failureThreshold: 60 + +podAnnotations: {} +podLabels: {} + +# The release image runs as UID/GID 65532 and owns /data; fsGroup makes a PVC writable by that user. +podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + +resources: {} + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/cmd/gendocs/categories.go b/cmd/gendocs/categories.go index 873a2f8d1b..9c2e10520f 100644 --- a/cmd/gendocs/categories.go +++ b/cmd/gendocs/categories.go @@ -33,6 +33,7 @@ const ( slugAzureFunctions = "azurefunctions" slugCosmosDB = "cosmosdb" slugAzureRedis = "azureredis" + slugAzureSearch = "azuresearch" slugAzureSQL = "azuresql" slugGCPStorage = "gcpstorage" slugOCIObjectStore = "ociobjectstorage" @@ -53,6 +54,9 @@ const ( slugOCILoadBalancer = "ociloadbalancer" slugOCIArtifacts = "ociartifacts" slugOCIInstances = "ocicontainerinstances" + slugOCIDNS = "ocidns" + slugOCIDatabase = "ocidatabase" + slugOCIFileStorage = "ocifilestorage" slugGCPPubSub = "gcppubsub" slugGCPSecretMgr = "gcpsecretmanager" slugGCPKMS = "gcpkms" @@ -96,7 +100,7 @@ func ociSlugs() []string { slugOCIObjectStore, slugOCIVault, slugOCIStreaming, slugOCIIdentity, slugOCIFunctions, slugOCICore, slugOCIQueue, slugOCINotify, slugOCIContainerEng, slugOCIMySQL, slugOCILogging, slugOCIMonitoring, slugOCINoSQL, slugOCIAPIGateway, slugOCIEvents, slugOCILoadBalancer, slugOCIArtifacts, - slugOCIInstances, + slugOCIInstances, slugOCIDNS, slugOCIDatabase, slugOCIFileStorage, } } @@ -170,22 +174,9 @@ func categoryGroups() []categoryGroup { { "Azure", []string{ - slugAzureBlob, - slugAzureKeyVault, - slugAzureQueue, - slugAzureFile, - slugAzureMonitor, - slugAzureGraph, - slugAzureTable, - slugAzureServiceBus, - slugAzureEventHubs, - slugAzureEventGrid, - slugAzureAppConfig, - slugAzureACR, - slugAzureFunctions, - slugCosmosDB, - slugAzureRedis, - slugAzureSQL, + slugAzureBlob, slugAzureKeyVault, slugAzureQueue, slugAzureFile, slugAzureMonitor, slugAzureGraph, + slugAzureTable, slugAzureServiceBus, slugAzureEventHubs, slugAzureEventGrid, slugAzureAppConfig, + slugAzureACR, slugAzureFunctions, slugCosmosDB, slugAzureRedis, slugAzureSearch, slugAzureSQL, }, }, {"GCP", []string{ @@ -277,6 +268,7 @@ func displayNamesA() map[string]string { slugAzureACR: "Azure Container Registry", slugAzureFunctions: "Azure Functions", slugAzureRedis: "Azure Cache for Redis", + slugAzureSearch: "Azure AI Search", slugAzureSQL: "Azure SQL Database", "backup": "Backup", "batch": "Batch", @@ -326,6 +318,9 @@ func displayNamesB() map[string]string { slugOCILoadBalancer: "OCI Load Balancer", slugOCIArtifacts: "OCI Artifacts and Container Registry", slugOCIInstances: "OCI Container Instances", + slugOCIDNS: "OCI DNS", + slugOCIDatabase: "OCI Autonomous Database", + slugOCIFileStorage: "OCI File Storage", slugGCPPubSub: "Google Cloud Pub/Sub", slugGCPSecretMgr: "Google Secret Manager", slugGCPKMS: "Google Cloud KMS", diff --git a/dashboard/azureacr_proxy.go b/dashboard/azureacr_proxy.go new file mode 100644 index 0000000000..7cf3e9072b --- /dev/null +++ b/dashboard/azureacr_proxy.go @@ -0,0 +1,296 @@ +package dashboard + +import ( + "errors" + "net/http" + "regexp" + "time" + + "github.com/labstack/echo/v5" + + acrbackend "github.com/blackbirdworks/gopherstack/services/azurecontainerregistry" +) + +const ( + azureACRProxyPrefix = "/dashboard/api/v1/azure/acr" + acrMaxTags = 200 + acrMask = "••••••••" + acrKeyPassword = "password" +) + +var ( + acrRepoRe = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)*(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*$`) + acrTagRe = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`) +) + +type acrRegistryJSON struct { + CreatedAt time.Time `json:"createdAt"` + Name string `json:"name"` + LoginServer string `json:"loginServer"` + Repositories int `json:"repositoryCount"` + AdminEnabled bool `json:"adminEnabled"` +} + +type acrRepoJSON struct { + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Name string `json:"name"` + Manifests int `json:"manifestCount"` + Tags int `json:"tagCount"` +} + +type acrManifestJSON struct { + Created time.Time `json:"created"` + Updated time.Time `json:"updated,omitzero"` + Name string `json:"name,omitempty"` + Digest string `json:"digest"` + MediaType string `json:"mediaType"` + Architecture string `json:"architecture"` + OS string `json:"os"` + Tags []string `json:"tags"` + Size int64 `json:"sizeBytes"` + Children int `json:"childManifests"` + Index bool `json:"index"` +} + +// registerAzureACRRoutes serves the Container Registry page from the backend in-process: repositories, tags and +// manifests are read-only except for tag deletion, and the admin password leaves only through the reveal POST. +func (h *DashboardHandler) registerAzureACRRoutes() { + api := &azureACRAPI{} + if h.config.AzureACROps != nil { + api.h = h.config.AzureACROps + } + + r := h.SubRouter + p := azureACRProxyPrefix + "/registries" + r.Add(http.MethodGet, p, api.guard(api.registries)) + r.Add(http.MethodGet, p+"/:reg/repositories", api.guard(api.repositories)) + r.Add(http.MethodGet, p+"/:reg/tags", api.guard(api.tags)) + r.Add(http.MethodGet, p+"/:reg/manifests", api.guard(api.manifests)) + r.Add(http.MethodDelete, p+"/:reg/tags", api.guard(api.deleteTag)) + r.Add(http.MethodGet, p+"/:reg/credentials", api.guard(api.credentials)) + r.Add(http.MethodPost, p+"/:reg/credentials/reveal", api.guard(api.reveal)) +} + +type azureACRAPI struct { + h *acrbackend.Handler +} + +func (a *azureACRAPI) guard(next echo.HandlerFunc) echo.HandlerFunc { + return func(c *echo.Context) error { + if a.h == nil { + return cloudUnavailable(c, "Azure Container Registry") + } + + return next(c) + } +} + +func (a *azureACRAPI) fail(c *echo.Context, err error) error { + switch { + case errors.Is(err, acrbackend.ErrRegistryNotFound), errors.Is(err, acrbackend.ErrNotFound): + return cloudJSONFail(c, http.StatusNotFound, cloudStatusNotFnd, err.Error()) + case errors.Is(err, acrbackend.ErrDenied), errors.Is(err, acrbackend.ErrAdminDisabled): + return cloudJSONFail(c, http.StatusConflict, "FAILED_PRECONDITION", err.Error()) + default: + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, err.Error()) + } +} + +func (a *azureACRAPI) registries(c *echo.Context) error { + names := a.h.Backend.Names() + out := make([]acrRegistryJSON, 0, len(names)) + + for _, n := range names { + info, ok := a.h.Backend.Registry(n) + if !ok { + continue + } + + out = append(out, acrRegistryJSON{ + Name: info.Name, CreatedAt: info.CreatedAt, AdminEnabled: info.AdminEnabled, + Repositories: len(a.h.Backend.Repositories(n)), LoginServer: info.Name + ".azurecr.localhost", + }) + } + + return c.JSON(http.StatusOK, map[string]any{"registries": out}) +} + +func (a *azureACRAPI) known(c *echo.Context) (string, bool) { + reg := c.Param("reg") + if _, ok := a.h.Backend.Registry(reg); !ok { + _ = a.fail(c, acrbackend.ErrRegistryNotFound) + + return "", false + } + + return reg, true +} + +func (a *azureACRAPI) repoParam(c *echo.Context) (string, bool) { + repo := c.QueryParam("repo") + if len(repo) > 255 || !acrRepoRe.MatchString(repo) { + _ = cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "invalid repository name") + + return "", false + } + + return repo, true +} + +func (a *azureACRAPI) repositories(c *echo.Context) error { + reg, ok := a.known(c) + if !ok { + return nil + } + + repos := a.h.Backend.Repositories(reg) + out := make([]acrRepoJSON, 0, len(repos)) + + for _, r := range repos { + out = append(out, acrRepoJSON{ + Name: r.Name, Created: r.Created, Updated: r.Updated, Manifests: r.ManifestCount, Tags: r.TagCount, + }) + } + + return c.JSON(http.StatusOK, map[string]any{"repositories": out}) +} + +// detail describes a manifest; a manifest that cannot be read keeps just its digest. +func (a *azureACRAPI) detail(c *echo.Context, reg, repo, digest string) acrManifestJSON { + out := acrManifestJSON{Digest: digest, Tags: []string{}} + + d, err := a.h.Backend.DescribeManifest(c.Request().Context(), reg, repo, digest) + if err != nil { + return out + } + + out.MediaType, out.Size, out.Architecture, out.OS = d.MediaType, d.Size, d.Architecture, d.OS + out.Index, out.Children = d.Index, d.Children + + return out +} + +func (a *azureACRAPI) tags(c *echo.Context) error { + reg, ok := a.known(c) + if !ok { + return nil + } + + repo, ok := a.repoParam(c) + if !ok { + return nil + } + + tags, found := a.h.Backend.Tags(reg, repo) + if !found { + return a.fail(c, acrbackend.ErrNotFound) + } + + if len(tags) > acrMaxTags { + tags = tags[:acrMaxTags] + } + + cache := map[string]acrManifestJSON{} + out := make([]acrManifestJSON, 0, len(tags)) + + for _, t := range tags { + m, seen := cache[t.Digest] + if !seen { + m = a.detail(c, reg, repo, t.Digest) + cache[t.Digest] = m + } + + m.Created, m.Updated, m.Name = t.Created, t.Updated, t.Name + out = append(out, m) + } + + return c.JSON(http.StatusOK, map[string]any{"tags": out}) +} + +func (a *azureACRAPI) manifests(c *echo.Context) error { + reg, ok := a.known(c) + if !ok { + return nil + } + + repo, ok := a.repoParam(c) + if !ok { + return nil + } + + list, found := a.h.Backend.Manifests(reg, repo) + if !found { + return a.fail(c, acrbackend.ErrNotFound) + } + + if len(list) > acrMaxTags { + list = list[:acrMaxTags] + } + + out := make([]acrManifestJSON, 0, len(list)) + + for _, m := range list { + d := a.detail(c, reg, repo, m.Digest) + d.Created = m.Created + d.Tags = append(d.Tags, m.Tags...) + out = append(out, d) + } + + return c.JSON(http.StatusOK, map[string]any{"manifests": out}) +} + +func (a *azureACRAPI) deleteTag(c *echo.Context) error { + reg, ok := a.known(c) + if !ok { + return nil + } + + repo, ok := a.repoParam(c) + if !ok { + return nil + } + + tag := c.QueryParam("tag") + if !acrTagRe.MatchString(tag) { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "invalid tag name") + } + + if err := a.h.Backend.DeleteTag(c.Request().Context(), reg, repo, tag); err != nil { + return a.fail(c, err) + } + + return c.NoContent(http.StatusNoContent) +} + +func (a *azureACRAPI) credentials(c *echo.Context) error { + reg, ok := a.known(c) + if !ok { + return nil + } + + info, _ := a.h.Backend.Registry(reg) + + return c.JSON(http.StatusOK, map[string]any{ + "adminEnabled": info.AdminEnabled, "username": info.Name, "passwords": []string{acrMask, acrMask}, + }) +} + +// reveal is the only route that sends the admin passwords out. +func (a *azureACRAPI) reveal(c *echo.Context) error { + reg, ok := a.known(c) + if !ok { + return nil + } + + creds, err := a.h.Backend.Credentials(reg) + if err != nil { + return a.fail(c, err) + } + + c.Response().Header().Set("Cache-Control", "no-store") + + return c.JSON(http.StatusOK, map[string]string{ + "username": creds.Username, acrKeyPassword: creds.Password1, "password2": creds.Password2, + }) +} diff --git a/dashboard/azureacr_proxy_test.go b/dashboard/azureacr_proxy_test.go new file mode 100644 index 0000000000..7b465a5261 --- /dev/null +++ b/dashboard/azureacr_proxy_test.go @@ -0,0 +1,187 @@ +package dashboard_test + +import ( + "encoding/base64" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/pkgs/ociregistry/ocitest" + "github.com/blackbirdworks/gopherstack/services/azurecontainerregistry" +) + +const acrBase = "/dashboard/api/v1/azure/acr/registries" + +type acrEnv struct { + h *dashboard.DashboardHandler + creds azurecontainerregistry.Credentials +} + +func newACREnv(t *testing.T) *acrEnv { + t.Helper() + + b := azurecontainerregistry.NewInMemoryBackend() + b.SetDataDir(t.TempDir()) + b.SetContext(t.Context()) + + acr := azurecontainerregistry.NewHandler(b, azurecontainerregistry.DefaultSettings()) + t.Cleanup(func() { acr.Shutdown(t.Context()) }) + + srv := httptest.NewServer(acr) + t.Cleanup(srv.Close) + + _, err := b.PutRegistry("devregistry", true) + require.NoError(t, err) + _, err = b.PutRegistry("plainregistry", false) + require.NoError(t, err) + + creds, err := b.Credentials("devregistry") + require.NoError(t, err) + + auth := base64.StdEncoding.EncodeToString([]byte(creds.Username + ":" + creds.Password1)) + c := ocitest.Client{ + T: t, Base: srv.URL, Host: "devregistry.azurecr.localhost", + Prepare: func(r *http.Request) { r.Header.Set("Authorization", "Basic "+auth) }, + } + c.PushImage("team/app", "v1") + c.PushImage("team/app", "v2") + + return &acrEnv{ + h: dashboard.NewHandler(dashboard.Config{AzureACROps: acr}), + creds: creds, + } +} + +func TestAzureACRProxy(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + method string + path string + wantBody string + wantCode int + }{ + {name: "registries", method: http.MethodGet, path: acrBase, wantCode: 200, wantBody: `"devregistry"`}, + { + name: "repositories", method: http.MethodGet, path: acrBase + "/devregistry/repositories", + wantCode: 200, wantBody: `"team/app"`, + }, + { + name: "tags", method: http.MethodGet, path: acrBase + "/devregistry/tags?repo=team%2Fapp", + wantCode: 200, wantBody: `"name":"v1"`, + }, + { + name: "manifests", method: http.MethodGet, path: acrBase + "/devregistry/manifests?repo=team%2Fapp", + wantCode: 200, wantBody: `"digest":"sha256:`, + }, + { + name: "unknown registry", method: http.MethodGet, path: acrBase + "/nope/repositories", + wantCode: 404, wantBody: "NOT_FOUND", + }, + { + name: "unknown repository", method: http.MethodGet, path: acrBase + "/devregistry/tags?repo=ghost", + wantCode: 404, wantBody: "NOT_FOUND", + }, + { + name: "traversal repo", method: http.MethodGet, path: acrBase + "/devregistry/tags?repo=..%2Fx", + wantCode: 400, wantBody: "invalid repository name", + }, + { + name: "uppercase repo", method: http.MethodGet, path: acrBase + "/devregistry/tags?repo=Team", + wantCode: 400, wantBody: "invalid repository name", + }, + { + name: "bad tag", + method: http.MethodDelete, + path: acrBase + "/devregistry/tags?repo=team%2Fapp&tag=sha256:abc", + wantCode: 400, + wantBody: "invalid tag name", + }, + { + name: "delete missing tag", + method: http.MethodDelete, + path: acrBase + "/devregistry/tags?repo=team%2Fapp&tag=zzz", + wantCode: 404, + wantBody: "NOT_FOUND", + }, + { + name: "no write to repositories", method: http.MethodPost, path: acrBase + "/devregistry/repositories", + wantCode: 404, wantBody: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + env := newACREnv(t) + rec := doProxy(t, env.h, tt.method, tt.path, "") + + assert.Equal(t, tt.wantCode, rec.Code) + assert.Contains(t, rec.Body.String(), tt.wantBody) + }) + } +} + +func TestAzureACRTagDelete(t *testing.T) { + t.Parallel() + + env := newACREnv(t) + + rec := doProxy(t, env.h, http.MethodDelete, acrBase+"/devregistry/tags?repo=team%2Fapp&tag=v1", "") + require.Equal(t, http.StatusNoContent, rec.Code) + + rec = doProxy(t, env.h, http.MethodGet, acrBase+"/devregistry/tags?repo=team%2Fapp", "") + require.Equal(t, http.StatusOK, rec.Code) + assert.NotContains(t, rec.Body.String(), `"name":"v1"`) + assert.Contains(t, rec.Body.String(), `"name":"v2"`) +} + +func TestAzureACRCredentialsMasked(t *testing.T) { + t.Parallel() + + env := newACREnv(t) + + rec := doProxy(t, env.h, http.MethodGet, acrBase+"/devregistry/credentials", "") + require.Equal(t, http.StatusOK, rec.Code) + assert.NotContains(t, rec.Body.String(), env.creds.Password1) + assert.NotContains(t, rec.Body.String(), env.creds.Password2) + assert.Contains(t, rec.Body.String(), `"adminEnabled":true`) + + for _, path := range []string{"/registries", "/devregistry/repositories", "/devregistry/tags?repo=team%2Fapp"} { + rec = doProxy(t, env.h, http.MethodGet, "/dashboard/api/v1/azure/acr"+path, "") + assert.NotContains(t, rec.Body.String(), env.creds.Password1, path) + } +} + +func TestAzureACRReveal(t *testing.T) { + t.Parallel() + + env := newACREnv(t) + + rec := doProxy(t, env.h, http.MethodPost, acrBase+"/devregistry/credentials/reveal", "") + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + assert.Contains(t, rec.Body.String(), env.creds.Password1) + + rec = doProxy(t, env.h, http.MethodGet, acrBase+"/devregistry/credentials/reveal", "") + assert.NotContains(t, rec.Body.String(), env.creds.Password1) + + rec = doProxy(t, env.h, http.MethodPost, acrBase+"/plainregistry/credentials/reveal", "") + assert.Equal(t, http.StatusConflict, rec.Code) +} + +func TestAzureACRAbsent(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + rec := doProxy(t, h, http.MethodGet, acrBase, "") + + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Contains(t, rec.Body.String(), "UNAVAILABLE") +} diff --git a/dashboard/azureappconfig_extra.go b/dashboard/azureappconfig_extra.go new file mode 100644 index 0000000000..0a66633ef8 --- /dev/null +++ b/dashboard/azureappconfig_extra.go @@ -0,0 +1,269 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "strings" + "time" + + "github.com/labstack/echo/v5" + + appconfigbackend "github.com/blackbirdworks/gopherstack/services/azureappconfig" +) + +const ( + appConfigFlagPrefix = ".appconfig.featureflag/" + appConfigFlagType = "application/vnd.microsoft.appconfig.ff+json;charset=utf-8" + appConfigMaxRevision = 200 + appConfigMaxFlagID = 256 + appConfigMask = "••••••••" + appConfigItemsKey = "items" + appConfigKeysKey = "keys" + appConfigSecretKey = "secret" +) + +type appConfigSnapshotJSON struct { + Created time.Time `json:"created"` + Expires *time.Time `json:"expires,omitempty"` + Name string `json:"name"` + Status string `json:"status"` + Composition string `json:"compositionType"` + Filters []string `json:"filters"` + Retention int64 `json:"retentionSeconds"` + Items int64 `json:"itemCount"` +} + +type appConfigFlagJSON struct { + LastModified time.Time `json:"lastModified"` + ID string `json:"id"` + Description string `json:"description"` + Label string `json:"label"` + ETag string `json:"etag"` + Enabled bool `json:"enabled"` + Locked bool `json:"locked"` + Filters int `json:"filterCount"` +} + +type appConfigKeyJSON struct { + LastModified time.Time `json:"lastModified"` + ID string `json:"id"` + Name string `json:"name"` + Masked string `json:"masked"` + ReadOnly bool `json:"readOnly"` +} + +type appConfigToggleJSON struct { + ID string `json:"id"` + Label string `json:"label"` + Enabled bool `json:"enabled"` +} + +type appConfigRevealJSON struct { + ID string `json:"id"` +} + +type appConfigFlagBody struct { + ID string `json:"id"` + Description string `json:"description"` + Conditions struct { + ClientFilters []json.RawMessage `json:"client_filters"` + } `json:"conditions"` + Enabled bool `json:"enabled"` +} + +func (a *azureAppConfigAPI) registerExtra(r *echo.Echo) { + p := azureAppConfigProxyPrefix + "/stores/:store" + r.Add(http.MethodGet, p+"/snapshots", a.guard(a.snapshots)) + r.Add(http.MethodGet, p+"/snapshots/:name/kv", a.guard(a.snapshotItems)) + r.Add(http.MethodGet, p+"/revisions", a.guard(a.revisions)) + r.Add(http.MethodGet, p+"/features", a.guard(a.features)) + r.Add(http.MethodPost, p+"/features/toggle", a.guard(a.toggleFeature)) + r.Add(http.MethodGet, p+"/keys", a.guard(a.accessKeys)) + r.Add(http.MethodPost, p+"/keys/reveal", a.guard(a.revealKey)) +} + +func appConfigKVs(kvs []appconfigbackend.KeyValue) []appConfigKVJSON { + items := make([]appConfigKVJSON, 0, len(kvs)) + for _, kv := range kvs { + items = append(items, appConfigKV(kv)) + } + + return items +} + +func (a *azureAppConfigAPI) snapshots(c *echo.Context) error { + snaps, err := a.b.ListSnapshots(c.Param("store"), "", nil) + if err != nil { + return a.fail(c, err) + } + + items := make([]appConfigSnapshotJSON, 0, len(snaps)) + + for _, s := range snaps { + filters := make([]string, 0, len(s.Filters)) + for _, f := range s.Filters { + filters = append(filters, "key="+f.Key+" label="+f.Label) + } + + items = append(items, appConfigSnapshotJSON{ + Name: s.Name, Status: s.Status, Composition: s.Composition, Created: s.Created, Expires: s.Expires, + Filters: filters, Retention: s.Retention, Items: int64(len(s.Items)), + }) + } + + return c.JSON(http.StatusOK, map[string]any{"snapshots": items}) +} + +func (a *azureAppConfigAPI) snapshotItems(c *echo.Context) error { + sel, err := appconfigbackend.NewSelector("", "", nil) + if err != nil { + return a.fail(c, err) + } + + kvs, err := a.b.SnapshotKeyValues(c.Param("store"), c.Param("name"), sel) + if err != nil { + return a.fail(c, err) + } + + return c.JSON(http.StatusOK, map[string]any{appConfigItemsKey: appConfigKVs(kvs)}) +} + +// revisions returns the newest recorded states of the matching key-values, newest first. +func (a *azureAppConfigAPI) revisions(c *echo.Context) error { + sel, err := appconfigbackend.NewSelector(c.QueryParam("key"), c.QueryParam("label"), nil) + if err != nil { + return a.fail(c, err) + } + + kvs, err := a.b.ListRevisions(c.Param("store"), sel, time.Time{}) + if err != nil { + return a.fail(c, err) + } + + items := appConfigKVs(kvs) + for i, j := 0, len(items)-1; i < j; i, j = i+1, j-1 { + items[i], items[j] = items[j], items[i] + } + + if len(items) > appConfigMaxRevision { + items = items[:appConfigMaxRevision] + } + + return c.JSON(http.StatusOK, map[string]any{appConfigItemsKey: items}) +} + +func (a *azureAppConfigAPI) features(c *echo.Context) error { + sel, err := appconfigbackend.NewSelector(appConfigFlagPrefix+"*", "", nil) + if err != nil { + return a.fail(c, err) + } + + kvs, err := a.b.ListKeyValues(c.Param("store"), sel, time.Time{}) + if err != nil { + return a.fail(c, err) + } + + flags := make([]appConfigFlagJSON, 0, len(kvs)) + + for _, kv := range kvs { + var body appConfigFlagBody + if kv.Value != nil { + _ = json.Unmarshal([]byte(*kv.Value), &body) + } + + flags = append(flags, appConfigFlagJSON{ + ID: strings.TrimPrefix(kv.Key, appConfigFlagPrefix), Description: body.Description, Label: kv.Label, + Enabled: body.Enabled, Locked: kv.Locked, ETag: kv.ETag, LastModified: kv.LastModified, + Filters: len(body.Conditions.ClientFilters), + }) + } + + return c.JSON(http.StatusOK, map[string]any{"features": flags}) +} + +// toggleFeature flips one flag's enabled field; every other field of the stored document is kept. +func (a *azureAppConfigAPI) toggleFeature(c *echo.Context) error { + var in appConfigToggleJSON + if !cloudJSONDecode(c, &in) { + return nil + } + + if in.ID == "" || len(in.ID) > appConfigMaxFlagID { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "invalid feature flag id") + } + + store, key := c.Param("store"), appConfigFlagPrefix+in.ID + + kv, err := a.b.GetKeyValue(store, key, in.Label, time.Time{}) + if err != nil { + return a.fail(c, err) + } + + doc := map[string]json.RawMessage{} + if kv.Value != nil { + if err = json.Unmarshal([]byte(*kv.Value), &doc); err != nil { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "stored feature flag is not valid JSON") + } + } + + doc["enabled"], _ = json.Marshal(in.Enabled) + + raw, err := json.Marshal(doc) + if err != nil { + return a.fail(c, err) + } + + value, ct := string(raw), appConfigFlagType + if kv.ContentType != nil { + ct = *kv.ContentType + } + + updated, err := a.b.SetKeyValue(store, key, in.Label, + appconfigbackend.KVInput{Value: &value, ContentType: &ct, Tags: kv.Tags}, + appconfigbackend.Precondition{IfMatch: kv.ETag}) + if err != nil { + return a.fail(c, err) + } + + return c.JSON(http.StatusOK, appConfigKV(updated)) +} + +// accessKeys lists the store's credentials without their secrets. +func (a *azureAppConfigAPI) accessKeys(c *echo.Context) error { + keys, err := a.b.AccessKeys(c.Param("store")) + if err != nil { + return a.fail(c, err) + } + + items := make([]appConfigKeyJSON, 0, len(keys)) + for _, k := range keys { + items = append(items, appConfigKeyJSON{ + ID: k.ID, Name: k.Name, ReadOnly: k.ReadOnly, LastModified: k.LastModified, Masked: appConfigMask, + }) + } + + return c.JSON(http.StatusOK, map[string]any{appConfigKeysKey: items}) +} + +// revealKey is the only route that sends a credential secret out. +func (a *azureAppConfigAPI) revealKey(c *echo.Context) error { + var in appConfigRevealJSON + if !cloudJSONDecode(c, &in) { + return nil + } + + keys, err := a.b.AccessKeys(c.Param("store")) + if err != nil { + return a.fail(c, err) + } + + for _, k := range keys { + if k.ID == in.ID { + c.Response().Header().Set("Cache-Control", "no-store") + + return c.JSON(http.StatusOK, map[string]string{"id": k.ID, appConfigSecretKey: k.Secret}) + } + } + + return a.fail(c, appconfigbackend.ErrKeyIDNotFound) +} diff --git a/dashboard/azureappconfig_extra_test.go b/dashboard/azureappconfig_extra_test.go new file mode 100644 index 0000000000..98f7be9b0a --- /dev/null +++ b/dashboard/azureappconfig_extra_test.go @@ -0,0 +1,224 @@ +package dashboard_test + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/azureappconfig" +) + +const ( + acPxBase = "/dashboard/api/v1/azure/appconfig/stores/store1" + acFlag = `{"id":"beta","description":"Beta UI","enabled":false,"conditions":` + + `{"client_filters":[{"name":"Microsoft.Percentage","parameters":{"Value":50}}]}}` +) + +func newAppConfigEnv(t *testing.T) (*dashboard.DashboardHandler, string) { + t.Helper() + + b := azureappconfig.NewInMemoryBackend() + _, err := b.PutStore("store1") + require.NoError(t, err) + + set := func(key, label, value string, ct *string) { + _, setErr := b.SetKeyValue( + "store1", + key, + label, + azureappconfig.KVInput{Value: &value, ContentType: ct}, + azureappconfig.Precondition{}, + ) + require.NoError(t, setErr) + } + + ff := "application/vnd.microsoft.appconfig.ff+json;charset=utf-8" + set("app/key", "", "v1", nil) + set("app/key", "", "v2", nil) + set(".appconfig.featureflag/beta", "prod", acFlag, &ff) + set(".appconfig.featureflag/gamma", "", `{"id":"gamma","enabled":true}`, &ff) + + _, err = b.CreateSnapshot("store1", "snap1", azureappconfig.SnapshotSpec{ + Filters: []azureappconfig.Filter{{Key: "app/*"}}, Composition: "key", + }) + require.NoError(t, err) + + keys, err := b.AccessKeys("store1") + require.NoError(t, err) + + return dashboard.NewHandler( + dashboard.Config{AzureAppConfigOps: azureappconfig.NewHandler(b, azureappconfig.Settings{})}, + ), keys[0].ID +} + +func TestAzureAppConfigExtraReads(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + wantBody []string + wantCode int + }{ + { + name: "snapshots", + path: acPxBase + "/snapshots", + wantCode: 200, + wantBody: []string{`"snap1"`, `"itemCount":1`}, + }, + { + name: "snapshot items", + path: acPxBase + "/snapshots/snap1/kv", + wantCode: 200, + wantBody: []string{`"app/key"`, `"v2"`}, + }, + { + name: "missing snapshot", + path: acPxBase + "/snapshots/nope/kv", + wantCode: 404, + wantBody: []string{"NOT_FOUND"}, + }, + { + name: "revisions newest first", + path: acPxBase + "/revisions?key=app/key", + wantCode: 200, + wantBody: []string{`"v2"`, `"v1"`}, + }, + { + name: "features", + path: acPxBase + "/features", + wantCode: 200, + wantBody: []string{`"id":"beta"`, `"description":"Beta UI"`, `"filterCount":1`, `"id":"gamma"`}, + }, + { + name: "missing store", + path: "/dashboard/api/v1/azure/appconfig/stores/nope/features", + wantCode: 404, + wantBody: []string{"NOT_FOUND"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h, _ := newAppConfigEnv(t) + rec := doProxy(t, h, http.MethodGet, tt.path, "") + + assert.Equal(t, tt.wantCode, rec.Code, rec.Body.String()) + + for _, w := range tt.wantBody { + assert.Contains(t, rec.Body.String(), w) + } + }) + } +} + +func TestAzureAppConfigRevisionOrder(t *testing.T) { + t.Parallel() + + h, _ := newAppConfigEnv(t) + rec := doProxy(t, h, http.MethodGet, acPxBase+"/revisions?key=app/key", "") + require.Equal(t, http.StatusOK, rec.Code) + + var body struct { + Items []struct { + Value string `json:"value"` + } `json:"items"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body)) + require.Len(t, body.Items, 2) + assert.Equal(t, "v2", body.Items[0].Value) +} + +func TestAzureAppConfigToggleKeepsDocument(t *testing.T) { + t.Parallel() + + h, _ := newAppConfigEnv(t) + + rec := doProxy( + t, + h, + http.MethodPost, + acPxBase+"/features/toggle", + `{"id":"beta","label":"prod","enabled":true,"value":"IGNORED","key":"app/key"}`, + ) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + rec = doProxy(t, h, http.MethodGet, acPxBase+"/features", "") + assert.Contains(t, rec.Body.String(), `"enabled":true`) + assert.Contains(t, rec.Body.String(), `"description":"Beta UI"`) + assert.Contains(t, rec.Body.String(), `"filterCount":1`) + + rec = doProxy(t, h, http.MethodGet, acPxBase+"/kv?key=app/key", "") + assert.NotContains(t, rec.Body.String(), "IGNORED") + + rec = doProxy(t, h, http.MethodGet, acPxBase+"/kv?key=.appconfig.featureflag/beta", "") + assert.Contains(t, rec.Body.String(), "Microsoft.Percentage") + assert.NotContains(t, rec.Body.String(), "IGNORED") +} + +func TestAzureAppConfigToggleRejects(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body string + wantBody string + wantCode int + }{ + {name: "missing id", body: `{"enabled":true}`, wantCode: 400, wantBody: "invalid feature flag id"}, + {name: "unknown flag", body: `{"id":"ghost","enabled":true}`, wantCode: 404, wantBody: "NOT_FOUND"}, + {name: "wrong label", body: `{"id":"beta","label":"dev","enabled":true}`, wantCode: 404, wantBody: "NOT_FOUND"}, + {name: "bad json", body: `{`, wantCode: 400, wantBody: "invalid request body"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h, _ := newAppConfigEnv(t) + rec := doProxy(t, h, http.MethodPost, acPxBase+"/features/toggle", tt.body) + + assert.Equal(t, tt.wantCode, rec.Code) + assert.Contains(t, rec.Body.String(), tt.wantBody) + }) + } +} + +func TestAzureAppConfigKeysMaskedAndReveal(t *testing.T) { + t.Parallel() + + h, id := newAppConfigEnv(t) + + rec := doProxy(t, h, http.MethodGet, acPxBase+"/keys", "") + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), id) + assert.NotContains(t, rec.Body.String(), "secret") + + rec = doProxy(t, h, http.MethodPost, acPxBase+"/keys/reveal", `{"id":"`+id+`"}`) + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + assert.Contains(t, rec.Body.String(), `"secret":"`) + + rec = doProxy(t, h, http.MethodPost, acPxBase+"/keys/reveal", `{"id":"nope"}`) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = doProxy(t, h, http.MethodGet, acPxBase+"/keys/reveal", "") + assert.NotContains(t, rec.Body.String(), `"secret"`) +} + +func TestAzureAppConfigExtraAbsent(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + for _, p := range []string{"/snapshots", "/features", "/keys", "/revisions"} { + rec := doProxy(t, h, http.MethodGet, acPxBase+p, "") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code, p) + } +} diff --git a/dashboard/azureappconfig_proxy.go b/dashboard/azureappconfig_proxy.go index 16d315a80c..ca6f15eaeb 100644 --- a/dashboard/azureappconfig_proxy.go +++ b/dashboard/azureappconfig_proxy.go @@ -47,6 +47,7 @@ func (h *DashboardHandler) registerAzureAppConfigRoutes() { r.Add(http.MethodGet, azureAppConfigProxyPrefix+"/stores/:store/labels", api.guard(api.labels)) r.Add(http.MethodPut, azureAppConfigProxyPrefix+"/stores/:store/kv", api.guard(api.put)) r.Add(http.MethodDelete, azureAppConfigProxyPrefix+"/stores/:store/kv", api.guard(api.remove)) + api.registerExtra(r) } type azureAppConfigAPI struct { @@ -65,7 +66,8 @@ func (a *azureAppConfigAPI) guard(next echo.HandlerFunc) echo.HandlerFunc { func (a *azureAppConfigAPI) fail(c *echo.Context, err error) error { switch { - case errors.Is(err, appconfigbackend.ErrStoreNotFound), errors.Is(err, appconfigbackend.ErrKeyNotFound): + case errors.Is(err, appconfigbackend.ErrStoreNotFound), errors.Is(err, appconfigbackend.ErrKeyNotFound), + errors.Is(err, appconfigbackend.ErrSnapshotNotFound), errors.Is(err, appconfigbackend.ErrKeyIDNotFound): return cloudJSONFail(c, http.StatusNotFound, cloudStatusNotFnd, err.Error()) case errors.Is(err, appconfigbackend.ErrLocked): return cloudJSONFail(c, http.StatusConflict, "FAILED_PRECONDITION", err.Error()) diff --git a/dashboard/azureappservice_proxy_test.go b/dashboard/azureappservice_proxy_test.go index b7819d7d36..85d3e7aad4 100644 --- a/dashboard/azureappservice_proxy_test.go +++ b/dashboard/azureappservice_proxy_test.go @@ -111,6 +111,12 @@ func (f *fakeFunctionApps) SwapContent(string, string) error { return nil } func (f *fakeFunctionApps) WarmUp(context.Context, string) error { return nil } +func (f *fakeFunctionApps) DeleteFunction(context.Context, string, string) error { return nil } + +func (f *fakeFunctionApps) AdminToken(string) (string, error) { return "", nil } + +func (f *fakeFunctionApps) SyncTriggers(context.Context, string) error { return nil } + func newAppServiceDashboard(t *testing.T) *dashboard.DashboardHandler { t.Helper() diff --git a/dashboard/azurearm_inproc_proxy.go b/dashboard/azurearm_inproc_proxy.go index 6b036ebd61..e5d12c7ef1 100644 --- a/dashboard/azurearm_inproc_proxy.go +++ b/dashboard/azurearm_inproc_proxy.go @@ -30,7 +30,7 @@ const ( var armPxSecretKeys = []string{ //nolint:gochecknoglobals // immutable allowlist "adminpassword", "customdata", "sharedkey", "daprainstrumentationkey", "dapraiconnectionstring", - "primarykey", "secondarykey", "protectedsettings", "secret", + "primarykey", "secondarykey", "protectedsettings", "secret", "administratorloginpassword", } // armProxy describes one in-process dispatch to the ARM handler: which resource types and operations the page may @@ -41,8 +41,10 @@ type armProxy struct { view func(doc any) name string prefix string + version string queryKeys []string methods []string + relayOps bool } func (h *DashboardHandler) registerARMProxy(px armProxy) { @@ -73,6 +75,10 @@ func (h *DashboardHandler) registerARMProxy(px armProxy) { } } + if px.version != "" { + q.Set(armQueryVersion, px.version) + } + fwd := r.Clone(r.Context()) fwd.URL.Path, fwd.URL.RawPath = unescapedPath(armPath), armPath fwd.URL.RawQuery = q.Encode() @@ -91,6 +97,10 @@ func (h *DashboardHandler) registerARMProxy(px armProxy) { rec := httptest.NewRecorder() inner.ServeHTTP(rec, fwd) + if px.relayOps { + netRelayOps(c, rec.Header(), px.prefix) + } + return armProxyRelay(c, rec, px.view) } diff --git a/dashboard/azureauthz_proxy.go b/dashboard/azureauthz_proxy.go new file mode 100644 index 0000000000..66b5027d18 --- /dev/null +++ b/dashboard/azureauthz_proxy.go @@ -0,0 +1,165 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "regexp" + "slices" + "strings" +) + +const ( + azureAuthzPrefix = "/dashboard/api/v1/azure/authz" + authzPxSubScope = 2 + authzPxRGScope = 4 + authzPxNS = "microsoft.authorization" + authzPxRoleDefs = "roledefinitions" + authzPxRoleAssign = "roleassignments" + authzPxLocks = "locks" + authzPxMaxNotes = 512 +) + +//nolint:gochecknoglobals // immutable allow-lists and validators +var ( + authzPxReadOnly = []string{"policydefinitions", "policysetdefinitions", "policyassignments", authzPxRoleDefs} + authzPxLevels = []string{"CanNotDelete", "ReadOnly"} + authzPxTypes = []string{"User", "Group", "ServicePrincipal"} + authzPxLockName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,89}$`) + authzPxRoleDef = regexp.MustCompile( + `^/subscriptions/[^/]+/providers/microsoft\.authorization/roledefinitions/[0-9a-f-]{36}$`) +) + +// registerAzureAuthzRoutes serves the access half of the Entra page: role definitions and assignments, locks and +// policy at subscription or resource-group scope, by dispatching in-process to the ARM handler. Role assignment and +// lock writes are rebuilt from fixed fields; policy and role definitions are read-only. +func (h *DashboardHandler) registerAzureAuthzRoutes() { + h.registerARMProxy(armProxy{ + name: "Azure Authorization", prefix: azureAuthzPrefix, + queryKeys: []string{armQueryVersion, "$filter"}, + methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete}, + allowed: authzProxyAllowed, + body: authzProxyBody, + }) +} + +// authzSplit splits a lower-cased path ({scope}/providers/microsoft.authorization/{type}[/{name}]) where the scope +// is a subscription or a resource group. +func authzSplit(p []string) (string, string, bool) { + scope := authzPxSubScope + if len(p) > authzPxSubScope && p[0] == armPxSubsSeg && p[2] == armPxRGsSeg { + scope = authzPxRGScope + } + + if len(p) < scope+3 || len(p) > scope+4 || p[0] != armPxSubsSeg || p[scope] != armPxProvSeg || + p[scope+1] != authzPxNS { + return "", "", false + } + + name := "" + if len(p) == scope+4 { + name = p[scope+3] + } + + return p[scope+2], name, true +} + +func authzProxyAllowed(method string, p []string) bool { + typ, name, ok := authzSplit(p) + if !ok { + return false + } + + switch { + case method == http.MethodGet: + return typ == authzPxRoleAssign || typ == authzPxLocks || slices.Contains(authzPxReadOnly, typ) + case name == "": + return false + case typ == authzPxRoleAssign: + return authzPxGUIDName(name) + case typ == authzPxLocks: + return authzPxLockName.MatchString(name) + } + + return false +} + +func authzPxGUIDName(n string) bool { return graphPxGUID.MatchString(n) } + +func authzProxyBody(method string, p []string, in []byte) ([]byte, bool) { + if method != http.MethodPut { + return in, true + } + + typ, _, _ := authzSplit(p) + + var props map[string]any + + switch typ { + case authzPxRoleAssign: + props = authzAssignmentProps(in) + case authzPxLocks: + props = authzLockProps(in) + } + + if props == nil { + return nil, false + } + + b, err := json.Marshal(map[string]any{"properties": props}) + + return b, err == nil +} + +type authzPropsIn struct { + Properties map[string]any `json:"properties"` +} + +func (r authzPropsIn) text(key string) string { + v, _ := r.Properties[key].(string) + + return v +} + +func authzAssignmentProps(in []byte) map[string]any { + var req authzPropsIn + if json.Unmarshal(in, &req) != nil { + return nil + } + + def, pid, typ := req.text("roleDefinitionId"), req.text("principalId"), req.text("principalType") + + if !authzPxRoleDef.MatchString(strings.ToLower(def)) || !graphPxGUID.MatchString(strings.ToLower(pid)) { + return nil + } + + out := map[string]any{"roleDefinitionId": def, "principalId": pid} + + if typ != "" { + if !slices.Contains(authzPxTypes, typ) { + return nil + } + + out["principalType"] = typ + } + + return out +} + +func authzLockProps(in []byte) map[string]any { + var req authzPropsIn + if json.Unmarshal(in, &req) != nil { + return nil + } + + level, notes := req.text("level"), req.text("notes") + if !slices.Contains(authzPxLevels, level) || len(notes) > authzPxMaxNotes { + return nil + } + + out := map[string]any{"level": level} + if notes != "" { + out["notes"] = notes + } + + return out +} diff --git a/dashboard/azureentra_proxy_test.go b/dashboard/azureentra_proxy_test.go new file mode 100644 index 0000000000..a22e5727a3 --- /dev/null +++ b/dashboard/azureentra_proxy_test.go @@ -0,0 +1,362 @@ +package dashboard_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/pkgs/testclient" + "github.com/blackbirdworks/gopherstack/services/azurearm" + "github.com/blackbirdworks/gopherstack/services/azuregraph" +) + +const ( + entraGraph = "/dashboard/api/v1/azure/graph" + entraAuthz = "/dashboard/api/v1/azure/authz" + entraARM = "/dashboard/api/v1/azure/arm" + entraSub = "/subscriptions/" + armSub + entraRG = entraSub + "/resourceGroups/entra" + entraAPI = "?api-version=2022-04-01" + entraAuth = "/providers/Microsoft.Authorization/" + entraGUID1 = "11111111-1111-1111-1111-111111111111" + entraGUID2 = "22222222-2222-2222-2222-222222222222" + entraRole = "/subscriptions/" + armSub + + "/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7" +) + +type entraEnv struct { + client *http.Client + url string +} + +func newEntraEnv(t *testing.T) *entraEnv { + t.Helper() + + armSvc, err := (&azurearm.Provider{}).Init(&service.AppContext{}) + require.NoError(t, err) + + armH, ok := armSvc.(*azurearm.Handler) + require.True(t, ok) + t.Cleanup(func() { armH.Shutdown(context.Background()) }) + + gs := azuregraph.DefaultSettings() + graphH := azuregraph.NewHandler(azuregraph.NewBackend(gs), gs) + + e := echo.New() + e.Any("/*", dashboard.NewHandler(dashboard.Config{AzureARMOps: armH, AzureGraphOps: graphH}).Handler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + env := &entraEnv{url: srv.URL, client: testclient.New(t)} + + code, _ := env.do(t, http.MethodPut, entraARM+entraRG+"?api-version=2021-04-01", `{"location":"eastus"}`) + require.Less(t, code, http.StatusBadRequest) + + return env +} + +func (e *entraEnv) do(t *testing.T, method, path, body string) (int, string) { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), method, e.url+path, strings.NewReader(body)) + require.NoError(t, err) + + req.Header.Set("Content-Type", "application/json") + + resp, err := e.client.Do(req) + require.NoError(t, err) + + defer resp.Body.Close() + + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + return resp.StatusCode, string(b) +} + +func (e *entraEnv) create(t *testing.T, path, body string) map[string]any { + t.Helper() + + code, out := e.do(t, http.MethodPost, path, body) + require.Less(t, code, http.StatusBadRequest, out) + + var m map[string]any + require.NoError(t, json.Unmarshal([]byte(out), &m)) + + return m +} + +func TestAzureGraphProxyAllowList(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + app := entraGraph + "/applications/" + entraGUID1 + + tests := []struct { + name, method, path, body string + }{ + {"delete user", http.MethodDelete, entraGraph + "/users/" + entraGUID1, ""}, + {"delete group", http.MethodDelete, entraGraph + "/groups/" + entraGUID1, ""}, + {"delete sp", http.MethodDelete, entraGraph + "/servicePrincipals/" + entraGUID1, ""}, + {"patch app", http.MethodPatch, app, `{}`}, + {"put app", http.MethodPut, app, `{}`}, + {"add key", http.MethodPost, app + "/addKey", `{}`}, + {"non guid id", http.MethodGet, entraGraph + "/applications/not-a-guid", ""}, + {"me", http.MethodGet, entraGraph + "/me", ""}, + {"grants", http.MethodGet, entraGraph + "/oauth2PermissionGrants", ""}, + {"organization", http.MethodGet, entraGraph + "/organization", ""}, + {"traversal", http.MethodGet, entraGraph + "/applications/%2e%2e/users", ""}, + {"slash", http.MethodGet, entraGraph + "/applications/" + entraGUID1 + "%2Fowners", ""}, + {"sp password", http.MethodPost, entraGraph + "/servicePrincipals/" + entraGUID1 + "/addPassword", `{}`}, + {"member ref get", http.MethodGet, entraGraph + "/groups/" + entraGUID1 + "/members/$ref", ""}, + {"remove member bad id", http.MethodDelete, entraGraph + "/groups/" + entraGUID1 + "/members/x/$ref", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + code, body := env.do(t, tt.method, tt.path, tt.body) + assert.Contains(t, []int{http.StatusBadRequest, http.StatusMethodNotAllowed, http.StatusNotFound}, code) + assert.NotContains(t, body, "Request_") + }) + } +} + +func TestAzureGraphProxyBadBodies(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + + tests := []struct{ name, path, body string }{ + {"app no name", "/applications", `{}`}, + {"app not json", "/applications", `nope`}, + {"sp bad appid", "/servicePrincipals", `{"appId":"x"}`}, + {"user no password", "/users", `{"displayName":"A","mailNickname":"a",` + + `"userPrincipalName":"a@gopherstack.onmicrosoft.com"}`}, + {"user bad upn", "/users", `{"displayName":"A","mailNickname":"a","userPrincipalName":"a b",` + + `"password":"Xx1!xxxxxx"}`}, + {"group bad nick", "/groups", `{"displayName":"G","mailNickname":"a b"}`}, + {"remove password bad key", "/applications/" + entraGUID1 + "/removePassword", `{"keyId":"x"}`}, + {"member bad id", "/groups/" + entraGUID1 + "/members/$ref", `{"id":"http://evil/x"}`}, + {"getbyids empty", "/directoryObjects/getByIds", `{"ids":[]}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + code, body := env.do(t, http.MethodPost, entraGraph+tt.path, tt.body) + assert.Equal(t, http.StatusBadRequest, code) + assert.Contains(t, body, "InvalidRequestContent") + }) + } +} + +func TestAzureGraphProxySecretOnce(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + + app := env.create(t, entraGraph+"/applications", + `{"displayName":"web","passwordCredentials":[{"secretText":"smuggled"}],`+ + `"signInAudience":"AzureADMultipleOrgs"}`) + id, _ := app["id"].(string) + require.NotEmpty(t, id) + assert.Equal(t, "AzureADMyOrg", app["signInAudience"]) + + pw := env.create(t, entraGraph+"/applications/"+id+"/addPassword", `{"displayName":"ci","extra":"ignored"}`) + secret, _ := pw["secretText"].(string) + require.Len(t, secret, 40) + assert.NotEmpty(t, pw["hint"]) + + paths := []string{"/applications", "/applications/" + id, "/applications?$select=id,passwordCredentials"} + + for _, path := range paths { + code, body := env.do(t, http.MethodGet, entraGraph+path, "") + require.Equal(t, http.StatusOK, code, body) + assert.NotContains(t, body, secret, path) + assert.NotContains(t, body, "secretText", path) + assert.Contains(t, body, `"hint"`, path) + } + + keyID, _ := pw["keyId"].(string) + code, body := env.do(t, http.MethodPost, entraGraph+"/applications/"+id+"/removePassword", `{"keyId":"`+keyID+`"}`) + require.Equal(t, http.StatusNoContent, code, body) + + _, body = env.do(t, http.MethodGet, entraGraph+"/applications/"+id, "") + assert.NotContains(t, body, keyID) + + code, _ = env.do(t, http.MethodDelete, entraGraph+"/applications/"+id, "") + assert.Equal(t, http.StatusNoContent, code) +} + +func TestAzureGraphProxyDirectory(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + const pass = "Sup3r-Secret-Pw!" + + app := env.create(t, entraGraph+"/applications", `{"displayName":"api"}`) + appID, _ := app["appId"].(string) + sp := env.create(t, entraGraph+"/servicePrincipals", `{"appId":"`+appID+`","accountEnabled":false}`) + spID, _ := sp["id"].(string) + assert.Equal(t, true, sp["accountEnabled"]) + + user := env.create(t, entraGraph+"/users", `{"displayName":"Ann","mailNickname":"ann",`+ + `"userPrincipalName":"ann@gopherstack.onmicrosoft.com","password":"`+pass+`","accountEnabled":false}`) + userID, _ := user["id"].(string) + assert.Equal(t, true, user["accountEnabled"]) + + group := env.create(t, entraGraph+"/groups", `{"displayName":"ops","mailNickname":"ops","mailEnabled":true}`) + groupID, _ := group["id"].(string) + assert.Equal(t, false, group["mailEnabled"]) + + code, body := env.do(t, http.MethodPost, entraGraph+"/groups/"+groupID+"/members/$ref", `{"id":"`+userID+`"}`) + require.Equal(t, http.StatusNoContent, code, body) + + _, body = env.do(t, http.MethodGet, entraGraph+"/groups/"+groupID+"/members", "") + assert.Contains(t, body, "ann@gopherstack.onmicrosoft.com") + + code, body = env.do(t, http.MethodPost, entraGraph+"/directoryObjects/getByIds", + `{"ids":["`+userID+`","`+spID+`"],"types":["application"]}`) + require.Equal(t, http.StatusOK, code, body) + assert.Contains(t, body, "Ann") + + code, body = env.do(t, http.MethodDelete, entraGraph+"/groups/"+groupID+"/members/"+userID+"/$ref", "") + require.Equal(t, http.StatusNoContent, code, body) + + code, body = env.do(t, http.MethodGet, entraGraph+"/servicePrincipals/"+spID+"/appRoleAssignments", "") + require.Equal(t, http.StatusOK, code, body) + + for _, path := range []string{"/users", "/users/" + userID, "/groups", "/servicePrincipals"} { + _, out := env.do(t, http.MethodGet, entraGraph+path, "") + assert.NotContains(t, out, pass, path) + assert.NotContains(t, strings.ToLower(out), "passwordprofile", path) + } + + _, out := env.do(t, http.MethodPost, entraGraph+"/users", `{"displayName":"Bob","mailNickname":"bob",`+ + `"userPrincipalName":"bob@gopherstack.onmicrosoft.com","password":"`+pass+`"}`) + assert.NotContains(t, out, pass) + + code, out = env.do(t, http.MethodPost, entraGraph+"/users", `{"displayName":"Weak","mailNickname":"weak",`+ + `"userPrincipalName":"weak@gopherstack.onmicrosoft.com","password":"short"}`) + assert.Equal(t, http.StatusBadRequest, code) + assert.NotContains(t, out, "short\"") +} + +func TestAzureAuthzProxy(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + assignPath := entraAuthz + entraRG + "/providers/Microsoft.Authorization/roleAssignments/" + entraGUID2 + assignBody := `{"properties":{"roleDefinitionId":"` + entraRole + `","principalId":"` + entraGUID1 + + `","principalType":"User","condition":"true","description":"smuggled","scope":"/"}}` + + code, body := env.do(t, http.MethodPut, assignPath+entraAPI, assignBody) + require.Less(t, code, http.StatusBadRequest, body) + assert.NotContains(t, body, "smuggled") + assert.Contains(t, body, entraRG) + + code, body = env.do(t, http.MethodGet, entraAuthz+entraRG+entraAuth+"roleAssignments"+entraAPI, "") + require.Equal(t, http.StatusOK, code, body) + assert.Contains(t, body, entraGUID2) + + code, body = env.do(t, http.MethodGet, entraAuthz+entraSub+entraAuth+"roleDefinitions"+entraAPI, "") + require.Equal(t, http.StatusOK, code, body) + assert.Contains(t, body, "Contributor") + + code, body = env.do(t, http.MethodGet, entraAuthz+entraSub+entraAuth+"policyDefinitions?api-version=2021-06-01", "") + require.Equal(t, http.StatusOK, code, body) + + lock := entraAuthz + entraRG + "/providers/Microsoft.Authorization/locks/keep?api-version=2020-05-01" + lockBody := `{"properties":{"level":"CanNotDelete","notes":"n","owners":[{"applicationId":"x"}]}}` + code, body = env.do(t, http.MethodPut, lock, lockBody) + require.Less(t, code, http.StatusBadRequest, body) + assert.NotContains(t, body, "applicationId") + + code, body = env.do(t, http.MethodDelete, entraARM+entraRG+"?api-version=2021-04-01", "") + assert.Equal(t, http.StatusConflict, code) + assert.Contains(t, body, "ScopeLocked") + + code, _ = env.do(t, http.MethodDelete, lock, "") + assert.Less(t, code, http.StatusMultipleChoices) + + code, _ = env.do(t, http.MethodDelete, assignPath+entraAPI, "") + assert.Less(t, code, http.StatusMultipleChoices) +} + +func TestAzureAuthzProxyRejects(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + loc := entraRG + "/providers/Microsoft.Authorization/" + + tests := []struct { + name, method, path, body string + }{ + {"put role definition", http.MethodPut, entraSub + entraAuth + "roleDefinitions/" + entraGUID1, `{}`}, + {"put policy", http.MethodPut, loc + "policyAssignments/x", `{}`}, + {"delete policy", http.MethodDelete, loc + "policyAssignments/x", ""}, + {"other namespace", http.MethodGet, entraRG + "/providers/Microsoft.Storage/storageAccounts", ""}, + { + "resource scope", + http.MethodGet, + entraRG + "/providers/Microsoft.Compute/virtualMachines/a" + entraAuth + "locks", + "", + }, + {"assignment bad name", http.MethodPut, loc + "roleAssignments/not-a-guid", `{}`}, + { + "assignment bad role", http.MethodPut, loc + "roleAssignments/" + entraGUID2, + `{"properties":{"roleDefinitionId":"/providers/x","principalId":"` + entraGUID1 + `"}}`, + }, + { + "assignment bad principal", http.MethodPut, loc + "roleAssignments/" + entraGUID2, + `{"properties":{"roleDefinitionId":"` + entraRole + `","principalId":"nope"}}`, + }, + {"lock bad level", http.MethodPut, loc + "locks/a", `{"properties":{"level":"Nope"}}`}, + {"lock bad name", http.MethodPut, loc + "locks/a%20b", `{"properties":{"level":"ReadOnly"}}`}, + {"post", http.MethodPost, loc + "locks/a", `{}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + code, _ := env.do(t, tt.method, entraAuthz+tt.path+entraAPI, tt.body) + assert.Contains(t, []int{http.StatusBadRequest, http.StatusMethodNotAllowed}, code) + }) + } +} + +func TestEntraProxiesUnavailable(t *testing.T) { + t.Parallel() + + e := echo.New() + e.Any("/*", dashboard.NewHandler(dashboard.Config{}).Handler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + env := &entraEnv{url: srv.URL, client: testclient.New(t)} + + for _, path := range []string{ + entraGraph + "/applications", + entraAuthz + entraSub + "/providers/Microsoft.Authorization/roleDefinitions" + entraAPI, + } { + code, body := env.do(t, http.MethodGet, path, "") + assert.Equal(t, http.StatusServiceUnavailable, code, path) + assert.Contains(t, body, "ServiceUnavailable") + } +} diff --git a/dashboard/azureeventgrid_proxy.go b/dashboard/azureeventgrid_proxy.go index 403019e94f..674096fd25 100644 --- a/dashboard/azureeventgrid_proxy.go +++ b/dashboard/azureeventgrid_proxy.go @@ -1,8 +1,6 @@ package dashboard import ( - "errors" - "io" "net/http" "github.com/labstack/echo/v5" @@ -10,13 +8,31 @@ import ( eventgridbackend "github.com/blackbirdworks/gopherstack/services/azureeventgrid" ) -const azureEventGridProxyPrefix = "/dashboard/api/v1/azure/eventgrid" +const ( + azureEventGridProxyPrefix = "/dashboard/api/v1/azure/eventgrid" + eventGridKindTopics = "topics" + eventGridKindDomains = "domains" + eventGridKindNamespaces = "namespaces" +) +// eventGridSubJSON is the secret-free view of an event subscription: the endpoint +// type only, never its URL, headers or delivery attribute values. type eventGridSubJSON struct { - Name string `json:"name"` - EndpointType string `json:"endpointType"` - State string `json:"state"` - Schema string `json:"deliverySchema"` + Name string `json:"name"` + EndpointType string `json:"endpointType"` + State string `json:"state"` + Schema string `json:"deliverySchema"` + SubjectBeginsWith string `json:"subjectBeginsWith,omitempty"` + SubjectEndsWith string `json:"subjectEndsWith,omitempty"` + DeadLetterType string `json:"deadLetterType,omitempty"` + DeadLetterContainer string `json:"deadLetterContainer,omitempty"` + QueueEventTimeToLive string `json:"queueEventTimeToLive,omitempty"` + IncludedEventTypes []string `json:"includedEventTypes"` + AdvancedFilterCount int `json:"advancedFilterCount"` + MaxDeliveryAttempts int `json:"maxDeliveryAttempts"` + EventTTLMinutes int `json:"eventTimeToLiveMinutes"` + QueueLockSeconds int `json:"queueLockSeconds,omitempty"` + QueueMaxDelivery int `json:"queueMaxDelivery,omitempty"` } type eventGridTopicJSON struct { @@ -25,16 +41,47 @@ type eventGridTopicJSON struct { Subscriptions []eventGridSubJSON `json:"subscriptions"` } -// registerAzureEventGridRoutes serves the Event Grid page from the backend in-process; -// topics and subscriptions are created through ARM, so the page lists and publishes. +type eventGridDomainTopicJSON struct { + Name string `json:"name"` + Subscriptions []eventGridSubJSON `json:"subscriptions"` +} + +type eventGridDomainJSON struct { + Name string `json:"name"` + InputSchema string `json:"inputSchema"` + Subscriptions []eventGridSubJSON `json:"subscriptions"` + Topics []eventGridDomainTopicJSON `json:"topics"` + AutoCreate bool `json:"autoCreateTopics"` + AutoDelete bool `json:"autoDeleteTopics"` +} + +type eventGridNamespaceJSON struct { + Name string `json:"name"` + Topics []eventGridDomainTopicJSON `json:"topics"` +} + +type eventGridScopeJSON struct { + ID string `json:"id"` + Subscriptions []eventGridSubJSON `json:"subscriptions"` +} + +// registerAzureEventGridRoutes serves the Event Grid page from the backend in-process. +// Topics, domains and subscriptions are created through ARM; the page lists, publishes +// test events and reveals access keys on an explicit POST. func (h *DashboardHandler) registerAzureEventGridRoutes() { api := &azureEventGridAPI{} if h.config.AzureEventGridOps != nil { api.b = h.config.AzureEventGridOps.Backend } - h.SubRouter.Add(http.MethodGet, azureEventGridProxyPrefix+"/topics", api.guard(api.topics)) - h.SubRouter.Add(http.MethodPost, azureEventGridProxyPrefix+"/topics/:topic/events", api.guard(api.publish)) + r := h.SubRouter + p := azureEventGridProxyPrefix + r.Add(http.MethodGet, p+"/"+eventGridKindTopics, api.guard(api.topics)) + r.Add(http.MethodGet, p+"/overview", api.guard(api.overview)) + r.Add(http.MethodPost, p+"/"+eventGridKindTopics+"/:topic/events", api.guard(api.publishTopic)) + r.Add(http.MethodPost, p+"/"+eventGridKindDomains+"/:domain/events", api.guard(api.publishDomain)) + r.Add(http.MethodPost, p+"/"+eventGridKindNamespaces+"/:ns/topics/:topic/events", api.guard(api.publishNamespace)) + r.Add(http.MethodPost, p+"/:kind/:name/keys/reveal", api.guard(api.revealKeys)) } type azureEventGridAPI struct { @@ -52,6 +99,19 @@ func (a *azureEventGridAPI) guard(next echo.HandlerFunc) echo.HandlerFunc { } func (a *azureEventGridAPI) topics(c *echo.Context) error { + return c.JSON(http.StatusOK, map[string]any{"topics": a.topicList()}) +} + +func (a *azureEventGridAPI) overview(c *echo.Context) error { + return c.JSON(http.StatusOK, map[string]any{ + "topics": a.topicList(), + "domains": a.domainList(), + "namespaces": a.namespaceList(), + "scopes": a.scopeList(), + }) +} + +func (a *azureEventGridAPI) topicList() []eventGridTopicJSON { out := []eventGridTopicJSON{} for _, name := range a.b.TopicNames() { @@ -61,63 +121,201 @@ func (a *azureEventGridAPI) topics(c *echo.Context) error { } out = append(out, eventGridTopicJSON{ - Name: name, InputSchema: info.InputSchema, Subscriptions: a.subscriptions(name), + Name: name, InputSchema: info.InputSchema, + Subscriptions: a.subscriptions(eventgridbackend.Scope{Topic: name}), + }) + } + + return out +} + +func (a *azureEventGridAPI) domainList() []eventGridDomainJSON { + out := []eventGridDomainJSON{} + + for _, name := range a.b.DomainNames() { + info, err := a.b.Domain(name) + if err != nil { + continue + } + + d := eventGridDomainJSON{ + Name: name, InputSchema: info.InputSchema, AutoCreate: info.AutoCreate, AutoDelete: info.AutoDelete, + Subscriptions: a.subscriptions(eventgridbackend.Scope{Domain: name}), + Topics: []eventGridDomainTopicJSON{}, + } + + topics, _ := a.b.DomainTopics(name) + for _, t := range topics { + d.Topics = append(d.Topics, eventGridDomainTopicJSON{ + Name: t.Name, + Subscriptions: a.subscriptions(eventgridbackend.Scope{Domain: name, Topic: t.Name}), + }) + } + + out = append(out, d) + } + + return out +} + +func (a *azureEventGridAPI) namespaceList() []eventGridNamespaceJSON { + names := a.b.NamespaceNames() + out := make([]eventGridNamespaceJSON, 0, len(names)) + + for _, ns := range names { + item := eventGridNamespaceJSON{Name: ns, Topics: []eventGridDomainTopicJSON{}} + topics, _ := a.b.NamespaceTopicNames(ns) + + for _, t := range topics { + item.Topics = append(item.Topics, eventGridDomainTopicJSON{ + Name: t, Subscriptions: a.pullSubscriptions(ns, t), + }) + } + + out = append(out, item) + } + + return out +} + +func (a *azureEventGridAPI) scopeList() []eventGridScopeJSON { + scopes := a.b.ARMScopes() + out := make([]eventGridScopeJSON, 0, len(scopes)) + + for _, s := range scopes { + out = append(out, eventGridScopeJSON{ + ID: s.ID, Subscriptions: a.subscriptions(eventgridbackend.Scope{Resource: s.ID}), }) } - return c.JSON(http.StatusOK, map[string]any{"topics": out}) + return out } -func (a *azureEventGridAPI) subscriptions(topic string) []eventGridSubJSON { - names, _ := a.b.SubscriptionNames(topic) +func (a *azureEventGridAPI) subscriptions(sc eventgridbackend.Scope) []eventGridSubJSON { + names, _ := a.b.ScopedSubscriptionNames(sc) out := make([]eventGridSubJSON, 0, len(names)) for _, name := range names { - props, err := a.b.Subscription(topic, name) + props, err := a.b.ScopedSubscription(sc, name) if err != nil { continue } - sub := eventGridSubJSON{Name: name} - sub.State, _ = props["provisioningState"].(string) - sub.Schema, _ = props["eventDeliverySchema"].(string) + out = append(out, subscriptionJSON(name, props)) + } - if dest, ok := props["destination"].(map[string]any); ok { - sub.EndpointType, _ = dest["endpointType"].(string) + return out +} + +func (a *azureEventGridAPI) pullSubscriptions(ns, topic string) []eventGridSubJSON { + names, _ := a.b.PullSubscriptionNames(ns, topic) + out := make([]eventGridSubJSON, 0, len(names)) + + for _, name := range names { + props, _, err := a.b.PullSubscription(ns, topic, name) + if err != nil { + continue } - out = append(out, sub) + out = append(out, pullSubscriptionJSON(name, props)) } return out } -func (a *azureEventGridAPI) publish(c *echo.Context) error { - topic := c.Param("topic") +func subMap(m map[string]any, key string) map[string]any { + v, _ := m[key].(map[string]any) - info, err := a.b.Topic(topic) - if err != nil { - return cloudJSONFail(c, http.StatusNotFound, cloudStatusNotFnd, err.Error()) + return v +} + +func subNum(m map[string]any, key string) int { + f, _ := m[key].(float64) + + return int(f) +} + +func subStr(m map[string]any, key string) string { + s, _ := m[key].(string) + + return s +} + +func subStrings(m map[string]any, key string) []string { + raw, _ := m[key].([]any) + out := make([]string, 0, len(raw)) + + for _, v := range raw { + if s, ok := v.(string); ok { + out = append(out, s) + } } - body, err := io.ReadAll(http.MaxBytesReader(c.Response(), c.Request().Body, eventgridbackend.MaxRequestBytes)) - if err != nil { - return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "request body too large or unreadable") + return out +} + +func subscriptionJSON(name string, props map[string]any) eventGridSubJSON { + filter, retry := subMap(props, "filter"), subMap(props, "retryPolicy") + adv, _ := filter["advancedFilters"].([]any) + + sub := eventGridSubJSON{ + Name: name, State: subStr(props, "provisioningState"), Schema: subStr(props, "eventDeliverySchema"), + EndpointType: subStr(subMap(props, "destination"), "endpointType"), + SubjectBeginsWith: subStr(filter, "subjectBeginsWith"), + SubjectEndsWith: subStr(filter, "subjectEndsWith"), + IncludedEventTypes: subStrings(filter, "includedEventTypes"), + AdvancedFilterCount: len(adv), + MaxDeliveryAttempts: subNum(retry, "maxDeliveryAttempts"), + EventTTLMinutes: subNum(retry, "eventTimeToLiveInMinutes"), } - events, err := eventgridbackend.ParseEvents(body, info.InputSchema) - if err == nil { - err = a.b.Publish(topic, events) + if dl := subMap(props, "deadLetterDestination"); dl != nil { + sub.DeadLetterType = subStr(dl, "endpointType") + sub.DeadLetterContainer = subStr(subMap(dl, "properties"), "blobContainerName") } - if err != nil { - status := http.StatusBadRequest - if errors.Is(err, eventgridbackend.ErrTopicNotFound) { - status = http.StatusNotFound - } + return sub +} - return cloudJSONFail(c, status, cloudStatusBadArg, err.Error()) +func pullSubscriptionJSON(name string, props map[string]any) eventGridSubJSON { + filters := subMap(props, "filtersConfiguration") + queue := subMap(subMap(props, "deliveryConfiguration"), "queue") + adv, _ := filters["filters"].([]any) + + return eventGridSubJSON{ + Name: name, State: subStr(props, "provisioningState"), Schema: subStr(props, "eventDeliverySchema"), + EndpointType: "Queue (pull)", + IncludedEventTypes: subStrings(filters, "includedEventTypes"), + AdvancedFilterCount: len(adv), + QueueLockSeconds: subNum(queue, "receiveLockDurationInSeconds"), + QueueMaxDelivery: subNum(queue, "maxDeliveryCount"), + QueueEventTimeToLive: subStr(queue, "eventTimeToLive"), + } +} + +// revealKeys returns a topic, domain or namespace key pair; it is the only route that sends keys out. +func (a *azureEventGridAPI) revealKeys(c *echo.Context) error { + var ( + keys eventgridbackend.Keys + err error + ) + + switch name := c.Param("name"); c.Param("kind") { + case eventGridKindTopics: + keys, err = a.b.TopicKeys(name) + case eventGridKindDomains: + keys, err = a.b.DomainKeys(name) + case eventGridKindNamespaces: + keys, err = a.b.NamespaceKeys(name) + default: + return cloudJSONFail(c, http.StatusNotFound, cloudStatusNotFnd, "unknown resource kind") } - return c.JSON(http.StatusOK, map[string]int{"published": len(events)}) + if err != nil { + return cloudJSONFail(c, http.StatusNotFound, cloudStatusNotFnd, err.Error()) + } + + c.Response().Header().Set("Cache-Control", "no-store") + + return c.JSON(http.StatusOK, map[string]string{"key1": keys.Key1, "key2": keys.Key2}) } diff --git a/dashboard/azureeventgrid_proxy_test.go b/dashboard/azureeventgrid_proxy_test.go new file mode 100644 index 0000000000..e945a8758b --- /dev/null +++ b/dashboard/azureeventgrid_proxy_test.go @@ -0,0 +1,346 @@ +package dashboard_test + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/azureeventgrid" + "github.com/blackbirdworks/gopherstack/services/azureeventhubs" +) + +const ( + egBase = "/dashboard/api/v1/azure/eventgrid" + ehBase = "/dashboard/api/v1/azure/eventhubs" + egStore = "/subscriptions/s/resourceGroups/r/providers/Microsoft.Storage/storageAccounts/acct" +) + +func newEventGridBackend(t *testing.T) *azureeventgrid.InMemoryBackend { + t.Helper() + + b := azureeventgrid.NewInMemoryBackend() + _, err := b.PutTopic("topic1", "/subscriptions/s/topics/topic1", "") + require.NoError(t, err) + _, err = b.PutTopic("ce1", "/subscriptions/s/topics/ce1", azureeventgrid.SchemaCloudEvent) + require.NoError(t, err) + _, err = b.PutDomain("dom1", "/subscriptions/s/domains/dom1", "", nil, nil) + require.NoError(t, err) + _, err = b.PutDomainTopic("dom1", "orders") + require.NoError(t, err) + require.NoError(t, b.PutNamespace("ns1")) + _, err = b.PutNamespaceTopic("ns1", "nt1") + require.NoError(t, err) + _, err = b.PutPullSubscription("ns1", "nt1", "pull1", map[string]any{}) + require.NoError(t, err) + + return b +} + +func eventGridHandler(t *testing.T, b *azureeventgrid.InMemoryBackend) *dashboard.DashboardHandler { + t.Helper() + + return dashboard.NewHandler( + dashboard.Config{AzureEventGridOps: azureeventgrid.NewHandler(b, azureeventgrid.Settings{})}, + ) +} + +func doProxy(t *testing.T, h *dashboard.DashboardHandler, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + + return rec +} + +func TestAzureEventGridOverview(t *testing.T) { + t.Parallel() + + b := newEventGridBackend(t) + _, err := b.PutSubscription(t.Context(), "topic1", "sub1", map[string]any{ + "destination": map[string]any{ + "endpointType": "StorageQueue", + "properties": map[string]any{"resourceId": egStore, "queueName": "q"}, + }, + "filter": map[string]any{ + "subjectBeginsWith": "/a", "includedEventTypes": []any{"T1"}, + "advancedFilters": []any{ + map[string]any{"operatorType": "StringIn", "key": "subject", "values": []any{"x"}}, + }, + }, + "retryPolicy": map[string]any{"maxDeliveryAttempts": 7, "eventTimeToLiveInMinutes": 90}, + "deadLetterDestination": map[string]any{ + "endpointType": "StorageBlob", + "properties": map[string]any{"resourceId": egStore, "blobContainerName": "dead"}, + }, + }) + require.NoError(t, err) + + rec := doProxy(t, eventGridHandler(t, b), http.MethodGet, egBase+"/overview", "") + require.Equal(t, http.StatusOK, rec.Code) + + body := rec.Body.String() + for _, want := range []string{ + `"name":"topic1"`, `"name":"dom1"`, `"name":"orders"`, `"name":"ns1"`, `"name":"nt1"`, `"name":"pull1"`, + `"endpointType":"StorageQueue"`, `"subjectBeginsWith":"/a"`, `"advancedFilterCount":1`, + `"maxDeliveryAttempts":7`, `"eventTimeToLiveMinutes":90`, `"deadLetterContainer":"dead"`, + `"endpointType":"Queue (pull)"`, + } { + assert.Contains(t, body, want) + } + + assert.NotContains(t, body, egStore) + assert.NotContains(t, strings.ToLower(body), "key1") +} + +func TestAzureEventGridRevealKeys(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + wantCode int + }{ + {name: "topic", path: "/topics/topic1/keys/reveal", wantCode: http.StatusOK}, + {name: "domain", path: "/domains/dom1/keys/reveal", wantCode: http.StatusOK}, + {name: "namespace", path: "/namespaces/ns1/keys/reveal", wantCode: http.StatusOK}, + {name: "missing", path: "/topics/nope/keys/reveal", wantCode: http.StatusNotFound}, + {name: "unknown kind", path: "/widgets/x/keys/reveal", wantCode: http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := doProxy(t, eventGridHandler(t, newEventGridBackend(t)), http.MethodPost, egBase+tt.path, "") + assert.Equal(t, tt.wantCode, rec.Code) + + if tt.wantCode == http.StatusOK { + assert.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + assert.Contains(t, rec.Body.String(), `"key1":"`) + } + }) + } + + t.Run("get is not routed", func(t *testing.T) { + t.Parallel() + + rec := doProxy( + t, + eventGridHandler(t, newEventGridBackend(t)), + http.MethodGet, + egBase+"/topics/topic1/keys/reveal", + "", + ) + assert.NotEqual(t, http.StatusOK, rec.Code) + assert.NotContains(t, rec.Body.String(), "key1") + }) +} + +func TestAzureEventGridPublishRebuild(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + body string + wantBody string + wantCode int + }{ + { + name: "topic", + path: "/topics/topic1/events", + body: `{"eventType":"T","subject":"/s","data":"{\"a\":1}"}`, + wantCode: 200, + wantBody: `"published":1`, + }, + { + name: "cloud topic", + path: "/topics/ce1/events", + body: `{"eventType":"T","data":"plain"}`, + wantCode: 200, + wantBody: `"published":1`, + }, + { + name: "domain", + path: "/domains/dom1/events", + body: `{"eventType":"T","domainTopic":"orders"}`, + wantCode: 200, + wantBody: `"published":1`, + }, + { + name: "domain needs topic", + path: "/domains/dom1/events", + body: `{"eventType":"T"}`, + wantCode: 400, + wantBody: "domainTopic is required", + }, + { + name: "namespace", + path: "/namespaces/ns1/topics/nt1/events", + body: `{"eventType":"T"}`, + wantCode: 200, + wantBody: `"published":1`, + }, + { + name: "missing ns topic", + path: "/namespaces/ns1/topics/nope/events", + body: `{"eventType":"T"}`, + wantCode: 404, + wantBody: "NOT_FOUND", + }, + { + name: "missing topic", + path: "/topics/nope/events", + body: `{"eventType":"T"}`, + wantCode: 404, + wantBody: "NOT_FOUND", + }, + { + name: "no event type", + path: "/topics/topic1/events", + body: `{"subject":"/s"}`, + wantCode: 400, + wantBody: "eventType is required", + }, + { + name: "long subject", + path: "/topics/topic1/events", + body: `{"eventType":"T","subject":"` + strings.Repeat("a", 300) + `"}`, + wantCode: 400, + wantBody: "subject", + }, + { + name: "oversized data", + path: "/topics/topic1/events", + body: `{"eventType":"T","data":"` + strings.Repeat("a", 70<<10) + `"}`, + wantCode: 400, + wantBody: "data is at most", + }, + { + name: "oversized body", + path: "/topics/topic1/events", + body: `{"eventType":"T","data":"` + strings.Repeat("a", 200<<10) + `"}`, + wantCode: 400, + }, + {name: "bad json", path: "/topics/topic1/events", body: `{`, wantCode: 400, wantBody: "INVALID_ARGUMENT"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := doProxy(t, eventGridHandler(t, newEventGridBackend(t)), http.MethodPost, egBase+tt.path, tt.body) + assert.Equal(t, tt.wantCode, rec.Code) + assert.Contains(t, rec.Body.String(), tt.wantBody) + }) + } + + t.Run("client schema fields ignored", func(t *testing.T) { + t.Parallel() + + rec := doProxy(t, eventGridHandler(t, newEventGridBackend(t)), http.MethodPost, egBase+"/topics/topic1/events", + `{"eventType":"T","id":"evil","topic":"/other","specversion":"1.0"}`) + assert.Equal(t, http.StatusOK, rec.Code) + }) +} + +func TestAzureEventGridPublishAllowList(t *testing.T) { + t.Parallel() + + h := eventGridHandler(t, newEventGridBackend(t)) + + for _, tt := range []struct{ method, path string }{ + {http.MethodDelete, "/topics/topic1"}, + {http.MethodPut, "/topics/topic1"}, + {http.MethodGet, "/topics/topic1/events"}, + } { + rec := doProxy(t, h, tt.method, egBase+tt.path, "") + assert.NotEqual(t, http.StatusOK, rec.Code, tt.method+" "+tt.path) + } +} + +func TestAzureEventGridUnavailable(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + for _, tt := range []struct{ method, path string }{ + {http.MethodGet, egBase + "/overview"}, + {http.MethodPost, egBase + "/topics/t/events"}, + {http.MethodPost, egBase + "/domains/d/events"}, + {http.MethodPost, egBase + "/namespaces/n/topics/t/events"}, + {http.MethodPost, egBase + "/topics/t/keys/reveal"}, + {http.MethodGet, ehBase + "/namespaces/n/policies"}, + {http.MethodPost, ehBase + "/namespaces/n/policies/reveal"}, + } { + rec := doProxy(t, h, tt.method, tt.path, `{}`) + assert.Equal(t, http.StatusServiceUnavailable, rec.Code, tt.path) + } +} + +func TestAzureEventHubsPolicies(t *testing.T) { + t.Parallel() + + b := azureeventhubs.NewInMemoryBackend() + require.NoError(t, b.EnsureNamespace("ns1")) + _, err := b.CreateHub(azureeventhubs.HubRef{Namespace: "ns1", Hub: "hub1"}, + azureeventhubs.HubConfig{PartitionCount: 1, Retention: azureeventhubs.DefaultRetention}) + require.NoError(t, err) + rule, err := b.PutRule("ns1", "hub1", "sender", []string{azureeventhubs.RightSend}) + require.NoError(t, err) + + h := dashboard.NewHandler(dashboard.Config{AzureEventHubsOps: azureeventhubs.NewHandler(b)}) + + list := doProxy(t, h, http.MethodGet, ehBase+"/namespaces/ns1/policies", "") + require.Equal(t, http.StatusOK, list.Code) + assert.Contains(t, list.Body.String(), `"name":"sender"`) + assert.Contains(t, list.Body.String(), `"hub":"hub1"`) + assert.Contains(t, list.Body.String(), `"name":"`+azureeventhubs.DefaultKeyName+`"`) + assert.NotContains(t, list.Body.String(), rule.PrimaryKey) + assert.NotContains(t, list.Body.String(), azureeventhubs.DefaultKeyValue) + + tests := []struct { + name string + path string + body string + wantCode int + }{ + { + name: "hub rule", + path: "/namespaces/ns1/policies/reveal", + body: `{"hub":"hub1","name":"sender"}`, + wantCode: 200, + }, + { + name: "root rule", + path: "/namespaces/ns1/policies/reveal", + body: `{"name":"` + azureeventhubs.DefaultKeyName + `"}`, + wantCode: 200, + }, + {name: "missing rule", path: "/namespaces/ns1/policies/reveal", body: `{"name":"nope"}`, wantCode: 404}, + {name: "missing ns", path: "/namespaces/zz/policies/reveal", body: `{"name":"x"}`, wantCode: 404}, + {name: "bad json", path: "/namespaces/ns1/policies/reveal", body: `{`, wantCode: 400}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := doProxy(t, h, http.MethodPost, ehBase+tt.path, tt.body) + assert.Equal(t, tt.wantCode, rec.Code) + + if tt.wantCode == http.StatusOK { + assert.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + assert.Contains(t, rec.Body.String(), `"primaryKey":"`) + } + }) + } +} diff --git a/dashboard/azureeventgrid_publish.go b/dashboard/azureeventgrid_publish.go new file mode 100644 index 0000000000..c5b0f6f7e9 --- /dev/null +++ b/dashboard/azureeventgrid_publish.go @@ -0,0 +1,229 @@ +package dashboard + +import ( + "encoding/json" + "errors" + "net/http" + "slices" + "time" + + "github.com/google/uuid" + "github.com/labstack/echo/v5" + + eventgridbackend "github.com/blackbirdworks/gopherstack/services/azureeventgrid" +) + +const ( + eventGridMaxBody = 128 << 10 + eventGridMaxData = 64 << 10 + eventGridMaxSubject = 256 + eventGridMaxType = 128 + eventGridTestSource = "/gopherstack/dashboard" + eventGridVersion = "1.0" +) + +// eventGridTestEvent holds the only fields the page may set; the event itself is rebuilt here. +type eventGridTestEvent struct { + Subject string `json:"subject"` + EventType string `json:"eventType"` + Data string `json:"data"` + DomainTopic string `json:"domainTopic"` +} + +func (e eventGridTestEvent) validate(needTopic bool) string { + switch { + case e.EventType == "" || len(e.EventType) > eventGridMaxType: + return "eventType is required (at most 128 characters)" + case len(e.Subject) > eventGridMaxSubject: + return "subject is at most 256 characters" + case len(e.Data) > eventGridMaxData: + return "data is at most 64 KiB" + case needTopic && (e.DomainTopic == "" || len(e.DomainTopic) > eventGridMaxSubject): + return "domainTopic is required" + } + + return "" +} + +func (e eventGridTestEvent) dataValue() any { + var v any + if json.Unmarshal([]byte(e.Data), &v) == nil && e.Data != "" { + return v + } + + return e.Data +} + +type eventGridSchemaEvent struct { + Data any `json:"data"` + ID string `json:"id"` + Subject string `json:"subject"` + EventType string `json:"eventType"` + EventTime string `json:"eventTime"` + DataVersion string `json:"dataVersion"` + Topic string `json:"topic,omitempty"` +} + +type eventGridCloudEvent struct { + Data any `json:"data"` + SpecVersion string `json:"specversion"` + ID string `json:"id"` + Source string `json:"source"` + Type string `json:"type"` + Subject string `json:"subject"` + Time string `json:"time"` +} + +type eventGridPublished struct { + Published int `json:"published"` +} + +// build returns the event in the given input schema; domainTopic routes it inside a domain. +func (e eventGridTestEvent) build(schema, domainTopic string, now time.Time) []byte { + id, ts := uuid.NewString(), now.UTC().Format(time.RFC3339Nano) + + if e.Subject == "" { + e.Subject = eventGridTestSource + "/test" + } + + var ev any + + if schema == eventgridbackend.SchemaCloudEvent { + source := eventGridTestSource + if domainTopic != "" { + source = domainTopic + } + + ev = eventGridCloudEvent{ + SpecVersion: eventGridVersion, ID: id, Source: source, Type: e.EventType, Subject: e.Subject, Time: ts, + Data: e.dataValue(), + } + } else { + ev = eventGridSchemaEvent{ + ID: id, Subject: e.Subject, EventType: e.EventType, EventTime: ts, DataVersion: eventGridVersion, + Topic: domainTopic, Data: e.dataValue(), + } + } + + raw, _ := json.Marshal(ev) + + return raw +} + +func (a *azureEventGridAPI) decodeTest(c *echo.Context, needTopic bool) (eventGridTestEvent, bool) { + var in eventGridTestEvent + + c.Request().Body = http.MaxBytesReader(c.Response(), c.Request().Body, eventGridMaxBody) + + if !cloudJSONDecode(c, &in) { + return in, false + } + + if msg := in.validate(needTopic); msg != "" { + _ = cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, msg) + + return in, false + } + + return in, true +} + +func (a *azureEventGridAPI) publishFail(c *echo.Context, err error) error { + status := http.StatusBadRequest + + for _, nf := range []error{ + eventgridbackend.ErrTopicNotFound, eventgridbackend.ErrDomainNotFound, + eventgridbackend.ErrNamespaceNotFound, eventgridbackend.ErrNamespaceTopicNotFound, + } { + if errors.Is(err, nf) { + status = http.StatusNotFound + } + } + + code := cloudStatusBadArg + if status == http.StatusNotFound { + code = cloudStatusNotFnd + } + + return cloudJSONFail(c, status, code, err.Error()) +} + +func (a *azureEventGridAPI) publishTopic(c *echo.Context) error { + topic := c.Param("topic") + + info, err := a.b.Topic(topic) + if err != nil { + return a.publishFail(c, err) + } + + in, ok := a.decodeTest(c, false) + if !ok { + return nil + } + + events, err := eventgridbackend.ParseEvents(in.build(info.InputSchema, "", time.Now()), info.InputSchema) + if err == nil { + err = a.b.Publish(topic, events) + } + + if err != nil { + return a.publishFail(c, err) + } + + return c.JSON(http.StatusOK, eventGridPublished{Published: len(events)}) +} + +func (a *azureEventGridAPI) publishDomain(c *echo.Context) error { + domain := c.Param("domain") + + info, err := a.b.Domain(domain) + if err != nil { + return a.publishFail(c, err) + } + + in, ok := a.decodeTest(c, true) + if !ok { + return nil + } + + events, err := eventgridbackend.ParseEvents( + in.build(info.InputSchema, in.DomainTopic, time.Now()), info.InputSchema, + ) + if err == nil { + err = a.b.PublishDomain(domain, events) + } + + if err != nil { + return a.publishFail(c, err) + } + + return c.JSON(http.StatusOK, eventGridPublished{Published: len(events)}) +} + +func (a *azureEventGridAPI) publishNamespace(c *echo.Context) error { + ns, topic := c.Param("ns"), c.Param("topic") + + if names, err := a.b.NamespaceTopicNames(ns); err != nil { + return a.publishFail(c, err) + } else if !slices.Contains(names, topic) { + return a.publishFail(c, eventgridbackend.ErrNamespaceTopicNotFound) + } + + in, ok := a.decodeTest(c, false) + if !ok { + return nil + } + + events, err := eventgridbackend.ParseEvents( + in.build(eventgridbackend.SchemaCloudEvent, "", time.Now()), eventgridbackend.SchemaCloudEvent, + ) + if err == nil { + err = a.b.PublishNamespace(ns, topic, events) + } + + if err != nil { + return a.publishFail(c, err) + } + + return c.JSON(http.StatusOK, eventGridPublished{Published: len(events)}) +} diff --git a/dashboard/azureeventhubs_policies.go b/dashboard/azureeventhubs_policies.go new file mode 100644 index 0000000000..ca4d9db38d --- /dev/null +++ b/dashboard/azureeventhubs_policies.go @@ -0,0 +1,61 @@ +package dashboard + +import ( + "errors" + "net/http" + + "github.com/labstack/echo/v5" + + eventhubsbackend "github.com/blackbirdworks/gopherstack/services/azureeventhubs" +) + +type eventHubsPolicyJSON struct { + Hub string `json:"hub,omitempty"` + Name string `json:"name"` + Rights []string `json:"rights"` +} + +type eventHubsPoliciesJSON struct { + Policies []eventHubsPolicyJSON `json:"policies"` +} + +type eventHubsRevealJSON struct { + Hub string `json:"hub"` + Name string `json:"name"` +} + +// policies lists the shared access policies without their keys. +func (a *azureEventHubsAPI) policies(c *echo.Context) error { + rules, err := a.b.ListRules(c.Param("ns")) + if err != nil { + return a.fail(c, err) + } + + out := make([]eventHubsPolicyJSON, 0, len(rules)) + for _, r := range rules { + out = append(out, eventHubsPolicyJSON{Hub: r.Hub, Name: r.Rule.Name, Rights: r.Rule.Rights}) + } + + return c.JSON(http.StatusOK, eventHubsPoliciesJSON{Policies: out}) +} + +// revealPolicy returns one policy's key pair; it is the only route that sends keys out. +func (a *azureEventHubsAPI) revealPolicy(c *echo.Context) error { + var in eventHubsRevealJSON + if !cloudJSONDecode(c, &in) { + return nil + } + + rule, err := a.b.GetRule(c.Param("ns"), in.Hub, in.Name) + if err != nil { + if errors.Is(err, eventhubsbackend.ErrRuleNotFound) { + return cloudJSONFail(c, http.StatusNotFound, cloudStatusNotFnd, err.Error()) + } + + return a.fail(c, err) + } + + c.Response().Header().Set("Cache-Control", "no-store") + + return c.JSON(http.StatusOK, map[string]string{"primaryKey": rule.PrimaryKey, "secondaryKey": rule.SecondaryKey}) +} diff --git a/dashboard/azureeventhubs_proxy.go b/dashboard/azureeventhubs_proxy.go index ebbf89bc40..550492cce4 100644 --- a/dashboard/azureeventhubs_proxy.go +++ b/dashboard/azureeventhubs_proxy.go @@ -20,6 +20,7 @@ const ( eventHubsDefaultPeek = 20 eventHubsMaxPeek = 100 eventHubsMaxKeyLen = 128 + eventHubsNamespacesSeg = "/namespaces" ) type eventHubsHubJSON struct { @@ -69,13 +70,27 @@ func (h *DashboardHandler) registerAzureEventHubsRoutes() { } r := h.SubRouter - r.Add(http.MethodGet, azureEventHubsProxyPrefix+"/namespaces", api.guard(api.namespaces)) - r.Add(http.MethodGet, azureEventHubsProxyPrefix+"/namespaces/:ns/hubs/:hub/partitions", api.guard(api.partitions)) + r.Add(http.MethodGet, azureEventHubsProxyPrefix+eventHubsNamespacesSeg, api.guard(api.namespaces)) r.Add( - http.MethodGet, azureEventHubsProxyPrefix+"/namespaces/:ns/hubs/:hub/partitions/:pid/events", + http.MethodGet, + azureEventHubsProxyPrefix+eventHubsNamespacesSeg+"/:ns/hubs/:hub/partitions", + api.guard(api.partitions), + ) + r.Add( + http.MethodGet, azureEventHubsProxyPrefix+eventHubsNamespacesSeg+"/:ns/hubs/:hub/partitions/:pid/events", api.guard(api.events), ) - r.Add(http.MethodPost, azureEventHubsProxyPrefix+"/namespaces/:ns/hubs/:hub/events", api.guard(api.send)) + r.Add( + http.MethodPost, + azureEventHubsProxyPrefix+eventHubsNamespacesSeg+"/:ns/hubs/:hub/events", + api.guard(api.send), + ) + r.Add(http.MethodGet, azureEventHubsProxyPrefix+eventHubsNamespacesSeg+"/:ns/policies", api.guard(api.policies)) + r.Add( + http.MethodPost, + azureEventHubsProxyPrefix+eventHubsNamespacesSeg+"/:ns/policies/reveal", + api.guard(api.revealPolicy), + ) } type azureEventHubsAPI struct { diff --git a/dashboard/azurefunctions_keys.go b/dashboard/azurefunctions_keys.go new file mode 100644 index 0000000000..4c6a6d82bc --- /dev/null +++ b/dashboard/azurefunctions_keys.go @@ -0,0 +1,211 @@ +package dashboard + +import ( + "bytes" + "context" + "io" + "net/http" + "net/url" + "slices" + "strings" + "time" + + "github.com/labstack/echo/v5" + + functionsbackend "github.com/blackbirdworks/gopherstack/services/azurefunctions" +) + +const ( + azureFunctionsMasterKind = "masterKey" + azureFunctionsMasterName = "_master" + azureFunctionsKeyHeader = "X-Functions-Key" + azureFunctionsInvokeLimit = 1 << 20 + azureFunctionsInvokeTTL = 60 * time.Second + azureFunctionsAPIPrefix = "/api/" +) + +type azureFunctionKeyJSON struct { + Kind string `json:"kind"` + Name string `json:"name"` + Value string `json:"value"` +} + +type azureFunctionInvokeIn struct { + Function string `json:"function"` + Method string `json:"method"` + Query string `json:"query"` + Body string `json:"body"` + ContentType string `json:"contentType"` +} + +type azureFunctionInvokeOut struct { + Headers map[string]string `json:"headers"` + Body string `json:"body"` + Status int `json:"status"` +} + +func (a *azureFunctionsAPI) hostKeys(c *echo.Context) error { + keys, err := a.b.HostKeys(c.Param("name")) + if err != nil { + return a.fail(c, err) + } + + items := []azureFunctionKeyJSON{{ + Kind: azureFunctionsMasterKind, Name: azureFunctionsMasterName, Value: azureFunctionsMask, + }} + items = appendMaskedKeys(items, functionsbackend.KeyTypeFunction, keys.FunctionKeys) + items = appendMaskedKeys(items, functionsbackend.KeyTypeSystem, keys.SystemKeys) + + return c.JSON(http.StatusOK, map[string]any{"keys": items}) +} + +func appendMaskedKeys(items []azureFunctionKeyJSON, kind string, set map[string]string) []azureFunctionKeyJSON { + names := make([]string, 0, len(set)) + for n := range set { + names = append(names, n) + } + + slices.Sort(names) + + for _, n := range names { + items = append(items, azureFunctionKeyJSON{Kind: kind, Name: n, Value: azureFunctionsMask}) + } + + return items +} + +func (a *azureFunctionsAPI) revealKey(c *echo.Context) error { + var in struct { + Kind string `json:"kind"` + Name string `json:"name"` + } + + if !cloudJSONDecode(c, &in) { + return nil + } + + keys, err := a.b.HostKeys(c.Param("name")) + if err != nil { + return a.fail(c, err) + } + + var ( + value string + ok bool + ) + + switch in.Kind { + case azureFunctionsMasterKind: + value, ok = keys.MasterKey, true + case functionsbackend.KeyTypeFunction: + value, ok = keys.FunctionKeys[in.Name] + case functionsbackend.KeyTypeSystem: + value, ok = keys.SystemKeys[in.Name] + } + + if !ok { + return cloudJSONFail(c, http.StatusNotFound, cloudStatusNotFnd, "key not found") + } + + c.Response().Header().Set("Cache-Control", "no-store") + + return c.JSON(http.StatusOK, azureFunctionKeyJSON{Kind: in.Kind, Name: in.Name, Value: value}) +} + +// bufferedResponse captures a handler's reply with the body capped. +type bufferedResponse struct { + header http.Header + body bytes.Buffer + status int +} + +func (r *bufferedResponse) Header() http.Header { return r.header } + +func (r *bufferedResponse) WriteHeader(code int) { + if r.status == 0 { + r.status = code + } +} + +func (r *bufferedResponse) Write(p []byte) (int, error) { + r.WriteHeader(http.StatusOK) + + if room := azureFunctionsInvokeLimit - r.body.Len(); room > 0 { + r.body.Write(p[:min(len(p), room)]) + } + + return len(p), nil +} + +func invokeMethodAllowed(m string) bool { + switch m { + case http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete: + return true + } + + return false +} + +// invoke calls an HTTP-triggered function through the app's own emulator host. The path comes from the +// function's route, never from the request, so the call cannot reach any other app or URL. +func (a *azureFunctionsAPI) invoke(c *echo.Context) error { + var in azureFunctionInvokeIn + + if !cloudJSONDecode(c, &in) { + return nil + } + + in.Method = strings.ToUpper(in.Method) + if !invokeMethodAllowed(in.Method) { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "unsupported method") + } + + name := c.Param("name") + + ctx, cancel := context.WithTimeout(c.Request().Context(), azureFunctionsInvokeTTL) + defer cancel() + + fn, err := a.b.Function(ctx, name, in.Function) + if err != nil { + return a.fail(c, err) + } + + target, err := url.Parse(fn.InvokeURL) + if err != nil || fn.InvokeURL == "" || !strings.HasPrefix(target.Path, azureFunctionsAPIPrefix) { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "function has no HTTP trigger") + } + + keys, err := a.b.HostKeys(name) + if err != nil { + return a.fail(c, err) + } + + req, err := http.NewRequestWithContext(ctx, in.Method, "http://"+a.b.HostName(name)+target.Path, + strings.NewReader(in.Body)) + if err != nil { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "invalid request") + } + + req.URL.RawQuery = strings.TrimPrefix(in.Query, "?") + req.Host = a.b.HostName(name) + req.Header.Set(azureFunctionsKeyHeader, keys.MasterKey) + + if in.ContentType != "" && !strings.ContainsAny(in.ContentType, "\r\n") { + req.Header.Set("Content-Type", in.ContentType) + } + + rec := &bufferedResponse{header: http.Header{}} + a.b.ServeHTTP(rec, req) + + out := azureFunctionInvokeOut{Status: max(rec.status, http.StatusOK), Headers: map[string]string{}} + for k := range rec.header { + out.Headers[k] = rec.header.Get(k) + } + + body, _ := io.ReadAll(&rec.body) + out.Body = string(body) + + c.Response().Header().Set("Cache-Control", "no-store") + + return c.JSON(http.StatusOK, out) +} diff --git a/dashboard/azurefunctions_proxy.go b/dashboard/azurefunctions_proxy.go index ed2a6374ab..360eede0b3 100644 --- a/dashboard/azurefunctions_proxy.go +++ b/dashboard/azurefunctions_proxy.go @@ -26,6 +26,7 @@ type azureFunctionAppJSON struct { SCMHost string `json:"scmHost"` Image string `json:"image"` Worker string `json:"worker"` + Failure string `json:"failure,omitempty"` State string `json:"state"` Stopped bool `json:"stopped"` Deployed bool `json:"deployed"` @@ -68,6 +69,9 @@ func (h *DashboardHandler) registerAzureFunctionsRoutes() { r.Add(http.MethodPost, azureFunctionsProxyPrefix+"/apps/:name/settings/reveal", api.guard(api.reveal)) r.Add(http.MethodGet, azureFunctionsProxyPrefix+"/apps/:name/functions", api.guard(api.functions)) r.Add(http.MethodGet, azureFunctionsProxyPrefix+"/apps/:name/deployments", api.guard(api.deployments)) + r.Add(http.MethodGet, azureFunctionsProxyPrefix+"/apps/:name/keys", api.guard(api.hostKeys)) + r.Add(http.MethodPost, azureFunctionsProxyPrefix+"/apps/:name/keys/reveal", api.guard(api.revealKey)) + r.Add(http.MethodPost, azureFunctionsProxyPrefix+"/apps/:name/invoke", api.guard(api.invoke)) r.Add(http.MethodPost, azureFunctionsProxyPrefix+"/apps/:name/:action", api.guard(api.lifecycle)) } @@ -97,7 +101,8 @@ func (a *azureFunctionsAPI) fail(c *echo.Context, err error) error { func azureFunctionApp(name string, i functionsbackend.AppInfo) azureFunctionAppJSON { out := azureFunctionAppJSON{ Name: name, Created: i.Created, HostName: i.HostName, SCMHost: i.SCMHost, - Image: i.Image, Worker: i.Worker, Stopped: i.Stopped, Deployed: i.Deployed, State: "Running", + Image: i.Image, Worker: i.Worker, Failure: i.Failure, Stopped: i.Stopped, + Deployed: i.Deployed, State: "Running", } if i.Stopped { @@ -164,6 +169,8 @@ func (a *azureFunctionsAPI) reveal(c *echo.Context) error { return cloudJSONFail(c, http.StatusNotFound, cloudStatusNotFnd, "setting not found") } + c.Response().Header().Set("Cache-Control", "no-store") + return c.JSON(http.StatusOK, azureFunctionSettingJSON{Name: in.Name, Value: value}) } diff --git a/dashboard/azurefunctions_proxy_test.go b/dashboard/azurefunctions_proxy_test.go index ebcf705a1c..a8d90d6e8a 100644 --- a/dashboard/azurefunctions_proxy_test.go +++ b/dashboard/azurefunctions_proxy_test.go @@ -126,6 +126,56 @@ func TestAzureFunctionsProxy(t *testing.T) { name: "missing app action", method: http.MethodPost, path: base + "/apps/nope/stop", served: true, wantCode: http.StatusNotFound, wantBody: "NOT_FOUND", }, + { + name: "keys masked", method: http.MethodGet, path: base + "/apps/app1/keys", served: true, + wantCode: http.StatusOK, wantBody: `"kind":"masterKey"`, notWant: "==", + }, + { + name: "reveal master key", method: http.MethodPost, path: base + "/apps/app1/keys/reveal", + body: `{"kind":"masterKey"}`, served: true, wantCode: http.StatusOK, wantBody: "==", + }, + { + name: "reveal host key", method: http.MethodPost, path: base + "/apps/app1/keys/reveal", + body: `{"kind":"functionKeys","name":"default"}`, served: true, wantCode: http.StatusOK, wantBody: "==", + }, + { + name: "reveal missing key", method: http.MethodPost, path: base + "/apps/app1/keys/reveal", + body: `{"kind":"systemKeys","name":"nope"}`, served: true, wantCode: http.StatusNotFound, + wantBody: "NOT_FOUND", + }, + { + name: "keys missing app", method: http.MethodGet, path: base + "/apps/nope/keys", served: true, + wantCode: http.StatusNotFound, wantBody: "NOT_FOUND", + }, + { + name: "invoke bad method", method: http.MethodPost, path: base + "/apps/app1/invoke", + body: `{"function":"hello","method":"TRACE"}`, served: true, wantCode: http.StatusBadRequest, + wantBody: "unsupported method", + }, + { + name: "invoke bad body", method: http.MethodPost, path: base + "/apps/app1/invoke", + body: `{`, served: true, wantCode: http.StatusBadRequest, wantBody: "INVALID_ARGUMENT", + }, + { + name: "invoke unknown function", method: http.MethodPost, path: base + "/apps/app1/invoke", + body: `{"function":"nope","method":"GET"}`, served: true, wantCode: http.StatusNotFound, + wantBody: "NOT_FOUND", + }, + { + name: "invoke non http function", method: http.MethodPost, path: base + "/apps/app1/invoke", + body: `{"function":"tick","method":"GET"}`, served: true, wantCode: http.StatusBadRequest, + wantBody: "no HTTP trigger", + }, + { + name: "invoke without runtime", method: http.MethodPost, path: base + "/apps/app1/invoke", + body: `{"function":"hello","method":"GET"}`, served: true, wantCode: http.StatusOK, + wantBody: `"status":503`, + }, + { + name: "invoke missing app", method: http.MethodPost, path: base + "/apps/nope/invoke", + body: `{"function":"hello","method":"GET"}`, served: true, wantCode: http.StatusNotFound, + wantBody: "NOT_FOUND", + }, { name: "not served", method: http.MethodGet, path: base + "/apps", wantCode: http.StatusServiceUnavailable, wantBody: "UNAVAILABLE", @@ -164,3 +214,32 @@ func TestAzureFunctionsProxy(t *testing.T) { }) } } + +func TestAzureFunctionsRevealNoStore(t *testing.T) { + t.Parallel() + + b := azurefunctions.NewInMemoryBackend() + b.PutApp("app1", azurefunctions.AppConfig{Kind: "functionapp,linux", LinuxFx: "NODE|20"}) + require.NoError(t, b.SetSettings("app1", map[string]string{"FOO": "v"})) + + h := dashboard.NewHandler(dashboard.Config{ + AzureFunctionsOps: azurefunctions.NewHandler(b, azurefunctions.Settings{}), + }) + + for _, tc := range []struct{ name, path, body string }{ + {"setting", "/apps/app1/settings/reveal", `{"name":"FOO"}`}, + {"key", "/apps/app1/keys/reveal", `{"kind":"masterKey"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + req := httptest.NewRequest( + http.MethodPost, "/dashboard/api/v1/azure/functions"+tc.path, strings.NewReader(tc.body), + ) + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + }) + } +} diff --git a/dashboard/azuregraph_proxy.go b/dashboard/azuregraph_proxy.go new file mode 100644 index 0000000000..6201fee901 --- /dev/null +++ b/dashboard/azuregraph_proxy.go @@ -0,0 +1,442 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "slices" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/labstack/echo/v5" +) + +const ( + azureGraphPrefix = "/dashboard/api/v1/azure/graph" + graphPxMaxBody = 1 << 20 + graphPxMaxName = 120 + graphPxMaxPassword = 256 + graphPxMaxIDs = 100 + graphPxMaxTop = 999 + graphPxGraphHost = "https://graph.microsoft.com/v1.0/directoryObjects/" + graphPxApplication = "applications" + graphPxPrincipals = "serviceprincipals" + graphPxUsers = "users" + graphPxGroups = "groups" + graphPxAddPassword = "addpassword" + graphPxRemovePw = "removepassword" + graphPxMembers = "members" + graphPxRef = "$ref" + graphPxPassword = "password" + graphPxDisplayName = "displayName" + graphPxItem = 2 + graphPxSub = 3 +) + +//nolint:gochecknoglobals // immutable allow-lists and validators +var ( + graphPxRoutes = map[string]bool{ + "GET applications": true, "POST applications": true, "GET applications/*": true, + "DELETE applications/*": true, "GET applications/*/owners": true, + "POST applications/*/addpassword": true, "POST applications/*/removepassword": true, + "GET serviceprincipals": true, "POST serviceprincipals": true, "GET serviceprincipals/*": true, + "GET serviceprincipals/*/approleassignments": true, + "GET users": true, "POST users": true, "GET users/*": true, + "GET groups": true, "POST groups": true, "GET groups/*": true, "GET groups/*/members": true, + "POST groups/*/members/$ref": true, "DELETE groups/*/members/*/$ref": true, + "POST directoryobjects/getbyids": true, + } + graphPxGUID = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + graphPxSelectRE = regexp.MustCompile(`^[A-Za-z0-9_,]{1,200}$`) + graphPxTokenRE = regexp.MustCompile(`^[A-Za-z0-9_=.\-]{1,1000}$`) + graphPxNickRE = regexp.MustCompile(`^[A-Za-z0-9._-]{1,64}$`) + graphPxUPNRE = regexp.MustCompile(`^[A-Za-z0-9._'+-]{1,64}@[A-Za-z0-9.-]{1,253}$`) + graphPxSecrets = []string{"secrettext", "passwordprofile", graphPxPassword} +) + +type graphProxyErr struct { + Error graphProxyErrBody `json:"error"` +} + +type graphProxyErrBody struct { + Code string `json:"code"` + Message string `json:"message"` +} + +// registerAzureGraphRoutes serves the Entra ID page by dispatching in-process to the Microsoft Graph handler with a +// server-minted token. Paths and methods are allow-listed, every write body is rebuilt from fixed fields, and +// secret-bearing fields are scrubbed from every response except the one addPassword answer that carries a new secret. +func (h *DashboardHandler) registerAzureGraphRoutes() { + ops := h.config.AzureGraphOps + + var inner *echo.Echo + + if ops != nil { + inner = echo.New() + inner.Any("/*", ops.Handler()) + } + + serve := func(c *echo.Context) error { + if inner == nil { + return c.JSON(http.StatusServiceUnavailable, graphProxyErr{ + graphProxyErrBody{errServiceUnavailable, "Microsoft Graph is not enabled"}, + }) + } + + r := c.Request() + rest := strings.TrimPrefix(r.URL.EscapedPath(), azureGraphPrefix) + + segs, ok := graphProxySegments(r.Method, rest) + if !ok { + return c.JSON(http.StatusBadRequest, graphProxyErr{ + graphProxyErrBody{"InvalidRequestUri", "unsupported Microsoft Graph proxy request"}, + }) + } + + in, err := io.ReadAll(http.MaxBytesReader(c.Response(), r.Body, graphPxMaxBody)) + if err != nil { + return graphProxyBadBody(c) + } + + body, ok := graphProxyBody(r.Method, segs, in) + if !ok { + return graphProxyBadBody(c) + } + + token, err := ops.InProcessToken() + if err != nil { + return c.JSON(http.StatusInternalServerError, graphProxyErr{ + graphProxyErrBody{"InternalFailure", "could not authorise the Microsoft Graph call"}, + }) + } + + fwd := graphProxyRequest(r, segs, body, token) + rec := httptest.NewRecorder() + inner.ServeHTTP(rec, fwd) + + return graphProxyRelay(c, rec, graphIsAddPassword(r.Method, segs)) + } + + for _, m := range []string{http.MethodGet, http.MethodPost, http.MethodDelete} { + h.SubRouter.Add(m, azureGraphPrefix+"/*", serve) + } +} + +func graphProxyBadBody(c *echo.Context) error { + return c.JSON(http.StatusBadRequest, graphProxyErr{ + graphProxyErrBody{"InvalidRequestContent", "invalid Microsoft Graph proxy request body"}, + }) +} + +func graphProxyRequest(r *http.Request, segs []string, body []byte, token string) *http.Request { + fwd := r.Clone(r.Context()) + fwd.URL.Path = "/v1.0/" + strings.Join(segs, "/") + fwd.URL.RawPath = "" + fwd.URL.RawQuery = graphProxyQuery(r.URL.Query()).Encode() + fwd.RequestURI = "" + fwd.Header = http.Header{echo.HeaderAuthorization: {"Bearer " + token}} + fwd.Body, fwd.ContentLength = io.NopCloser(bytes.NewReader(body)), int64(len(body)) + + if len(body) > 0 { + fwd.Header.Set("Content-Type", "application/json") + } + + return fwd +} + +// graphProxySegments validates the proxied path and returns its decoded segments, keeping the case the caller sent. +func graphProxySegments(method, raw string) ([]string, bool) { + parts := strings.Split(strings.Trim(raw, "/"), "/") + + for i, p := range parts { + dec, err := url.PathUnescape(p) + if err != nil || dec == "" || strings.ContainsAny(dec, "/:?#\\") || dec == "." || dec == ".." { + return nil, false + } + + parts[i] = dec + } + + if !graphProxyAllowed(method, graphLower(parts)) { + return nil, false + } + + return parts, true +} + +func graphLower(parts []string) []string { + out := make([]string, len(parts)) + for i, p := range parts { + out[i] = strings.ToLower(p) + } + + return out +} + +// graphProxyAllowed matches the lower-cased path, with every GUID segment folded to "*", against graphPxRoutes. +func graphProxyAllowed(method string, p []string) bool { + shape := make([]string, len(p)) + + for i, seg := range p { + shape[i] = seg + if graphPxGUID.MatchString(seg) { + shape[i] = "*" + } + } + + return graphPxRoutes[method+" "+strings.Join(shape, "/")] +} + +func graphIsAddPassword(method string, segs []string) bool { + return method == http.MethodPost && len(segs) == graphPxSub && strings.EqualFold(segs[2], graphPxAddPassword) +} + +func graphProxyQuery(in url.Values) url.Values { + q := url.Values{} + + if v := in.Get("$select"); graphPxSelectRE.MatchString(v) { + q.Set("$select", v) + } + + if v := in.Get("$skiptoken"); graphPxTokenRE.MatchString(v) { + q.Set("$skiptoken", v) + } + + if n, err := strconv.Atoi(in.Get("$top")); err == nil && n >= 1 && n <= graphPxMaxTop { + q.Set("$top", strconv.Itoa(n)) + } + + return q +} + +// graphProxyBody rebuilds the body of every write from fixed fields; reads and deletes carry no body. +func graphProxyBody(method string, segs []string, in []byte) ([]byte, bool) { + if method != http.MethodPost { + return nil, true + } + + var out any + + var ok bool + + p := graphLower(segs) + + switch { + case len(p) == graphPxItem && p[0] == "directoryobjects": + out, ok = graphIDsBody(in) + case len(p) == 1: + out, ok = graphCreateBody(p[0], in) + case p[2] == graphPxAddPassword: + out, ok = graphAddPasswordBody(in) + case p[2] == graphPxRemovePw: + out, ok = graphKeyIDBody(in) + default: + out, ok = graphMemberBody(in) + } + + if !ok { + return nil, false + } + + b, err := json.Marshal(out) + + return b, err == nil +} + +func graphDecode(in []byte, into any) bool { + return json.Unmarshal(in, into) == nil +} + +func graphText(v string) (string, bool) { + v = strings.TrimSpace(v) + control := strings.ContainsFunc(v, func(r rune) bool { return r < ' ' }) + if v == "" || utf8.RuneCountInString(v) > graphPxMaxName || control { + return "", false + } + + return v, true +} + +func graphCreateBody(coll string, in []byte) (any, bool) { + var req struct { + DisplayName string `json:"displayName"` + AppID string `json:"appId"` + MailNickname string `json:"mailNickname"` + UserPrincipalName string `json:"userPrincipalName"` + Password string `json:"password"` + ForceChange bool `json:"forceChangePasswordNextSignIn"` + } + + if !graphDecode(in, &req) { + return nil, false + } + + switch coll { + case graphPxPrincipals: + if !graphPxGUID.MatchString(strings.ToLower(req.AppID)) { + return nil, false + } + + return map[string]any{"appId": req.AppID}, true + case graphPxApplication: + name, ok := graphText(req.DisplayName) + + return map[string]any{graphPxDisplayName: name}, ok + case graphPxGroups: + return graphGroupBody(req.DisplayName, req.MailNickname) + } + + return graphUserBody(req.DisplayName, req.MailNickname, req.UserPrincipalName, req.Password, req.ForceChange) +} + +func graphGroupBody(displayName, nick string) (any, bool) { + name, ok := graphText(displayName) + if !ok || !graphPxNickRE.MatchString(nick) { + return nil, false + } + + return map[string]any{ + graphPxDisplayName: name, "mailNickname": nick, "mailEnabled": false, "securityEnabled": true, + }, true +} + +func graphUserBody(displayName, nick, upn, password string, force bool) (any, bool) { + name, ok := graphText(displayName) + if !ok || !graphPxNickRE.MatchString(nick) || !graphPxUPNRE.MatchString(upn) || + password == "" || len(password) > graphPxMaxPassword { + return nil, false + } + + return map[string]any{ + "accountEnabled": true, graphPxDisplayName: name, "mailNickname": nick, "userPrincipalName": upn, + "passwordProfile": map[string]any{"password": password, "forceChangePasswordNextSignIn": force}, + }, true +} + +func graphAddPasswordBody(in []byte) (any, bool) { + var req struct { + DisplayName string `json:"displayName"` + EndDateTime string `json:"endDateTime"` + } + + if len(bytes.TrimSpace(in)) > 0 && !graphDecode(in, &req) { + return nil, false + } + + cred := map[string]any{} + + if req.DisplayName != "" { + name, ok := graphText(req.DisplayName) + if !ok { + return nil, false + } + + cred[graphPxDisplayName] = name + } + + if req.EndDateTime != "" { + t, err := time.Parse(time.RFC3339, req.EndDateTime) + if err != nil { + return nil, false + } + + cred["endDateTime"] = t.UTC().Format(time.RFC3339) + } + + return map[string]any{"passwordCredential": cred}, true +} + +func graphKeyIDBody(in []byte) (any, bool) { + var req struct { + KeyID string `json:"keyId"` + } + + if !graphDecode(in, &req) || !graphPxGUID.MatchString(strings.ToLower(req.KeyID)) { + return nil, false + } + + return map[string]any{"keyId": req.KeyID}, true +} + +func graphMemberBody(in []byte) (any, bool) { + var req struct { + ID string `json:"id"` + } + + if !graphDecode(in, &req) || !graphPxGUID.MatchString(strings.ToLower(req.ID)) { + return nil, false + } + + return map[string]any{"@odata.id": graphPxGraphHost + req.ID}, true +} + +func graphIDsBody(in []byte) (any, bool) { + var req struct { + IDs []string `json:"ids"` + } + + if !graphDecode(in, &req) || len(req.IDs) == 0 || len(req.IDs) > graphPxMaxIDs { + return nil, false + } + + for _, id := range req.IDs { + if !graphPxGUID.MatchString(strings.ToLower(id)) { + return nil, false + } + } + + return map[string]any{ + "ids": req.IDs, "types": []string{"user", "group", "servicePrincipal", "application"}, + }, true +} + +// graphProxyRelay copies the Graph answer after removing secret-bearing fields. The addPassword answer is the one +// place a freshly generated secret may reach the browser, and it is marked uncacheable. +func graphProxyRelay(c *echo.Context, rec *httptest.ResponseRecorder, keepSecret bool) error { + body := rec.Body.Bytes() + ok := rec.Code < http.StatusMultipleChoices + + if len(body) > 0 && !keepSecret || !ok { + var doc any + if json.Unmarshal(body, &doc) == nil { + graphScrub(doc) + + if out, err := json.Marshal(doc); err == nil { + body = out + } + } + } + + c.Response().Header().Set("Cache-Control", "no-store") + + if len(body) == 0 { + return c.NoContent(rec.Code) + } + + return c.JSONBlob(rec.Code, body) +} + +func graphScrub(v any) { + switch t := v.(type) { + case []any: + for _, x := range t { + graphScrub(x) + } + case map[string]any: + for k, x := range t { + if slices.Contains(graphPxSecrets, strings.ToLower(k)) { + delete(t, k) + + continue + } + + graphScrub(x) + } + } +} diff --git a/dashboard/azurelogic_proxy.go b/dashboard/azurelogic_proxy.go new file mode 100644 index 0000000000..4549f4398d --- /dev/null +++ b/dashboard/azurelogic_proxy.go @@ -0,0 +1,404 @@ +package dashboard + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "slices" + "strings" + + "github.com/labstack/echo/v5" +) + +const ( + azureLogicPrefix = "/dashboard/api/v1/azure/logic" + azureLogicAccessPrefix = "/dashboard/api/v1/azure/logicaccess" + logicPxNamespace = "microsoft.logic" + logicPxARMNamespace = "Microsoft.Logic" + logicPxAPIVersion = "2019-05-01" + logicPxWorkflows = "workflows" + logicPxTriggers = "triggers" + logicPxRuns = "runs" + logicPxActions = "actions" + logicPxDataPrefix = "/_logic/workflows/" + logicPxRGList = 7 + logicPxBasePath = 9 + logicPxKeyStatus = "status" + logicPxRunVerb = "run" + logicPxRestItem = 2 + logicPxMaxInvoke = 1 << 16 + logicPxMaxReply = 256 << 10 + logicPxMaxContent = 1 << 20 + logicPxKeyURI = "uri" + logicPxKeyValue = "value" + logicPxKindInputs = "inputs" + logicPxKindOutputs = "outputs" +) + +var ( + logicPxSubRe = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`) + logicPxNameRe = regexp.MustCompile(`^[A-Za-z0-9._()-]{1,128}$`) +) + +// logicPxQueryKeys are the only list parameters forwarded to the emulator. +// +//nolint:gochecknoglobals // immutable allow-list +var logicPxQueryKeys = []string{"$top", "$skiptoken", "$filter"} + +// registerAzureLogicRoutes exposes Microsoft.Logic workflows by dispatching in-process to the ARM handler. The +// generic proxy reaches only workflow reads, enable/disable, trigger and run reads, recurrence trigger runs and run +// cancellation, with the api-version pinned and the signed content links removed from every answer. The routes below +// the access prefix are the only ones that touch signed URLs, and each does so server side: the callback URL is +// returned only by an explicit POST, a trigger invocation and a content read use the signed URL without ever +// returning it. +func (h *DashboardHandler) registerAzureLogicRoutes() { + h.registerARMProxy(armProxy{ + name: "Azure Logic Apps", prefix: azureLogicPrefix, + queryKeys: logicPxQueryKeys, version: logicPxAPIVersion, + methods: []string{http.MethodGet, http.MethodPost}, + allowed: logicProxyAllowed, + body: logicProxyBody, + view: logicProxyView, + }) + + var inner *echo.Echo + + if h.config.AzureARMOps != nil { + inner = echo.New() + inner.Any("/*", h.config.AzureARMOps.Handler()) + } + + a := logicAccess{inner: inner} + guard := func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c *echo.Context) error { + if inner == nil { + return azureACIFail( + c, + http.StatusServiceUnavailable, + errServiceUnavailable, + "Azure Logic Apps is not enabled", + ) + } + + return next(c) + } + } + + const ids = "/:sub/:rg/:wf" + + h.SubRouter.Add(http.MethodPost, azureLogicAccessPrefix+"/callback"+ids+"/:trigger", guard(a.callback)) + h.SubRouter.Add(http.MethodPost, azureLogicAccessPrefix+"/invoke"+ids+"/:trigger", guard(a.invoke)) + h.SubRouter.Add(http.MethodGet, azureLogicAccessPrefix+"/content"+ids+"/:run/trigger/:kind", guard(a.content)) + h.SubRouter.Add(http.MethodGet, azureLogicAccessPrefix+"/content"+ids+"/:run/action/:action/:kind", + guard(a.content)) +} + +func logicProxyAllowed(method string, p []string) bool { + if len(p) == logicPxRGList { + return method == http.MethodGet && p[0] == armPxSubsSeg && p[2] == armPxRGsSeg && p[4] == armPxProvSeg && + p[5] == logicPxNamespace && p[6] == logicPxWorkflows + } + + typ, name, rest, ok := armScope(p, logicPxNamespace) + if !ok || typ != logicPxWorkflows || name == "" { + return false + } + + return logicProxyRest(method, rest) +} + +// logicProxyShapes are the workflow-relative calls the generic proxy allows; item names are written as *. +// +//nolint:gochecknoglobals // immutable allow-list +var logicProxyShapes = []string{ + "GET:", "GET:triggers", "GET:runs", "GET:triggers/*", "GET:runs/*", "GET:runs/*/actions", + "GET:runs/*/actions/*", "POST:enable", "POST:disable", "POST:triggers/*/run", "POST:runs/*/cancel", +} + +func logicProxyRest(method string, rest []string) bool { + parts := make([]string, len(rest)) + + for i, seg := range rest { + if i%logicPxRestItem == 1 { + parts[i] = "*" + } else { + parts[i] = seg + } + } + + return slices.Contains(logicProxyShapes, method+":"+strings.Join(parts, "/")) +} + +// logicProxyBody sends the actions that take no input an empty object. +func logicProxyBody(method string, _ []string, _ []byte) ([]byte, bool) { + if method == http.MethodPost { + return []byte("{}"), true + } + + return nil, true +} + +// logicProxyView drops the signed URI of every content link; the page reads content through the access routes. +func logicProxyView(doc any) { + switch t := doc.(type) { + case []any: + for _, x := range t { + logicProxyView(x) + } + case map[string]any: + for k, x := range t { + if m, ok := x.(map[string]any); ok && (k == "inputsLink" || k == "outputsLink" || k == "contentLink") { + delete(m, logicPxKeyURI) + + continue + } + + logicProxyView(x) + } + } +} + +type logicAccess struct { + inner *echo.Echo + host string +} + +// withHost makes the emulator build URLs on the caller's host name; the port is left out so it uses its own. +func (a logicAccess) withHost(c *echo.Context) logicAccess { + host := c.Request().Host + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + + a.host = host + + return a +} + +type logicIDs struct{ sub, rg, wf string } + +func (i logicIDs) armPath(rest ...string) string { + parts := make([]string, 0, logicPxBasePath+len(rest)) + parts = append(parts, "", armPxSubsSeg, url.PathEscape(i.sub), "resourceGroups", url.PathEscape(i.rg), armPxProvSeg, + logicPxARMNamespace, logicPxWorkflows, url.PathEscape(i.wf)) + + for _, r := range rest { + parts = append(parts, url.PathEscape(r)) + } + + return strings.Join(parts, "/") +} + +func logicParams(c *echo.Context, names ...string) (logicIDs, []string, bool) { + ids := logicIDs{sub: c.Param("sub"), rg: c.Param("rg"), wf: c.Param("wf")} + if !logicPxSubRe.MatchString(ids.sub) || !logicPxNameRe.MatchString(ids.rg) || !logicPxNameRe.MatchString(ids.wf) { + return ids, nil, false + } + + extra := make([]string, 0, len(names)) + + for _, n := range names { + v := c.Param(n) + if !logicPxNameRe.MatchString(v) { + return ids, nil, false + } + + extra = append(extra, v) + } + + return ids, extra, true +} + +func (a logicAccess) do(ctx context.Context, method, path, query string, body []byte) *httptest.ResponseRecorder { + target := path + if query != "" { + target += "?" + query + } + + req := httptest.NewRequestWithContext(ctx, method, target, bytes.NewReader(body)) + if a.host != "" { + req.Host = a.host + } + + if len(body) > 0 { + req.Header.Set("Content-Type", "application/json") + } + + rec := httptest.NewRecorder() + a.inner.ServeHTTP(rec, req) + + return rec +} + +func (a logicAccess) arm(ctx context.Context, method, path string) *httptest.ResponseRecorder { + var body []byte + if method == http.MethodPost { + body = []byte("{}") + } + + return a.do(ctx, method, path, "api-version="+logicPxAPIVersion, body) +} + +// signed resolves the callback URL of a Request trigger as a data plane path and query; it is never sent out. +func (a logicAccess) signed(ctx context.Context, ids logicIDs, trigger string) (map[string]any, string, string, int) { + rec := a.arm(ctx, http.MethodPost, ids.armPath(logicPxTriggers, trigger, "listCallbackUrl")) + if rec.Code != http.StatusOK { + return nil, "", "", rec.Code + } + + var cb map[string]any + if json.Unmarshal(rec.Body.Bytes(), &cb) != nil { + return nil, "", "", http.StatusBadGateway + } + + path, query, ok := logicDataTarget(logicStr(cb[logicPxKeyValue])) + if !ok { + return nil, "", "", http.StatusBadGateway + } + + return cb, path, query, http.StatusOK +} + +// logicDataTarget keeps only the path and query of a signed URL and requires the data plane prefix. +func logicDataTarget(raw string) (string, string, bool) { + u, err := url.Parse(raw) + if err != nil || !strings.HasPrefix(u.Path, logicPxDataPrefix) { + return "", "", false + } + + return u.Path, u.RawQuery, true +} + +func logicStr(v any) string { + s, _ := v.(string) + + return s +} + +func logicNoStore(c *echo.Context) { + c.Response().Header().Set("Cache-Control", "no-store") +} + +// callback returns the signed callback URL of one trigger. It is the only route that sends a signature out. +func (a logicAccess) callback(c *echo.Context) error { + a = a.withHost(c) + + ids, extra, ok := logicParams(c, "trigger") + if !ok { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestUri", "invalid Logic Apps request") + } + + cb, _, _, status := a.signed(c.Request().Context(), ids, extra[0]) + if cb == nil { + return azureACIFail(c, status, "CallbackUnavailable", "the trigger has no callback URL") + } + + logicNoStore(c) + + return c.JSON(http.StatusOK, cb) +} + +// invoke calls a Request trigger through its signed URL and returns only the result of the call. +func (a logicAccess) invoke(c *echo.Context) error { + ids, extra, ok := logicParams(c, "trigger") + if !ok { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestUri", "invalid Logic Apps request") + } + + body, err := io.ReadAll(http.MaxBytesReader(c.Response(), c.Request().Body, logicPxMaxInvoke)) + if err != nil || (len(bytes.TrimSpace(body)) > 0 && !json.Valid(body)) { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestContent", "the trigger body must be JSON") + } + + ctx := c.Request().Context() + + cb, path, query, status := a.signed(ctx, ids, extra[0]) + if cb == nil { + return azureACIFail(c, status, "CallbackUnavailable", "the trigger has no callback URL") + } + + method := strings.ToUpper(logicStr(cb["method"])) + if method == "" { + method = http.MethodPost + } + + rec := a.do(ctx, method, path, query, bytes.TrimSpace(body)) + reply := rec.Body.Bytes() + + if len(reply) > logicPxMaxReply { + reply = reply[:logicPxMaxReply] + } + + logicNoStore(c) + + return c.JSON(http.StatusOK, map[string]any{ + logicPxKeyStatus: rec.Code, "contentType": rec.Header().Get("Content-Type"), + "runId": rec.Header().Get("X-Ms-Workflow-Run-Id"), "body": string(reply), + }) +} + +// content reads the stored inputs or outputs of a trigger or an action of one run through its signed link. +func (a logicAccess) content(c *echo.Context) error { + names := []string{logicPxRunVerb, "kind"} + if c.Param("action") != "" { + names = []string{logicPxRunVerb, "action", "kind"} + } + + ids, extra, ok := logicParams(c, names...) + if !ok || (extra[len(extra)-1] != logicPxKindInputs && extra[len(extra)-1] != logicPxKindOutputs) { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestUri", "invalid Logic Apps request") + } + + ctx := c.Request().Context() + run, kind := extra[0], extra[len(extra)-1] + + forTrigger := len(extra) == logicPxRestItem + armPath := ids.armPath(logicPxRuns, run) + + if !forTrigger { + armPath = ids.armPath(logicPxRuns, run, logicPxActions, extra[1]) + } + + rec := a.arm(ctx, http.MethodGet, armPath) + + if rec.Code != http.StatusOK { + return azureACIFail(c, rec.Code, "ContentUnavailable", "the run or action was not found") + } + + var doc map[string]any + if json.Unmarshal(rec.Body.Bytes(), &doc) != nil { + return azureACIFail(c, http.StatusBadGateway, "ContentUnavailable", "unreadable run") + } + + props, _ := doc["properties"].(map[string]any) + if forTrigger { + props, _ = props["trigger"].(map[string]any) + } + + link, _ := props[kind+"Link"].(map[string]any) + + path, query, ok := logicDataTarget(logicStr(link[logicPxKeyURI])) + if !ok { + return azureACIFail(c, http.StatusNotFound, "ContentNotFound", "the "+kind+" are not stored") + } + + data := a.do(ctx, http.MethodGet, path, query, nil) + if data.Code != http.StatusOK { + return azureACIFail(c, data.Code, "ContentUnavailable", "the stored "+kind+" could not be read") + } + + out := data.Body.Bytes() + if len(out) > logicPxMaxContent { + return azureACIFail(c, http.StatusRequestEntityTooLarge, "ContentTooLarge", "the stored "+kind+" are too large") + } + + logicNoStore(c) + + return c.JSONBlob(http.StatusOK, out) +} diff --git a/dashboard/azurelogic_proxy_test.go b/dashboard/azurelogic_proxy_test.go new file mode 100644 index 0000000000..0c9e6edeba --- /dev/null +++ b/dashboard/azurelogic_proxy_test.go @@ -0,0 +1,373 @@ +package dashboard_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" +) + +const ( + logicAPI = "?api-version=2019-05-01" + logicRoot = "/dashboard/api/v1/azure/logic" + logicAccess = "/dashboard/api/v1/azure/logicaccess" + logicNS = armBase + "Microsoft.Logic" + logicWF = logicNS + "/workflows/greeter" + logicTicker = logicNS + "/workflows/ticker" + logicIDs = "/" + armSub + "/demo/greeter" + + logicSchema = `"$schema":"https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/` + + `workflowdefinition.json#","contentVersion":"1.0.0.0"` + logicGreeter = `{"location":"westus","properties":{"definition":{` + logicSchema + `,` + + `"triggers":{"manual":{"type":"Request","kind":"Http","inputs":{"method":"POST"}}},` + + `"actions":{"Compose":{"type":"Compose","inputs":"@concat('hello ', triggerBody()?['name'])","runAfter":{}},` + + `"Response":{"type":"Response","kind":"Http","runAfter":{"Compose":["Succeeded"]},` + + `"inputs":{"statusCode":200,"body":{"greeting":"@outputs('Compose')"}}}}}}}` + logicRecurrence = `{"location":"westus","properties":{"definition":{` + logicSchema + `,` + + `"triggers":{"tick":{"type":"Recurrence","recurrence":{"frequency":"Month","interval":1}}},` + + `"actions":{"Compose":{"type":"Compose","inputs":"tick","runAfter":{}}}}}}` +) + +func newLogicDashboard(t *testing.T) *dashboard.DashboardHandler { + t.Helper() + + return newARMDashboard(t, []struct{ path, body string }{ + {"/subscriptions/" + armSub + "/resourceGroups/demo?api-version=2021-04-01", `{"location":"westus"}`}, + {logicWF + logicAPI, logicGreeter}, + {logicTicker + logicAPI, logicRecurrence}, + }) +} + +func logicGet(t *testing.T, h *dashboard.DashboardHandler, path string) map[string]any { + t.Helper() + + rec := lbDo(t, h, http.MethodGet, path, "") + require.Equal(t, http.StatusOK, rec.Code, "%s: %s", path, rec.Body.String()) + + var out map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + + return out +} + +func logicValues(t *testing.T, doc map[string]any) []map[string]any { + t.Helper() + + raw, ok := doc["value"].([]any) + require.True(t, ok) + + out := make([]map[string]any, 0, len(raw)) + + for _, r := range raw { + m, isMap := r.(map[string]any) + require.True(t, isMap) + + out = append(out, m) + } + + return out +} + +// logicFirstRun invokes the greeter and waits until its run has finished. +func logicFirstRun(t *testing.T, h *dashboard.DashboardHandler) string { + t.Helper() + + rec := lbDo(t, h, http.MethodPost, logicAccess+"/invoke"+logicIDs+"/manual", `{"name":"bob"}`) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + var reply struct { + Body string `json:"body"` + RunID string `json:"runId"` + Status int `json:"status"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &reply)) + require.Equal(t, http.StatusOK, reply.Status, rec.Body.String()) + require.NotEmpty(t, reply.RunID) + assert.JSONEq(t, `{"greeting":"hello bob"}`, reply.Body) + + require.Eventually(t, func() bool { + runs := logicGet(t, h, logicRoot+logicWF+"/runs"+logicAPI) + for _, r := range logicValues(t, runs) { + props, _ := r["properties"].(map[string]any) + if r["name"] == reply.RunID && props["status"] == "Succeeded" { + return true + } + } + + return false + }, 5*time.Second, 10*time.Millisecond) + + return reply.RunID +} + +func TestAzureLogicProxyWorkflowsAndTriggers(t *testing.T) { + t.Parallel() + + h := newLogicDashboard(t) + + list := logicGet(t, h, logicRoot+armBase+"Microsoft.Logic/workflows"+logicAPI) + assert.Len(t, logicValues(t, list), 2) + + one := logicGet(t, h, logicRoot+logicWF+logicAPI) + props, _ := one["properties"].(map[string]any) + assert.Equal(t, "Enabled", props["state"]) + assert.Contains(t, props, "definition") + + for state, action := range map[string]string{"Disabled": "disable", "Enabled": "enable"} { + rec := lbDo(t, h, http.MethodPost, logicRoot+logicWF+"/"+action+logicAPI, "") + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + got, _ := logicGet(t, h, logicRoot+logicWF+logicAPI)["properties"].(map[string]any) + assert.Equal(t, state, got["state"]) + } + + trig := lbDo(t, h, http.MethodGet, logicRoot+logicWF+"/triggers"+logicAPI, "") + require.Equal(t, http.StatusOK, trig.Code) + assert.Contains(t, trig.Body.String(), `"name":"manual"`) + assert.NotContains(t, trig.Body.String(), "sig=") + assert.NotContains(t, trig.Body.String(), "callbackUrl") + + item := lbDo(t, h, http.MethodGet, logicRoot+logicWF+"/triggers/manual"+logicAPI, "") + require.Equal(t, http.StatusOK, item.Code) + assert.NotContains(t, item.Body.String(), "sig=") + + wf := lbDo(t, h, http.MethodGet, logicRoot+logicWF+logicAPI, "") + assert.NotContains(t, wf.Body.String(), "sig=") + + tick := lbDo(t, h, http.MethodPost, logicRoot+logicTicker+"/triggers/tick/run"+logicAPI, "") + require.Less(t, tick.Code, http.StatusMultipleChoices, tick.Body.String()) + + require.Eventually(t, func() bool { + runs := logicGet(t, h, logicRoot+logicTicker+"/runs"+logicAPI) + + return len(logicValues(t, runs)) == 1 + }, 5*time.Second, 10*time.Millisecond) +} + +func TestAzureLogicProxyRunsHideSignedLinks(t *testing.T) { + t.Parallel() + + h := newLogicDashboard(t) + run := logicFirstRun(t, h) + + list := lbDo(t, h, http.MethodGet, logicRoot+logicWF+"/runs"+logicAPI, "") + assert.NotContains(t, list.Body.String(), "sig=") + assert.NotContains(t, list.Body.String(), "/_logic/") + + detail := lbDo(t, h, http.MethodGet, logicRoot+logicWF+"/runs/"+run+logicAPI, "") + require.Equal(t, http.StatusOK, detail.Code) + assert.NotContains(t, detail.Body.String(), "sig=") + assert.NotContains(t, detail.Body.String(), `"uri"`) + assert.Contains(t, detail.Body.String(), `"outputsLink":{"contentSize":`) + + actions := lbDo(t, h, http.MethodGet, logicRoot+logicWF+"/runs/"+run+"/actions"+logicAPI, "") + require.Equal(t, http.StatusOK, actions.Code) + assert.NotContains(t, actions.Body.String(), "sig=") + assert.Contains(t, actions.Body.String(), `"name":"Compose"`) + assert.Contains(t, actions.Body.String(), `"name":"Response"`) + + action := lbDo(t, h, http.MethodGet, logicRoot+logicWF+"/runs/"+run+"/actions/Compose"+logicAPI, "") + require.Equal(t, http.StatusOK, action.Code) + assert.NotContains(t, action.Body.String(), "sig=") + + finished := lbDo(t, h, http.MethodPost, logicRoot+logicWF+"/runs/"+run+"/cancel"+logicAPI, "") + assert.Equal(t, http.StatusBadRequest, finished.Code, finished.Body.String()) + assert.Contains(t, finished.Body.String(), "RunNotRunning") +} + +func TestAzureLogicProxyContentThroughSignedLinks(t *testing.T) { + t.Parallel() + + h := newLogicDashboard(t) + run := logicFirstRun(t, h) + content := logicAccess + "/content" + logicIDs + "/" + run + + out := lbDo(t, h, http.MethodGet, content+"/action/Compose/outputs", "") + require.Equal(t, http.StatusOK, out.Code, out.Body.String()) + assert.JSONEq(t, `"hello bob"`, out.Body.String()) + assert.Equal(t, "no-store", out.Header().Get("Cache-Control")) + + in := lbDo(t, h, http.MethodGet, content+"/action/Compose/inputs", "") + require.Equal(t, http.StatusOK, in.Code, in.Body.String()) + assert.Contains(t, in.Body.String(), "hello bob") + + trig := lbDo(t, h, http.MethodGet, content+"/trigger/outputs", "") + require.Equal(t, http.StatusOK, trig.Code, trig.Body.String()) + assert.Contains(t, trig.Body.String(), `"name":"bob"`) + assert.NotContains(t, trig.Body.String(), "sig=") + + for _, p := range []string{ + content + "/action/Compose/secrets", content + "/action/Missing/outputs", content + "/trigger/nope", + logicAccess + "/content" + logicIDs + "/missing/trigger/outputs", + logicAccess + "/content" + logicIDs + "/" + run + "/action/..%2f..%2fx/outputs", + } { + got := lbDo(t, h, http.MethodGet, p, "") + assert.GreaterOrEqual(t, got.Code, http.StatusBadRequest, p) + assert.NotContains(t, got.Body.String(), "sig=", p) + } +} + +func TestAzureLogicProxyCallbackIsExplicit(t *testing.T) { + t.Parallel() + + h := newLogicDashboard(t) + + rec := lbDo(t, h, http.MethodPost, logicAccess+"/callback"+logicIDs+"/manual", "") + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + + var cb struct { + Value string `json:"value"` + Method string `json:"method"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cb)) + assert.Equal(t, http.MethodPost, cb.Method) + assert.Contains(t, cb.Value, "/_logic/workflows/") + assert.Contains(t, cb.Value, "sig=") + + hostReq := httptest.NewRequest(http.MethodPost, logicAccess+"/callback"+logicIDs+"/manual", http.NoBody) + hostReq.Host = "files.test:18400" + hostRec := httptest.NewRecorder() + require.NoError(t, h.Handler()(echo.NewContext(hostReq, hostRec))) + require.Equal(t, http.StatusOK, hostRec.Code, hostRec.Body.String()) + assert.Contains(t, hostRec.Body.String(), `"value":"https://files.test:`) + assert.NotContains(t, hostRec.Body.String(), "files.test:18400") + + generic := lbDo(t, h, http.MethodPost, logicRoot+logicWF+"/triggers/manual/listCallbackUrl"+logicAPI, "{}") + assert.Equal(t, http.StatusBadRequest, generic.Code) + assert.NotContains(t, generic.Body.String(), "sig=") + + invoke := lbDo(t, h, http.MethodPost, logicAccess+"/invoke"+logicIDs+"/manual", `{"name":"eve"}`) + require.Equal(t, http.StatusOK, invoke.Code) + assert.NotContains(t, invoke.Body.String(), "sig=") + assert.NotContains(t, invoke.Body.String(), "/_logic/") + assert.Equal(t, "no-store", invoke.Header().Get("Cache-Control")) + + missing := lbDo(t, h, http.MethodPost, logicAccess+"/callback"+logicIDs+"/nope", "") + assert.GreaterOrEqual(t, missing.Code, http.StatusBadRequest) + assert.NotContains(t, missing.Body.String(), "sig=") +} + +func TestAzureLogicProxyRejectsOutsideAllowList(t *testing.T) { + t.Parallel() + + h := newLogicDashboard(t) + raw := logicRoot + logicWF + + tests := []struct { + name string + method string + path string + body string + }{ + {"put workflow", http.MethodPut, raw + logicAPI, logicGreeter}, + {"delete workflow", http.MethodDelete, raw + logicAPI, ""}, + {"patch workflow", http.MethodPatch, raw + logicAPI, "{}"}, + {"regenerate key", http.MethodPost, raw + "/regenerateAccessKey" + logicAPI, `{"keyType":"Primary"}`}, + {"callback via generic", http.MethodPost, raw + "/triggers/manual/listCallbackUrl" + logicAPI, "{}"}, + {"version callback", http.MethodPost, raw + "/versions/v1/triggers/manual/listCallbackUrl" + logicAPI, "{}"}, + {"versions", http.MethodGet, raw + "/versions" + logicAPI, ""}, + {"histories", http.MethodGet, raw + "/triggers/manual/histories" + logicAPI, ""}, + {"set state", http.MethodPost, raw + "/triggers/manual/setState" + logicAPI, "{}"}, + {"reset trigger", http.MethodPost, raw + "/triggers/manual/reset" + logicAPI, ""}, + {"validate", http.MethodPost, raw + "/validate" + logicAPI, "{}"}, + {"get run action content", http.MethodGet, raw + "/runs/r/actions/a/contents/ActionOutputs" + logicAPI, ""}, + {"delete run", http.MethodDelete, raw + "/runs/r" + logicAPI, ""}, + {"other provider", http.MethodGet, logicRoot + armBase + "Microsoft.Network/virtualNetworks" + logicAPI, ""}, + {"other type", http.MethodGet, logicRoot + logicNS + "/integrationAccounts" + logicAPI, ""}, + {"subscription list", http.MethodGet, logicRoot + "/subscriptions/" + armSub + + "/providers/Microsoft.Logic/workflows" + logicAPI, ""}, + { + "resource group put", + http.MethodPut, + logicRoot + "/subscriptions/" + armSub + "/resourceGroups/x" + logicAPI, + "{}", + }, + {"traversal", http.MethodGet, logicRoot + armBase + "Microsoft.Logic/workflows/%2e%2e" + logicAPI, ""}, + {"encoded slash", http.MethodGet, logicRoot + armBase + "Microsoft.Logic/workflows/a%2Fb" + logicAPI, ""}, + {"empty", http.MethodGet, logicRoot + "/", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := lbDo(t, h, tt.method, tt.path, tt.body) + assert.GreaterOrEqual(t, got.Code, http.StatusBadRequest, "%s %s", tt.method, tt.path) + assert.NotContains(t, got.Body.String(), "sig=") + assert.NotEqual(t, http.StatusOK, got.Code) + }) + } + + state := logicGet(t, h, raw+logicAPI) + assert.Equal(t, "greeter", state["name"], "rejected calls changed nothing") +} + +func TestAzureLogicProxyAccessRejectsBadInput(t *testing.T) { + t.Parallel() + + h := newLogicDashboard(t) + + tests := []struct { + name string + method string + path string + body string + }{ + {"invoke bad sub", http.MethodPost, logicAccess + "/invoke/a%20b/demo/greeter/manual", "{}"}, + {"invoke bad trigger", http.MethodPost, logicAccess + "/invoke" + logicIDs + "/a$b", "{}"}, + {"invoke not json", http.MethodPost, logicAccess + "/invoke" + logicIDs + "/manual", "not json"}, + { + "invoke huge", http.MethodPost, logicAccess + "/invoke" + logicIDs + "/manual", + `{"a":"` + strings.Repeat("x", 1<<17) + `"}`, + }, + {"callback bad wf", http.MethodPost, logicAccess + "/callback/" + armSub + "/demo/a$b/manual", ""}, + {"content get on callback", http.MethodGet, logicAccess + "/callback" + logicIDs + "/manual", ""}, + {"invoke get", http.MethodGet, logicAccess + "/invoke" + logicIDs + "/manual", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := lbDo(t, h, tt.method, tt.path, tt.body) + assert.GreaterOrEqual( + t, + got.Code, + http.StatusBadRequest, + "%s %s: %s", + tt.method, + tt.path, + got.Body.String(), + ) + assert.NotContains(t, got.Body.String(), "sig=") + }) + } +} + +func TestAzureLogicProxyUnavailableWithoutService(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + for _, p := range []struct{ method, path string }{ + {http.MethodGet, logicRoot + armBase + "Microsoft.Logic/workflows" + logicAPI}, + {http.MethodPost, logicRoot + logicWF + "/enable" + logicAPI}, + {http.MethodPost, logicAccess + "/callback" + logicIDs + "/manual"}, + {http.MethodPost, logicAccess + "/invoke" + logicIDs + "/manual"}, + {http.MethodGet, logicAccess + "/content" + logicIDs + "/r/trigger/outputs"}, + } { + got := lbDo(t, h, p.method, p.path, `{}`) + assert.Equal(t, http.StatusServiceUnavailable, got.Code, "%s %s", p.method, p.path) + } +} diff --git a/dashboard/azurenetwork_body.go b/dashboard/azurenetwork_body.go new file mode 100644 index 0000000000..07a007de8b --- /dev/null +++ b/dashboard/azurenetwork_body.go @@ -0,0 +1,268 @@ +package dashboard + +import ( + "encoding/json" + "math" + "net/http" + "net/netip" + "regexp" + "slices" + "strings" +) + +const ( + netPxMinPriority = 100 + netPxMaxPriority = 4096 + netPxMaxDesc = 140 + netPxMaxList = 50 + netPxVNetPutIdx = 8 + netPxProps = "properties" + netPxNameKey = "name" + netPxProtocolKey = "protocol" +) + +var ( + netPxLocationRE = regexp.MustCompile(`^[a-z0-9]{2,40}$`) + netPxNameRE = regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9._-]{0,78}[A-Za-z0-9_])?$`) + netPxPortRE = regexp.MustCompile(`^(\*|\d{1,5}(-\d{1,5})?)$`) + netPxTagRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.]{0,63}$`) + netPxDescRE = regexp.MustCompile(`^[^<>"\\]*$`) + netPxServiceRE = regexp.MustCompile(`^Microsoft\.[A-Za-z]{2,40}/[A-Za-z]{2,40}$`) + netPxAssocRE = regexp.MustCompile( + `(?i)^/subscriptions/[0-9a-f-]{1,40}/resourceGroups/[A-Za-z0-9._()-]{1,90}/providers/Microsoft\.Network/` + + `(networkSecurityGroups|natGateways)/[A-Za-z0-9._-]{1,80}$`) + netPxDirections = []string{"Inbound", "Outbound"} //nolint:gochecknoglobals // lookup + netPxAccesses = []string{"Allow", "Deny"} //nolint:gochecknoglobals // lookup + netPxProtocols = []string{"Tcp", "Udp", "Icmp", "Esp", "Ah", "*"} //nolint:gochecknoglobals // lookup +) + +type netPxObj = map[string]any + +// netProxyBody rebuilds the body of every write from fixed fields; anything the caller adds is dropped. +func netProxyBody(method string, p []string, in []byte) ([]byte, bool) { + if method == http.MethodGet || method == http.MethodDelete { + return in, true + } + + if method == http.MethodPost { + return []byte("{}"), true + } + + var req netPxObj + if json.Unmarshal(in, &req) != nil { + return nil, false + } + + props, _ := req[netPxProps].(netPxObj) + + var out netPxObj + + switch { + case len(p) == netPxVNetPutIdx && p[netPxRGTypeIdx] == netPxVNets: + out = netVNetBody(req, props) + case len(p) > netPxVNetPutIdx && p[netPxRGTypeIdx] == netPxVNets && p[netPxVNetPutIdx] == netPxSubnets: + out = netSubnetBody(props) + case len(p) > netPxVNetPutIdx && p[netPxRGTypeIdx] == netPxNSGs && p[netPxVNetPutIdx] == netPxRules: + out = netRuleBody(props) + } + + if out == nil { + return nil, false + } + + b, err := json.Marshal(out) + + return b, err == nil +} + +func netVNetBody(req, props netPxObj) netPxObj { + loc, _ := req["location"].(string) + space, _ := props["addressSpace"].(netPxObj) + prefixes := netCIDRList(space["addressPrefixes"]) + + if !netPxLocationRE.MatchString(loc) || len(prefixes) == 0 { + return nil + } + + out := netPxObj{"addressSpace": netPxObj{"addressPrefixes": prefixes}} + + if raw, present := props["subnets"]; present { + list, _ := raw.([]any) + subnets := make([]any, 0, len(list)) + + for _, e := range list { + m, _ := e.(netPxObj) + name, _ := m[netPxNameKey].(string) + prefix, _ := propsOfAny(m)["addressPrefix"].(string) + + if !netPxNameRE.MatchString(name) || !netValidCIDR(prefix) { + return nil + } + + subnets = append(subnets, netPxObj{netPxNameKey: name, netPxProps: netPxObj{"addressPrefix": prefix}}) + } + + out["subnets"] = subnets + } + + return netPxObj{"location": loc, netPxProps: out} +} + +func propsOfAny(m netPxObj) netPxObj { + p, _ := m[netPxProps].(netPxObj) + + return p +} + +func netSubnetBody(props netPxObj) netPxObj { + prefix, _ := props["addressPrefix"].(string) + if !netValidCIDR(prefix) { + return nil + } + + out := netPxObj{"addressPrefix": prefix} + + for _, k := range []string{"networkSecurityGroup", "natGateway"} { + ref, present := props[k].(netPxObj) + if !present { + continue + } + + id, _ := ref["id"].(string) + if !netPxAssocRE.MatchString(id) { + return nil + } + + out[k] = netPxObj{"id": id} + } + + if raw, present := props["delegations"]; present { + list, _ := raw.([]any) + dels := make([]any, 0, len(list)) + + for _, e := range list { + m, _ := e.(netPxObj) + name, _ := m[netPxNameKey].(string) + svc, _ := propsOfAny(m)["serviceName"].(string) + + if !netPxNameRE.MatchString(name) || !netPxServiceRE.MatchString(svc) { + return nil + } + + dels = append(dels, netPxObj{netPxNameKey: name, netPxProps: netPxObj{"serviceName": svc}}) + } + + out["delegations"] = dels + } + + return netPxObj{netPxProps: out} +} + +func netRuleBody(props netPxObj) netPxObj { + prio, ok := props["priority"].(float64) + if !ok || prio != math.Trunc(prio) || prio < netPxMinPriority || prio > netPxMaxPriority { + return nil + } + + out := netPxObj{"priority": int(prio)} + + for k, allowed := range map[string][]string{ + "direction": netPxDirections, "access": netPxAccesses, netPxProtocolKey: netPxProtocols, + } { + v, _ := props[k].(string) + if !slices.Contains(allowed, v) { + return nil + } + + out[k] = v + } + + if d, present := props["description"].(string); present { + if len(d) > netPxMaxDesc || !netPxDescRE.MatchString(d) { + return nil + } + + out["description"] = d + } + + for _, f := range []struct { + valid func(string) bool + one, many string + }{ + {netPxPortRE.MatchString, "sourcePortRange", "sourcePortRanges"}, + {netPxPortRE.MatchString, "destinationPortRange", "destinationPortRanges"}, + {netValidPrefix, "sourceAddressPrefix", "sourceAddressPrefixes"}, + {netValidPrefix, "destinationAddressPrefix", "destinationAddressPrefixes"}, + } { + if !netCopyValues(out, props, f.one, f.many, f.valid) { + return nil + } + } + + return netPxObj{netPxProps: out} +} + +// netCopyValues copies exactly one of the singular or plural fields once every value validates. +func netCopyValues(out, props netPxObj, one, many string, valid func(string) bool) bool { + if v, ok := props[one].(string); ok && props[many] == nil { + out[one] = v + + return valid(v) + } + + list, ok := props[many].([]any) + if !ok || len(list) == 0 || len(list) > netPxMaxList || props[one] != nil { + return false + } + + vals := make([]string, 0, len(list)) + + for _, e := range list { + s, isStr := e.(string) + if !isStr || !valid(s) { + return false + } + + vals = append(vals, s) + } + + out[many] = vals + + return true +} + +func netValidPrefix(s string) bool { + return s == "*" || netValidCIDR(s) || netPxTagRE.MatchString(s) || netValidAddr(s) +} + +func netValidAddr(s string) bool { + _, err := netip.ParseAddr(s) + + return err == nil +} + +func netValidCIDR(s string) bool { + p, err := netip.ParsePrefix(s) + + return err == nil && !strings.Contains(s, "%") && p.IsValid() +} + +func netCIDRList(v any) []string { + list, _ := v.([]any) + if len(list) == 0 || len(list) > netPxMaxList { + return nil + } + + out := make([]string, 0, len(list)) + + for _, e := range list { + s, ok := e.(string) + if !ok || !netValidCIDR(s) { + return nil + } + + out = append(out, s) + } + + return out +} diff --git a/dashboard/azurenetwork_proxy.go b/dashboard/azurenetwork_proxy.go new file mode 100644 index 0000000000..e8bf19addf --- /dev/null +++ b/dashboard/azurenetwork_proxy.go @@ -0,0 +1,139 @@ +package dashboard + +import ( + "net/http" + "net/url" + "slices" + "strings" + + "github.com/labstack/echo/v5" +) + +const ( + azureNetworkPrefix = "/dashboard/api/v1/azure/network" + netPxNamespace = "microsoft.network" + netPxAPIVersion = "2024-05-01" + netPxVNets = "virtualnetworks" + netPxSubnets = "subnets" + netPxNSGs = "networksecuritygroups" + netPxRules = "securityrules" + netPxLBs = "loadbalancers" + netPxAGWs = "applicationgateways" + netPxPolicies = "firewallpolicies" + netPxRCGs = "rulecollectiongroups" + netPxLocations = "locations" + netPxRGList = 7 + netPxRGTypeIdx = 6 + netPxOpSegs = 8 + netPxOpKindIdx = 6 + netPxOpLocIdx = 4 + netPxActionOnly = 1 + netPxChildItem = 2 +) + +var ( + netPxReadTypes = []string{ //nolint:gochecknoglobals // immutable allow-list + netPxVNets, netPxNSGs, "publicipaddresses", "networkinterfaces", "natgateways", netPxLBs, + "privateendpoints", "privatednszones", netPxAGWs, "azurefirewalls", netPxPolicies, "ipgroups", + "applicationsecuritygroups", + } + netPxAGWActions = []string{aciPxStart, aciPxStop, "backendhealth"} //nolint:gochecknoglobals // lookup + netPxOpKinds = []string{"operations", "operationresults"} //nolint:gochecknoglobals // lookup +) + +// registerAzureNetworkRoutes exposes Microsoft.Network resources by dispatching in-process to the ARM handler. +// Reads cover the whole network surface; writes are limited to virtual networks, subnets, security rules, load +// balancer deletion and application gateway start/stop/backend health. Every write forwards a body rebuilt from +// fixed fields, the api-version is pinned, and async operation headers are relayed as proxy-relative URLs. +func (h *DashboardHandler) registerAzureNetworkRoutes() { + h.registerARMProxy(armProxy{ + name: "Azure Networking", prefix: azureNetworkPrefix, + queryKeys: []string{}, version: netPxAPIVersion, relayOps: true, + methods: []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete}, + allowed: netProxyAllowed, + body: netProxyBody, + }) +} + +func netProxyAllowed(method string, p []string) bool { + switch { + case len(p) == netPxOpSegs && p[2] == armPxProvSeg: + return netProxyOperation(method, p) + case len(p) == netPxRGList: + return netProxyRGList(method, p) + } + + typ, name, rest, ok := armScope(p, netPxNamespace) + if !ok || !slices.Contains(netPxReadTypes, typ) { + return false + } + + if name == "" { + return method == http.MethodGet + } + + switch len(rest) { + case 0: + return netProxyItemOp(method, typ) + case netPxActionOnly: + return (method == http.MethodGet && typ == netPxPolicies && rest[0] == netPxRCGs) || + (method == http.MethodPost && typ == netPxAGWs && slices.Contains(netPxAGWActions, rest[0])) + case netPxChildItem: + return netProxyChildOp(method, typ, rest[0]) + } + + return false +} + +func netProxyOperation(method string, p []string) bool { + return method == http.MethodGet && p[0] == armPxSubsSeg && p[2] == armPxProvSeg && p[3] == netPxNamespace && + p[netPxOpLocIdx] == netPxLocations && slices.Contains(netPxOpKinds, p[netPxOpKindIdx]) +} + +func netProxyRGList(method string, p []string) bool { + return method == http.MethodGet && p[0] == armPxSubsSeg && p[2] == armPxRGsSeg && p[4] == armPxProvSeg && + p[5] == netPxNamespace && slices.Contains(netPxReadTypes, p[netPxRGTypeIdx]) +} + +func netProxyItemOp(method, typ string) bool { + switch method { + case http.MethodGet: + return true + case http.MethodPut: + return typ == netPxVNets + case http.MethodDelete: + return typ == netPxVNets || typ == netPxLBs + } + + return false +} + +func netProxyChildOp(method, typ, kind string) bool { + switch { + case typ == netPxVNets && kind == netPxSubnets, typ == netPxNSGs && kind == netPxRules: + return method == http.MethodGet || method == http.MethodPut || method == http.MethodDelete + case typ == netPxPolicies && kind == netPxRCGs: + return method == http.MethodGet + } + + return false +} + +// netRelayOps exposes the async operation URLs of a response as paths below the proxy prefix. +func netRelayOps(c *echo.Context, hdr http.Header, prefix string) { + for _, k := range []string{"Azure-Asyncoperation", "Location"} { + raw := hdr.Get(k) + if raw == "" { + continue + } + + u, err := url.Parse(raw) + if err != nil || !strings.HasPrefix(strings.ToLower(u.Path), "/subscriptions/") { + continue + } + + u.Scheme, u.Host = "", "" + u.Path = prefix + u.Path + c.Response().Header().Set(k, u.String()) + } +} diff --git a/dashboard/azurenetwork_proxy_test.go b/dashboard/azurenetwork_proxy_test.go new file mode 100644 index 0000000000..19ea5fb787 --- /dev/null +++ b/dashboard/azurenetwork_proxy_test.go @@ -0,0 +1,234 @@ +package dashboard_test + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" +) + +const ( + netAPI = "?api-version=2024-05-01" + netRoot = "/dashboard/api/v1/azure/network" + netNS = armBase + "Microsoft.Network" + netVNetID = netNS + "/virtualNetworks/hub" + netNSGID = netNS + "/networkSecurityGroups/web" + netPolID = netNS + "/firewallPolicies/pol" + netRGList = "/subscriptions/" + armSub + "/resourceGroups/demo/providers/Microsoft.Network/" + netOps = netRoot + "/subscriptions/" + armSub + "/providers/Microsoft.Network/locations/eastus/" + netAGWs = netRoot + netNS + "/applicationGateways/g" + + netBadVNetPrefix = `{"location":"eastus","properties":{"addressSpace":{"addressPrefixes":["nope"]}}}` + netBadVNetRegion = `{"location":"East US","properties":{"addressSpace":{"addressPrefixes":["10.0.0.0/8"]}}}` + netBadNAT = `{"properties":{"addressPrefix":"10.0.5.0/24","natGateway":{"id":"/subscriptions/x/evil"}}}` + netBadDelegation = `{"properties":{"addressPrefix":"10.0.5.0/24",` + + `"delegations":[{"name":"d","properties":{"serviceName":"nope"}}]}}` +) + +func newNetworkDashboard(t *testing.T) *dashboard.DashboardHandler { + t.Helper() + + rule := `{"name":"ssh","properties":{"priority":100,"direction":"Inbound","access":"Allow","protocol":"Tcp",` + + `"sourcePortRange":"*","destinationPortRange":"22","sourceAddressPrefix":"*","destinationAddressPrefix":"*"}}` + rcg := `{"properties":{"priority":200,"ruleCollections":[{"ruleCollectionType":"FirewallPolicyFilterRuleCollection",` + + `"name":"allow","priority":100,"action":{"type":"Allow"},"rules":[{"ruleType":"NetworkRule","name":"r1",` + + `"ipProtocols":["TCP"],"sourceAddresses":["10.0.0.0/24"],"destinationAddresses":["*"],` + + `"destinationPorts":["443"]}]}]}}` + + return newARMDashboard(t, []struct{ path, body string }{ + {"/subscriptions/" + armSub + "/resourceGroups/demo?api-version=2021-04-01", `{"location":"eastus"}`}, + {netNSGID + netAPI, `{"location":"eastus","properties":{"securityRules":[` + rule + `]}}`}, + {netVNetID + netAPI, `{"location":"eastus","properties":{"addressSpace":{"addressPrefixes":["10.0.0.0/16"]},` + + `"subnets":[{"name":"default","properties":{"addressPrefix":"10.0.0.0/24"}}]}}`}, + {netPolID + netAPI, `{"location":"eastus","properties":{"sku":{"tier":"Standard"}}}`}, + {netPolID + "/ruleCollectionGroups/grp" + netAPI, rcg}, + }) +} + +func TestAzureNetworkProxy(t *testing.T) { + t.Parallel() + + px := netRoot + netVNetID + nsg := netRoot + netNSGID + rules := nsg + "/securityRules/" + rule := func(extra string) string { + return `{"properties":{"priority":300,"direction":"Inbound","access":"Deny","protocol":"Udp",` + + `"sourcePortRange":"*","destinationPortRanges":["53","1000-2000"],"sourceAddressPrefix":"Internet",` + + `"destinationAddressPrefix":"10.0.0.0/24"` + extra + `}}` + } + badRule := func(old, repl string) string { return strings.Replace(rule(""), old, repl, 1) } + sub := netRoot + "/subscriptions/" + armSub + "/providers/Microsoft.Network/" + + tests := []struct { + name string + steps []gcpStep + }{ + {"read", []gcpStep{ + {http.MethodGet, netRoot + netRGList + "virtualNetworks" + netAPI, "", `"10.0.0.0/16"`, 200}, + {http.MethodGet, sub + "networkSecurityGroups" + netAPI, "", `"ssh"`, 200}, + {http.MethodGet, px + netAPI, "", `"name":"default"`, 200}, + {http.MethodGet, px + "/subnets/default" + netAPI, "", `"10.0.0.0/24"`, 200}, + {http.MethodGet, nsg + "/securityRules/ssh" + netAPI, "", `"destinationPortRange":"22"`, 200}, + {http.MethodGet, netRoot + netPolID + "/ruleCollectionGroups" + netAPI, "", `"grp"`, 200}, + {http.MethodGet, netRoot + netPolID + "/ruleCollectionGroups/grp" + netAPI, "", `"r1"`, 200}, + {http.MethodGet, netRoot + netRGList + "applicationGateways" + netAPI, "", `"value":[]`, 200}, + {http.MethodGet, netRoot + netRGList + "azureFirewalls" + netAPI, "", `"value":[]`, 200}, + {http.MethodGet, netRoot + netRGList + "ipGroups" + netAPI, "", `"value":[]`, 200}, + {http.MethodGet, netRoot + netRGList + "loadBalancers" + netAPI, "", `"value":[]`, 200}, + }}, + {"pinned version", []gcpStep{ + {http.MethodGet, px + "?api-version=1999-01-01", "", `"name":"default"`, 200}, + {http.MethodGet, px, "", `"name":"default"`, 200}, + }}, + {"writes", []gcpStep{ + { + http.MethodPut, netRoot + netNS + "/virtualNetworks/spoke" + netAPI, + `{"location":"westus","tags":{"x":"y"},"properties":{"dhcpOptions":{"dnsServers":["8.8.8.8"]},` + + `"addressSpace":{"addressPrefixes":["10.1.0.0/16"],"evil":1},` + + `"subnets":[{"name":"app","properties":{"addressPrefix":"10.1.0.0/24","serviceEndpoints":[{}]}}]}}`, + `"10.1.0.0/16"`, 201, + }, + {http.MethodGet, netRoot + netNS + "/virtualNetworks/spoke" + netAPI, "", `"name":"app"`, 200}, + { + http.MethodPut, netRoot + netNS + "/virtualNetworks/spoke/subnets/db" + netAPI, + `{"properties":{"addressPrefix":"10.1.1.0/24","networkSecurityGroup":{"id":"` + netNSGID + `"},` + + `"delegations":[{"name":"d","properties":{"serviceName":"Microsoft.Web/serverFarms"}}]}}`, + `Microsoft.Web/serverFarms`, 201, + }, + { + http.MethodDelete, + netRoot + netNS + "/virtualNetworks/spoke/subnets/db" + netAPI, + "", + "", + http.StatusAccepted, + }, + {http.MethodPut, rules + "https" + netAPI, rule(`,"description":"dns"`), `"priority":300`, 201}, + {http.MethodGet, nsg + netAPI, "", `"https"`, 200}, + {http.MethodDelete, rules + "https" + netAPI, "", "", http.StatusAccepted}, + {http.MethodDelete, netRoot + netNS + "/virtualNetworks/spoke" + netAPI, "", "", http.StatusAccepted}, + {http.MethodGet, netRoot + netNS + "/virtualNetworks/spoke" + netAPI, "", "NotFound", http.StatusNotFound}, + }}, + {"body validation", []gcpStep{ + {http.MethodPut, rules + "r" + netAPI, badRule(`"priority":300`, `"priority":99`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"priority":300`, `"priority":4097`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"priority":300`, `"priority":300.5`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"Inbound"`, `"Sideways"`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"Deny"`, `"Maybe"`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"Udp"`, `"Gre"`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"53"`, `"http"`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"Internet"`, `"bad prefix"`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, `{"properties":{"priority":300}}`, "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, `[]`, "invalid", 400}, + {http.MethodPut, px + netAPI, netBadVNetPrefix, "invalid", 400}, + {http.MethodPut, px + netAPI, netBadVNetRegion, "invalid", 400}, + {http.MethodPut, px + "/subnets/x" + netAPI, netBadNAT, "invalid", 400}, + {http.MethodPut, px + "/subnets/x" + netAPI, netBadDelegation, "invalid", 400}, + }}, + {"rejects", []gcpStep{ + {http.MethodDelete, nsg + netAPI, "", "unsupported", 400}, + {http.MethodPut, nsg + netAPI, `{}`, "unsupported", 400}, + {http.MethodPost, px + netAPI, `{}`, "unsupported", 400}, + {http.MethodPut, netRoot + netNS + "/publicIPAddresses/p" + netAPI, `{}`, "unsupported", 400}, + {http.MethodDelete, netAGWs + netAPI, "", "unsupported", 400}, + {http.MethodPost, netAGWs + "/restart" + netAPI, "", "unsupported", 400}, + {http.MethodPut, netRoot + netPolID + "/ruleCollectionGroups/grp" + netAPI, `{}`, "unsupported", 400}, + {http.MethodDelete, netRoot + netPolID + netAPI, "", "unsupported", 400}, + { + http.MethodGet, + netRoot + armBase + "Microsoft.Compute/virtualMachines/x" + netAPI, + "", + "unsupported", + 400, + }, + {http.MethodGet, netRoot + netNS + "/routeTables" + netAPI, "", "unsupported", 400}, + {http.MethodGet, px + "/../x" + netAPI, "", "unsupported", 400}, + {http.MethodGet, netOps + "foo/1", "", "unsupported", 400}, + {http.MethodPost, netOps + "operations/1", "", "unsupported", 400}, + }}, + {"appgw actions reach the RP", []gcpStep{ + {http.MethodPost, netAGWs + "/stop" + netAPI, "", "NotFound", http.StatusNotFound}, + {http.MethodPost, netAGWs + "/backendHealth" + netAPI, "", "NotFound", http.StatusNotFound}, + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + runGCPSteps(t, newNetworkDashboard(t), tt.steps) + }) + } + + t.Run("async operations are relayed under the proxy", func(t *testing.T) { + t.Parallel() + + h := newNetworkDashboard(t) + do := func(method, path, body string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + + return rec + } + + del := do(http.MethodDelete, px+netAPI, "") + require.Equal(t, http.StatusAccepted, del.Code) + + op := del.Header().Get("Azure-Asyncoperation") + require.True( + t, + strings.HasPrefix(op, netRoot+"/subscriptions/"+armSub+"/providers/Microsoft.Network/locations/"), + op, + ) + assert.NotContains(t, op, "https://") + + status := do(http.MethodGet, op, "") + assert.Equal(t, http.StatusOK, status.Code) + assert.Contains(t, status.Body.String(), "Succeeded") + + loc := del.Header().Get("Location") + require.True(t, strings.HasPrefix(loc, netRoot+"/subscriptions/"), loc) + assert.Equal(t, http.StatusNoContent, do(http.MethodGet, loc, "").Code) + }) + + t.Run("extra fields are dropped", func(t *testing.T) { + t.Parallel() + + h := newNetworkDashboard(t) + do := func(method, path, body string) string { + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + require.Less(t, rec.Code, http.StatusBadRequest, rec.Body.String()) + + return rec.Body.String() + } + + vnet := netRoot + netNS + "/virtualNetworks/spoke" + netAPI + do( + http.MethodPut, + vnet, + `{"location":"westus","tags":{"marker":"y"},"properties":{"dhcpOptions":{"dnsServers":["8.8.8.8"]},`+ + `"addressSpace":{"addressPrefixes":["10.1.0.0/16"]}}}`, + ) + + got := do(http.MethodGet, vnet, "") + assert.NotContains(t, got, "marker") + assert.NotContains(t, got, "8.8.8.8") + assert.Contains(t, got, "10.1.0.0/16") + }) + + t.Run("unserved", func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, sub+"virtualNetworks"+netAPI, nil) + require.NoError(t, dashboard.NewHandler(dashboard.Config{}).Handler()(echo.NewContext(req, rec))) + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + }) +} diff --git a/dashboard/azureredis_proxy.go b/dashboard/azureredis_proxy.go new file mode 100644 index 0000000000..822d4e308d --- /dev/null +++ b/dashboard/azureredis_proxy.go @@ -0,0 +1,211 @@ +package dashboard + +import ( + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/labstack/echo/v5" + + redisbackend "github.com/blackbirdworks/gopherstack/services/azureredis" +) + +const ( + azureRedisProxyPrefix = "/dashboard/api/v1/azure/redis" + redisMaxPattern = 256 + redisMaxKeyName = 1024 + redisMaxCacheID = 512 + redisMask = "••••••••" + redisKeysField = "keys" + redisKeyField = "key" + redisTypeField = "type" + redisTruncatedField = "truncated" +) + +type redisCacheJSON struct { + ID string `json:"id"` + Name string `json:"name"` + SKU string `json:"sku"` + MinTLS string `json:"minTlsVersion"` + Host string `json:"host"` + AccessKeysMasked string `json:"accessKeys"` + Port int `json:"port"` + SSLPort int `json:"sslPort"` + Keys int `json:"keyCount"` + NonSSLEnabled bool `json:"enableNonSslPort"` + Persistence bool `json:"persistence"` + AuthNotRequired bool `json:"authNotRequired"` +} + +type redisKeyJSON struct { + Key string `json:"key"` + Type string `json:"type"` + TTLSeconds int64 `json:"ttlSeconds"` +} + +type redisRevealJSON struct { + Cache string `json:"cache"` +} + +// registerAzureRedisRoutes serves the Cache for Redis page from the backend in-process. The keys browser is +// read-only; access keys leave the server only through the reveal POST. +func (h *DashboardHandler) registerAzureRedisRoutes() { + api := &azureRedisAPI{} + if h.config.AzureRedisOps != nil { + api.b = h.config.AzureRedisOps.Backend + } + + r := h.SubRouter + p := azureRedisProxyPrefix + r.Add(http.MethodGet, p+"/caches", api.guard(api.caches)) + r.Add(http.MethodGet, p+"/keys", api.guard(api.scan)) + r.Add(http.MethodGet, p+"/value", api.guard(api.value)) + r.Add(http.MethodPost, p+"/access-keys/reveal", api.guard(api.reveal)) +} + +type azureRedisAPI struct { + b *redisbackend.InMemoryBackend +} + +func (a *azureRedisAPI) guard(next echo.HandlerFunc) echo.HandlerFunc { + return func(c *echo.Context) error { + if a.b == nil { + return cloudUnavailable(c, "Azure Cache for Redis") + } + + return next(c) + } +} + +func (a *azureRedisAPI) fail(c *echo.Context, err error) error { + if errors.Is(err, redisbackend.ErrNotFound) { + return cloudJSONFail(c, http.StatusNotFound, cloudStatusNotFnd, err.Error()) + } + + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, err.Error()) +} + +func (a *azureRedisAPI) caches(c *echo.Context) error { + infos := a.b.Caches() + out := make([]redisCacheJSON, 0, len(infos)) + + for _, i := range infos { + name := i.ID + if idx := strings.LastIndex(name, "/"); idx >= 0 { + name = name[idx+1:] + } + + out = append(out, redisCacheJSON{ + ID: i.ID, Name: name, SKU: i.Spec.SKU, MinTLS: i.Spec.MinTLS, Host: i.Endpoint.Host, + Port: i.Endpoint.Port, SSLPort: i.Endpoint.SSLPort, Keys: i.Keys, NonSSLEnabled: i.Spec.EnableNonSSLPort, + Persistence: i.Spec.Persistence, AuthNotRequired: i.Spec.AuthNotRequired, AccessKeysMasked: redisMask, + }) + } + + return c.JSON(http.StatusOK, map[string]any{"caches": out}) +} + +func redisCacheParam(c *echo.Context) (string, bool) { + id := c.QueryParam("cache") + if id == "" || len(id) > redisMaxCacheID { + _ = cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "cache is required") + + return "", false + } + + return id, true +} + +func redisDBParam(c *echo.Context) int { + db, err := strconv.Atoi(c.QueryParam("db")) + if err != nil { + return 0 + } + + return db +} + +func (a *azureRedisAPI) scan(c *echo.Context) error { + id, ok := redisCacheParam(c) + if !ok { + return nil + } + + match, cursor := c.QueryParam("match"), c.QueryParam("cursor") + if len(match) > redisMaxPattern || len(cursor) > redisMaxKeyName { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "pattern or cursor is too long") + } + + count, _ := strconv.Atoi(c.QueryParam("count")) + + entries, next, err := a.b.Scan(id, redisDBParam(c), match, cursor, count) + if err != nil { + return a.fail(c, err) + } + + keys := make([]redisKeyJSON, 0, len(entries)) + for _, e := range entries { + keys = append(keys, redisKeyJSON{Key: e.Key, Type: e.Type, TTLSeconds: redisTTL(e.TTL)}) + } + + return c.JSON(http.StatusOK, map[string]any{redisKeysField: keys, "next": next}) +} + +// redisTTL reports -1 for a key without expiry, as Redis does. +func redisTTL(d time.Duration) int64 { + if d <= 0 { + return -1 + } + + return int64((d + time.Second - 1) / time.Second) +} + +func (a *azureRedisAPI) value(c *echo.Context) error { + id, ok := redisCacheParam(c) + if !ok { + return nil + } + + key := c.QueryParam("key") + if key == "" || len(key) > redisMaxKeyName { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "key is required") + } + + v, err := a.b.Value(id, redisDBParam(c), key) + if err != nil { + return a.fail(c, err) + } + + elems := v.Elements + if elems == nil { + elems = []string{} + } + + return c.JSON(http.StatusOK, map[string]any{ + redisKeyField: v.Key, redisTypeField: v.Type, "text": v.Text, "elements": elems, "total": v.Total, + redisTruncatedField: v.Truncated, "ttlSeconds": redisTTL(v.TTL), + }) +} + +// reveal is the only route that sends the access keys out. +func (a *azureRedisAPI) reveal(c *echo.Context) error { + var in redisRevealJSON + if !cloudJSONDecode(c, &in) { + return nil + } + + if in.Cache == "" || len(in.Cache) > redisMaxCacheID { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "cache is required") + } + + keys, err := a.b.Keys(in.Cache) + if err != nil { + return a.fail(c, err) + } + + c.Response().Header().Set("Cache-Control", "no-store") + + return c.JSON(http.StatusOK, map[string]string{"primary": keys.Primary, "secondary": keys.Secondary}) +} diff --git a/dashboard/azureredis_proxy_test.go b/dashboard/azureredis_proxy_test.go new file mode 100644 index 0000000000..a1cadab35d --- /dev/null +++ b/dashboard/azureredis_proxy_test.go @@ -0,0 +1,249 @@ +package dashboard_test + +import ( + "encoding/json" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "testing" + + goredis "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/azureredis" +) + +const ( + redisPxBase = "/dashboard/api/v1/azure/redis" + redisCacheID = "/subscriptions/s/resourcegroups/rg/providers/microsoft.cache/redis/cache1" +) + +type redisEnv struct { + h *dashboard.DashboardHandler + keys azureredis.Keys +} + +func newRedisEnv(t *testing.T) *redisEnv { + t.Helper() + + b := azureredis.NewInMemoryBackend(azureredis.DefaultSettings()) + t.Cleanup(b.Close) + + ep, err := b.Ensure(redisCacheID, azureredis.Spec{ + SKU: azureredis.SKUStandard, MinTLS: "1.2", EnableNonSSLPort: true, + }) + require.NoError(t, err) + + keys, err := b.Keys(redisCacheID) + require.NoError(t, err) + + c := goredis.NewClient(&goredis.Options{ + Addr: net.JoinHostPort(ep.Host, strconv.Itoa(ep.Port)), Password: keys.Primary, + }) + t.Cleanup(func() { _ = c.Close() }) + + ctx := t.Context() + require.NoError(t, c.Set(ctx, "user:1", "alice", 0).Err()) + require.NoError(t, c.Set(ctx, "user:2", "bob", 0).Err()) + require.NoError(t, c.Set(ctx, "session:9", strings.Repeat("x", 6000), 0).Err()) + require.NoError(t, c.RPush(ctx, "queue", "a", "b", "c").Err()) + require.NoError(t, c.HSet(ctx, "profile", "name", "alice", "city", "x").Err()) + require.NoError(t, c.ZAdd(ctx, "board", goredis.Z{Score: 2, Member: "m"}).Err()) + require.NoError(t, c.Set(ctx, "ttl:key", "v", 0).Err()) + require.NoError(t, c.Expire(ctx, "ttl:key", 3600*1e9).Err()) + + c3 := goredis.NewClient(&goredis.Options{ + Addr: net.JoinHostPort(ep.Host, strconv.Itoa(ep.Port)), Password: keys.Primary, DB: 3, + }) + t.Cleanup(func() { _ = c3.Close() }) + require.NoError(t, c3.Set(ctx, "other", "db3", 0).Err()) + + return &redisEnv{ + h: dashboard.NewHandler(dashboard.Config{AzureRedisOps: azureredis.NewHandler(b)}), + keys: keys, + } +} + +func redisURL(path string, q url.Values) string { + if q == nil { + q = url.Values{} + } + + if q.Get("cache") == "" { + q.Set("cache", redisCacheID) + } + + return redisPxBase + path + "?" + q.Encode() +} + +func TestAzureRedisProxy(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + wantBody []string + notBody []string + wantCode int + }{ + { + name: "caches", path: redisPxBase + "/caches", wantCode: 200, + wantBody: []string{`"name":"cache1"`, `"sku":"Standard"`, `"enableNonSslPort":true`}, + }, + { + name: "scan all", path: redisURL("/keys", nil), wantCode: 200, + wantBody: []string{`"user:1"`, `"queue"`, `"type":"list"`}, + }, + { + name: "scan pattern", path: redisURL("/keys", url.Values{"match": {"user:*"}}), wantCode: 200, + wantBody: []string{`"user:1"`, `"user:2"`}, notBody: []string{`"queue"`}, + }, + { + name: "scan db3", path: redisURL("/keys", url.Values{"db": {"3"}}), wantCode: 200, + wantBody: []string{`"other"`}, notBody: []string{`"user:1"`}, + }, + { + name: "scan pages", path: redisURL("/keys", url.Values{"count": {"2"}}), wantCode: 200, + wantBody: []string{`"next":"`}, + }, + { + name: "ttl", path: redisURL("/keys", url.Values{"match": {"ttl:*"}}), wantCode: 200, + wantBody: []string{`"ttlSeconds":3600`}, + }, + { + name: "no expiry", path: redisURL("/keys", url.Values{"match": {"user:1"}}), wantCode: 200, + wantBody: []string{`"ttlSeconds":-1`}, + }, + { + name: "string value", path: redisURL("/value", url.Values{"key": {"user:1"}}), wantCode: 200, + wantBody: []string{`"text":"alice"`, `"type":"string"`}, + }, + { + name: "capped string", path: redisURL("/value", url.Values{"key": {"session:9"}}), wantCode: 200, + wantBody: []string{`"truncated":true`, `"total":6000`}, + }, + { + name: "list value", path: redisURL("/value", url.Values{"key": {"queue"}}), wantCode: 200, + wantBody: []string{`"elements":["a","b","c"]`}, + }, + { + name: "hash value", path: redisURL("/value", url.Values{"key": {"profile"}}), wantCode: 200, + wantBody: []string{`city = x`, `name = alice`}, + }, + { + name: "zset value", path: redisURL("/value", url.Values{"key": {"board"}}), wantCode: 200, + wantBody: []string{`m (score 2)`}, + }, + { + name: "missing key", path: redisURL("/value", url.Values{"key": {"nope"}}), wantCode: 404, + wantBody: []string{"NOT_FOUND"}, + }, + { + name: "bad database", path: redisURL("/keys", url.Values{"db": {"16"}}), wantCode: 400, + wantBody: []string{"database must be"}, + }, + { + name: "unknown cache", path: redisURL("/keys", url.Values{"cache": {"/x/y"}}), wantCode: 404, + wantBody: []string{"NOT_FOUND"}, + }, + { + name: "missing cache", path: redisPxBase + "/keys", wantCode: 400, + wantBody: []string{"cache is required"}, + }, + { + name: "long pattern", + path: redisURL("/keys", url.Values{"match": {strings.Repeat("a", 300)}}), + wantCode: 400, + wantBody: []string{"too long"}, + }, + { + name: "missing key name", path: redisURL("/value", nil), wantCode: 400, + wantBody: []string{"key is required"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + env := newRedisEnv(t) + rec := doProxy(t, env.h, http.MethodGet, tt.path, "") + + assert.Equal(t, tt.wantCode, rec.Code, rec.Body.String()) + + for _, w := range tt.wantBody { + assert.Contains(t, rec.Body.String(), w) + } + + for _, w := range tt.notBody { + assert.NotContains(t, rec.Body.String(), w) + } + }) + } +} + +func TestAzureRedisKeysNeverInGets(t *testing.T) { + t.Parallel() + + env := newRedisEnv(t) + + paths := []string{ + redisPxBase + "/caches", redisURL("/keys", nil), redisURL("/value", url.Values{"key": {"user:1"}}), + } + + for _, path := range paths { + rec := doProxy(t, env.h, http.MethodGet, path, "") + assert.NotContains(t, rec.Body.String(), env.keys.Primary, path) + assert.NotContains(t, rec.Body.String(), env.keys.Secondary, path) + } + + rec := doProxy(t, env.h, http.MethodGet, redisPxBase+"/access-keys/reveal", "") + assert.NotContains(t, rec.Body.String(), env.keys.Primary) +} + +func TestAzureRedisReveal(t *testing.T) { + t.Parallel() + + env := newRedisEnv(t) + + rec := doProxy(t, env.h, http.MethodPost, redisPxBase+"/access-keys/reveal", `{"cache":"`+redisCacheID+`"}`) + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + + var got map[string]string + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, env.keys.Primary, got["primary"]) + assert.Equal(t, env.keys.Secondary, got["secondary"]) + + rec = doProxy(t, env.h, http.MethodPost, redisPxBase+"/access-keys/reveal", `{"cache":"/nope"}`) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = doProxy(t, env.h, http.MethodPost, redisPxBase+"/access-keys/reveal", `{}`) + assert.Equal(t, http.StatusBadRequest, rec.Code) +} + +func TestAzureRedisIsReadOnly(t *testing.T) { + t.Parallel() + + env := newRedisEnv(t) + + for _, m := range []string{http.MethodPut, http.MethodDelete, http.MethodPost} { + rec := doProxy(t, env.h, m, redisURL("/keys", nil), `{"key":"x"}`) + assert.NotEqual(t, http.StatusOK, rec.Code, m) + } + + rec := doProxy(t, env.h, http.MethodGet, redisURL("/keys", url.Values{"match": {"user:*"}}), "") + assert.Contains(t, rec.Body.String(), `"user:1"`) +} + +func TestAzureRedisAbsent(t *testing.T) { + t.Parallel() + + rec := doProxy(t, dashboard.NewHandler(dashboard.Config{}), http.MethodGet, redisPxBase+"/caches", "") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Contains(t, rec.Body.String(), "UNAVAILABLE") +} diff --git a/dashboard/azuresearch_proxy.go b/dashboard/azuresearch_proxy.go new file mode 100644 index 0000000000..32cb61dc44 --- /dev/null +++ b/dashboard/azuresearch_proxy.go @@ -0,0 +1,623 @@ +package dashboard + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "slices" + "strings" + + "github.com/labstack/echo/v5" +) + +const ( + azureSearchPrefix = "/dashboard/api/v1/azure/search" + azureSearchKeysPrefix = "/dashboard/api/v1/azure/searchkeys" + azureSearchDataPrefix = "/dashboard/api/v1/azure/searchdata" + + srchPxNamespace = "microsoft.search" + srchPxARMNS = "Microsoft.Search" + srchPxServices = "searchservices" + srchPxARMServices = "searchServices" + srchPxARMVersion = "2025-05-01" + srchPxDataVersion = "2024-07-01" + srchPxHostSuffix = ".search.windows.net" + srchPxRGList = 7 + srchPxBasePath = 9 + srchPxMaxBody = 1 << 16 + srchPxMaxReply = 8 << 20 + srchPxMaxText = 4096 + srchPxMaxList = 32 + srchPxMaxTop = 1000 + srchPxMaxSkip = 100000 + srchPxMaskTail = 4 + srchPxFPBytes = 4 + srchPxNone = "" + srchPxMask = "••••••••" + srchPxDocsSearch = "docs/search.post.search" + srchPxKeyQuery = "query" + srchPxKeyFP = "fp" +) + +var ( + srchPxSubRe = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`) + srchPxNameRe = regexp.MustCompile(`^[A-Za-z0-9._()-]{1,128}$`) + srchPxItemRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,128}$`) + srchPxKeyRe = regexp.MustCompile(`^[A-Za-z0-9 _-]{1,64}$`) + srchPxFPRe = regexp.MustCompile(`^[0-9a-f]{8}$`) +) + +// srchPxSecretFields are the string properties of data plane definitions that never leave the server. +// +//nolint:gochecknoglobals // immutable allow-list +var srchPxSecretFields = []string{ + "connectionstring", "apikey", "key", "accountkey", "sharedaccesssignature", "secret", "clientsecret", + "applicationsecret", "authsecret", "sastoken", +} + +// srchPxReadShapes are the data plane calls the explorer may make; item names are written as *. +// +//nolint:gochecknoglobals // immutable allow-list +var srchPxReadShapes = []string{ + "indexes", "indexes/*", "indexes/*/search.stats", "indexers", "indexers/*", "indexers/*/search.status", + "datasources", "datasources/*", "skillsets", "skillsets/*", "synonymmaps", "synonymmaps/*", "servicestats", +} + +//nolint:gochecknoglobals // immutable allow-list +var srchPxPostShapes = []string{"indexers/*/search.run", "indexers/*/search.reset", "indexes/*/" + srchPxDocsSearch} + +// registerAzureSearchRoutes exposes Azure AI Search by dispatching in-process. Service reads go to the Microsoft.Search +// ARM provider; the data plane is reached only through allow-listed shapes with the admin key held on the server, and +// keys leave the server only through the explicit reveal routes. +func (h *DashboardHandler) registerAzureSearchRoutes() { + h.registerARMProxy(armProxy{ + name: "Azure AI Search", prefix: azureSearchPrefix, version: srchPxARMVersion, + methods: []string{http.MethodGet}, allowed: searchProxyAllowed, + }) + + a := searchAccess{} + + if h.config.AzureARMOps != nil && h.config.AzureSearchOps != nil { + a.arm = echo.New() + a.arm.Any("/*", h.config.AzureARMOps.Handler()) + a.data = echo.New() + a.data.Any("/*", h.config.AzureSearchOps.Handler()) + } + + guard := func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c *echo.Context) error { + if a.arm == nil { + return azureACIFail(c, http.StatusServiceUnavailable, errServiceUnavailable, + "Azure AI Search is not enabled") + } + + return next(c) + } + } + + const ids = "/:sub/:rg/:svc" + + h.SubRouter.Add(http.MethodGet, azureSearchKeysPrefix+ids, guard(a.listKeys)) + h.SubRouter.Add(http.MethodPost, azureSearchKeysPrefix+ids+"/admin/reveal", guard(a.revealAdmin)) + h.SubRouter.Add(http.MethodPost, azureSearchKeysPrefix+ids+"/query", guard(a.createQueryKey)) + h.SubRouter.Add(http.MethodPost, azureSearchKeysPrefix+ids+"/query/:fp/reveal", guard(a.revealQueryKey)) + h.SubRouter.Add(http.MethodDelete, azureSearchKeysPrefix+ids+"/query/:fp", guard(a.deleteQueryKey)) + h.SubRouter.Add(http.MethodGet, azureSearchDataPrefix+ids+"/*", guard(a.dataPlane)) + h.SubRouter.Add(http.MethodPost, azureSearchDataPrefix+ids+"/*", guard(a.dataPlane)) +} + +func searchProxyAllowed(method string, p []string) bool { + if method != http.MethodGet { + return false + } + + if len(p) == srchPxRGList { + return p[0] == armPxSubsSeg && p[2] == armPxRGsSeg && p[4] == armPxProvSeg && p[5] == srchPxNamespace && + p[6] == srchPxServices + } + + typ, name, rest, ok := armScope(p, srchPxNamespace) + + return ok && typ == srchPxServices && (name == "" || len(rest) == 0) +} + +type searchAccess struct { + arm *echo.Echo + data *echo.Echo +} + +type searchIDs struct{ sub, rg, svc string } + +func (i searchIDs) armPath(rest ...string) string { + parts := make([]string, 0, srchPxBasePath+len(rest)) + parts = append(parts, "", armPxSubsSeg, url.PathEscape(i.sub), "resourceGroups", url.PathEscape(i.rg), armPxProvSeg, + srchPxARMNS, srchPxARMServices, url.PathEscape(i.svc)) + + for _, r := range rest { + parts = append(parts, url.PathEscape(r)) + } + + return strings.Join(parts, "/") +} + +func searchParams(c *echo.Context) (searchIDs, bool) { + ids := searchIDs{sub: c.Param("sub"), rg: c.Param("rg"), svc: c.Param("svc")} + + return ids, srchPxSubRe.MatchString(ids.sub) && srchPxNameRe.MatchString(ids.rg) && + srchPxNameRe.MatchString(ids.svc) +} + +func (a searchAccess) armDo(ctx context.Context, method, path string, body []byte) *httptest.ResponseRecorder { + req := httptest.NewRequestWithContext(ctx, method, path+"?api-version="+srchPxARMVersion, bytes.NewReader(body)) + if len(body) > 0 { + req.Header.Set("Content-Type", "application/json") + } + + rec := httptest.NewRecorder() + a.arm.ServeHTTP(rec, req) + + return rec +} + +func searchBad(c *echo.Context) error { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestUri", "invalid Azure AI Search request") +} + +func searchNoStore(c *echo.Context) { c.Response().Header().Set("Cache-Control", "no-store") } + +// service checks that the service exists in the given subscription and resource group. +func (a searchAccess) service(c *echo.Context, ids searchIDs) bool { + rec := a.armDo(c.Request().Context(), http.MethodGet, ids.armPath(), nil) + if rec.Code == http.StatusOK { + return true + } + + _ = azureACIFail(c, rec.Code, "ResourceNotFound", "the search service was not found") + + return false +} + +type searchKey struct{ name, key string } + +func (k searchKey) fingerprint() string { + sum := sha256.Sum256([]byte(k.key)) + + return hex.EncodeToString(sum[:srchPxFPBytes]) +} + +func searchMask(key string) string { + if len(key) <= srchPxMaskTail { + return srchPxMask + } + + return srchPxMask + key[len(key)-srchPxMaskTail:] +} + +func (k searchKey) view() map[string]string { + return map[string]string{netPxNameKey: k.name, srchPxKeyFP: k.fingerprint(), "masked": searchMask(k.key)} +} + +func (a searchAccess) adminKeys(c *echo.Context, ids searchIDs) (string, string, bool) { + rec := a.armDo(c.Request().Context(), http.MethodPost, ids.armPath("listAdminKeys"), []byte("{}")) + if rec.Code != http.StatusOK { + _ = azureACIFail(c, rec.Code, "KeysUnavailable", "the admin keys could not be read") + + return "", "", false + } + + var doc struct { + Primary string `json:"primaryKey"` + Secondary string `json:"secondaryKey"` + } + if json.Unmarshal(rec.Body.Bytes(), &doc) != nil { + _ = azureACIFail(c, http.StatusBadGateway, "KeysUnavailable", "unreadable admin keys") + + return "", "", false + } + + return doc.Primary, doc.Secondary, true +} + +func (a searchAccess) queryKeys(c *echo.Context, ids searchIDs) ([]searchKey, bool) { + rec := a.armDo(c.Request().Context(), http.MethodPost, ids.armPath("listQueryKeys"), []byte("{}")) + if rec.Code != http.StatusOK { + _ = azureACIFail(c, rec.Code, "KeysUnavailable", "the query keys could not be read") + + return nil, false + } + + var doc struct { + Value []struct { + Name string `json:"name"` + Key string `json:"key"` + } `json:"value"` + } + if json.Unmarshal(rec.Body.Bytes(), &doc) != nil { + _ = azureACIFail(c, http.StatusBadGateway, "KeysUnavailable", "unreadable query keys") + + return nil, false + } + + out := make([]searchKey, 0, len(doc.Value)) + for _, k := range doc.Value { + out = append(out, searchKey{name: k.Name, key: k.Key}) + } + + return out, true +} + +// listKeys answers masked keys only. +func (a searchAccess) listKeys(c *echo.Context) error { + ids, ok := searchParams(c) + if !ok { + return searchBad(c) + } + + if !a.service(c, ids) { + return nil + } + + primary, secondary, ok := a.adminKeys(c, ids) + if !ok { + return nil + } + + keys, ok := a.queryKeys(c, ids) + if !ok { + return nil + } + + views := make([]map[string]string, 0, len(keys)) + for _, k := range keys { + views = append(views, k.view()) + } + + searchNoStore(c) + + return c.JSON(http.StatusOK, map[string]any{ + "admin": map[string]string{"primary": searchMask(primary), "secondary": searchMask(secondary)}, + srchPxKeyQuery: views, + }) +} + +// revealAdmin returns the admin key pair. It is the only route that sends them out. +func (a searchAccess) revealAdmin(c *echo.Context) error { + ids, ok := searchParams(c) + if !ok { + return searchBad(c) + } + + if !a.service(c, ids) { + return nil + } + + primary, secondary, ok := a.adminKeys(c, ids) + if !ok { + return nil + } + + searchNoStore(c) + + return c.JSON(http.StatusOK, map[string]string{"primaryKey": primary, "secondaryKey": secondary}) +} + +func (a searchAccess) createQueryKey(c *echo.Context) error { + ids, ok := searchParams(c) + if !ok { + return searchBad(c) + } + + var in struct { + Name string `json:"name"` + } + + dec := json.NewDecoder(http.MaxBytesReader(c.Response(), c.Request().Body, srchPxMaxBody)) + dec.DisallowUnknownFields() + + if dec.Decode(&in) != nil || !srchPxKeyRe.MatchString(in.Name) { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestContent", "the key name is invalid") + } + + if !a.service(c, ids) { + return nil + } + + rec := a.armDo(c.Request().Context(), http.MethodPost, ids.armPath("createQueryKey", in.Name), []byte("{}")) + if rec.Code != http.StatusOK && rec.Code != http.StatusCreated { + return azureACIFail(c, rec.Code, "CreateFailed", "the query key could not be created") + } + + var made struct { + Name string `json:"name"` + Key string `json:"key"` + } + if json.Unmarshal(rec.Body.Bytes(), &made) != nil { + return azureACIFail(c, http.StatusBadGateway, "CreateFailed", "unreadable query key") + } + + searchNoStore(c) + + return c.JSON(http.StatusCreated, searchKey{name: made.Name, key: made.Key}.view()) +} + +func (a searchAccess) keyByFingerprint(c *echo.Context, ids searchIDs) (searchKey, bool) { + fp := c.Param(srchPxKeyFP) + if !srchPxFPRe.MatchString(fp) { + _ = searchBad(c) + + return searchKey{}, false + } + + if !a.service(c, ids) { + return searchKey{}, false + } + + keys, ok := a.queryKeys(c, ids) + if !ok { + return searchKey{}, false + } + + for _, k := range keys { + if k.fingerprint() == fp { + return k, true + } + } + + _ = azureACIFail(c, http.StatusNotFound, "KeyNotFound", "the query key was not found") + + return searchKey{}, false +} + +// revealQueryKey returns one query key value. It is the only route that sends one out. +func (a searchAccess) revealQueryKey(c *echo.Context) error { + ids, ok := searchParams(c) + if !ok { + return searchBad(c) + } + + k, found := a.keyByFingerprint(c, ids) + if !found { + return nil + } + + searchNoStore(c) + + return c.JSON(http.StatusOK, map[string]string{netPxNameKey: k.name, "key": k.key}) +} + +func (a searchAccess) deleteQueryKey(c *echo.Context) error { + ids, ok := searchParams(c) + if !ok { + return searchBad(c) + } + + k, found := a.keyByFingerprint(c, ids) + if !found { + return nil + } + + rec := a.armDo(c.Request().Context(), http.MethodDelete, ids.armPath("deleteQueryKey", k.key), nil) + if rec.Code >= http.StatusMultipleChoices { + return azureACIFail(c, rec.Code, "DeleteFailed", "the query key could not be deleted") + } + + return c.NoContent(http.StatusNoContent) +} + +// searchShape turns data plane segments into a shape with item names written as *. +func searchShape(segs []string) string { + parts := make([]string, len(segs)) + copy(parts, segs) + + if len(parts) > 1 { + parts[1] = "*" + } + + return strings.Join(parts, "/") +} + +func searchDataAllowed(method string, segs []string) bool { + if len(segs) == 0 { + return false + } + + for _, s := range segs { + if !srchPxItemRe.MatchString(s) { + return false + } + } + + shape := searchShape(segs) + + switch method { + case http.MethodGet: + return slices.Contains(srchPxReadShapes, shape) + case http.MethodPost: + return slices.Contains(srchPxPostShapes, shape) + } + + return false +} + +type searchQueryIn struct { + Top *int `json:"top"` + Skip *int `json:"skip"` + Search string `json:"search"` + Filter string `json:"filter"` + OrderBy string `json:"orderby"` + Select string `json:"select"` + QueryType string `json:"queryType"` + SearchMode string `json:"searchMode"` + SearchFields string `json:"searchFields"` + Highlight string `json:"highlight"` + Facets []string `json:"facets"` + Count bool `json:"count"` +} + +func (q searchQueryIn) valid() bool { + text := []string{q.Search, q.Filter, q.OrderBy, q.Select, q.SearchFields, q.Highlight} + if slices.ContainsFunc(text, func(s string) bool { return len(s) > srchPxMaxText }) { + return false + } + + if len(q.Facets) > srchPxMaxList || + slices.ContainsFunc(q.Facets, func(s string) bool { return len(s) > srchPxMaxText }) { + return false + } + + if q.Top != nil && (*q.Top < 0 || *q.Top > srchPxMaxTop) || + q.Skip != nil && (*q.Skip < 0 || *q.Skip > srchPxMaxSkip) { + return false + } + + return slices.Contains([]string{"", "simple", "full"}, q.QueryType) && + slices.Contains([]string{"", "any", "all"}, q.SearchMode) +} + +// searchQueryBody rebuilds an explorer request from fixed fields; anything else the caller sent is rejected. +func searchQueryBody(raw []byte) ([]byte, bool) { + var in searchQueryIn + + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + + if dec.Decode(&in) != nil || !in.valid() { + return nil, false + } + + out := map[string]any{"search": in.Search, "count": in.Count} + + for k, v := range map[string]string{ + "filter": in.Filter, "orderby": in.OrderBy, "select": in.Select, "queryType": in.QueryType, + "searchMode": in.SearchMode, "searchFields": in.SearchFields, "highlight": in.Highlight, + } { + if v != "" { + out[k] = v + } + } + + if len(in.Facets) > 0 { + out["facets"] = in.Facets + } + + if in.Top != nil { + out["top"] = *in.Top + } + + if in.Skip != nil { + out["skip"] = *in.Skip + } + + body, err := json.Marshal(out) + + return body, err == nil +} + +// searchScrub hides the secret strings of a data plane definition (data sources, skillsets, indexers). +func searchScrub(v any) { + switch t := v.(type) { + case []any: + for _, x := range t { + searchScrub(x) + } + case map[string]any: + for k, x := range t { + if _, isStr := x.(string); isStr && slices.Contains(srchPxSecretFields, strings.ToLower(k)) { + t[k] = srchPxNone + + continue + } + + searchScrub(x) + } + } +} + +func (a searchAccess) dataPlane(c *echo.Context) error { + ids, ok := searchParams(c) + if !ok { + return searchBad(c) + } + + r := c.Request() + segs := strings.Split(strings.Trim(c.Param("*"), "/"), "/") + + if !searchDataAllowed(r.Method, segs) { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestUri", "unsupported Azure AI Search request") + } + + var body []byte + + isQuery := r.Method == http.MethodPost && strings.HasSuffix(strings.Join(segs, "/"), srchPxDocsSearch) + + if isQuery { + raw, err := io.ReadAll(http.MaxBytesReader(c.Response(), r.Body, srchPxMaxBody)) + if err != nil { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestContent", "the query is too large") + } + + if body, ok = searchQueryBody(raw); !ok { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestContent", "the query is invalid") + } + } + + if !a.service(c, ids) { + return nil + } + + primary, _, ok := a.adminKeys(c, ids) + if !ok { + return nil + } + + req := httptest.NewRequestWithContext(r.Context(), r.Method, + "/"+strings.Join(segs, "/")+"?api-version="+srchPxDataVersion, bytes.NewReader(body)) + req.Host = ids.svc + srchPxHostSuffix + req.Header.Set("Api-Key", primary) + + if len(body) > 0 { + req.Header.Set("Content-Type", "application/json") + } + + rec := httptest.NewRecorder() + a.data.ServeHTTP(rec, req) + + return searchRelay(c, rec, isQuery) +} + +func searchRelay(c *echo.Context, rec *httptest.ResponseRecorder, isQuery bool) error { + reply := rec.Body.Bytes() + if len(reply) > srchPxMaxReply { + return azureACIFail(c, http.StatusBadGateway, "ReplyTooLarge", "the reply is too large") + } + + if len(reply) == 0 { + return c.NoContent(rec.Code) + } + + if !isQuery && rec.Code < http.StatusMultipleChoices { + var doc any + if json.Unmarshal(reply, &doc) == nil { + searchScrub(doc) + + if out, err := json.Marshal(doc); err == nil { + reply = out + } + } + } + + return c.JSONBlob(rec.Code, reply) +} + +// registerSearchAndWebRoutes registers the Azure AI Search and Lambda Web Functions pages. +func (h *DashboardHandler) registerSearchAndWebRoutes() { + h.registerAzureSearchRoutes() + h.registerLambdaWebRoutes() +} diff --git a/dashboard/azuresearch_proxy_test.go b/dashboard/azuresearch_proxy_test.go new file mode 100644 index 0000000000..4c63d45d36 --- /dev/null +++ b/dashboard/azuresearch_proxy_test.go @@ -0,0 +1,597 @@ +package dashboard_test + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/azurearm" + "github.com/blackbirdworks/gopherstack/services/azuresearch" +) + +const ( + srchAPI = "?api-version=2025-05-01" + srchRoot = "/dashboard/api/v1/azure/search" + srchKeys = "/dashboard/api/v1/azure/searchkeys" + srchData = "/dashboard/api/v1/azure/searchdata" + srchIDs = "/" + armSub + "/demo/hotels" + srchService = armBase + "Microsoft.Search/searchServices/hotels" + srchDataAPI = "api-version=2024-07-01" + srchSecret = "Server=db;Password=hunter2" +) + +var errSrchOffline = errors.New("storage offline") + +// srchPlane adapts the data plane backend to the Microsoft.Search ARM provider. +type srchPlane struct{ b *azuresearch.InMemoryBackend } + +func (p srchPlane) Port() int { return azuresearch.DefaultPort } + +func (p srchPlane) PutService(name string, cfg azurearm.SearchServiceConfig) (time.Time, error) { + return p.b.PutService(name, azuresearch.ServiceConfig(cfg)) +} + +func (p srchPlane) DeleteService(name string) error { return p.b.DeleteService(name) } + +func (p srchPlane) AdminKeys(name string) (azurearm.SearchAdminKeys, error) { + k, err := p.b.Keys(name) + + return azurearm.SearchAdminKeys{Primary: k.Primary, Secondary: k.Secondary}, err +} + +func (p srchPlane) RegenerateAdminKey(name, kind string) (azurearm.SearchAdminKeys, error) { + k, err := p.b.RegenerateAdminKey(name, kind) + + return azurearm.SearchAdminKeys{Primary: k.Primary, Secondary: k.Secondary}, err +} + +func (p srchPlane) QueryKeys(name string) ([]azurearm.SearchQueryKey, error) { + keys, err := p.b.ListQueryKeys(name) + out := make([]azurearm.SearchQueryKey, 0, len(keys)) + + for _, k := range keys { + out = append(out, azurearm.SearchQueryKey(k)) + } + + return out, err +} + +func (p srchPlane) CreateQueryKey(name, keyName string) (azurearm.SearchQueryKey, error) { + k, err := p.b.CreateQueryKey(name, keyName) + + return azurearm.SearchQueryKey(k), err +} + +func (p srchPlane) DeleteQueryKey(name, key string) error { return p.b.DeleteQueryKey(name, key) } + +type srchBlobs struct{} + +func (srchBlobs) ListBlobs(container string) ([]azuresearch.BlobEntry, error) { + if container != "docs" { + return nil, errSrchOffline + } + + return []azuresearch.BlobEntry{{Name: "a.txt", ETag: "e1", Size: 5}}, nil +} + +func (srchBlobs) ReadBlob(string, string) ([]byte, error) { return []byte("hello"), nil } + +type srchFixture struct { + h *dashboard.DashboardHandler + b *azuresearch.InMemoryBackend + admin string +} + +func newSearchFixture(t *testing.T) srchFixture { + t.Helper() + + svc, err := (&azurearm.Provider{}).Init(&service.AppContext{}) + require.NoError(t, err) + + armH, ok := svc.(*azurearm.Handler) + require.True(t, ok) + t.Cleanup(func() { armH.Shutdown(t.Context()) }) + + b := azuresearch.NewInMemoryBackend() + b.SetBlobSource(srchBlobs{}) + armH.Registry.SetSearch(srchPlane{b: b}) + + sh := azuresearch.NewHandler(b, azuresearch.DefaultSettings()) + h := dashboard.NewHandler(dashboard.Config{AzureARMOps: armH, AzureSearchOps: sh}) + + for _, s := range []struct{ path, body string }{ + {"/subscriptions/" + armSub + "/resourceGroups/demo?api-version=2021-04-01", `{"location":"westus"}`}, + {srchService + srchAPI, `{"location":"westus","sku":{"name":"standard"},` + + `"properties":{"replicaCount":2,"partitionCount":1}}`}, + } { + rec := lbDo(t, h, http.MethodPut, "/dashboard/api/v1/azure/arm"+s.path, s.body) + require.Less(t, rec.Code, http.StatusBadRequest, "%s: %s", s.path, rec.Body.String()) + } + + keys, err := b.Keys("hotels") + require.NoError(t, err) + + f := srchFixture{h: h, b: b, admin: keys.Primary} + f.seed(t) + + return f +} + +// seed fills the service through the data plane handler, the way an SDK client would. +func (f srchFixture) seed(t *testing.T) { + t.Helper() + + e := echo.New() + e.Any("/*", azuresearch.NewHandler(f.b, azuresearch.DefaultSettings()).Handler()) + + send := func(method, path, body string, want int) { + req := httptest.NewRequest(method, path+"?"+srchDataAPI, strings.NewReader(body)) + req.Host = "hotels.search.windows.net" + req.Header.Set("Api-Key", f.admin) + req.Header.Set("Content-Type", "application/json") + + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + require.Equal(t, want, rec.Code, "%s %s: %s", method, path, rec.Body.String()) + } + + send(http.MethodPost, "/indexes", `{"name":"hotels","fields":[`+ + `{"name":"id","type":"Edm.String","key":true},`+ + `{"name":"name","type":"Edm.String","searchable":true},`+ + `{"name":"category","type":"Edm.String","filterable":true,"facetable":true},`+ + `{"name":"rating","type":"Edm.Double","filterable":true,"sortable":true}]}`, http.StatusCreated) + send(http.MethodPost, "/indexes/hotels/docs/search.index", `{"value":[`+ + `{"@search.action":"upload","id":"1","name":"Grand Hotel","category":"hotel","rating":4.5},`+ + `{"@search.action":"upload","id":"2","name":"Budget Inn","category":"motel","rating":3.0},`+ + `{"@search.action":"upload","id":"3","name":"Royal Hotel","category":"hotel","rating":4.8}]}`, http.StatusOK) + send(http.MethodPut, "/datasources/ds", `{"name":"ds","type":"azureblob","credentials":`+ + `{"connectionString":"`+srchSecret+`"},"container":{"name":"docs"}}`, http.StatusCreated) + send(http.MethodPut, "/indexers/ix", `{"name":"ix","dataSourceName":"ds","targetIndexName":"hotels",`+ + `"fieldMappings":[{"sourceFieldName":"metadata_storage_path","targetFieldName":"id",`+ + `"mappingFunction":{"name":"base64Encode"}}]}`, http.StatusCreated) +} + +func srchGet(t *testing.T, h *dashboard.DashboardHandler, path string) map[string]any { + t.Helper() + + rec := lbDo(t, h, http.MethodGet, path, "") + require.Equal(t, http.StatusOK, rec.Code, "%s: %s", path, rec.Body.String()) + + var out map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + + return out +} + +func TestAzureSearchProxyServices(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + + list := srchGet(t, f.h, srchRoot+armBase+"Microsoft.Search/searchServices"+srchAPI) + items, _ := list["value"].([]any) + require.Len(t, items, 1) + + one := srchGet(t, f.h, srchRoot+srchService+srchAPI) + props, _ := one["properties"].(map[string]any) + assert.EqualValues(t, 2, props["replicaCount"]) + assert.EqualValues(t, 1, props["partitionCount"]) + assert.Contains(t, props["endpoint"], "hotels") +} + +func TestAzureSearchProxyServicesAllowList(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + + tests := []struct { + name string + method string + path string + want int + }{ + { + "admin keys over get", + http.MethodGet, + srchRoot + srchService + "/listAdminKeys" + srchAPI, + http.StatusBadRequest, + }, + { + "query keys over get", + http.MethodGet, + srchRoot + srchService + "/listQueryKeys" + srchAPI, + http.StatusBadRequest, + }, + { + "other provider", + http.MethodGet, + srchRoot + armBase + "Microsoft.Logic/workflows" + srchAPI, + http.StatusBadRequest, + }, + { + "resource groups", + http.MethodGet, + srchRoot + "/subscriptions/" + armSub + "/resourceGroups" + srchAPI, + http.StatusBadRequest, + }, + {"traversal", http.MethodGet, srchRoot + "/subscriptions/" + armSub + "/../x" + srchAPI, http.StatusBadRequest}, + {"put", http.MethodPut, srchRoot + srchService + srchAPI, http.StatusNotFound}, + {"delete", http.MethodDelete, srchRoot + srchService + srchAPI, http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := lbDo(t, f.h, tt.method, tt.path, `{}`) + assert.Contains(t, []int{tt.want, http.StatusMethodNotAllowed}, rec.Code, rec.Body.String()) + assert.NotContains(t, rec.Body.String(), f.admin) + }) + } +} + +func TestAzureSearchKeysAreMaskedUntilRevealed(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + keys, err := f.b.Keys("hotels") + require.NoError(t, err) + + query, err := f.b.ListQueryKeys("hotels") + require.NoError(t, err) + require.NotEmpty(t, query) + + list := lbDo(t, f.h, http.MethodGet, srchKeys+srchIDs, "") + require.Equal(t, http.StatusOK, list.Code, list.Body.String()) + assert.Equal(t, "no-store", list.Header().Get("Cache-Control")) + + for _, secret := range []string{keys.Primary, keys.Secondary, query[0].Key} { + assert.NotContains(t, list.Body.String(), secret) + } + + type masked struct { + Name string `json:"name"` + Fp string `json:"fp"` + Masked string `json:"masked"` + } + + var shown struct { + Admin struct { + Primary string `json:"primary"` + Secondary string `json:"secondary"` + } `json:"admin"` + Query []masked `json:"query"` + } + + require.NoError(t, json.Unmarshal(list.Body.Bytes(), &shown)) + assert.True(t, strings.HasSuffix(keys.Primary, strings.TrimPrefix(shown.Admin.Primary, "••••••••"))) + assert.NotEqual(t, keys.Primary, shown.Admin.Primary) + require.NotEmpty(t, shown.Query) + assert.Len(t, shown.Query[0].Fp, 8) + + reveal := lbDo(t, f.h, http.MethodPost, srchKeys+srchIDs+"/admin/reveal", "") + require.Equal(t, http.StatusOK, reveal.Code) + assert.Equal(t, "no-store", reveal.Header().Get("Cache-Control")) + assert.JSONEq(t, `{"primaryKey":"`+keys.Primary+`","secondaryKey":"`+keys.Secondary+`"}`, reveal.Body.String()) + + one := lbDo(t, f.h, http.MethodPost, srchKeys+srchIDs+"/query/"+shown.Query[0].Fp+"/reveal", "") + require.Equal(t, http.StatusOK, one.Code, one.Body.String()) + assert.Contains(t, one.Body.String(), query[0].Key) +} + +func TestAzureSearchQueryKeyLifecycle(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + + made := lbDo(t, f.h, http.MethodPost, srchKeys+srchIDs+"/query", `{"name":"web front"}`) + require.Equal(t, http.StatusCreated, made.Code, made.Body.String()) + + var created struct { + Name string `json:"name"` + Fp string `json:"fp"` + } + + require.NoError(t, json.Unmarshal(made.Body.Bytes(), &created)) + assert.Equal(t, "web front", created.Name) + + all, err := f.b.ListQueryKeys("hotels") + require.NoError(t, err) + + for _, k := range all { + assert.NotContains(t, made.Body.String(), k.Key, "creating a key does not echo the value") + } + + del := lbDo(t, f.h, http.MethodDelete, srchKeys+srchIDs+"/query/"+created.Fp, "") + require.Equal(t, http.StatusNoContent, del.Code, del.Body.String()) + + after, err := f.b.ListQueryKeys("hotels") + require.NoError(t, err) + assert.Len(t, after, len(all)-1) + + gone := lbDo(t, f.h, http.MethodDelete, srchKeys+srchIDs+"/query/"+created.Fp, "") + assert.Equal(t, http.StatusNotFound, gone.Code) +} + +func TestAzureSearchKeysRejectBadInput(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + + tests := []struct { + name string + method string + path string + body string + want int + }{ + {"bad name", http.MethodPost, srchKeys + srchIDs + "/query", `{"name":"a/b"}`, http.StatusBadRequest}, + { + "extra field", + http.MethodPost, + srchKeys + srchIDs + "/query", + `{"name":"ok","key":"mine"}`, + http.StatusBadRequest, + }, + {"empty name", http.MethodPost, srchKeys + srchIDs + "/query", `{}`, http.StatusBadRequest}, + {"bad fingerprint", http.MethodDelete, srchKeys + srchIDs + "/query/NOTHEX", "", http.StatusBadRequest}, + { + "unknown fingerprint", + http.MethodPost, + srchKeys + srchIDs + "/query/00000000/reveal", + "", + http.StatusNotFound, + }, + {"unknown service", http.MethodGet, srchKeys + "/" + armSub + "/demo/ghost", "", http.StatusNotFound}, + { + "unknown group", + http.MethodPost, + srchKeys + "/" + armSub + "/other/hotels/admin/reveal", + "", + http.StatusNotFound, + }, + {"bad subscription", http.MethodGet, srchKeys + "/a%20b/demo/hotels", "", http.StatusBadRequest}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := lbDo(t, f.h, tt.method, tt.path, tt.body) + assert.Equal(t, tt.want, rec.Code, rec.Body.String()) + assert.NotContains(t, rec.Body.String(), f.admin) + }) + } +} + +func TestAzureSearchDataPlaneReads(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + + idx := srchGet(t, f.h, srchData+srchIDs+"/indexes") + list, _ := idx["value"].([]any) + require.Len(t, list, 1) + + stats := srchGet(t, f.h, srchData+srchIDs+"/indexes/hotels/search.stats") + assert.EqualValues(t, 3, stats["documentCount"]) + + svc := srchGet(t, f.h, srchData+srchIDs+"/servicestats") + assert.Contains(t, svc, "counters") + + paths := []string{ + "/indexes/hotels", "/indexers", "/indexers/ix", "/indexers/ix/search.status", "/datasources", "/skillsets", + "/synonymmaps", + } + + for _, p := range paths { + srchGet(t, f.h, srchData+srchIDs+p) + } + + body := lbDo(t, f.h, http.MethodGet, srchData+srchIDs+"/datasources", "").Body.String() + assert.NotContains(t, body, "hunter2", "connection strings never reach the browser") + assert.NotContains(t, body, f.admin) +} + +func TestAzureSearchExplorerFilterAndFacets(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + path := srchData + srchIDs + "/indexes/hotels/docs/search.post.search" + + rec := lbDo(t, f.h, http.MethodPost, path, `{"search":"hotel","filter":"rating ge 4","orderby":"rating desc",`+ + `"select":"id,name","facets":["category"],"top":5,"count":true}`) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + type facet struct { + Value string `json:"value"` + Count int `json:"count"` + } + + var out struct { + Facets map[string][]facet `json:"@search.facets"` + Value []map[string]any `json:"value"` + Count int `json:"@odata.count"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + assert.Equal(t, 2, out.Count) + require.Len(t, out.Value, 2) + assert.Equal(t, "Royal Hotel", out.Value[0]["name"], "ordered by rating descending") + require.Len(t, out.Facets["category"], 1) + assert.Equal(t, "hotel", out.Facets["category"][0].Value) + assert.NotContains(t, rec.Body.String(), f.admin) +} + +func TestAzureSearchExplorerRebuildsTheBody(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + path := srchData + srchIDs + "/indexes/hotels/docs/search.post.search" + + tests := []struct { + name string + body string + want int + }{ + {"minimal", `{"search":"*"}`, http.StatusOK}, + {"empty", `{}`, http.StatusOK}, + {"unknown field", `{"search":"*","vectorQueries":[{"kind":"vector"}]}`, http.StatusBadRequest}, + {"scoring parameters", `{"search":"*","scoringParameters":["a-1"]}`, http.StatusBadRequest}, + {"top too large", `{"search":"*","top":1001}`, http.StatusBadRequest}, + {"negative skip", `{"search":"*","skip":-1}`, http.StatusBadRequest}, + {"bad query type", `{"search":"*","queryType":"semantic"}`, http.StatusBadRequest}, + {"bad search mode", `{"search":"*","searchMode":"some"}`, http.StatusBadRequest}, + { + "too many facets", + `{"search":"*","facets":[` + strings.Repeat(`"category",`, 40) + `"x"]}`, + http.StatusBadRequest, + }, + {"oversize text", `{"search":"` + strings.Repeat("a", 5000) + `"}`, http.StatusBadRequest}, + {"not json", `search=*`, http.StatusBadRequest}, + {"wrong types", `{"search":5}`, http.StatusBadRequest}, + {"bad filter reaches the service", `{"search":"*","filter":"nope nope"}`, http.StatusBadRequest}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := lbDo(t, f.h, http.MethodPost, path, tt.body) + assert.Equal(t, tt.want, rec.Code, rec.Body.String()) + }) + } +} + +func TestAzureSearchDataPlaneAllowList(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + base := srchData + srchIDs + + tests := []struct { + name string + method string + path string + }{ + {"create index", http.MethodPost, base + "/indexes"}, + {"put index", http.MethodPut, base + "/indexes/hotels"}, + {"delete index", http.MethodDelete, base + "/indexes/hotels"}, + {"upload documents", http.MethodPost, base + "/indexes/hotels/docs/search.index"}, + {"read documents", http.MethodGet, base + "/indexes/hotels/docs"}, + {"document by key", http.MethodGet, base + "/indexes/hotels/docs/1"}, + {"count", http.MethodGet, base + "/indexes/hotels/docs/$count"}, + {"analyze", http.MethodPost, base + "/indexes/hotels/search.analyze"}, + {"put datasource", http.MethodPut, base + "/datasources/ds"}, + {"delete indexer", http.MethodDelete, base + "/indexers/ix"}, + {"odata key syntax", http.MethodGet, base + "/indexes('hotels')"}, + {"traversal", http.MethodGet, base + "/indexes/../servicestats"}, + {"empty", http.MethodGet, base + "/"}, + {"unknown root", http.MethodGet, base + "/admin"}, + {"search over get", http.MethodGet, base + "/indexes/hotels/docs/search.post.search"}, + {"run over get", http.MethodGet, base + "/indexers/ix/search.run"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := lbDo(t, f.h, tt.method, tt.path, `{"name":"x"}`) + assert.GreaterOrEqual(t, rec.Code, http.StatusBadRequest, rec.Body.String()) + assert.NotEqual(t, http.StatusOK, rec.Code) + }) + } + + still := srchGet(t, f.h, base+"/indexes") + list, _ := still["value"].([]any) + assert.Len(t, list, 1, "nothing was created or removed") +} + +func TestAzureSearchIndexerRunAndReset(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + base := srchData + srchIDs + "/indexers/ix" + + before := srchGet(t, f.h, base+"/search.status") + assert.Equal(t, "unknown", before["status"]) + + run := lbDo(t, f.h, http.MethodPost, base+"/search.run", "") + require.Equal(t, http.StatusAccepted, run.Code, run.Body.String()) + + after := srchGet(t, f.h, base+"/search.status") + last, _ := after["lastResult"].(map[string]any) + require.NotNil(t, last) + assert.Equal(t, "success", last["status"], "%v", last) + + reset := lbDo(t, f.h, http.MethodPost, base+"/search.reset", "") + assert.Equal(t, http.StatusNoContent, reset.Code, reset.Body.String()) +} + +func TestAzureSearchAbsentBackendsAnswer503(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + for _, tt := range []struct{ name, method, path string }{ + {"arm", http.MethodGet, srchRoot + armBase + "Microsoft.Search/searchServices" + srchAPI}, + {"keys", http.MethodGet, srchKeys + srchIDs}, + {"reveal", http.MethodPost, srchKeys + srchIDs + "/admin/reveal"}, + {"data", http.MethodGet, srchData + srchIDs + "/indexes"}, + {"query", http.MethodPost, srchData + srchIDs + "/indexes/hotels/docs/search.post.search"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := lbDo(t, h, tt.method, tt.path, `{}`) + assert.Equal(t, http.StatusServiceUnavailable, rec.Code, rec.Body.String()) + }) + } +} + +func TestAzureSearchNoSecretInAnyRead(t *testing.T) { + t.Parallel() + + f := newSearchFixture(t) + keys, err := f.b.Keys("hotels") + require.NoError(t, err) + + query, err := f.b.ListQueryKeys("hotels") + require.NoError(t, err) + + secrets := make([]string, 0, 3+len(query)) + secrets = append(secrets, keys.Primary, keys.Secondary, "hunter2") + for _, q := range query { + secrets = append(secrets, q.Key) + } + + paths := []string{ + srchRoot + srchService + srchAPI, + srchRoot + armBase + "Microsoft.Search/searchServices" + srchAPI, + srchKeys + srchIDs, + srchData + srchIDs + "/indexes", + srchData + srchIDs + "/indexers", + srchData + srchIDs + "/indexers/ix", + srchData + srchIDs + "/datasources", + srchData + srchIDs + "/datasources/ds", + srchData + srchIDs + "/skillsets", + srchData + srchIDs + "/servicestats", + } + + for _, p := range paths { + rec := lbDo(t, f.h, http.MethodGet, p, "") + for _, s := range secrets { + assert.NotContains(t, rec.Body.String(), s, p) + } + } +} diff --git a/dashboard/azuresql_proxy.go b/dashboard/azuresql_proxy.go new file mode 100644 index 0000000000..3fbc2787d0 --- /dev/null +++ b/dashboard/azuresql_proxy.go @@ -0,0 +1,166 @@ +package dashboard + +import ( + "encoding/json" + "errors" + "net/http" + "net/netip" + "regexp" + "strings" + "time" + + "github.com/labstack/echo/v5" + + sqlbackend "github.com/blackbirdworks/gopherstack/services/azuresql" +) + +const ( + azureSQLPrefix = "/dashboard/api/v1/azure/sql" + azureSQLQueryPrefix = "/dashboard/api/v1/azure/sqlquery" + sqlPxNamespace = "microsoft.sql" + sqlPxServers = "servers" + sqlPxDatabases = "databases" + sqlPxFirewall = "firewallrules" + sqlPxRGList = 7 + sqlPxChildItem = 2 + sqlPxMaxServerID = 512 + sqlPxRowsField = "rows" + sqlPxPropsField = "properties" +) + +var sqlPxRuleRE = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,128}$`) + +// registerAzureSQLRoutes exposes logical servers, databases and firewall rules by dispatching in-process to the ARM +// handler (firewall rules are the only writes, with a body rebuilt from fixed fields; the administrator password +// is scrubbed from every response), and serves the read-only query runner from the SQL backend. +func (h *DashboardHandler) registerAzureSQLRoutes() { + h.registerARMProxy(armProxy{ + name: "Azure SQL", prefix: azureSQLPrefix, + queryKeys: []string{armQueryVersion}, + methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete}, + allowed: sqlProxyAllowed, + body: sqlProxyBody, + }) + + q := &azureSQLQueryAPI{} + if h.config.AzureSQLOps != nil { + q.b = h.config.AzureSQLOps.Backend + } + + h.SubRouter.Add(http.MethodPost, azureSQLQueryPrefix, q.guard(q.run)) +} + +func sqlProxyAllowed(method string, p []string) bool { + if len(p) == sqlPxRGList { + return method == http.MethodGet && p[0] == armPxSubsSeg && p[flexPxRGIdx] == armPxRGsSeg && + p[flexPxRGProvIdx] == armPxProvSeg && p[flexPxRGNSIdx] == sqlPxNamespace && + p[flexPxRGTypeIdx] == sqlPxServers + } + + typ, name, rest, ok := armScope(p, sqlPxNamespace) + if !ok || typ != sqlPxServers { + return false + } + + if name == "" || len(rest) == 0 { + return method == http.MethodGet + } + + return sqlProxyChild(method, rest) +} + +func sqlProxyChild(method string, rest []string) bool { + readable := rest[0] == sqlPxDatabases || rest[0] == sqlPxFirewall + + switch { + case len(rest) == 1: + return method == http.MethodGet && readable + case len(rest) != sqlPxChildItem: + return false + case method == http.MethodGet: + return readable + case method == http.MethodPut || method == http.MethodDelete: + return rest[0] == sqlPxFirewall && sqlPxRuleRE.MatchString(rest[1]) + } + + return false +} + +type sqlRuleIn struct { + Properties map[string]string `json:"properties"` +} + +func sqlProxyBody(method string, _ []string, in []byte) ([]byte, bool) { + if method != http.MethodPut { + return in, true + } + + var req sqlRuleIn + if json.Unmarshal(in, &req) != nil { + return nil, false + } + + start, err1 := netip.ParseAddr(strings.TrimSpace(req.Properties["startIpAddress"])) + end, err2 := netip.ParseAddr(strings.TrimSpace(req.Properties["endIpAddress"])) + + if err1 != nil || err2 != nil || !start.Is4() || !end.Is4() { + return nil, false + } + + b, err := json.Marshal(map[string]any{sqlPxPropsField: map[string]any{ + "startIpAddress": start.String(), "endIpAddress": end.String(), + }}) + + return b, err == nil +} + +type sqlQueryIn struct { + Server string `json:"server"` + Database string `json:"database"` + Query string `json:"query"` +} + +type azureSQLQueryAPI struct { + b *sqlbackend.InMemoryBackend +} + +func (a *azureSQLQueryAPI) guard(next echo.HandlerFunc) echo.HandlerFunc { + return func(c *echo.Context) error { + if a.b == nil { + return cloudUnavailable(c, "Azure SQL") + } + + return next(c) + } +} + +// run executes one validated SELECT; the backend caps rows, cell size and time. +func (a *azureSQLQueryAPI) run(c *echo.Context) error { + var in sqlQueryIn + if !cloudJSONDecode(c, &in) { + return nil + } + + if in.Server == "" || len(in.Server) > sqlPxMaxServerID { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, "server is required") + } + + started := time.Now() + + res, err := a.b.Query(c.Request().Context(), strings.ToLower(in.Server), in.Database, in.Query) + if err != nil { + switch { + case errors.Is(err, sqlbackend.ErrNotFound): + return cloudJSONFail(c, http.StatusNotFound, cloudStatusNotFnd, err.Error()) + case errors.Is(err, sqlbackend.ErrNoEngine): + return cloudJSONFail(c, http.StatusConflict, "FAILED_PRECONDITION", err.Error()) + default: + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, err.Error()) + } + } + + return c.JSON(http.StatusOK, map[string]any{ + "columns": res.Columns, sqlPxRowsField: res.Rows, "truncated": res.Truncated, + "maxRows": sqlbackend.MaxQueryRows, "elapsedMs": time.Since(started).Milliseconds(), + }) +} diff --git a/dashboard/azuresql_proxy_test.go b/dashboard/azuresql_proxy_test.go new file mode 100644 index 0000000000..fb6e079c5d --- /dev/null +++ b/dashboard/azuresql_proxy_test.go @@ -0,0 +1,291 @@ +package dashboard_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/azurearm" + "github.com/blackbirdworks/gopherstack/services/azuresql" +) + +const ( + sqlAPI = "?api-version=2023-08-01-preview" + sqlPxBase = "/dashboard/api/v1/azure/sql" + sqlQueryURL = "/dashboard/api/v1/azure/sqlquery" + sqlPw = "Adm1n-Secret-pw!" + sqlServer = armBase + "Microsoft.Sql/servers/srv1" +) + +func newSQLARMDashboard(t *testing.T) *dashboard.DashboardHandler { + t.Helper() + + svc, err := (&azurearm.Provider{}).Init(&service.AppContext{}) + require.NoError(t, err) + + armH, ok := svc.(*azurearm.Handler) + require.True(t, ok) + t.Cleanup(func() { armH.Shutdown(context.Background()) }) + + h := dashboard.NewHandler(dashboard.Config{AzureARMOps: armH}) + seeds := []struct{ path, body string }{ + {"/subscriptions/" + armSub + "/resourceGroups/demo?api-version=2021-04-01", `{"location":"eastus"}`}, + { + sqlServer + sqlAPI, + `{"location":"eastus","properties":{"administratorLogin":"sqladmin","administratorLoginPassword":"` + sqlPw + `"}}`, + }, + { + sqlServer + "/databases/orders" + sqlAPI, + `{"location":"eastus","sku":{"name":"S0","tier":"Standard"},"properties":{"maxSizeBytes":2147483648}}`, + }, + } + + for _, s := range seeds { + rec := doProxy(t, h, http.MethodPut, "/dashboard/api/v1/azure/arm"+s.path, s.body) + require.Less(t, rec.Code, http.StatusBadRequest, "%s: %s", s.path, rec.Body.String()) + } + + return h +} + +func TestAzureSQLProxy(t *testing.T) { + t.Parallel() + + px := sqlPxBase + sqlServer + list := sqlPxBase + "/subscriptions/" + armSub + "/providers/Microsoft.Sql/servers" + + tests := []struct { + name string + steps []gcpStep + }{ + {"read", []gcpStep{ + {http.MethodGet, list + sqlAPI, "", `"name":"srv1"`, 200}, + {http.MethodGet, sqlPxBase + armBase + "Microsoft.Sql/servers" + sqlAPI, "", `"name":"srv1"`, 200}, + {http.MethodGet, px + sqlAPI, "", `"administratorLogin":"sqladmin"`, 200}, + {http.MethodGet, px + "/databases" + sqlAPI, "", `"name":"orders"`, 200}, + {http.MethodGet, px + "/databases" + sqlAPI, "", `"maxSizeBytes":2147483648`, 200}, + {http.MethodGet, px + "/databases" + sqlAPI, "", `"status":"Online"`, 200}, + {http.MethodGet, px + "/databases/orders" + sqlAPI, "", `"name":"S0"`, 200}, + {http.MethodGet, px + "/firewallRules" + sqlAPI, "", `value`, 200}, + }}, + {"firewall writes", []gcpStep{ + { + http.MethodPut, px + "/firewallRules/office" + sqlAPI, + `{"properties":{"startIpAddress":"10.0.0.1","endIpAddress":"10.0.0.9"},"extra":"dropped"}`, + `"startIpAddress":"10.0.0.1"`, 200, + }, + {http.MethodGet, px + "/firewallRules" + sqlAPI, "", `"name":"office"`, 200}, + {http.MethodDelete, px + "/firewallRules/office" + sqlAPI, "", "", 200}, + }}, + {"rejects", []gcpStep{ + {http.MethodPut, px + sqlAPI, `{}`, "unsupported", 400}, + {http.MethodDelete, px + sqlAPI, "", "unsupported", 400}, + {http.MethodPut, px + "/databases/x" + sqlAPI, `{}`, "unsupported", 400}, + {http.MethodDelete, px + "/databases/orders" + sqlAPI, "", "unsupported", 400}, + {http.MethodGet, px + "/administrators" + sqlAPI, "", "unsupported", 400}, + {http.MethodGet, px + "/auditingSettings" + sqlAPI, "", "unsupported", 400}, + {http.MethodPost, px + "/databases" + sqlAPI, `{}`, "Method Not Allowed", 405}, + {http.MethodGet, px + "/databases/../x" + sqlAPI, "", "unsupported", 400}, + {http.MethodPut, list + sqlAPI, `{}`, "unsupported", 400}, + { + http.MethodPut, px + "/firewallRules/bad" + sqlAPI, + `{"properties":{"startIpAddress":"nope","endIpAddress":"1.1.1.1"}}`, "invalid", 400, + }, + { + http.MethodPut, px + "/firewallRules/bad" + sqlAPI, + `{"properties":{"startIpAddress":"::1","endIpAddress":"::2"}}`, "invalid", 400, + }, + {http.MethodPut, px + "/firewallRules/bad" + sqlAPI, `not json`, "invalid", 400}, + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + runGCPSteps(t, newSQLARMDashboard(t), tt.steps) + }) + } +} + +func TestAzureSQLProxyNeverReturnsPassword(t *testing.T) { + t.Parallel() + + h := newSQLARMDashboard(t) + px := sqlPxBase + sqlServer + + for _, p := range []string{ + px + sqlAPI, sqlPxBase + "/subscriptions/" + armSub + "/providers/Microsoft.Sql/servers" + sqlAPI, + px + "/databases" + sqlAPI, + } { + rec := doProxy(t, h, http.MethodGet, p, "") + assert.NotContains(t, rec.Body.String(), sqlPw, p) + assert.NotContains(t, strings.ToLower(rec.Body.String()), "administratorloginpassword", p) + } +} + +func TestAzureSQLProxyAbsent(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + rec := doProxy(t, h, http.MethodGet, sqlPxBase+"/subscriptions/s/providers/Microsoft.Sql/servers"+sqlAPI, "") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + + rec = doProxy(t, h, http.MethodPost, sqlQueryURL, `{"server":"x","database":"master","query":"SELECT 1"}`) + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Contains(t, rec.Body.String(), "UNAVAILABLE") +} + +type sqlTestEngine struct { + gotDB string + gotQ string +} + +func (*sqlTestEngine) Endpoint() (string, int) { return "127.0.0.1", 1 } +func (*sqlTestEngine) SetLogin(context.Context, string, string) error { return nil } +func (*sqlTestEngine) CreateDatabase(context.Context, string, string) error { return nil } +func (*sqlTestEngine) DropDatabase(context.Context, string) error { return nil } +func (*sqlTestEngine) ListDatabases(context.Context) ([]string, error) { return nil, nil } +func (*sqlTestEngine) Close(context.Context) {} + +func (e *sqlTestEngine) Query(_ context.Context, db, text string, _ int) (azuresql.QueryResult, error) { + e.gotDB, e.gotQ = db, text + + return azuresql.QueryResult{Columns: []string{"one", "two"}, Rows: [][]string{{"1", "x"}}, Truncated: true}, nil +} + +type sqlTestLauncher struct{ engine *sqlTestEngine } + +func (l *sqlTestLauncher) Launch(context.Context, azuresql.LaunchSpec) (azuresql.Engine, error) { + return l.engine, nil +} + +func newSQLQueryDashboard(t *testing.T, withEngine bool) (*dashboard.DashboardHandler, *sqlTestEngine) { + t.Helper() + + b := azuresql.NewInMemoryBackend() + t.Cleanup(b.Close) + + eng := &sqlTestEngine{} + if withEngine { + b.SetLauncher(&sqlTestLauncher{engine: eng}, azuresql.DefaultStartupTimeout) + } + + _, err := b.Ensure(strings.ToLower(sqlServer), azuresql.Spec{AdminLogin: "a", AdminPassword: sqlPw}) + require.NoError(t, err) + + return dashboard.NewHandler(dashboard.Config{AzureSQLOps: azuresql.NewHandler(b)}), eng +} + +func TestAzureSQLQuery(t *testing.T) { + t.Parallel() + + body := func(server, db, q string) string { + return `{"server":"` + server + `","database":"` + db + `","query":"` + q + `"}` + } + + tests := []struct { + name string + body string + wantBody string + wantCode int + withEngine bool + }{ + { + name: "select", withEngine: true, body: body(sqlServer, "master", "SELECT 1"), + wantCode: 200, wantBody: `"columns":["one","two"]`, + }, + { + name: "truncated flag", withEngine: true, body: body(sqlServer, "master", "SELECT 1"), + wantCode: 200, wantBody: `"truncated":true`, + }, + { + name: "row cap", withEngine: true, body: body(sqlServer, "master", "SELECT 1"), + wantCode: 200, wantBody: `"maxRows":500`, + }, + { + name: "delete rejected", withEngine: true, body: body(sqlServer, "master", "DELETE FROM t"), + wantCode: 400, wantBody: "only SELECT", + }, + { + name: "batch rejected", withEngine: true, body: body(sqlServer, "master", "SELECT 1; DROP TABLE t"), + wantCode: 400, wantBody: "single statement", + }, + { + name: "exec rejected", withEngine: true, body: body(sqlServer, "master", "SELECT 1 EXEC x"), + wantCode: 400, wantBody: "EXEC", + }, + { + name: "bad database", withEngine: true, body: body(sqlServer, "a]b", "SELECT 1"), + wantCode: 400, wantBody: "database name", + }, + { + name: "unknown server", withEngine: true, body: body("/subscriptions/x/servers/none", "master", "SELECT 1"), + wantCode: 404, wantBody: "NOT_FOUND", + }, + { + name: "missing server", withEngine: true, body: `{"database":"master","query":"SELECT 1"}`, + wantCode: 400, wantBody: "server is required", + }, + { + name: "state only", withEngine: false, body: body(sqlServer, "master", "SELECT 1"), + wantCode: 409, wantBody: "state-only", + }, + {name: "bad json", withEngine: true, body: `{`, wantCode: 400, wantBody: "invalid request body"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h, _ := newSQLQueryDashboard(t, tt.withEngine) + rec := doProxy(t, h, http.MethodPost, sqlQueryURL, tt.body) + + assert.Equal(t, tt.wantCode, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), tt.wantBody) + }) + } +} + +func TestAzureSQLQueryReachesEngineOnlyWhenValid(t *testing.T) { + t.Parallel() + + h, eng := newSQLQueryDashboard(t, true) + + rec := doProxy( + t, + h, + http.MethodPost, + sqlQueryURL, + `{"server":"`+sqlServer+`","database":"app","query":"DROP TABLE t"}`, + ) + require.Equal(t, http.StatusBadRequest, rec.Code) + assert.Empty(t, eng.gotQ) + + rec = doProxy(t, h, http.MethodPost, sqlQueryURL, `{"server":"`+sqlServer+`","database":"app","query":"SELECT 2"}`) + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "app", eng.gotDB) + assert.Equal(t, "SELECT 2", eng.gotQ) +} + +func TestAzureSQLQueryMethodAllowList(t *testing.T) { + t.Parallel() + + h, _ := newSQLQueryDashboard(t, true) + + for _, m := range []string{http.MethodGet, http.MethodPut, http.MethodDelete} { + req := httptest.NewRequest(m, sqlQueryURL, strings.NewReader(`{}`)) + rec := httptest.NewRecorder() + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + assert.NotEqual(t, http.StatusOK, rec.Code, m) + } +} diff --git a/dashboard/cloud_extras_proxy_test.go b/dashboard/cloud_extras_proxy_test.go index 3cae256ab3..3c2af363f1 100644 --- a/dashboard/cloud_extras_proxy_test.go +++ b/dashboard/cloud_extras_proxy_test.go @@ -127,8 +127,6 @@ func TestAzureEventHubsProxyPeek(t *testing.T) { func TestAzureEventGridProxy(t *testing.T) { t.Parallel() - const base = "/dashboard/api/v1/azure/eventgrid" - cfg := func() dashboard.Config { b := azureeventgrid.NewInMemoryBackend() _, err := b.PutTopic("topic1", "/subscriptions/s/topics/topic1", "") @@ -137,25 +135,11 @@ func TestAzureEventGridProxy(t *testing.T) { return dashboard.Config{AzureEventGridOps: azureeventgrid.NewHandler(b, azureeventgrid.Settings{})} } - const event = `{"id":"1","subject":"/s","eventType":"T","eventTime":"2026-01-01T00:00:00Z","dataVersion":"1"}` - runProxyCases(t, cfg, []proxyCase{ - {name: "topics", method: http.MethodGet, path: base + "/topics", wantCode: 200, wantBody: `"topic1"`}, - { - name: "publish", method: http.MethodPost, path: base + "/topics/topic1/events", - body: event, wantCode: 200, wantBody: `"published":1`, - }, - { - name: "publish array", method: http.MethodPost, path: base + "/topics/topic1/events", - body: "[" + event + "," + event + "]", wantCode: 200, wantBody: `"published":2`, - }, - { - name: "publish invalid", method: http.MethodPost, path: base + "/topics/topic1/events", - body: `{"id":"1"}`, wantCode: 400, wantBody: "required property", - }, + {name: "topics", method: http.MethodGet, path: egBase + "/topics", wantCode: 200, wantBody: `"topic1"`}, { - name: "publish missing topic", method: http.MethodPost, path: base + "/topics/nope/events", - body: event, wantCode: 404, wantBody: "NOT_FOUND", + name: "publish", method: http.MethodPost, path: egBase + "/topics/topic1/events", + body: `{"eventType":"T","subject":"/s"}`, wantCode: 200, wantBody: `"published":1`, }, }) } diff --git a/dashboard/gcp_proxy_common.go b/dashboard/gcp_proxy_common.go index 336efbbd6e..4d1689f859 100644 --- a/dashboard/gcp_proxy_common.go +++ b/dashboard/gcp_proxy_common.go @@ -128,6 +128,10 @@ func (h *DashboardHandler) registerGCPRESTProxyVersion(prefix, version string, r r := c.Request() + if r.Method == http.MethodPost { + c.Response().Header().Set("Cache-Control", "no-store") + } + segs, verb, ok := parseGCPRESTPath(strings.TrimPrefix(r.URL.EscapedPath(), prefix), version) if !ok { return c.String(http.StatusBadRequest, "unsupported GCP proxy request") @@ -239,6 +243,8 @@ func (h *DashboardHandler) registerConsolePageRoutes() { h.registerAzureContainerAppsRoutes() h.registerAzureVMRoutes() h.registerAzureAKSRoutes() + h.registerAzureNetworkRoutes() + h.registerStorageAndWorkflowRoutes() h.registerOCIContainerEngineRoutes() h.registerOCIMySQLRoutes() h.registerOCILoggingRoutes() @@ -249,10 +255,14 @@ func (h *DashboardHandler) registerConsolePageRoutes() { h.registerOCILoadBalancerRoutes() h.registerOCIArtifactsRoutes() h.registerOCIContainerInstancesRoutes() + h.registerOCIDNSRoutes() + h.registerOCIDatabaseRoutes() h.registerGCPIdentityToolkitRoutes() h.registerAzureFlexDBRoutes() h.registerGCPAPIGatewayRoutes() h.registerGCPBatchRoutes() h.registerGCPResourceManagerRoutes() h.registerAzureAPIMRoutes() + h.registerAzureGraphRoutes() + h.registerAzureAuthzRoutes() } diff --git a/dashboard/gcpartifactregistry_proxy.go b/dashboard/gcpartifactregistry_proxy.go index 956b380b57..5a0d9dca7c 100644 --- a/dashboard/gcpartifactregistry_proxy.go +++ b/dashboard/gcpartifactregistry_proxy.go @@ -4,43 +4,105 @@ import ( "context" "net/http" "net/url" + "strconv" "strings" "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb" + "github.com/labstack/echo/v5" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + + "github.com/blackbirdworks/gopherstack/services/gcpartifactregistry" ) const ( gcpArtifactRegistryProxyPrefix = "/dashboard/api/v1/gcp/artifactregistry" - arLocationSegs = 4 - arReposSegs = 5 - arRepoSegs = 6 - arRepoSubSegs = 7 - arPkgSegs = 8 - arPkgSubSegs = 9 - arVersionSegs = 10 - arLocationsIdx = 2 - arReposIdx = 4 - arSubIdx = 6 - arPkgSubIdx = 8 - arReposKind = "repositories" - arPackagesKind = "packages" - arFilesKind = "files" - arImagesKind = "dockerImages" - arVersionsKind = "versions" - arTagsKind = "tags" + arLocationSegs = 4 + arReposSegs = 5 + arRepoSegs = 6 + arRepoSubSegs = 7 + arPkgSegs = 8 + arPkgSubSegs = 9 + arVersionSegs = 10 + arLocationsIdx = 2 + arReposIdx = 4 + arSubIdx = 6 + arPkgSubIdx = 8 + arReposKind = "repositories" + arPackagesKind = "packages" + arFilesKind = "files" + arImagesKind = "dockerImages" + arVersionsKind = "versions" + arTagsKind = "tags" + arVerbCleanup = "cleanupPreview" + arLocationsKind = "locations" + + arDownloadMaxBytes = 32 << 20 + arFileSegs = 8 ) // registerGCPArtifactRegistryRoutes exposes repositories, packages, versions, tags and files to the SPA. func (h *DashboardHandler) registerGCPArtifactRegistryRoutes() { var route gcpRESTRoute + api := (*gcpArtifactRegistryAPI)(nil) + if ops := h.config.GCPArtifactRegistryOps; ops != nil { - route = (&gcpArtifactRegistryAPI{ar: ops.API()}).route + api = &gcpArtifactRegistryAPI{ar: ops.API()} + route = api.route } h.registerGCPRESTProxy(gcpArtifactRegistryProxyPrefix, route) + h.SubRouter.Add(http.MethodGet, gcpArtifactRegistryProxyPrefix+"/download", func(c *echo.Context) error { + if api == nil { + return c.JSON(http.StatusServiceUnavailable, map[string]string{keyError: errServiceUnavailable}) + } + + return api.download(c) + }) +} + +// arFileNameOK accepts only projects/P/locations/L/repositories/R/files/F names, with F still path-escaped. +func arFileNameOK(name string) bool { + segs := strings.Split(name, "/") + if len(segs) != arFileSegs || segs[0] != gcpRESTProjectsSeg || segs[arLocationsIdx] != arLocationsKind || + segs[arReposIdx] != arReposKind || segs[arSubIdx] != arFilesKind { + return false + } + + for _, s := range segs { + if s == "" || s == "." || s == ".." { + return false + } + } + + return true +} + +// download streams one repository file as an attachment, size-capped and never rendered by the browser. +func (a *gcpArtifactRegistryAPI) download(c *echo.Context) error { + name := c.QueryParam("name") + if !arFileNameOK(name) { + return writeGCPRESTError(c, status.Error(codes.InvalidArgument, "invalid file name")) + } + + data, err := a.ar.FileContent(c.Request().Context(), name, arDownloadMaxBytes) + if err != nil { + return writeGCPRESTError(c, err) + } + + leaf, _ := url.PathUnescape(name[strings.LastIndex(name, "/")+1:]) + leaf = strings.NewReplacer("\"", "_", "\r", "_", "\n", "_", "/", "_").Replace(leaf) + + h := c.Response().Header() + h.Set("Content-Type", "application/octet-stream") + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Content-Disposition", "attachment; filename="+strconv.Quote(leaf)) + + return c.Blob(http.StatusOK, "application/octet-stream", data) } // arName joins the first n segments, re-escaping package, version and file IDs @@ -60,13 +122,14 @@ func arName(r gcpRESTCall, n int) string { } type gcpArtifactRegistryAPI struct { - ar artifactregistrypb.ArtifactRegistryServer + ar *gcpartifactregistry.API } func (a *gcpArtifactRegistryAPI) route(ctx context.Context, r gcpRESTCall) (proto.Message, error) { n := len(r.segs) if n < arReposSegs || n > arVersionSegs || r.segs[0] != gcpRESTProjectsSeg || - r.segs[arLocationsIdx] != "locations" || r.segs[arReposIdx] != arReposKind || r.verb != "" { + r.segs[arLocationsIdx] != arLocationsKind || r.segs[arReposIdx] != arReposKind || + (r.verb != "" && (r.verb != arVerbCleanup || n != arRepoSegs)) { return nil, errGCPUnsupported() } @@ -99,6 +162,10 @@ func (a *gcpArtifactRegistryAPI) repositories(ctx context.Context, r gcpRESTCall func (a *gcpArtifactRegistryAPI) repository(ctx context.Context, r gcpRESTCall) (proto.Message, error) { name := r.name(arRepoSegs) + if r.verb == arVerbCleanup { + return a.cleanupPreview(ctx, r, name) + } + switch r.method { case http.MethodGet: return a.ar.GetRepository(ctx, &artifactregistrypb.GetRepositoryRequest{Name: name}) @@ -109,6 +176,41 @@ func (a *gcpArtifactRegistryAPI) repository(ctx context.Context, r gcpRESTCall) return nil, errGCPUnsupported() } +// cleanupPreview reports what the repository's cleanup policies would delete now; nothing is deleted. +func (a *gcpArtifactRegistryAPI) cleanupPreview( + ctx context.Context, r gcpRESTCall, name string, +) (proto.Message, error) { + if r.method != http.MethodGet { + return nil, errGCPUnsupported() + } + + repo, err := a.ar.GetRepository(ctx, &artifactregistrypb.GetRepositoryRequest{Name: name}) + if err != nil { + return nil, err //nolint:wrapcheck // backend status passes through + } + + names, err := a.ar.PreviewCleanup(ctx, name) + if err != nil { + return nil, err //nolint:wrapcheck // backend status passes through + } + + list := make([]any, 0, len(names)) + for _, n := range names { + list = append(list, n) + } + + out, sErr := structpb.NewStruct(map[string]any{ + "dryRun": repo.GetCleanupPolicyDryRun(), + "policies": float64(len(repo.GetCleanupPolicies())), + arVersionsKind: list, + }) + if sErr != nil { + return nil, status.Error(codes.Internal, sErr.Error()) + } + + return out, nil +} + func (a *gcpArtifactRegistryAPI) repoChildren(ctx context.Context, r gcpRESTCall) (proto.Message, error) { if r.method != http.MethodGet { return nil, errGCPUnsupported() @@ -175,12 +277,27 @@ func (a *gcpArtifactRegistryAPI) pkgChildren(ctx context.Context, r gcpRESTCall) } func (a *gcpArtifactRegistryAPI) version(ctx context.Context, r gcpRESTCall) (proto.Message, error) { - if r.segs[arSubIdx] != arPackagesKind || r.segs[arPkgSubIdx] != arVersionsKind { + if r.segs[arSubIdx] != arPackagesKind { return nil, errGCPUnsupported() } name := arName(r, arVersionSegs) + if r.segs[arPkgSubIdx] == arTagsKind { + switch r.method { + case http.MethodGet: + return a.ar.GetTag(ctx, &artifactregistrypb.GetTagRequest{Name: name}) + case http.MethodDelete: + return a.ar.DeleteTag(ctx, &artifactregistrypb.DeleteTagRequest{Name: name}) + } + + return nil, errGCPUnsupported() + } + + if r.segs[arPkgSubIdx] != arVersionsKind { + return nil, errGCPUnsupported() + } + switch r.method { case http.MethodGet: view := artifactregistrypb.VersionView(artifactregistrypb.VersionView_value[r.query.Get("view")]) diff --git a/dashboard/gcpartifactregistry_proxy_test.go b/dashboard/gcpartifactregistry_proxy_test.go new file mode 100644 index 0000000000..79fd410a30 --- /dev/null +++ b/dashboard/gcpartifactregistry_proxy_test.go @@ -0,0 +1,238 @@ +package dashboard_test + +import ( + "bytes" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/gcpartifactregistry" +) + +type upstreamPassword = artifactregistrypb.RemoteRepositoryConfig_UpstreamCredentials_UsernamePasswordCredentials + +const ( + arTestRoot = "/dashboard/api/v1/gcp/artifactregistry" + arTestParent = "projects/p/locations/us-central1" + arTestRepo = arTestRoot + "/v1/" + arTestParent + "/repositories/gen" +) + +func newArtifactRegistryDashboard(t *testing.T) *dashboard.DashboardHandler { + t.Helper() + + ar := gcpartifactregistry.NewHandler(gcpartifactregistry.NewInMemoryBackend()) + + _, err := ar.Backend.CreateRepository(&artifactregistrypb.CreateRepositoryRequest{ + Parent: arTestParent, RepositoryId: "gen", + Repository: &artifactregistrypb.Repository{ + Format: artifactregistrypb.Repository_GENERIC, + CleanupPolicyDryRun: true, + CleanupPolicies: map[string]*artifactregistrypb.CleanupPolicy{"old": { + Id: "old", Action: artifactregistrypb.CleanupPolicy_DELETE, + ConditionType: &artifactregistrypb.CleanupPolicy_Condition{ + Condition: &artifactregistrypb.CleanupPolicyCondition{OlderThan: durationpb.New(time.Nanosecond)}, + }, + }}, + }, + }) + require.NoError(t, err) + + _, err = ar.Backend.CreateRepository(&artifactregistrypb.CreateRepositoryRequest{ + Parent: arTestParent, RepositoryId: "remote", + Repository: &artifactregistrypb.Repository{ + Format: artifactregistrypb.Repository_DOCKER, Mode: artifactregistrypb.Repository_REMOTE_REPOSITORY, + ModeConfig: &artifactregistrypb.Repository_RemoteRepositoryConfig{ + RemoteRepositoryConfig: &artifactregistrypb.RemoteRepositoryConfig{ + Description: "hub", + DisableUpstreamValidation: true, + UpstreamCredentials: &artifactregistrypb.RemoteRepositoryConfig_UpstreamCredentials{ + Credentials: &artifactregistrypb.RemoteRepositoryConfig_UpstreamCredentials_UsernamePasswordCredentials_{ + UsernamePasswordCredentials: &upstreamPassword{ + Username: "bot", + PasswordSecretVersion: "projects/p/secrets/hub/versions/1", + }, + }, + }, + }, + }, + }, + }) + require.NoError(t, err) + + var body bytes.Buffer + + mw := multipart.NewWriter(&body) + meta, err := mw.CreatePart(map[string][]string{"Content-Type": {"application/json"}}) + require.NoError(t, err) + _, err = meta.Write([]byte(`{"packageId":"pkg","versionId":"1.0.0","filename":"a.txt"}`)) + require.NoError(t, err) + media, err := mw.CreatePart(map[string][]string{"Content-Type": {"application/octet-stream"}}) + require.NoError(t, err) + _, err = media.Write([]byte("hello-artifact")) + require.NoError(t, err) + require.NoError(t, mw.Close()) + + req := httptest.NewRequest(http.MethodPost, + "/upload/v1/"+arTestParent+"/repositories/gen/genericArtifacts:create?uploadType=multipart", &body) + req.Header.Set("Content-Type", "multipart/related; boundary="+mw.Boundary()) + + rec := httptest.NewRecorder() + ar.Backend.FileExtras().Handler.ServeHTTP(rec, req) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + return dashboard.NewHandler(dashboard.Config{GCPArtifactRegistryOps: ar}) +} + +func TestGCPArtifactRegistryProxyRoutes(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + method string + path string + wantBody string + wantCode int + }{ + {"cleanup preview", http.MethodGet, arTestRepo + ":cleanupPreview", `"dryRun":true`, http.StatusOK}, + { + "cleanup preview lists", + http.MethodGet, + arTestRepo + ":cleanupPreview", + "packages/pkg/versions/1.0.0", + http.StatusOK, + }, + { + "cleanup preview is read only", + http.MethodPost, + arTestRepo + ":cleanupPreview", + "unsupported", + http.StatusNotFound, + }, + { + "cleanup preview on a list", + http.MethodGet, + arTestRoot + "/v1/" + arTestParent + ":cleanupPreview", + "unsupported", + http.StatusNotFound, + }, + { + "remote detail names the secret", + http.MethodGet, + arTestRoot + "/v1/" + arTestParent + "/repositories/remote", + "projects/p/secrets/hub/versions/1", + http.StatusOK, + }, + { + "tag get in non docker repo", + http.MethodGet, + arTestRepo + "/packages/pkg/tags/t1", + "FAILED_PRECONDITION", + http.StatusBadRequest, + }, + { + "tag delete in non docker repo", + http.MethodDelete, + arTestRepo + "/packages/pkg/tags/t1", + "FAILED_PRECONDITION", + http.StatusBadRequest, + }, + {"tag create refused", http.MethodPost, arTestRepo + "/packages/pkg/tags", "unsupported", http.StatusNotFound}, + { + "tag patch refused", + http.MethodPatch, + arTestRepo + "/packages/pkg/tags/t1", + "unsupported", + http.StatusNotFound, + }, + { + "version delete", + http.MethodDelete, + arTestRepo + "/packages/pkg/versions/1.0.0", + `"done":true`, + http.StatusOK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := serveGCP(t, newArtifactRegistryDashboard(t), tt.method, tt.path, "") + assert.Equal(t, tt.wantCode, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), tt.wantBody) + }) + } +} + +func TestGCPArtifactRegistryProxyRemoteNeverLeaksSecretValue(t *testing.T) { + t.Parallel() + + rec := serveGCP(t, newArtifactRegistryDashboard(t), http.MethodGet, + arTestRoot+"/v1/"+arTestParent+"/repositories/remote", "") + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.NotContains(t, rec.Body.String(), "hunter2") + assert.Contains(t, rec.Body.String(), "passwordSecretVersion") +} + +func TestGCPArtifactRegistryProxyDownload(t *testing.T) { + t.Parallel() + + h := newArtifactRegistryDashboard(t) + files := serveGCP(t, h, http.MethodGet, arTestRepo+"/files", "") + require.Equal(t, http.StatusOK, files.Code, files.Body.String()) + + name := extractJSONString(t, files.Body.String(), `"name":"`) + require.Contains(t, name, "/files/") + + tests := []struct { + name string + query string + wantBody string + wantCode int + }{ + {"file", name, "hello-artifact", http.StatusOK}, + {"empty", "", "invalid file name", http.StatusBadRequest}, + {"traversal", arTestParent + "/repositories/gen/files/..", "invalid file name", http.StatusBadRequest}, + {"wrong kind", arTestParent + "/repositories/gen/packages/pkg", "invalid file name", http.StatusBadRequest}, + {"too short", arTestParent + "/repositories/gen", "invalid file name", http.StatusBadRequest}, + {"missing file", arTestParent + "/repositories/gen/files/nope", "NOT_FOUND", http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := serveGCP( + t, + h, + http.MethodGet, + arTestRoot+"/download?name="+strings.ReplaceAll(tt.query, "%", "%25"), + "", + ) + assert.Equal(t, tt.wantCode, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), tt.wantBody) + + if tt.wantCode == http.StatusOK { + assert.Equal(t, "application/octet-stream", rec.Header().Get("Content-Type")) + assert.Equal(t, "nosniff", rec.Header().Get("X-Content-Type-Options")) + assert.Contains(t, rec.Header().Get("Content-Disposition"), "attachment") + } + }) + } +} + +func TestGCPArtifactRegistryProxyDownloadUnavailable(t *testing.T) { + t.Parallel() + + rec := serveGCP(t, dashboard.NewHandler(dashboard.Config{}), http.MethodGet, arTestRoot+"/download?name=x", "") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) +} diff --git a/dashboard/gcpcloudbuild_mask.go b/dashboard/gcpcloudbuild_mask.go new file mode 100644 index 0000000000..0f21396e8a --- /dev/null +++ b/dashboard/gcpcloudbuild_mask.go @@ -0,0 +1,143 @@ +package dashboard + +import ( + "strings" + + "cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb" + "cloud.google.com/go/longrunning/autogen/longrunningpb" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +const cbMasked = "***" + +// cbSecretName reports whether a substitution or environment variable name looks like it holds a secret. +func cbSecretName(name string) bool { + u := strings.ToUpper(name) + + for _, w := range []string{"SECRET", "PASSWORD", "PASSWD", "TOKEN", "KEY", "CREDENTIAL", "PRIVATE"} { + if strings.Contains(u, w) { + return true + } + } + + return false +} + +func cbMaskSubstitutions(subs map[string]string) { + for k := range subs { + if cbSecretName(k) { + subs[k] = cbMasked + } + } +} + +// cbMaskBuild clears KMS-encrypted secretEnv ciphertexts (names stay) and the values of secret-looking +// substitutions and step environment variables. +func cbMaskBuild(b *cloudbuildpb.Build) *cloudbuildpb.Build { + if b == nil { + return nil + } + + out, _ := proto.Clone(b).(*cloudbuildpb.Build) + + for _, s := range out.GetSecrets() { + for k := range s.GetSecretEnv() { + s.SecretEnv[k] = nil + } + } + + for _, in := range out.GetAvailableSecrets().GetInline() { + for k := range in.GetEnvMap() { + in.EnvMap[k] = nil + } + } + + cbMaskSubstitutions(out.GetSubstitutions()) + + for _, step := range out.GetSteps() { + for i, e := range step.GetEnv() { + if k, _, ok := strings.Cut(e, "="); ok && cbSecretName(k) { + step.Env[i] = k + "=" + cbMasked + } + } + } + + return out +} + +func cbMaskTrigger(t *cloudbuildpb.BuildTrigger) *cloudbuildpb.BuildTrigger { + if t == nil { + return nil + } + + out, _ := proto.Clone(t).(*cloudbuildpb.BuildTrigger) + cbMaskSubstitutions(out.GetSubstitutions()) + + if b := out.GetBuild(); b != nil { + out.BuildTemplate = &cloudbuildpb.BuildTrigger_Build{Build: cbMaskBuild(b)} + } + + return out +} + +func cbMaskAny(a *anypb.Any) *anypb.Any { + if a == nil { + return nil + } + + m, err := a.UnmarshalNew() + if err != nil { + return a + } + + masked := cbMaskMessage(m) + + out, err := anypb.New(masked) + if err != nil { + return a + } + + return out +} + +// cbMaskMessage returns a copy of m that is safe to show: Build, BuildTrigger, their list responses and the +// operations wrapping them. Other messages pass through unchanged. +func cbMaskMessage(m proto.Message) proto.Message { + switch t := m.(type) { + case *cloudbuildpb.Build: + return cbMaskBuild(t) + case *cloudbuildpb.BuildTrigger: + return cbMaskTrigger(t) + case *cloudbuildpb.BuildOperationMetadata: + out, _ := proto.Clone(t).(*cloudbuildpb.BuildOperationMetadata) + out.Build = cbMaskBuild(t.GetBuild()) + + return out + case *cloudbuildpb.ListBuildsResponse: + out, _ := proto.Clone(t).(*cloudbuildpb.ListBuildsResponse) + for i, b := range out.GetBuilds() { + out.Builds[i] = cbMaskBuild(b) + } + + return out + case *cloudbuildpb.ListBuildTriggersResponse: + out, _ := proto.Clone(t).(*cloudbuildpb.ListBuildTriggersResponse) + for i, tr := range out.GetTriggers() { + out.Triggers[i] = cbMaskTrigger(tr) + } + + return out + case *longrunningpb.Operation: + out, _ := proto.Clone(t).(*longrunningpb.Operation) + out.Metadata = cbMaskAny(t.GetMetadata()) + + if r, ok := t.GetResult().(*longrunningpb.Operation_Response); ok { + out.Result = &longrunningpb.Operation_Response{Response: cbMaskAny(r.Response)} + } + + return out + } + + return m +} diff --git a/dashboard/gcpcloudbuild_proxy.go b/dashboard/gcpcloudbuild_proxy.go index 192b66751f..2637d81269 100644 --- a/dashboard/gcpcloudbuild_proxy.go +++ b/dashboard/gcpcloudbuild_proxy.go @@ -3,6 +3,7 @@ package dashboard import ( "context" "net/http" + "regexp" "cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb" "google.golang.org/grpc/codes" @@ -24,12 +25,21 @@ const ( cbLocationsKind = "locations" cbBuildsKind = "builds" cbTriggersKind = "triggers" + cbPoolsKind = "workerPools" cbLogLeaf = "log" cbVerbCancel = "cancel" cbVerbRetry = "retry" cbVerbRun = "run" + cbVerbApprove = "approve" + + cbMaxSubs = 20 + cbMaxSubValue = 1024 + cbMaxComment = 1000 + cbMaxRevisionID = 256 ) +var cbSubKeyRE = regexp.MustCompile(`^_[A-Z0-9_]{1,64}$`) + // registerGCPCloudBuildRoutes exposes Cloud Build builds, logs and triggers to the SPA. func (h *DashboardHandler) registerGCPCloudBuildRoutes() { var route gcpRESTRoute @@ -45,7 +55,21 @@ type gcpCloudBuildAPI struct { api *gcpcloudbuildbackend.API } +// route dispatches an allow-listed request and hides secret material in whatever comes back. func (a *gcpCloudBuildAPI) route(ctx context.Context, r gcpRESTCall) (proto.Message, error) { + out, err := a.dispatch(ctx, r) + if err != nil { + return nil, err + } + + if out == nil { + return nil, nil //nolint:nilnil // void result renders as an empty object + } + + return cbMaskMessage(out), nil +} + +func (a *gcpCloudBuildAPI) dispatch(ctx context.Context, r gcpRESTCall) (proto.Message, error) { n := len(r.segs) if n < cbListSegs || n > cbLogSegs || r.segs[0] != gcpRESTProjectsSeg || r.segs[cbLocationsIdx] != cbLocationsKind { @@ -59,6 +83,10 @@ func (a *gcpCloudBuildAPI) route(ctx context.Context, r gcpRESTCall) (proto.Mess if n <= cbItemSegs { return a.triggers(ctx, r) } + case cbPoolsKind: + if n <= cbItemSegs { + return a.pools(ctx, r) + } } return nil, errGCPUnsupported() @@ -92,11 +120,100 @@ func (a *gcpCloudBuildAPI) build(ctx context.Context, r gcpRESTCall) (proto.Mess return a.api.CancelBuild(ctx, &cloudbuildpb.CancelBuildRequest{Name: name}) case r.method == http.MethodPost && r.verb == cbVerbRetry: return a.api.RetryBuild(ctx, &cloudbuildpb.RetryBuildRequest{Name: name}) + case r.method == http.MethodPost && r.verb == cbVerbApprove: + return a.approve(ctx, r, name) } return nil, errGCPUnsupported() } +// approve rebuilds the request from the decision and comment alone. +func (a *gcpCloudBuildAPI) approve(ctx context.Context, r gcpRESTCall, name string) (proto.Message, error) { + var in cloudbuildpb.ApproveBuildRequest + if err := r.unmarshal(&in); err != nil { + return nil, err + } + + decision := in.GetApprovalResult().GetDecision() + if decision != cloudbuildpb.ApprovalResult_APPROVED && decision != cloudbuildpb.ApprovalResult_REJECTED { + return nil, status.Error(codes.InvalidArgument, "decision must be APPROVED or REJECTED") + } + + comment := in.GetApprovalResult().GetComment() + if len(comment) > cbMaxComment { + return nil, status.Error(codes.InvalidArgument, "comment is too long") + } + + return a.api.ApproveBuild(ctx, &cloudbuildpb.ApproveBuildRequest{ + Name: name, + ApprovalResult: &cloudbuildpb.ApprovalResult{Decision: decision, Comment: comment}, + }) +} + +// pools is read-only: worker pools are listed and fetched, never changed from the dashboard. +func (a *gcpCloudBuildAPI) pools(ctx context.Context, r gcpRESTCall) (proto.Message, error) { + if r.method != http.MethodGet || r.verb != "" { + return nil, errGCPUnsupported() + } + + if len(r.segs) == cbListSegs { + return a.api.ListWorkerPools(ctx, &cloudbuildpb.ListWorkerPoolsRequest{ + Parent: r.name(cbListSegs - 1), PageSize: r.pageSize(), PageToken: r.query.Get("pageToken"), + }) + } + + return a.api.GetWorkerPool(ctx, &cloudbuildpb.GetWorkerPoolRequest{Name: r.name(cbItemSegs)}) +} + +// runRequest rebuilds the run request from the revision and plain substitutions alone. +func cbRunRequest(r gcpRESTCall, name string) (*cloudbuildpb.RunBuildTriggerRequest, error) { + var in cloudbuildpb.RunBuildTriggerRequest + if err := r.unmarshal(&in); err != nil { + return nil, err + } + + src := in.GetSource() + out := &cloudbuildpb.RunBuildTriggerRequest{Name: name} + + if src == nil { + return out, nil + } + + if len(src.GetSubstitutions()) > cbMaxSubs { + return nil, status.Error(codes.InvalidArgument, "too many substitutions") + } + + subs := map[string]string{} + + for k, v := range src.GetSubstitutions() { + if !cbSubKeyRE.MatchString(k) || len(v) > cbMaxSubValue { + return nil, status.Error(codes.InvalidArgument, "invalid substitution "+k) + } + + subs[k] = v + } + + rs := &cloudbuildpb.RepoSource{Substitutions: subs} + + switch { + case src.GetBranchName() != "": + rs.Revision = &cloudbuildpb.RepoSource_BranchName{BranchName: src.GetBranchName()} + case src.GetTagName() != "": + rs.Revision = &cloudbuildpb.RepoSource_TagName{TagName: src.GetTagName()} + case src.GetCommitSha() != "": + rs.Revision = &cloudbuildpb.RepoSource_CommitSha{CommitSha: src.GetCommitSha()} + } + + if len(src.GetBranchName()) > cbMaxRevisionID || len(src.GetTagName()) > cbMaxRevisionID || + len(src.GetCommitSha()) > cbMaxRevisionID { + return nil, status.Error(codes.InvalidArgument, "revision is too long") + } + + out.Source = rs + + return out, nil +} + func (a *gcpCloudBuildAPI) buildLog(ctx context.Context, r gcpRESTCall) (proto.Message, error) { if r.method != http.MethodGet || r.verb != "" || r.segs[cbItemSegs] != cbLogLeaf { return nil, errGCPUnsupported() @@ -136,13 +253,11 @@ func (a *gcpCloudBuildAPI) triggers(ctx context.Context, r gcpRESTCall) (proto.M case r.method == http.MethodDelete && r.verb == "": return a.api.DeleteBuildTrigger(ctx, &cloudbuildpb.DeleteBuildTriggerRequest{Name: name}) case r.method == http.MethodPost && r.verb == cbVerbRun: - req := &cloudbuildpb.RunBuildTriggerRequest{Name: name} - if err := r.unmarshal(req); err != nil { + req, err := cbRunRequest(r, name) + if err != nil { return nil, err } - req.Name = name - return a.api.RunBuildTrigger(ctx, req) } diff --git a/dashboard/gcpcloudbuild_proxy_test.go b/dashboard/gcpcloudbuild_proxy_test.go new file mode 100644 index 0000000000..8063163a90 --- /dev/null +++ b/dashboard/gcpcloudbuild_proxy_test.go @@ -0,0 +1,251 @@ +package dashboard_test + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb" + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/gcpcloudbuild" +) + +const ( + cbTestLoc = "/dashboard/api/v1/gcp/cloudbuild/v1/projects/p/locations/global" + cbTestParent = "projects/p/locations/global" + cbTestCipher = "c3VwZXItc2VjcmV0LWNpcGhlcnRleHQ=" + cbTestPlain = "super-secret-ciphertext" + cbTestSubPass = "hunter2-substitution" +) + +func newCloudBuildDashboard(t *testing.T) (*dashboard.DashboardHandler, string) { + t.Helper() + + h := gcpcloudbuild.NewHandler(gcpcloudbuild.NewInMemoryBackend()) + + trig, err := h.API().CreateBuildTrigger(t.Context(), &cloudbuildpb.CreateBuildTriggerRequest{ + Parent: cbTestParent, + Trigger: &cloudbuildpb.BuildTrigger{ + Name: "gated", + Substitutions: map[string]string{"_API_TOKEN": cbTestSubPass, "_REGION": "us"}, + BuildTemplate: &cloudbuildpb.BuildTrigger_Build{Build: &cloudbuildpb.Build{ + Steps: []*cloudbuildpb.BuildStep{ + {Name: "busybox", Args: []string{"true"}, Env: []string{"DB_PASSWORD=hunter2", "MODE=ci"}}, + }, + Approval: &cloudbuildpb.BuildApproval{ + Config: &cloudbuildpb.ApprovalConfig{ApprovalRequired: true}, + }, + Substitutions: map[string]string{"_API_TOKEN": cbTestSubPass}, + Secrets: []*cloudbuildpb.Secret{{ + KmsKeyName: "projects/p/locations/global/keyRings/r/cryptoKeys/k", + SecretEnv: map[string][]byte{"TOKEN": []byte(cbTestPlain)}, + }}, + AvailableSecrets: &cloudbuildpb.Secrets{Inline: []*cloudbuildpb.InlineSecret{{ + KmsKeyName: "projects/p/locations/global/keyRings/r/cryptoKeys/k", + EnvMap: map[string][]byte{"INLINE": []byte(cbTestPlain)}, + }}}, + }}, + }, + }) + require.NoError(t, err) + + return dashboard.NewHandler(dashboard.Config{GCPCloudBuildOps: h}), trig.GetId() +} + +func serveGCP(t *testing.T, h *dashboard.DashboardHandler, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + + return rec +} + +func TestGCPCloudBuildProxyMasksSecrets(t *testing.T) { + t.Parallel() + + h, trigID := newCloudBuildDashboard(t) + + trig := serveGCP(t, h, http.MethodGet, cbTestLoc+"/triggers/"+trigID, "") + require.Equal(t, http.StatusOK, trig.Code, trig.Body.String()) + + run := serveGCP(t, h, http.MethodPost, cbTestLoc+"/triggers/"+trigID+":run", "{}") + require.Equal(t, http.StatusOK, run.Code, run.Body.String()) + + builds := serveGCP(t, h, http.MethodGet, cbTestLoc+"/builds", "") + require.Equal(t, http.StatusOK, builds.Code, builds.Body.String()) + + bodies := map[string]string{"trigger": trig.Body.String(), "run": run.Body.String(), "builds": builds.Body.String()} + + for name, body := range bodies { + assert.NotContains(t, body, cbTestCipher, name) + assert.NotContains(t, body, cbTestPlain, name) + assert.NotContains(t, body, cbTestSubPass, name) + assert.NotContains(t, body, "hunter2", name) + } + + assert.Contains(t, trig.Body.String(), `"_REGION":"us"`) + assert.Contains(t, trig.Body.String(), `"_API_TOKEN":"***"`) + assert.Contains(t, builds.Body.String(), `"TOKEN"`) + assert.Contains(t, builds.Body.String(), "MODE=ci") +} + +func pendingBuild(t *testing.T) (*dashboard.DashboardHandler, string) { + t.Helper() + + h, trigID := newCloudBuildDashboard(t) + run := serveGCP(t, h, http.MethodPost, cbTestLoc+"/triggers/"+trigID+":run", + `{"source":{"branchName":"main","projectId":"evil","dir":"/etc"}}`) + require.Equal(t, http.StatusOK, run.Code, run.Body.String()) + + list := serveGCP(t, h, http.MethodGet, cbTestLoc+"/builds", "") + require.Equal(t, http.StatusOK, list.Code, list.Body.String()) + require.Contains(t, list.Body.String(), `"PENDING"`) + + return h, cbTestLoc + "/builds/" + extractJSONString(t, list.Body.String(), `"id":"`) +} + +func TestGCPCloudBuildProxyApproval(t *testing.T) { + t.Parallel() + + const reject = `{"approvalResult":{"decision":"REJECTED","comment":"no","approverAccount":"x@y"}}` + + tests := []struct { + name string + prior string + body string + wantBody string + wantCode int + }{ + {"missing decision", "", `{}`, "decision must be", http.StatusBadRequest}, + { + "unspecified decision", + "", + `{"approvalResult":{"decision":"DECISION_UNSPECIFIED"}}`, + "decision must be", + http.StatusBadRequest, + }, + { + "long comment", + "", + `{"approvalResult":{"decision":"APPROVED","comment":"` + strings.Repeat("x", 1001) + `"}}`, + "comment is too long", + http.StatusBadRequest, + }, + {"bad body", "", `{bad`, "invalid request body", http.StatusBadRequest}, + {"reject", "", reject, "operations", http.StatusOK}, + {"approve", "", `{"approvalResult":{"decision":"APPROVED"}}`, "operations", http.StatusOK}, + { + "already decided", + reject, + `{"approvalResult":{"decision":"APPROVED"}}`, + "not awaiting approval", + http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h, build := pendingBuild(t) + + if tt.prior != "" { + require.Equal(t, http.StatusOK, serveGCP(t, h, http.MethodPost, build+":approve", tt.prior).Code) + } + + rec := serveGCP(t, h, http.MethodPost, build+":approve", tt.body) + assert.Equal(t, tt.wantCode, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), tt.wantBody) + assert.NotContains(t, rec.Body.String(), "x@y") + }) + } +} + +func extractJSONString(t *testing.T, body, key string) string { + t.Helper() + + _, rest, ok := strings.Cut(body, key) + require.True(t, ok, body) + + v, _, _ := strings.Cut(rest, `"`) + + return v +} + +func TestGCPCloudBuildProxyRoutes(t *testing.T) { + t.Parallel() + + h, trigID := newCloudBuildDashboard(t) + trig := cbTestLoc + "/triggers/" + trigID + + tests := []struct { + name string + method string + path string + body string + wantBody string + wantCode int + }{ + {"pools list", http.MethodGet, cbTestLoc + "/workerPools", "", "{", http.StatusOK}, + {"pool missing", http.MethodGet, cbTestLoc + "/workerPools/nope", "", "NOT_FOUND", http.StatusNotFound}, + {"pool create refused", http.MethodPost, cbTestLoc + "/workerPools", "{}", "unsupported", http.StatusNotFound}, + { + "pool delete refused", + http.MethodDelete, + cbTestLoc + "/workerPools/p", + "", + "unsupported", + http.StatusNotFound, + }, + {"trigger list", http.MethodGet, cbTestLoc + "/triggers", "", `"gated"`, http.StatusOK}, + {"trigger create refused", http.MethodPost, cbTestLoc + "/triggers", "{}", "unsupported", http.StatusNotFound}, + {"trigger patch refused", http.MethodPatch, trig, "{}", "unsupported", http.StatusNotFound}, + {"build create refused", http.MethodPost, cbTestLoc + "/builds", "{}", "unsupported", http.StatusNotFound}, + { + "bad substitution key", + http.MethodPost, + trig + ":run", + `{"source":{"substitutions":{"bad key":"v"}}}`, + "invalid substitution", + http.StatusBadRequest, + }, + { + "bad substitution value", + http.MethodPost, + trig + ":run", + `{"source":{"substitutions":{"_A":"` + strings.Repeat("v", 1025) + `"}}}`, + "invalid substitution", + http.StatusBadRequest, + }, + {"unknown verb", http.MethodPost, trig + ":bogus", "", "unsupported", http.StatusNotFound}, + {"build bogus verb", http.MethodPost, cbTestLoc + "/builds/b1:bogus", "", "unsupported", http.StatusNotFound}, + {"build missing", http.MethodGet, cbTestLoc + "/builds/nope", "", "NOT_FOUND", http.StatusNotFound}, + {"cancel missing", http.MethodPost, cbTestLoc + "/builds/nope:cancel", "", "NOT_FOUND", http.StatusNotFound}, + {"other kind", http.MethodGet, cbTestLoc + "/other", "", "unsupported", http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := serveGCP(t, h, tt.method, tt.path, tt.body) + assert.Equal(t, tt.wantCode, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), tt.wantBody) + }) + } +} + +func TestGCPCloudBuildProxyUnavailable(t *testing.T) { + t.Parallel() + + rec := serveGCP(t, dashboard.NewHandler(dashboard.Config{}), http.MethodGet, cbTestLoc+"/builds", "") + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) +} diff --git a/dashboard/gcpcompute_body.go b/dashboard/gcpcompute_body.go new file mode 100644 index 0000000000..d6cc52801a --- /dev/null +++ b/dashboard/gcpcompute_body.go @@ -0,0 +1,223 @@ +package dashboard + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "regexp" + "strconv" +) + +const ( + computeMaxLabels = 64 + computeMaxLabelLen = 63 + computeMaxDiskGB = 65536 + computeNICAccessName = "External NAT" + computeNICAccessType = "ONE_TO_ONE_NAT" + computeKeyName = "name" +) + +var ( + errComputeBody = errors.New("invalid request body") + errComputeSnapName = errors.New("snapshot name must be 1-63 lowercase letters, digits or dashes") + errComputeInstFields = errors.New("a valid instance name and machine type are required") + errComputeOneEach = errors.New("exactly one boot disk and one network interface are required") + errComputeImage = errors.New("a boot image (projects//global/images/[family/]) is required") + errComputeDiskSize = errors.New("disk size is out of range") + errComputeNetwork = errors.New("invalid network") + errComputeSubnet = errors.New("invalid subnetwork") + errComputeLabels = errors.New("too many labels") + errComputeLabel = errors.New("invalid label") + + computeNameRE = regexp.MustCompile(`^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`) + computeMachineRE = regexp.MustCompile(`^(?:zones/[a-z0-9-]+/machineTypes/)?[a-z0-9][-a-z0-9]{0,62}$`) + computeImageRE = regexp.MustCompile( + `^projects/[a-z][-a-z0-9]{0,61}[a-z0-9]/global/images/(?:family/)?[a-z0-9][-a-z0-9]{0,62}$`, + ) + computeNetworkRE = regexp.MustCompile(`^(?:global/networks/)?[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`) + computeSubnetRE = regexp.MustCompile(`^(?:regions/[a-z0-9-]+/subnetworks/)?[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`) + computeLabelRE = regexp.MustCompile(`^[a-z0-9_-]{0,63}$`) +) + +type computeInstanceIn struct { + Labels map[string]string `json:"labels"` + Name string `json:"name"` + MachineType string `json:"machineType"` + Disks []computeDiskIn `json:"disks"` + NetworkInterfaces []computeNICIn `json:"networkInterfaces"` +} + +type computeDiskIn struct { + InitializeParams struct { + SourceImage string `json:"sourceImage"` + DiskSizeGb string `json:"diskSizeGb"` + } `json:"initializeParams"` +} + +type computeNICIn struct { + Network string `json:"network"` + Subnetwork string `json:"subnetwork"` + AccessConfigs []struct { + Type string `json:"type"` + } `json:"accessConfigs"` +} + +// gcpComputeBody returns the body forwarded upstream: nothing for reads and actions, and a body rebuilt from fixed +// fields for creating an instance or a snapshot, so metadata, service accounts and every other field are dropped. +func gcpComputeBody(method string, parts []string, in io.Reader) ([]byte, error) { + if method != http.MethodPost { + return nil, nil + } + + switch { + case len(parts) == computeScopedList && parts[computeScopedColl] == computeInstances: + return rebuildComputeInstance(in, parts[computeScopedList-2]) + case len(parts) == computeScopedVerb && parts[computeScopedVerb-1] == computeSnapshotVerb: + return rebuildComputeSnapshot(in) + } + + return nil, nil +} + +func decodeComputeBody(in io.Reader, out any) error { + if err := json.NewDecoder(in).Decode(out); err != nil { + return errComputeBody + } + + return nil +} + +func rebuildComputeSnapshot(in io.Reader) ([]byte, error) { + var req struct { + Name string `json:"name"` + } + + if err := decodeComputeBody(in, &req); err != nil { + return nil, err + } + + if !computeNameRE.MatchString(req.Name) { + return nil, errComputeSnapName + } + + return json.Marshal(map[string]string{computeKeyName: req.Name}) +} + +func rebuildComputeInstance(in io.Reader, zone string) ([]byte, error) { + var req computeInstanceIn + + if err := decodeComputeBody(in, &req); err != nil { + return nil, err + } + + if !computeNameRE.MatchString(req.Name) || !computeMachineRE.MatchString(req.MachineType) { + return nil, errComputeInstFields + } + + if len(req.Disks) != 1 || len(req.NetworkInterfaces) != 1 { + return nil, errComputeOneEach + } + + disk, err := rebuildComputeDisk(req.Disks[0]) + if err != nil { + return nil, err + } + + nic, err := rebuildComputeNIC(req.NetworkInterfaces[0]) + if err != nil { + return nil, err + } + + labels, err := rebuildComputeLabels(req.Labels) + if err != nil { + return nil, err + } + + machine := req.MachineType + if machine == shortComputeName(machine) { + machine = "zones/" + zone + "/machineTypes/" + machine + } + + out := map[string]any{ + computeKeyName: req.Name, "machineType": machine, + "disks": []any{disk}, "networkInterfaces": []any{nic}, + } + if len(labels) > 0 { + out["labels"] = labels + } + + return json.Marshal(out) +} + +func shortComputeName(s string) string { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == '/' { + return s[i+1:] + } + } + + return s +} + +func rebuildComputeDisk(d computeDiskIn) (map[string]any, error) { + p := d.InitializeParams + if !computeImageRE.MatchString(p.SourceImage) { + return nil, errComputeImage + } + + init := map[string]any{"sourceImage": p.SourceImage} + + if p.DiskSizeGb != "" { + n, err := strconv.Atoi(p.DiskSizeGb) + if err != nil || n < 1 || n > computeMaxDiskGB { + return nil, errComputeDiskSize + } + + init["diskSizeGb"] = strconv.Itoa(n) + } + + return map[string]any{"boot": true, "autoDelete": true, "initializeParams": init}, nil +} + +func rebuildComputeNIC(n computeNICIn) (map[string]any, error) { + out := map[string]any{} + + if n.Network != "" { + if !computeNetworkRE.MatchString(n.Network) { + return nil, errComputeNetwork + } + + out["network"] = n.Network + } + + if n.Subnetwork != "" { + if !computeSubnetRE.MatchString(n.Subnetwork) { + return nil, errComputeSubnet + } + + out["subnetwork"] = n.Subnetwork + } + + if len(n.AccessConfigs) > 0 { + out["accessConfigs"] = []any{ + map[string]string{computeKeyName: computeNICAccessName, "type": computeNICAccessType}, + } + } + + return out, nil +} + +func rebuildComputeLabels(in map[string]string) (map[string]string, error) { + if len(in) > computeMaxLabels { + return nil, errComputeLabels + } + + for k, v := range in { + if !computeNameRE.MatchString(k) || !computeLabelRE.MatchString(v) { + return nil, errComputeLabel + } + } + + return in, nil +} diff --git a/dashboard/gcpcompute_proxy.go b/dashboard/gcpcompute_proxy.go index 9a35cb8adb..235e3bac69 100644 --- a/dashboard/gcpcompute_proxy.go +++ b/dashboard/gcpcompute_proxy.go @@ -1,6 +1,8 @@ package dashboard import ( + "bytes" + "io" "net/http" "net/url" "slices" @@ -13,6 +15,7 @@ const ( gcpComputeProxyPrefix = "/dashboard/api/v1/gcp/compute" gcpComputeAPIPrefix = "/compute/v1" gcpComputeMaxBodyBytes = 1 << 20 + gcpComputeRevealParam = "reveal" computeMinSegs = 3 computeScopeIdx = 2 @@ -31,22 +34,42 @@ const ( computeDisks = "disks" computeAddresses = "addresses" computeRouters = "routers" + computeSnapshots = "snapshots" + computeSubnets = "subnetworks" + computeBackendSvcs = "backendServices" + computeURLMaps = "urlMaps" + computeHTTPProxies = "targetHttpProxies" + computeHTTPSProxies = "targetHttpsProxies" + computeFwdRules = "forwardingRules" computeStart = "start" computeStop = "stop" computeReset = "reset" + computeSuspend = "suspend" + computeResume = "resume" + computeSnapshotVerb = "createSnapshot" ) //nolint:gochecknoglobals // read-only lookup tables var ( - computeAggregated = []string{ - computeInstances, computeDisks, "subnetworks", computeAddresses, computeRouters, computeOperations, + computeLB = []string{ + computeBackendSvcs, + computeURLMaps, + computeHTTPProxies, + computeHTTPSProxies, + computeFwdRules, } - computeGlobal = []string{ - "networks", "firewalls", "routes", "snapshots", computeOperations, computeAddresses, - } - computeZonal = []string{computeInstances, computeDisks, computeOperations} - computeRegional = []string{"subnetworks", computeAddresses, computeRouters, computeOperations} - computeInstanceActions = []string{computeStart, computeStop, computeReset} + computeAggregated = append([]string{ + computeInstances, computeDisks, computeSubnets, computeAddresses, computeRouters, computeOperations, + }, computeLB...) + computeGlobal = append([]string{ + "networks", "firewalls", "routes", computeSnapshots, computeOperations, computeAddresses, "images", + "securityPolicies", + }, computeLB...) + computeZonal = []string{computeInstances, computeDisks, computeOperations, "machineTypes"} + computeRegional = append( + []string{computeSubnets, computeAddresses, computeRouters, computeOperations}, + computeLB...) + computeInstanceActions = []string{computeStart, computeStop, computeReset, computeSuspend, computeResume} computeQueryKeys = []string{ "filter", "maxResults", @@ -55,8 +78,8 @@ var ( } ) -// registerGCPComputeRoutes exposes Compute Engine browsing and instance, disk, snapshot and address -// management to the SPA by dispatching in-process to the emulator's handler. +// registerGCPComputeRoutes exposes Compute Engine browsing, instance and snapshot creation and resource management to +// the SPA by dispatching in-process to the emulator's handler. Metadata values never leave it unredacted. func (h *DashboardHandler) registerGCPComputeRoutes() { var inner *echo.Echo @@ -72,7 +95,10 @@ func (h *DashboardHandler) registerGCPComputeRoutes() { r := c.Request() - apiPath, ok := gcpComputeProxyPath(r.Method, strings.TrimPrefix(r.URL.EscapedPath(), gcpComputeProxyPrefix)) + apiPath, parts, ok := gcpComputeProxyPath( + r.Method, + strings.TrimPrefix(r.URL.EscapedPath(), gcpComputeProxyPrefix), + ) if !ok { return cloudJSONFail( c, @@ -82,14 +108,30 @@ func (h *DashboardHandler) registerGCPComputeRoutes() { ) } + body, err := gcpComputeBody(r.Method, parts, http.MaxBytesReader(c.Response(), r.Body, gcpComputeMaxBodyBytes)) + if err != nil { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, err.Error()) + } + fwd := r.Clone(r.Context()) fwd.URL.Path, fwd.URL.RawPath = unescapedPath(apiPath), apiPath fwd.URL.RawQuery = gcpComputeQuery(r.URL.Query()) fwd.RequestURI = "" - fwd.Body = http.MaxBytesReader(c.Response(), r.Body, gcpComputeMaxBodyBytes) - inner.ServeHTTP(c.Response(), fwd) + fwd.Body = io.NopCloser(bytes.NewReader(body)) + fwd.ContentLength = int64(len(body)) + + if r.Method != http.MethodGet { + inner.ServeHTTP(c.Response(), fwd) + + return nil + } + + reveal := "" + if len(parts) == computeScopedItem && parts[computeScopedColl] == computeInstances { + reveal = r.URL.Query().Get(gcpComputeRevealParam) + } - return nil + return serveRedacted(c, inner, fwd, reveal) } for _, method := range []string{http.MethodGet, http.MethodPost, http.MethodDelete} { @@ -110,18 +152,19 @@ func gcpComputeQuery(in url.Values) string { } // gcpComputeProxyPath maps /projects/

/... below the proxy prefix to the emulator's /compute/v1 path. Reads cover -// instances, disks, snapshots, networks, subnetworks, firewalls, routes, routers, addresses and operations; writes are -// limited to instance start/stop/reset and deleting instances, disks, snapshots and addresses. -func gcpComputeProxyPath(method, raw string) (string, bool) { +// the compute, network, load balancing, Cloud Armor and operation collections; writes are limited to instance +// start/stop/reset/suspend/resume, creating an instance or a disk snapshot (bodies rebuilt from fixed fields), and +// deleting instances, disks, snapshots and addresses. +func gcpComputeProxyPath(method, raw string) (string, []string, bool) { parts := strings.Split(strings.TrimPrefix(raw, "/"), "/") if len(parts) < computeMinSegs || parts[0] != gcpRESTProjectsSeg { - return "", false + return "", nil, false } for _, p := range parts { dec, err := url.PathUnescape(p) if err != nil || dec == "" || dec == "." || dec == ".." || strings.ContainsAny(dec, "/:") { - return "", false + return "", nil, false } } @@ -137,10 +180,10 @@ func gcpComputeProxyPath(method, raw string) (string, bool) { } if !allowed { - return "", false + return "", nil, false } - return gcpComputeAPIPrefix + "/" + strings.Join(parts, "/"), true + return gcpComputeAPIPrefix + "/" + strings.Join(parts, "/"), parts, true } func computeReadable(parts []string) bool { @@ -172,8 +215,21 @@ func computeScopedRead(parts, collections []string) bool { } func computeActionable(parts []string) bool { - return len(parts) == computeScopedVerb && parts[computeScopeIdx] == computeScopeZones && - parts[computeScopedColl] == computeInstances && slices.Contains(computeInstanceActions, parts[computeScopedVerb-1]) + if parts[computeScopeIdx] != computeScopeZones { + return false + } + + switch len(parts) { + case computeScopedList: + return parts[computeScopedColl] == computeInstances + case computeScopedVerb: + verb := parts[computeScopedVerb-1] + + return (parts[computeScopedColl] == computeInstances && slices.Contains(computeInstanceActions, verb)) || + (parts[computeScopedColl] == computeDisks && verb == computeSnapshotVerb) + } + + return false } func computeDeletable(parts []string) bool { diff --git a/dashboard/gcpcompute_proxy_test.go b/dashboard/gcpcompute_proxy_test.go index 2365e3a629..644f886f44 100644 --- a/dashboard/gcpcompute_proxy_test.go +++ b/dashboard/gcpcompute_proxy_test.go @@ -41,7 +41,8 @@ func seedComputeHandler(t *testing.T) *gcpcompute.Handler { net = `{"name":"vpc","autoCreateSubnetworks":false}` sub = `{"name":"sub","network":"global/networks/vpc","ipCidrRange":"10.10.0.0/24"}` vm = `{"name":"vm1","machineType":"zones/us-central1-a/machineTypes/e2-medium","labels":{"env":"dev"},` + - `"metadata":{"items":[{"key":"startup-script","value":"echo secret-ish"}]},"disks":[{"boot":true,` + + `"metadata":{"items":[{"key":"startup-script","value":"echo secret-ish"},` + + `{"key":"ssh-keys","value":"alice:ssh-rsa AAAAprivate"}]},"disks":[{"boot":true,` + `"autoDelete":true,"initializeParams":{"sourceImage":"projects/debian-cloud/global/images/family/debian-12",` + `"diskSizeGb":"10"}}],"networkInterfaces":[{"subnetwork":"regions/us-central1/subnetworks/sub",` + `"accessConfigs":[{"name":"External NAT","type":"ONE_TO_ONE_NAT"}]}]}` @@ -99,23 +100,45 @@ func TestGCPComputeProxy(t *testing.T) { {http.MethodGet, root + "/global/snapshots", "", "snapshotList", 200}, {http.MethodGet, root + "/aggregated/operations", "", "operation", 200}, {http.MethodGet, root + "/regions/us-central1/addresses/addr1", "", "addr1", 200}, + {http.MethodGet, zone + "/machineTypes", "", "e2-medium", 200}, + {http.MethodGet, root + "/global/images", "", "imageList", 200}, + {http.MethodGet, root + "/global/securityPolicies", "", "securityPolicyList", 200}, + {http.MethodGet, root + "/global/backendServices", "", "backendServiceList", 200}, + {http.MethodGet, root + "/aggregated/backendServices", "", "AggregatedList", 200}, + {http.MethodGet, root + "/aggregated/forwardingRules", "", "AggregatedList", 200}, + {http.MethodGet, root + "/aggregated/urlMaps", "", "AggregatedList", 200}, + {http.MethodGet, root + "/aggregated/targetHttpProxies", "", "AggregatedList", 200}, + {http.MethodGet, root + "/global/forwardingRules", "", "forwardingRuleList", 200}, + {http.MethodGet, root + "/regions/us-central1/urlMaps", "", "urlMapList", 200}, + {http.MethodGet, root + "/global/operations", "", "operationList", 200}, + {http.MethodGet, zone + "/operations", "", "operationList", 200}, }}, {"actions", []gcpStep{ {http.MethodPost, vm + "/stop", "", `"operationType":"stop"`, 200}, {http.MethodGet, vm, "", "TERMINATED", 200}, {http.MethodPost, vm + "/start", "", `"operationType":"start"`, 200}, {http.MethodPost, vm + "/reset", "", `"operationType":"reset"`, 200}, + {http.MethodPost, vm + "/suspend", "", `"operationType":"suspend"`, 200}, + {http.MethodPost, vm + "/resume", "", `"operationType":"resume"`, 200}, {http.MethodDelete, root + "/regions/us-central1/addresses/addr1", "", `"operationType":"delete"`, 200}, {http.MethodDelete, vm, "", `"operationType":"delete"`, 200}, {http.MethodGet, vm, "", "notFound", 404}, }}, {"rejects", []gcpStep{ - {http.MethodPost, zone + "/instances", `{"name":"x"}`, "unsupported", 400}, + {http.MethodPost, zone + "/instances", `{"name":"x"}`, "machine type are required", 400}, + {http.MethodPost, zone + "/instances", `not json`, "invalid request body", 400}, + {http.MethodPost, zone + "/disks", `{}`, "unsupported", 400}, + {http.MethodPost, zone + "/disks/vm1/resize", `{}`, "unsupported", 400}, + {http.MethodPost, root + "/global/networks", `{"name":"x"}`, "unsupported", 400}, + {http.MethodPost, root + "/global/firewalls", `{"name":"x"}`, "unsupported", 400}, + {http.MethodDelete, root + "/global/firewalls/x", "", "unsupported", 400}, + {http.MethodGet, root + "/global/instanceTemplates", "", "unsupported", 400}, + {http.MethodGet, root + "/projects", "", "unsupported", 400}, {http.MethodPost, vm + "/setMetadata", "{}", "unsupported", 400}, {http.MethodPost, vm + "/stop%3Fx", "", "unsupported", 400}, {http.MethodPut, vm, "{}", "Method Not Allowed", 405}, {http.MethodDelete, root + "/global/networks/vpc", "", "unsupported", 400}, - {http.MethodGet, root + "/global/images", "", "unsupported", 400}, + {http.MethodGet, root + "/global/sslCertificates", "", "unsupported", 400}, {http.MethodGet, root + "/zones/%2e%2e/instances", "", "unsupported", 400}, {http.MethodGet, root + "/global/networks/vpc/extra", "", "unsupported", 400}, {http.MethodGet, "/dashboard/api/v1/gcp/compute/other/p/zones", "", "unsupported", 400}, @@ -144,3 +167,149 @@ func TestGCPComputeProxyUnserved(t *testing.T) { assert.Equal(t, http.StatusServiceUnavailable, rec.Code) assert.Contains(t, rec.Body.String(), "not enabled") } + +func TestGCPComputeProxyMetadataRedaction(t *testing.T) { + t.Parallel() + + const ( + zone = "/dashboard/api/v1/gcp/compute/projects/p/zones/us-central1-a" + vm = zone + "/instances/vm1" + ) + + get := func(t *testing.T, h *dashboard.DashboardHandler, path string) string { + t.Helper() + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, path, nil) + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + return rec.Body.String() + } + + tests := []struct { + name string + path string + wantValue []string + wantAbsent []string + }{ + {"item default", vm, []string{`"startup-script"`, `"valueLength":15`, `"redacted":true`}, + []string{"secret-ish", "AAAAprivate"}}, + {"zonal list", zone + "/instances", []string{`"valueLength":15`}, []string{"secret-ish", "AAAAprivate"}}, + {"aggregated list", "/dashboard/api/v1/gcp/compute/projects/p/aggregated/instances", + []string{`"valueLength":15`}, []string{"secret-ish", "AAAAprivate"}}, + {"reveal startup script", vm + "?reveal=startup-script", []string{"echo secret-ish"}, []string{"AAAAprivate"}}, + {"reveal ssh keys refused", vm + "?reveal=ssh-keys", []string{`"redacted":true`}, []string{"AAAAprivate"}}, + {"reveal ignored on list", zone + "/instances?reveal=startup-script", []string{`"redacted":true`}, + []string{"secret-ish"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{GCPComputeOps: seedComputeHandler(t)}) + body := get(t, h, tt.path) + + for _, w := range tt.wantValue { + assert.Contains(t, body, w) + } + + for _, a := range tt.wantAbsent { + assert.NotContains(t, body, a) + } + }) + } +} + +func TestGCPComputeProxyCreate(t *testing.T) { + t.Parallel() + + const ( + root = "/dashboard/api/v1/gcp/compute/projects/p" + zone = root + "/zones/us-central1-a" + img = "projects/debian-cloud/global/images/family/debian-12" + good = `{"name":"web","machineType":"e2-small","labels":{"env":"dev"},` + + `"metadata":{"items":[{"key":"startup-script","value":"leak-me"}]},` + + `"serviceAccounts":[{"email":"x@y.z","scopes":["a"]}],` + + `"disks":[{"boot":true,"initializeParams":{"sourceImage":"` + img + `","diskSizeGb":"12"}}],` + + `"networkInterfaces":[{"network":"global/networks/vpc","subnetwork":"regions/us-central1/subnetworks/sub",` + + `"accessConfigs":[{"type":"ONE_TO_ONE_NAT","natIP":"1.2.3.4"}]}]}` + ) + + tests := []struct { + name string + steps []gcpStep + }{ + {"rebuilds body", []gcpStep{ + {http.MethodPost, zone + "/instances", good, `"operationType":"insert"`, 200}, + {http.MethodGet, zone + "/instances/web", "", "zones/us-central1-a/machineTypes/e2-small", 200}, + {http.MethodGet, zone + "/instances/web", "", `"env":"dev"`, 200}, + }}, + { + "snapshot", + []gcpStep{ + { + http.MethodPost, + zone + "/disks/vm1/createSnapshot", + `{"name":"snap1","labels":{"a":"b"},"storageLocations":["x"]}`, + `"operationType":"createSnapshot"`, + 200, + }, + {http.MethodGet, root + "/global/snapshots/snap1", "", `"sourceDisk"`, 200}, + {http.MethodPost, zone + "/disks/vm1/createSnapshot", `{"name":"Bad_Name"}`, "snapshot name", 400}, + {http.MethodPost, zone + "/disks/vm1/createSnapshot", `{"name":"snap1"}`, "already exists", 409}, + {http.MethodDelete, root + "/global/snapshots/snap1", "", `"operationType":"delete"`, 200}, + }, + }, + { + "rejects bad fields", + []gcpStep{ + {http.MethodPost, zone + "/instances", `{"name":"Web","machineType":"e2-small"}`, "required", 400}, + { + http.MethodPost, + zone + "/instances", + `{"name":"web","machineType":"e2-small","disks":[],"networkInterfaces":[]}`, + "exactly one", + 400, + }, + {http.MethodPost, zone + "/instances", `{"name":"web","machineType":"e2-small",` + + `"disks":[{"initializeParams":{"sourceImage":"https://evil/x"}}],"networkInterfaces":[{}]}`, "boot image", 400}, + {http.MethodPost, zone + "/instances", `{"name":"web","machineType":"e2-small",` + + `"disks":[{"initializeParams":{"sourceImage":"` + img + `","diskSizeGb":"0"}}],"networkInterfaces":[{}]}`, + "disk size", 400}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{GCPComputeOps: seedComputeHandler(t)}) + runGCPSteps(t, h, tt.steps) + }) + } +} + +func TestGCPComputeProxyCreateDropsUnlistedFields(t *testing.T) { + t.Parallel() + + const zone = "/dashboard/api/v1/gcp/compute/projects/p/zones/us-central1-a" + + body := `{"name":"web","machineType":"e2-small","metadata":{"items":[{"key":"startup-script","value":"leak-me"}]},` + + `"serviceAccounts":[{"email":"x@y.z","scopes":["a"]}],"tags":{"items":["t"]},` + + `"disks":[{"initializeParams":{"sourceImage":"projects/debian-cloud/global/images/family/debian-12"}}],` + + `"networkInterfaces":[{"network":"vpc","subnetwork":"sub"}]}` + + h := dashboard.NewHandler(dashboard.Config{GCPComputeOps: seedComputeHandler(t)}) + runGCPSteps(t, h, []gcpStep{{http.MethodPost, zone + "/instances", body, "insert", 200}}) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, zone+"/instances/web?reveal=startup-script", nil) + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + require.Equal(t, http.StatusOK, rec.Code) + assert.NotContains(t, rec.Body.String(), "leak-me") + assert.NotContains(t, rec.Body.String(), "x@y.z") + assert.NotContains(t, rec.Body.String(), `"items":["t"]`) +} diff --git a/dashboard/gcpcompute_redact.go b/dashboard/gcpcompute_redact.go new file mode 100644 index 0000000000..5283f07bae --- /dev/null +++ b/dashboard/gcpcompute_redact.go @@ -0,0 +1,111 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "strings" + + "github.com/labstack/echo/v5" +) + +const ( + computeMetaKey = "key" + computeMetaValue = "value" + computeMetaLength = "valueLength" + computeMetaHidden = "redacted" + computeMetaField = "metadata" + computeMetaItems = "items" +) + +// serveRedacted runs the upstream request and hides every metadata value in a successful JSON answer. Startup scripts +// and SSH keys live in metadata, so only an explicit reveal of one key that is not SSH-related keeps its value. +func serveRedacted(c *echo.Context, inner *echo.Echo, fwd *http.Request, reveal string) error { + rec := httptest.NewRecorder() + inner.ServeHTTP(rec, fwd) + + body := rec.Body.Bytes() + if rec.Code == http.StatusOK { + body = redactComputeMetadata(body, reveal) + } + + h := c.Response().Header() + for k, v := range rec.Header() { + if k != "Content-Length" { + h[k] = v + } + } + + h.Set("Content-Length", strconv.Itoa(len(body))) + c.Response().WriteHeader(rec.Code) + + _, err := c.Response().Write(body) + + return err +} + +func computeRevealable(key string) bool { + return key != "" && !strings.Contains(strings.ToLower(key), "ssh") +} + +func redactComputeMetadata(body []byte, reveal string) []byte { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + + var doc any + if err := dec.Decode(&doc); err != nil { + return body + } + + redactWalk(doc, reveal) + + out, err := json.Marshal(doc) + if err != nil { + return body + } + + return out +} + +func redactWalk(v any, reveal string) { + switch t := v.(type) { + case map[string]any: + if meta, ok := t[computeMetaField].(map[string]any); ok { + redactItems(meta, reveal) + } + + for _, child := range t { + redactWalk(child, reveal) + } + case []any: + for _, child := range t { + redactWalk(child, reveal) + } + } +} + +func redactItems(meta map[string]any, reveal string) { + items, ok := meta[computeMetaItems].([]any) + if !ok { + return + } + + for _, it := range items { + m, isMap := it.(map[string]any) + if !isMap { + continue + } + + val, _ := m[computeMetaValue].(string) + m[computeMetaLength] = len(val) + + if key, _ := m[computeMetaKey].(string); key == reveal && computeRevealable(key) { + continue + } + + m[computeMetaValue] = "" + m[computeMetaHidden] = true + } +} diff --git a/dashboard/gcpcontainer_kubeconfig.go b/dashboard/gcpcontainer_kubeconfig.go new file mode 100644 index 0000000000..9ad20eaf8c --- /dev/null +++ b/dashboard/gcpcontainer_kubeconfig.go @@ -0,0 +1,54 @@ +package dashboard + +import ( + "fmt" + "strconv" + "strings" + + "cloud.google.com/go/container/apiv1/containerpb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" +) + +const gkeVerbKubeconfig = "kubeconfig" + +// kubeconfig renders a kubeconfig for one cluster. It carries the cluster-admin token on docker-engine clusters, so +// it is only reachable through this POST verb, which the REST proxy answers with Cache-Control: no-store. +func (a *gcpContainerAPI) kubeconfig(r gcpRESTCall) (proto.Message, error) { + c, err := a.b.GetCluster(&containerpb.GetClusterRequest{Name: r.name(gkeItem)}) + if err != nil { + return nil, err + } + + if c.GetEndpoint() == "" { + return nil, status.Error(codes.FailedPrecondition, "cluster has no endpoint yet") + } + + token := a.b.EngineToken() + name := fmt.Sprintf("gke_%s_%s_%s", r.segs[1], r.segs[gkeLocationsIdx+1], c.GetName()) + + var user string + + if token != "" { + user = " token: " + strconv.Quote(token) + "\n" + } else { + user = " exec:\n apiVersion: client.authentication.k8s.io/v1beta1\n" + + " command: gke-gcloud-auth-plugin\n provideClusterInfo: true\n" + + " installHint: Install gke-gcloud-auth-plugin to authenticate against this cluster\n" + } + + var sb strings.Builder + + sb.WriteString("apiVersion: v1\nkind: Config\ncurrent-context: " + strconv.Quote(name) + "\n") + sb.WriteString("clusters:\n- name: " + strconv.Quote(name) + "\n cluster:\n") + sb.WriteString(" server: " + strconv.Quote("https://"+c.GetEndpoint()) + "\n") + sb.WriteString(" certificate-authority-data: " + + strconv.Quote(c.GetMasterAuth().GetClusterCaCertificate()) + "\n") + sb.WriteString("contexts:\n- name: " + strconv.Quote(name) + "\n context:\n cluster: " + + strconv.Quote(name) + "\n user: " + strconv.Quote(name) + "\n") + sb.WriteString("users:\n- name: " + strconv.Quote(name) + "\n user:\n" + user) + + return structpb.NewStruct(map[string]any{"kubeconfig": sb.String(), "hasToken": token != ""}) +} diff --git a/dashboard/gcpcontainer_proxy.go b/dashboard/gcpcontainer_proxy.go index 703988b307..49ae5503a4 100644 --- a/dashboard/gcpcontainer_proxy.go +++ b/dashboard/gcpcontainer_proxy.go @@ -28,7 +28,7 @@ const ( ) // registerGCPContainerRoutes exposes the Kubernetes Engine API to the SPA as REST JSON, calling the backend -// in-process. Only reads, node pool resize and cluster deletion are reachable; cluster credentials are removed. +// in-process. Only reads, node pool resize, cluster deletion and an explicit kubeconfig reveal are reachable. func (h *DashboardHandler) registerGCPContainerRoutes() { var route gcpRESTRoute @@ -128,6 +128,8 @@ func (a *gcpContainerAPI) cluster(r gcpRESTCall) (proto.Message, error) { } return structpb.NewStruct(fields) + case r.method == http.MethodPost && r.verb == gkeVerbKubeconfig: + return a.kubeconfig(r) case r.method == http.MethodDelete && r.verb == "": return a.b.DeleteCluster(&containerpb.DeleteClusterRequest{Name: name}) } diff --git a/dashboard/gcpcontainer_proxy_test.go b/dashboard/gcpcontainer_proxy_test.go index c004d38eb1..66f13a9003 100644 --- a/dashboard/gcpcontainer_proxy_test.go +++ b/dashboard/gcpcontainer_proxy_test.go @@ -80,6 +80,12 @@ func TestGCPContainerProxy(t *testing.T) { }, {http.MethodGet, cluster + ":poolSizes", "", `"default-pool":4`, 200}, }}, + {"kubeconfig", []gcpStep{ + {http.MethodPost, cluster + ":kubeconfig", ``, "gke-gcloud-auth-plugin", 200}, + {http.MethodPost, cluster + ":kubeconfig", ``, "gke_p_us-central1-a_c1", 200}, + {http.MethodGet, cluster + ":kubeconfig", ``, "NOT_FOUND", 404}, + {http.MethodPost, gkeBase + "/us-central1-a/clusters/nope:kubeconfig", ``, "NOT_FOUND", 404}, + }}, {"operations", []gcpStep{ {http.MethodGet, gkeBase + "/-/operations", "", "CREATE_CLUSTER", 200}, }}, @@ -117,6 +123,18 @@ func TestGCPContainerProxy(t *testing.T) { } }) + t.Run("kubeconfig headers", func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, cluster+":kubeconfig", nil) + require.NoError(t, newGKEDashboard(t).Handler()(echo.NewContext(req, rec))) + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "no-store", rec.Header().Get("Cache-Control")) + assert.NotContains(t, rec.Body.String(), "correct-horse-battery") + assert.NotContains(t, rec.Body.String(), "private-key") + }) + t.Run("unavailable", func(t *testing.T) { t.Parallel() diff --git a/dashboard/gcpexplorers_proxy_test.go b/dashboard/gcpexplorers_proxy_test.go index 6b8e3cfaaa..4c422ba5a8 100644 --- a/dashboard/gcpexplorers_proxy_test.go +++ b/dashboard/gcpexplorers_proxy_test.go @@ -108,6 +108,7 @@ func TestGCPEventarcProxy(t *testing.T) { t.Helper() ea := gcpeventarc.NewHandler(gcpeventarc.NewInMemoryBackend()) + ea.Backend.SetOperationDelay(0) _, err := ea.Backend.CreateTrigger(&eventarcpb.CreateTriggerRequest{ Parent: "projects/p/locations/us-central1", TriggerId: "trig1", Trigger: &eventarcpb.Trigger{ diff --git a/dashboard/lambdaweb_proxy.go b/dashboard/lambdaweb_proxy.go new file mode 100644 index 0000000000..77703eb29a --- /dev/null +++ b/dashboard/lambdaweb_proxy.go @@ -0,0 +1,333 @@ +package dashboard + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/services/lambdaweb" +) + +const ( + lambdaWebPrefix = "/dashboard/api/v1/lambdaweb" + lambdaWebAPI = "/2025-03-07/web-functions" + lambdaWebMaxBody = 1 << 17 + lambdaWebMaxInvoke = 2 * lambdaWebMaxBody + lambdaWebMaxReply = 256 << 10 + lambdaWebMaxPath = 2048 + lambdaWebMaxHeader = 24 + lambdaWebMaxValue = 4096 + lambdaWebPageSize = 50 + lambdaWebHidden = "" +) + +var ( + lambdaWebNameRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,140}$`) + lambdaWebRegionRe = regexp.MustCompile(`^[a-z0-9-]{1,32}$`) + lambdaWebHeaderRe = regexp.MustCompile("^[A-Za-z0-9!#$%&'*+.^_`|~-]{1,64}$") + lambdaWebTokenRe = regexp.MustCompile(`^[A-Za-z0-9+/=_.:-]{0,1000}$`) +) + +// lambdaWebBlocked are request headers the invoke panel never forwards. +// +//nolint:gochecknoglobals // immutable deny-list +var lambdaWebBlocked = []string{ + "host", "connection", "content-length", "transfer-encoding", "upgrade", "expect", "te", "trailer", + "keep-alive", "proxy-authorization", "proxy-connection", "x-real-ip", "x-forwarded-for", "x-forwarded-host", +} + +// registerLambdaWebRoutes exposes Lambda Web Functions by dispatching in-process to its handler. Reads are fixed +// control plane calls; the invoke panel calls an endpoint domain through the same handler, so no network is touched. +func (h *DashboardHandler) registerLambdaWebRoutes() { + var inner *echo.Echo + + if h.config.LambdaWebOps != nil { + inner = echo.New() + inner.Any("/*", h.config.LambdaWebOps.Handler()) + } + + lw := lambdaWebAccess{inner: inner} + guard := func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c *echo.Context) error { + if inner == nil { + return azureACIFail(c, http.StatusServiceUnavailable, errServiceUnavailable, + "Lambda Web Functions is not enabled") + } + + return next(c) + } + } + + h.SubRouter.Add(http.MethodGet, lambdaWebPrefix+"/functions", guard(lw.list("", ""))) + h.SubRouter.Add(http.MethodGet, lambdaWebPrefix+"/functions/:fn", guard(lw.get(""))) + h.SubRouter.Add( + http.MethodGet, + lambdaWebPrefix+"/functions/:fn/revisions", + guard(lw.list("revisions", "list-revisions")), + ) + h.SubRouter.Add(http.MethodGet, lambdaWebPrefix+"/functions/:fn/revisions/:child", guard(lw.get("revisions"))) + h.SubRouter.Add( + http.MethodGet, + lambdaWebPrefix+"/functions/:fn/endpoints", + guard(lw.list("endpoints", "list-endpoints")), + ) + h.SubRouter.Add(http.MethodGet, lambdaWebPrefix+"/functions/:fn/endpoints/:child", guard(lw.get("endpoints"))) + h.SubRouter.Add(http.MethodPost, lambdaWebPrefix+"/invoke", guard(lw.invoke)) +} + +type lambdaWebAccess struct{ inner *echo.Echo } + +func lambdaWebBad(c *echo.Context) error { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestUri", "invalid Lambda Web Functions request") +} + +func (lw lambdaWebAccess) do(c *echo.Context, region, method, path string, body []byte) *httptest.ResponseRecorder { + req := httptest.NewRequestWithContext(c.Request().Context(), method, path, bytes.NewReader(body)) + if region != "" { + req.Header.Set("X-Amz-Region", region) + } + + if len(body) > 0 { + req.Header.Set("Content-Type", "application/json") + } + + rec := httptest.NewRecorder() + lw.inner.ServeHTTP(rec, req) + + return rec +} + +func lambdaWebRegion(c *echo.Context) (string, bool) { + region := c.QueryParam("region") + + return region, region == "" || lambdaWebRegionRe.MatchString(region) +} + +// lambdaWebView hides environment variable values, which commonly hold secrets. +func lambdaWebView(v any) { + switch t := v.(type) { + case []any: + for _, x := range t { + lambdaWebView(x) + } + case map[string]any: + for k, x := range t { + if m, ok := x.(map[string]any); ok && k == "environmentVariables" { + for name := range m { + m[name] = lambdaWebHidden + } + + continue + } + + lambdaWebView(x) + } + } +} + +func lambdaWebRelay(c *echo.Context, rec *httptest.ResponseRecorder) error { + body := rec.Body.Bytes() + if len(body) == 0 { + return c.NoContent(rec.Code) + } + + if rec.Code < http.StatusMultipleChoices { + body = armScrubBody(body, lambdaWebView) + } else if t := rec.Header().Get("X-Amzn-Errortype"); t != "" { + c.Response().Header().Set("X-Amzn-Errortype", t) + } + + return c.JSONBlob(rec.Code, body) +} + +// list serves a function list (child "") or a list of a function's revisions or endpoints. +func (lw lambdaWebAccess) list(child, op string) echo.HandlerFunc { + return func(c *echo.Context) error { + region, ok := lambdaWebRegion(c) + token := c.QueryParam("nextToken") + + if !ok || !lambdaWebTokenRe.MatchString(token) { + return lambdaWebBad(c) + } + + path := lambdaWebAPI + + if child != "" { + fn := c.Param("fn") + if !lambdaWebNameRe.MatchString(fn) { + return lambdaWebBad(c) + } + + path += "/" + url.PathEscape(fn) + "/" + op + } + + in := map[string]any{"maxResults": lambdaWebPageSize} + if token != "" { + in["nextToken"] = token + } + + body, err := json.Marshal(in) + if err != nil { + return lambdaWebBad(c) + } + + return lambdaWebRelay(c, lw.do(c, region, http.MethodPost, path, body)) + } +} + +// get serves one function (child "") or one revision or endpoint of it. +func (lw lambdaWebAccess) get(child string) echo.HandlerFunc { + return func(c *echo.Context) error { + region, ok := lambdaWebRegion(c) + fn := c.Param("fn") + + if !ok || !lambdaWebNameRe.MatchString(fn) { + return lambdaWebBad(c) + } + + path := lambdaWebAPI + "/" + url.PathEscape(fn) + + if child != "" { + name := c.Param("child") + if !lambdaWebNameRe.MatchString(name) { + return lambdaWebBad(c) + } + + path += "/" + child + "/" + url.PathEscape(name) + } + + return lambdaWebRelay(c, lw.do(c, region, http.MethodGet, path, nil)) + } +} + +type lambdaWebHeaderIn struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type lambdaWebInvokeIn struct { + Region string `json:"region"` + Function string `json:"function"` + Endpoint string `json:"endpoint"` + Method string `json:"method"` + Path string `json:"path"` + Body string `json:"body"` + Headers []lambdaWebHeaderIn `json:"headers"` +} + +func (in lambdaWebInvokeIn) valid() bool { + if !lambdaWebNameRe.MatchString(in.Function) || !lambdaWebNameRe.MatchString(in.Endpoint) || + in.Region != "" && !lambdaWebRegionRe.MatchString(in.Region) { + return false + } + + methods := []string{ + http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete, + http.MethodOptions, + } + + if !slices.Contains(methods, in.Method) || len(in.Body) > lambdaWebMaxBody { + return false + } + + if !strings.HasPrefix(in.Path, "/") || strings.HasPrefix(in.Path, "//") || len(in.Path) > lambdaWebMaxPath || + strings.ContainsFunc(in.Path, func(r rune) bool { return r <= ' ' || r == 0x7f }) { + return false + } + + return in.headersValid() +} + +func (in lambdaWebInvokeIn) headersValid() bool { + if len(in.Headers) > lambdaWebMaxHeader { + return false + } + + for _, h := range in.Headers { + if !lambdaWebHeaderRe.MatchString(h.Name) || slices.Contains(lambdaWebBlocked, strings.ToLower(h.Name)) || + len(h.Value) > lambdaWebMaxValue || strings.ContainsAny(h.Value, "\r\n\x00") { + return false + } + } + + return true +} + +type lambdaWebReply struct { + Headers map[string][]string `json:"headers"` + Body string `json:"body"` + Encoding string `json:"encoding"` + Domain string `json:"domain"` + DurationMs int64 `json:"durationMs"` + Status int `json:"status"` + Truncated bool `json:"truncated"` +} + +// invoke sends one request to the domain of an endpoint and returns what came back. +func (lw lambdaWebAccess) invoke(c *echo.Context) error { + var in lambdaWebInvokeIn + + dec := json.NewDecoder(http.MaxBytesReader(c.Response(), c.Request().Body, lambdaWebMaxInvoke)) + dec.DisallowUnknownFields() + + if dec.Decode(&in) != nil || !in.valid() { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestContent", "the invoke request is invalid") + } + + epPath := lambdaWebAPI + "/" + url.PathEscape(in.Function) + "/endpoints/" + url.PathEscape(in.Endpoint) + + ep := lw.do(c, in.Region, http.MethodGet, epPath, nil) + if ep.Code != http.StatusOK { + return azureACIFail(c, ep.Code, "ResourceNotFound", "the endpoint was not found") + } + + var info struct { + Domain string `json:"domainName"` + } + if json.Unmarshal(ep.Body.Bytes(), &info) != nil || !lambdaweb.IsEndpointHost(info.Domain) { + return azureACIFail(c, http.StatusBadGateway, "EndpointUnavailable", "the endpoint has no domain") + } + + req, err := http.NewRequestWithContext(c.Request().Context(), in.Method, "http://"+info.Domain+in.Path, + strings.NewReader(in.Body)) + if err != nil { + return azureACIFail(c, http.StatusBadRequest, "InvalidRequestContent", "the request path is invalid") + } + + for _, h := range in.Headers { + req.Header.Add(h.Name, h.Value) + } + + start := time.Now() + rec := httptest.NewRecorder() + lw.inner.ServeHTTP(rec, req) + + out := lambdaWebReply{ + Status: rec.Code, Headers: rec.Header().Clone(), DurationMs: time.Since(start).Milliseconds(), + Domain: info.Domain, + } + + raw, _ := io.ReadAll(io.LimitReader(rec.Body, lambdaWebMaxReply+1)) + if len(raw) > lambdaWebMaxReply { + raw, out.Truncated = raw[:lambdaWebMaxReply], true + } + + if utf8.Valid(raw) { + out.Body, out.Encoding = string(raw), "text" + } else { + out.Body, out.Encoding = base64.StdEncoding.EncodeToString(raw), "base64" + } + + return c.JSON(http.StatusOK, out) +} diff --git a/dashboard/lambdaweb_proxy_test.go b/dashboard/lambdaweb_proxy_test.go new file mode 100644 index 0000000000..46ba989242 --- /dev/null +++ b/dashboard/lambdaweb_proxy_test.go @@ -0,0 +1,358 @@ +package dashboard_test + +import ( + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/lambdaweb" +) + +const ( + lwRoot = "/dashboard/api/v1/lambdaweb" + lwRole = "arn:aws:iam::123456789012:role/web-role" + lwReg = "?region=us-east-1" + lwCtl = "/2025-03-07/web-functions" + lwEnvKV = "DB_PASSWORD" +) + +type lwCodes struct{} + +func (lwCodes) ResolveObject(_ context.Context, bucket, _, _ string) (string, error) { + if bucket != "code" { + return "", lambdaweb.ErrBucketNotFound + } + + return "v1", nil +} + +func (lwCodes) GetObject(context.Context, string, string, string) ([]byte, error) { + return []byte("zip"), nil +} + +type lwEnv struct{ srv *httptest.Server } + +func (e lwEnv) Addr() (string, int) { + host, port, _ := net.SplitHostPort(e.srv.Listener.Addr().String()) + n, _ := strconv.Atoi(port) + + return host, n +} + +func (e lwEnv) Close(context.Context) { e.srv.Close() } + +// lwLauncher serves an in-process app in place of a container. +type lwLauncher struct{} + +func (lwLauncher) Launch(context.Context, lambdaweb.LaunchSpec) (lambdaweb.Environment, error) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + w.Header().Set("X-App", "yes") + + if r.URL.Path == "/fail" { + http.Error(w, "boom", http.StatusInternalServerError) + + return + } + + _ = json.NewEncoder(w).Encode(map[string]string{ + "method": r.Method, "path": r.URL.RequestURI(), "body": string(body), "accept": r.Header.Get("Accept"), + }) + })) + + return lwEnv{srv: srv}, nil +} + +type lwFixture struct { + h *dashboard.DashboardHandler + backend *lambdaweb.InMemoryBackend +} + +func newLambdaWebFixture(t *testing.T, withApp bool) lwFixture { + t.Helper() + + var now atomic.Int64 + + now.Store(time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC).UnixNano()) + + backend := lambdaweb.NewInMemoryBackend("123456789012") + backend.SetClock(func() time.Time { return time.Unix(0, now.Add(int64(time.Second))).UTC() }) + backend.SetCodeStore(lwCodes{}) + t.Cleanup(backend.Close) + + if withApp { + backend.SetLauncher(lwLauncher{}) + } + + lh := lambdaweb.NewHandler(backend) + lh.AccountID, lh.DefaultRegion = "123456789012", "us-east-1" + + ctl := echo.New() + ctl.Any("/*", lh.Handler()) + + revision := `{"buildConfig":{"codeConfig":{"s3Object":{"bucket":"code","key":"app.zip"}},` + + `"runtimeConfig":{"runtime":"nodejs24.x"}},"serviceConfig":{"executionRoleArn":"` + lwRole + `",` + + `"environmentVariables":{"` + lwEnvKV + `":"hunter2"}}}` + + create := `{"functionName":"app","revisionConfig":` + revision + `,"endpointConfig":{"endpointName":"prod",` + + `"endpointType":"HomeRegion","authType":"ApplicationManaged"}}` + + req := httptest.NewRequest(http.MethodPut, lwCtl, strings.NewReader(create)) + req.Header.Set("X-Amz-Region", "us-east-1") + + rec := httptest.NewRecorder() + ctl.ServeHTTP(rec, req) + require.Less(t, rec.Code, http.StatusBadRequest, rec.Body.String()) + + return lwFixture{h: dashboard.NewHandler(dashboard.Config{LambdaWebOps: lh}), backend: backend} +} + +func lwGet(t *testing.T, f lwFixture, path string) map[string]any { + t.Helper() + + var out map[string]any + + require.Eventually(t, func() bool { + rec := lbDo(t, f.h, http.MethodGet, lwRoot+path, "") + if rec.Code != http.StatusOK { + return false + } + + out = map[string]any{} + + return json.Unmarshal(rec.Body.Bytes(), &out) == nil + }, 5*time.Second, 5*time.Millisecond, path) + + return out +} + +func lwActive(t *testing.T, f lwFixture) { + t.Helper() + + require.Eventually(t, func() bool { + ep := lwGet(t, f, "/functions/app/endpoints/prod"+lwReg) + + return ep["state"] == "Active" + }, 5*time.Second, 5*time.Millisecond) +} + +func TestLambdaWebProxyReads(t *testing.T) { + t.Parallel() + + f := newLambdaWebFixture(t, false) + + fns := lwGet(t, f, "/functions"+lwReg) + list, _ := fns["functions"].([]any) + require.Len(t, list, 1) + + one := lwGet(t, f, "/functions/app"+lwReg) + assert.Equal(t, "app", one["functionName"]) + + revs := lwGet(t, f, "/functions/app/revisions"+lwReg) + items, _ := revs["revisions"].([]any) + require.Len(t, items, 1) + + rev, _ := items[0].(map[string]any) + id, _ := rev["revisionId"].(string) + require.NotEmpty(t, id) + + full := lwGet(t, f, "/functions/app/revisions/"+id+lwReg) + assert.Contains(t, jsonText(t, full), "app.zip") + + eps := lwGet(t, f, "/functions/app/endpoints"+lwReg) + list, _ = eps["endpoints"].([]any) + require.Len(t, list, 1) + assert.Contains(t, jsonText(t, list[0]), "domainName") +} + +func jsonText(t *testing.T, v any) string { + t.Helper() + + raw, err := json.Marshal(v) + require.NoError(t, err) + + return string(raw) +} + +func TestLambdaWebProxyHidesEnvironmentValues(t *testing.T) { + t.Parallel() + + f := newLambdaWebFixture(t, false) + + revs := lwGet(t, f, "/functions/app/revisions"+lwReg) + items, _ := revs["revisions"].([]any) + rev, _ := items[0].(map[string]any) + + rec := lbDo(t, f.h, http.MethodGet, lwRoot+"/functions/app/revisions/"+rev["revisionId"].(string)+lwReg, "") + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), lwEnvKV, "names stay visible") + assert.NotContains(t, rec.Body.String(), "hunter2", "values never reach the browser") +} + +func TestLambdaWebProxyRejectsBadRequests(t *testing.T) { + t.Parallel() + + f := newLambdaWebFixture(t, false) + + tests := []struct { + name string + method string + path string + want int + }{ + {"bad region", http.MethodGet, "/functions?region=US%20EAST", http.StatusBadRequest}, + {"bad token", http.MethodGet, "/functions?nextToken=%3Cscript%3E", http.StatusBadRequest}, + {"bad function name", http.MethodGet, "/functions/a%20b", http.StatusBadRequest}, + {"bad revision name", http.MethodGet, "/functions/app/revisions/a%2Fb", http.StatusBadRequest}, + {"delete function", http.MethodDelete, "/functions/app", http.StatusNotFound}, + {"put function", http.MethodPut, "/functions", http.StatusNotFound}, + {"post revisions", http.MethodPost, "/functions/app/revisions", http.StatusNotFound}, + {"delete endpoint", http.MethodDelete, "/functions/app/endpoints/prod", http.StatusNotFound}, + {"patch endpoint", http.MethodPatch, "/functions/app/endpoints/prod", http.StatusNotFound}, + {"resource policy", http.MethodGet, "/resource-policy/x", http.StatusNotFound}, + {"unknown function", http.MethodGet, "/functions/ghost" + lwReg, http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := lbDo(t, f.h, tt.method, lwRoot+tt.path, `{}`) + assert.Contains(t, []int{tt.want, http.StatusMethodNotAllowed}, rec.Code, rec.Body.String()) + }) + } + + still := lwGet(t, f, "/functions"+lwReg) + list, _ := still["functions"].([]any) + assert.Len(t, list, 1, "nothing was removed") +} + +func lwInvoke(t *testing.T, f lwFixture, body string) *httptest.ResponseRecorder { + t.Helper() + + return lbDo(t, f.h, http.MethodPost, lwRoot+"/invoke", body) +} + +func TestLambdaWebInvokeReachesTheApp(t *testing.T) { + t.Parallel() + + f := newLambdaWebFixture(t, true) + lwActive(t, f) + + rec := lwInvoke( + t, + f, + `{"region":"us-east-1","function":"app","endpoint":"prod","method":"POST","path":"/hello?x=1",`+ + `"headers":[{"name":"Accept","value":"application/json"}],"body":"ping"}`, + ) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + var out struct { + Headers map[string][]string `json:"headers"` + Body string `json:"body"` + Encoding string `json:"encoding"` + Domain string `json:"domain"` + Status int `json:"status"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + assert.Equal(t, http.StatusOK, out.Status) + assert.Equal(t, "text", out.Encoding) + assert.Contains(t, out.Domain, ".lambda-web.") + assert.Contains(t, out.Body, `"path":"/hello?x=1"`) + assert.Contains(t, out.Body, `"body":"ping"`) + assert.Contains(t, out.Body, `"accept":"application/json"`) + assert.Equal(t, []string{"yes"}, out.Headers["X-App"]) + + failing := lwInvoke(t, f, `{"function":"app","endpoint":"prod","method":"GET","path":"/fail"}`) + require.Equal(t, http.StatusOK, failing.Code) + assert.Contains(t, failing.Body.String(), `"status":500`) +} + +func TestLambdaWebInvokeWithoutAnEngineShowsThePlatformError(t *testing.T) { + t.Parallel() + + f := newLambdaWebFixture(t, false) + lwActive(t, f) + + rec := lwInvoke(t, f, `{"function":"app","endpoint":"prod","method":"GET","path":"/"}`) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), `"status":503`) +} + +func TestLambdaWebInvokeValidation(t *testing.T) { + t.Parallel() + + f := newLambdaWebFixture(t, true) + lwActive(t, f) + + long := strings.Repeat("a", 2100) + + tests := []struct { + name string + body string + want int + }{ + {"unknown field", `{"function":"app","endpoint":"prod","method":"GET","path":"/","host":"evil"}`, 400}, + {"bad method", `{"function":"app","endpoint":"prod","method":"TRACE","path":"/"}`, 400}, + {"relative path", `{"function":"app","endpoint":"prod","method":"GET","path":"hello"}`, 400}, + {"scheme relative path", `{"function":"app","endpoint":"prod","method":"GET","path":"//evil.example/x"}`, 400}, + {"space in path", `{"function":"app","endpoint":"prod","method":"GET","path":"/a b"}`, 400}, + {"long path", `{"function":"app","endpoint":"prod","method":"GET","path":"/` + long + `"}`, 400}, + {"host header", `{"function":"app","endpoint":"prod","method":"GET","path":"/",` + + `"headers":[{"name":"Host","value":"x"}]}`, 400}, + {"content length header", `{"function":"app","endpoint":"prod","method":"GET","path":"/",` + + `"headers":[{"name":"content-length","value":"1"}]}`, 400}, + {"header injection", `{"function":"app","endpoint":"prod","method":"GET","path":"/",` + + `"headers":[{"name":"X-A","value":"a\r\nX-B: b"}]}`, 400}, + {"bad header name", `{"function":"app","endpoint":"prod","method":"GET","path":"/",` + + `"headers":[{"name":"X A","value":"a"}]}`, 400}, + {"bad function", `{"function":"a b","endpoint":"prod","method":"GET","path":"/"}`, 400}, + {"unknown endpoint", `{"function":"app","endpoint":"ghost","method":"GET","path":"/"}`, 404}, + {"unknown function", `{"function":"ghost","endpoint":"prod","method":"GET","path":"/"}`, 404}, + {"not json", `method=GET`, 400}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := lwInvoke(t, f, tt.body) + assert.Equal(t, tt.want, rec.Code, rec.Body.String()) + }) + } +} + +func TestLambdaWebAbsentBackendAnswers503(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + for _, tt := range []struct{ name, method, path string }{ + {"list", http.MethodGet, "/functions"}, + {"get", http.MethodGet, "/functions/app"}, + {"revisions", http.MethodGet, "/functions/app/revisions"}, + {"endpoints", http.MethodGet, "/functions/app/endpoints"}, + {"invoke", http.MethodPost, "/invoke"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := lbDo(t, h, tt.method, lwRoot+tt.path, `{}`) + assert.Equal(t, http.StatusServiceUnavailable, rec.Code, rec.Body.String()) + }) + } +} diff --git a/dashboard/oci_proxy_test.go b/dashboard/oci_proxy_test.go index 0e6075db5c..b6de56b51d 100644 --- a/dashboard/oci_proxy_test.go +++ b/dashboard/oci_proxy_test.go @@ -132,7 +132,10 @@ func TestOCIVaultProxy(t *testing.T) { seed := func(t *testing.T) seeded { t.Helper() - v := ocivault.NewHandler(ocivault.NewInMemoryBackend()) + vb := ocivault.NewInMemoryBackend() + vb.SetTransitionDelay(0) + + v := ocivault.NewHandler(vb) vault, _, err := v.Backend.CreateVault(ocivault.CreateVaultIn{ CompartmentID: ociComp, DisplayName: "vault-one", VaultType: "DEFAULT", }, "") diff --git a/dashboard/ocidatabase_proxy.go b/dashboard/ocidatabase_proxy.go new file mode 100644 index 0000000000..4884337e98 --- /dev/null +++ b/dashboard/ocidatabase_proxy.go @@ -0,0 +1,440 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "regexp" + "slices" + "strings" + + "github.com/labstack/echo/v5" + + idbackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +const ( + ociDBPrefix = ociProxyPrefix + "/database" + ociDBAPIPrefix = "/20160918" + ociDBDatabases = "autonomousDatabases" + ociDBBackups = "autonomousDatabaseBackups" + ociDBActions = "actions" + ociDBWallet = "wallet" + ociDBMaxBody = 1 << 14 + ociDBMaxName = 255 + ociDBMaxACL = 64 + ociDBMaxPassword = 256 + ociDBItemSegs = 2 + ociDBSubSegs = 3 + ociDBActionSegs = 4 + ociDBKeyComp = "compartmentId" + ociDBKeyDisplay = "displayName" + ociDBKeyLimit = "limit" + ociDBKeyPage = "page" + ociDBKeyState = "lifecycleState" + ociDBKeyPassword = "password" + ociDBKeyAdminPass = "adminPassword" + ociDBKeyACL = "whitelistedIps" + ociDBKeyCompute = "computeCount" + ociDBKeyStorage = "dataStorageSizeInTBs" + ociDBKeyDBID = "autonomousDatabaseId" + ociDBWalletFile = "Wallet_" +) + +var ( + ociDBIDRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,255}$`) + ociDBNameRe = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9]{0,29}$`) + ociDBACLRe = regexp.MustCompile(`^[A-Za-z0-9.:/_-]{1,255}$`) + ociDBFileRe = regexp.MustCompile(`[^A-Za-z0-9]`) +) + +// ociDBQueryKeys are the only query parameters each listing forwards; everything else is dropped. +// +//nolint:gochecknoglobals // immutable allow-list +var ociDBQueryKeys = map[string][]string{ + ociDBDatabases: { + ociDBKeyComp, ociDBKeyState, ociDBKeyDisplay, "dbWorkload", "isFreeTier", ociDBKeyLimit, ociDBKeyPage, + ociDNSKeySortBy, ociDNSKeySortOrd, + }, + ociDBBackups: { + ociDBKeyDBID, ociDBKeyComp, ociDBKeyState, ociDBKeyLimit, ociDBKeyPage, ociDNSKeySortBy, ociDNSKeySortOrd, + }, +} + +type ociDBCall struct { + body func(c *echo.Context) ([]byte, bool) + path string + coll string + wallet bool +} + +// registerOCIDatabaseRoutes serves the Autonomous Database page by dispatching in-process to the emulator's control +// plane without request signing. Only the database, action, backup and wallet operations the page uses are +// reachable, each write rebuilt from fixed fields. The admin and wallet passwords travel only in the request body +// and are stripped from every answer; the wallet zip is streamed back as an attachment. +func (h *DashboardHandler) registerOCIDatabaseRoutes() { + var inner *echo.Echo + + ops := h.config.OCIDatabaseOps + if ops != nil { + inner = echo.New() + inner.Any("/*", ops.Trusted()) + } + + guard := func(next echo.HandlerFunc) echo.HandlerFunc { + return ociGuard(inner != nil, "OCI Autonomous Database", next) + } + + h.SubRouter.Add(http.MethodGet, ociDBPrefix+"/compartments", guard(func(c *echo.Context) error { + comps := ops.Backend.Compartments() + if !slices.Contains(comps, idbackend.TenancyID) { + comps = slices.Insert(comps, 0, idbackend.TenancyID) + } + + return c.JSON(http.StatusOK, map[string]any{ociCompartmentsKey: comps}) + })) + + h.SubRouter.Add(http.MethodGet, ociDBPrefix+"/engine", guard(func(c *echo.Context) error { + return c.JSON(http.StatusOK, map[string]any{"engine": ops.Backend.EngineMode()}) + })) + + serve := guard(func(c *echo.Context) error { + r := c.Request() + + parts, ok := ociSplitPath(strings.TrimPrefix(r.URL.EscapedPath(), ociDBPrefix+"/api")) + if !ok { + return ociInvalid(c, "unsupported Autonomous Database proxy request") + } + + call, ok := ociDBResolve(r.Method, parts) + if !ok { + return ociInvalid(c, "unsupported Autonomous Database proxy request") + } + + var body []byte + + if call.body != nil { + if body, ok = call.body(c); !ok { + return ociInvalid(c, "invalid Autonomous Database request") + } + } + + query := ociArtQuery(r.URL.Query(), ociDBQueryKeys[call.coll]) + r.Header = http.Header{} + r.URL.RawQuery = query + + if call.wallet { + return ociDBWalletDownload(c, inner, parts[1], body) + } + + rec := ociArtForward(c, inner, call.path, body) + if rec.Code < http.StatusMultipleChoices { + rec.Body = bytes.NewBuffer(ociDBScrubJSON(rec.Body.Bytes())) + } + + return ociRelay(c, rec, "opc-next-page", "etag", "opc-request-id", "opc-work-request-id") + }) + + for _, m := range []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete} { + h.SubRouter.Add(m, ociDBPrefix+"/api/*", serve) + } +} + +// ociDBResolve maps a proxy request to its emulator call; false means the request is outside the allow-list. +func ociDBResolve(method string, parts []string) (ociDBCall, bool) { + if len(parts) > ociDBActionSegs || (len(parts) >= ociDBItemSegs && !ociDBIDOK(parts[1])) { + return ociDBCall{}, false + } + + var ( + call ociDBCall + ok bool + ) + + switch parts[0] { + case ociDBDatabases: + call, ok = ociDBDatabaseCall(method, parts) + case ociDBBackups: + call, ok = ociDBBackupCall(method, parts) + } + + if !ok { + return ociDBCall{}, false + } + + call.coll = parts[0] + call.path = ociDBAPIPrefix + call.path + + return call, true +} + +func ociDBDatabaseCall(method string, parts []string) (ociDBCall, bool) { + switch { + case len(parts) == 1 && method == http.MethodGet: + return ociDBCall{path: "/autonomousDatabases"}, true + case len(parts) == 1 && method == http.MethodPost: + return ociDBCall{path: "/autonomousDatabases", body: ociDBCreateBody}, true + case len(parts) == ociDBItemSegs: + item := "/autonomousDatabases/" + parts[1] + + switch method { + case http.MethodGet, http.MethodDelete: + return ociDBCall{path: item}, true + case http.MethodPut: + return ociDBCall{path: item, body: ociDBUpdateBody}, true + } + case len(parts) == ociDBSubSegs && parts[2] == ociDBWallet && method == http.MethodPost: + return ociDBCall{ + path: "/autonomousDatabases/" + parts[1] + "/actions/generateWallet", body: ociDBWalletBody, wallet: true, + }, true + case len(parts) == ociDBActionSegs && parts[2] == ociDBActions && method == http.MethodPost && ociDBActionOK(parts[3]): + return ociDBCall{path: "/autonomousDatabases/" + parts[1] + "/actions/" + parts[3]}, true + } + + return ociDBCall{}, false +} + +func ociDBBackupCall(method string, parts []string) (ociDBCall, bool) { + switch { + case len(parts) == 1 && method == http.MethodGet: + return ociDBCall{path: "/autonomousDatabaseBackups"}, true + case len(parts) == 1 && method == http.MethodPost: + return ociDBCall{path: "/autonomousDatabaseBackups", body: ociDBBackupBody}, true + } + + return ociDBCall{}, false +} + +func ociDBIDOK(seg string) bool { return ociDBIDRe.MatchString(unescapedPath(seg)) } + +func ociDBActionOK(a string) bool { return a == aciPxStart || a == aciPxStop || a == aciPxRestart } + +func ociDBACLBad(s string) bool { return !ociDBACLRe.MatchString(s) } + +func ociDBText(s string) bool { return s != "" && len(s) <= ociDBMaxName } + +func ociDBWorkloadOK(w string) bool { return slices.Contains([]string{"OLTP", "DW", "AJD", "APEX"}, w) } + +func ociDBModelOK(m string) bool { return m == "" || m == "ECPU" || m == "OCPU" } + +type ociDBCreateIn struct { + Compute *float64 `json:"computeCount"` + Storage *int `json:"dataStorageSizeInTBs"` + FreeTier *bool `json:"isFreeTier"` + Mtls *bool `json:"isMtlsConnectionRequired"` + Comp string `json:"compartmentId"` + Display string `json:"displayName"` + DBName string `json:"dbName"` + Workload string `json:"dbWorkload"` + Model string `json:"computeModel"` + Password string `json:"adminPassword"` +} + +func (in ociDBCreateIn) valid() bool { + return ociDBText(in.Comp) && len(in.Display) <= ociDBMaxName && ociDBNameRe.MatchString(in.DBName) && + ociDBWorkloadOK(in.Workload) && ociDBModelOK(in.Model) && in.Password != "" && + len(in.Password) <= ociDBMaxPassword +} + +func (in ociDBCreateIn) build() map[string]any { + out := map[string]any{ + ociDBKeyComp: in.Comp, "dbName": in.DBName, "dbWorkload": in.Workload, ociDBKeyAdminPass: in.Password, + } + + if in.Display != "" { + out[ociDBKeyDisplay] = in.Display + } + + if in.Model != "" { + out["computeModel"] = in.Model + } + + if in.Compute != nil { + out[ociDBKeyCompute] = *in.Compute + } + + if in.Storage != nil { + out[ociDBKeyStorage] = *in.Storage + } + + if in.FreeTier != nil { + out["isFreeTier"] = *in.FreeTier + } + + if in.Mtls != nil { + out["isMtlsConnectionRequired"] = *in.Mtls + } + + return out +} + +func ociDBCreateBody(c *echo.Context) ([]byte, bool) { + var in ociDBCreateIn + + if !ociDecodeBodyN(c, &in, ociDBMaxBody) || !in.valid() { + return nil, false + } + + return ociArtMarshal(in.build()) +} + +func ociDBUpdateBody(c *echo.Context) ([]byte, bool) { + var in struct { + Display *string `json:"displayName"` + ACL *[]string `json:"whitelistedIps"` + ACLOn *bool `json:"isAccessControlEnabled"` + Compute *float64 `json:"computeCount"` + } + + if !ociDecodeBodyN(c, &in, ociDBMaxBody) { + return nil, false + } + + out := map[string]any{} + + if in.Display != nil { + if !ociDBText(*in.Display) { + return nil, false + } + + out[ociDBKeyDisplay] = *in.Display + } + + if in.ACL != nil { + if len(*in.ACL) > ociDBMaxACL || slices.ContainsFunc(*in.ACL, ociDBACLBad) { + return nil, false + } + + out[ociDBKeyACL] = *in.ACL + } + + if in.ACLOn != nil { + out["isAccessControlEnabled"] = *in.ACLOn + } + + if in.Compute != nil { + out[ociDBKeyCompute] = *in.Compute + } + + if len(out) == 0 { + return nil, false + } + + return ociArtMarshal(out) +} + +func ociDBBackupBody(c *echo.Context) ([]byte, bool) { + var in struct { + DBID string `json:"autonomousDatabaseId"` + Display string `json:"displayName"` + } + + if !ociDecodeBodyN(c, &in, ociDBMaxBody) || !ociDBIDRe.MatchString(in.DBID) || len(in.Display) > ociDBMaxName { + return nil, false + } + + out := map[string]any{ociDBKeyDBID: in.DBID} + if in.Display != "" { + out[ociDBKeyDisplay] = in.Display + } + + return ociArtMarshal(out) +} + +func ociDBWalletBody(c *echo.Context) ([]byte, bool) { + var in struct { + Password string `json:"password"` + } + + if !ociDecodeBodyN(c, &in, ociDBMaxBody) || in.Password == "" || len(in.Password) > ociDBMaxPassword { + return nil, false + } + + return ociArtMarshal(map[string]any{ociDBKeyPassword: in.Password, "generateType": "ALL"}) +} + +// ociDBScrubJSON removes password fields from any answer. +func ociDBScrubJSON(raw []byte) []byte { + if len(bytes.TrimSpace(raw)) == 0 { + return raw + } + + var doc any + if json.Unmarshal(raw, &doc) != nil { + return []byte("{}") + } + + ociDBStrip(doc) + + out, err := json.Marshal(doc) + if err != nil { + return []byte("{}") + } + + return out +} + +func ociDBStrip(v any) { + switch t := v.(type) { + case []any: + for _, x := range t { + ociDBStrip(x) + } + case map[string]any: + delete(t, ociDBKeyPassword) + delete(t, ociDBKeyAdminPass) + + for _, x := range t { + ociDBStrip(x) + } + } +} + +// ociDBWalletDownload answers GenerateAutonomousDatabaseWallet as a zip attachment. The wallet password is only +// ever in the forwarded body; failures keep the emulator's JSON error envelope. +func ociDBWalletDownload(c *echo.Context, inner *echo.Echo, id string, body []byte) error { + c.Request().Method = http.MethodGet + + meta := ociArtForward(c, inner, ociDBAPIPrefix+"/autonomousDatabases/"+id, nil) + if meta.Code != http.StatusOK { + return ociRelay(c, meta, "opc-request-id") + } + + var db struct { + DBName string `json:"dbName"` + } + + _ = json.Unmarshal(meta.Body.Bytes(), &db) + + c.Request().Method = http.MethodPost + + rec := ociArtForward(c, inner, ociDBAPIPrefix+"/autonomousDatabases/"+id+"/actions/generateWallet", body) + if rec.Code != http.StatusOK { + return ociRelay(c, ociDBScrubRec(rec), "opc-request-id") + } + + h := c.Response().Header() + h.Set("Content-Disposition", `attachment; filename="`+ociDBWalletName(db.DBName)+`"`) + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Content-Security-Policy", "sandbox; default-src 'none'") + h.Set("Cache-Control", "no-store") + + return c.Blob(http.StatusOK, "application/zip", rec.Body.Bytes()) +} + +func ociDBScrubRec(rec *httptest.ResponseRecorder) *httptest.ResponseRecorder { + rec.Body = bytes.NewBuffer(ociDBScrubJSON(rec.Body.Bytes())) + + return rec +} + +// ociDBWalletName builds the download file name from the database name, reduced to letters and digits. +func ociDBWalletName(dbName string) string { + name := ociDBFileRe.ReplaceAllString(dbName, "") + if name == "" { + name = "database" + } + + return ociDBWalletFile + name + ".zip" +} diff --git a/dashboard/ocidatabase_proxy_test.go b/dashboard/ocidatabase_proxy_test.go new file mode 100644 index 0000000000..99e2f5011c --- /dev/null +++ b/dashboard/ocidatabase_proxy_test.go @@ -0,0 +1,327 @@ +package dashboard_test + +import ( + "archive/zip" + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/ocidatabase" +) + +const ( + dbComp = "ocid1.compartment.oc1..dbcomp" + dbBase = "/dashboard/api/v1/oci/database" + dbAPI = dbBase + "/api" + dbAdminPass = "Sup3rSecretPw#Value" + dbWalletPass = "Sup3rSecretWallet#1" +) + +type dbFixture struct { + h *dashboard.DashboardHandler + ops *ocidatabase.Handler +} + +func newDBDashboard(t *testing.T) dbFixture { + t.Helper() + + be := ocidatabase.NewInMemoryBackend() + be.SetTransition(0) + t.Cleanup(be.Close) + + ops := ocidatabase.NewHandler(be) + + return dbFixture{h: dashboard.NewHandler(dashboard.Config{OCIDatabaseOps: ops}), ops: ops} +} + +func (f dbFixture) direct(t *testing.T, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + e := echo.New() + e.Any("/*", f.ops.Trusted()) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + e.ServeHTTP(rec, req) + + return rec +} + +func (f dbFixture) proxy(t *testing.T, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + return lbDo(t, f.h, method, path, body) +} + +func (f dbFixture) settle(t *testing.T, id, state string) { + t.Helper() + + require.Eventually(t, func() bool { + f.ops.Backend.Sweep() + + rec := f.direct(t, http.MethodGet, "/20160918/autonomousDatabases/"+id, "") + + var got struct { + State string `json:"lifecycleState"` + } + + return json.Unmarshal(rec.Body.Bytes(), &got) == nil && got.State == state + }, 10*time.Second, 5*time.Millisecond) +} + +func (f dbFixture) create(t *testing.T, extra string) string { + t.Helper() + + rec := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabases", + `{"compartmentId":"`+dbComp+`","displayName":"orders","dbName":"ORDERS","dbWorkload":"OLTP", + "adminPassword":"`+dbAdminPass+`","computeModel":"ECPU","computeCount":2,"dataStorageSizeInTBs":1`+extra+`}`) + require.Equal(t, http.StatusAccepted, rec.Code, rec.Body.String()) + + var db struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &db)) + f.settle(t, db.ID, "AVAILABLE") + + return db.ID +} + +func TestOCIDatabaseProxyLifecycle(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + id := f.create(t, "") + item := dbAPI + "/autonomousDatabases/" + id + + list := f.proxy(t, http.MethodGet, dbAPI+"/autonomousDatabases?compartmentId="+dbComp, "") + require.Equal(t, http.StatusOK, list.Code, list.Body.String()) + assert.Contains(t, list.Body.String(), `"dbName":"ORDERS"`) + + for _, step := range []struct{ action, state string }{ + {"stop", "STOPPED"}, {"start", "AVAILABLE"}, {"restart", "AVAILABLE"}, + } { + got := f.proxy(t, http.MethodPost, item+"/actions/"+step.action, "") + require.Less(t, got.Code, http.StatusMultipleChoices, "%s: %s", step.action, got.Body.String()) + f.settle(t, id, step.state) + } + + upd := f.proxy(t, http.MethodPut, item, + `{"displayName":"orders-renamed","whitelistedIps":["203.0.113.0/24"],"isAccessControlEnabled":true, + "computeCount":4,"isFreeTier":true,"adminPassword":"Different#Passw0rd1"}`) + require.Less(t, upd.Code, http.StatusMultipleChoices, upd.Body.String()) + f.settle(t, id, "AVAILABLE") + + got := f.proxy(t, http.MethodGet, item, "") + require.Equal(t, http.StatusOK, got.Code) + assert.Contains(t, got.Body.String(), `"displayName":"orders-renamed"`) + assert.Contains(t, got.Body.String(), `"203.0.113.0/24"`) + assert.Contains(t, got.Body.String(), `"computeCount":4`) + assert.Contains(t, got.Body.String(), `"isFreeTier":false`, "fields outside the allow-list are dropped") + + const longTerm = `,"isLongTermBackup":true,"retentionPeriodInDays":365}` + + bk := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabaseBackups", + `{"autonomousDatabaseId":"`+id+`","displayName":"manual-1"`+longTerm) + require.Equal(t, http.StatusAccepted, bk.Code, bk.Body.String()) + assert.Contains(t, bk.Body.String(), `"displayName":"manual-1"`) + assert.NotContains(t, bk.Body.String(), "LONGTERM") + + query := "?compartmentId=" + dbComp + "&autonomousDatabaseId=" + id + bks := f.proxy(t, http.MethodGet, dbAPI+"/autonomousDatabaseBackups"+query, "") + require.Equal(t, http.StatusOK, bks.Code, bks.Body.String()) + assert.Contains(t, bks.Body.String(), `"manual-1"`) + + del := f.proxy(t, http.MethodDelete, item, "") + assert.Less(t, del.Code, http.StatusMultipleChoices, del.Body.String()) +} + +func TestOCIDatabaseProxyPasswordsNeverEchoed(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + id := f.create(t, "") + item := dbAPI + "/autonomousDatabases/" + id + + paths := []struct{ method, path, body string }{ + {http.MethodGet, item, ""}, + {http.MethodGet, dbAPI + "/autonomousDatabases?compartmentId=" + dbComp, ""}, + {http.MethodPost, item + "/actions/stop", ""}, + } + + for _, p := range paths { + got := f.proxy(t, p.method, p.path, p.body) + assert.NotContains(t, got.Body.String(), dbAdminPass, "%s %s", p.method, p.path) + assert.NotContains(t, got.Body.String(), "adminPassword", "%s %s", p.method, p.path) + } + + create := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabases", + `{"compartmentId":"`+dbComp+`","dbName":"SECOND","dbWorkload":"DW","adminPassword":"`+dbAdminPass+`", + "computeCount":2,"dataStorageSizeInTBs":1}`) + require.Equal(t, http.StatusAccepted, create.Code, create.Body.String()) + assert.NotContains(t, create.Body.String(), dbAdminPass) + + rejected := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabases", + `{"compartmentId":"`+dbComp+`","dbName":"THIRD","dbWorkload":"OLTP","adminPassword":"weak"}`) + assert.GreaterOrEqual(t, rejected.Code, http.StatusBadRequest) + assert.NotContains(t, rejected.Body.String(), "weak\"") +} + +func TestOCIDatabaseProxyCreateRebuild(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + id := f.create(t, `,"freeformTags":{"owner":"mallory"},"isDedicated":true,"isDevTier":true, + "secretId":"ocid1.secret","source":"BACKUP_FROM_ID","whitelistedIps":["1.2.3.4"],"subnetId":"ocid1.subnet.x"`) + + got := f.direct(t, http.MethodGet, "/20160918/autonomousDatabases/"+id, "") + require.Equal(t, http.StatusOK, got.Code) + assert.NotContains(t, got.Body.String(), "mallory") + assert.NotContains(t, got.Body.String(), "1.2.3.4") + assert.NotContains(t, got.Body.String(), "ocid1.subnet.x") +} + +func TestOCIDatabaseProxyWalletAttachment(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + id := f.create(t, "") + + got := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabases/"+id+"/wallet", + `{"password":"`+dbWalletPass+`","generateType":"SINGLE","isRegional":true}`) + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Equal(t, "application/zip", got.Header().Get("Content-Type")) + assert.Equal(t, `attachment; filename="Wallet_ORDERS.zip"`, got.Header().Get("Content-Disposition")) + assert.Equal(t, "nosniff", got.Header().Get("X-Content-Type-Options")) + assert.Equal(t, "no-store", got.Header().Get("Cache-Control")) + assert.Contains(t, got.Header().Get("Content-Security-Policy"), "sandbox") + + zr, err := zip.NewReader(bytes.NewReader(got.Body.Bytes()), int64(got.Body.Len())) + require.NoError(t, err) + + names := make([]string, 0, len(zr.File)) + + for _, zf := range zr.File { + names = append(names, zf.Name) + } + + assert.Contains(t, names, "tnsnames.ora") + assert.Contains(t, names, "ewallet.p12") + assert.NotContains(t, got.Header().Get("Content-Disposition"), dbWalletPass) +} + +func TestOCIDatabaseProxyWalletFailures(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + id := f.create(t, "") + wallet := dbAPI + "/autonomousDatabases/" + id + "/wallet" + + weak := f.proxy(t, http.MethodPost, wallet, `{"password":"short1!"}`) + assert.Equal(t, http.StatusBadRequest, weak.Code, weak.Body.String()) + assert.NotContains(t, weak.Body.String(), "short1!") + assert.Empty(t, weak.Header().Get("Content-Disposition")) + + for _, body := range []string{`{}`, `{"password":""}`, `{`, `{"password":"` + strings.Repeat("a1!", 100) + `"}`} { + got := f.proxy(t, http.MethodPost, wallet, body) + assert.Equal(t, http.StatusBadRequest, got.Code, body) + } + + missing := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabases/ocid1.nope/wallet", + `{"password":"`+dbWalletPass+`"}`) + assert.Equal(t, http.StatusNotFound, missing.Code, missing.Body.String()) + assert.NotContains(t, missing.Body.String(), dbWalletPass) + + assert.Equal(t, http.StatusBadRequest, f.proxy(t, http.MethodGet, wallet, "").Code) + assert.Equal(t, http.StatusBadRequest, f.proxy(t, http.MethodPut, wallet, `{"shouldRotate":true}`).Code) +} + +func TestOCIDatabaseProxyRejectsOutsideAllowList(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + item := dbAPI + "/autonomousDatabases/ocid1.db" + valid := `{"compartmentId":"` + dbComp + `","dbName":"X","dbWorkload":"OLTP","adminPassword":"` + dbAdminPass + `"}` + + tests := []struct { + name string + method string + path string + body string + }{ + {"restore", http.MethodPost, item + "/actions/restore", `{"latest":true}`}, + {"change compartment", http.MethodPost, item + "/actions/changeCompartment", `{"compartmentId":"x"}`}, + {"generate wallet action", http.MethodPost, item + "/actions/generateWallet", `{"password":"abcdefg1!"}`}, + {"unknown action", http.MethodPost, item + "/actions/terminate", ""}, + {"rotate wallet", http.MethodPut, item + "/wallet", `{"shouldRotate":true}`}, + {"delete backup", http.MethodDelete, dbAPI + "/autonomousDatabaseBackups/b1", ""}, + {"get backup", http.MethodGet, dbAPI + "/autonomousDatabaseBackups/b1", ""}, + {"work requests", http.MethodGet, dbAPI + "/workRequests", ""}, + {"patch database", http.MethodPatch, item, `{}`}, + {"unknown collection", http.MethodGet, dbAPI + "/cloudExadataInfrastructures", ""}, + {"traversal", http.MethodGet, dbAPI + "/autonomousDatabases/%2e%2e", ""}, + {"encoded slash", http.MethodGet, dbAPI + "/autonomousDatabases/a%2Fb", ""}, + {"admin path", http.MethodPost, dbAPI + "/_gopherstack/keys", `{}`}, + {"too deep", http.MethodGet, item + "/a/b/c", ""}, + {"bad db name", http.MethodPost, dbAPI + "/autonomousDatabases", strings.Replace(valid, `"X"`, `"1 bad"`, 1)}, + {"bad workload", http.MethodPost, dbAPI + "/autonomousDatabases", strings.Replace(valid, `"OLTP"`, `"EXA"`, 1)}, + { + "no password", http.MethodPost, dbAPI + "/autonomousDatabases", + `{"compartmentId":"` + dbComp + `","dbName":"X","dbWorkload":"OLTP"}`, + }, + {"empty update", http.MethodPut, item, `{"freeformTags":{"a":"b"}}`}, + {"bad acl", http.MethodPut, item, `{"whitelistedIps":["a b"]}`}, + {"empty display name", http.MethodPut, item, `{"displayName":""}`}, + {"backup without db", http.MethodPost, dbAPI + "/autonomousDatabaseBackups", `{"displayName":"x"}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := f.proxy(t, tt.method, tt.path, tt.body) + assert.Contains(t, []int{http.StatusBadRequest, http.StatusMethodNotAllowed}, got.Code, got.Body.String()) + }) + } +} + +func TestOCIDatabaseProxyReadRoutes(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + f.create(t, "") + + comps := f.proxy(t, http.MethodGet, dbBase+"/compartments", "") + require.Equal(t, http.StatusOK, comps.Code) + assert.Contains(t, comps.Body.String(), dbComp) + + engine := f.proxy(t, http.MethodGet, dbBase+"/engine", "") + require.Equal(t, http.StatusOK, engine.Code) + assert.Contains(t, engine.Body.String(), `"engine":"stub"`) +} + +func TestOCIDatabaseProxyUnavailableWithoutService(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + for _, p := range []struct{ method, path string }{ + {http.MethodGet, dbBase + "/compartments"}, + {http.MethodGet, dbBase + "/engine"}, + {http.MethodGet, dbAPI + "/autonomousDatabases?compartmentId=" + dbComp}, + {http.MethodPost, dbAPI + "/autonomousDatabases/x/wallet"}, + } { + got := lbDo(t, h, p.method, p.path, `{}`) + assert.Equal(t, http.StatusServiceUnavailable, got.Code, "%s %s", p.method, p.path) + } +} diff --git a/dashboard/ocidns_proxy.go b/dashboard/ocidns_proxy.go new file mode 100644 index 0000000000..b6751dfafe --- /dev/null +++ b/dashboard/ocidns_proxy.go @@ -0,0 +1,469 @@ +package dashboard + +import ( + "bytes" + "context" + "encoding/json" + "net" + "net/http" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "github.com/labstack/echo/v5" + mdns "github.com/miekg/dns" + + idbackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +const ( + ociDNSPrefix = ociProxyPrefix + "/dns" + ociDNSAPIPrefix = "/20180115" + ociDNSZones = "zones" + ociDNSZoneItem = "zone" + ociDNSViews = "views" + ociDNSResolvers = "resolvers" + ociDNSEndpoints = "endpoints" + ociDNSRecords = "records" + ociDNSTsigKeys = "tsigKeys" + ociDNSPolicies = "steeringPolicies" + ociDNSMaxBody = 1 << 16 + ociDNSMaxName = 255 + ociDNSMaxRdata = 4096 + ociDNSMaxItems = 50 + ociDNSMaxTTL = 2147483647 + ociDNSMaxFQDN = 253 + ociDNSMaxLabel = 63 + ociDNSItemSegs = 2 + ociDNSSubSegs = 3 + ociDNSQueryWait = 3 * time.Second + ociDNSKeyLimit = "limit" + ociDNSKeyPage = "page" + ociDNSKeyScope = "scope" + ociDNSKeyViewID = "viewId" + ociDNSKeySortBy = "sortBy" + ociDNSKeySortOrd = "sortOrder" + ociDNSKeyName = "name" + ociDNSKeyState = "lifecycleState" + ociDNSKeyDomain = "domain" + ociDNSKeyRtype = "rtype" + ociDNSKeyDisplay = "displayName" + ociDNSKeyComp = "compartmentId" + ociDNSKeySecret = "secret" + ociDNSKeyPort = "port" + ociDNSKeyType = "type" + ociDNSKeyEnabled = "enabled" + ociDNSOpAdd = "ADD" + ociDNSOpRemove = "REMOVE" + ociDNSScopeGlob = "GLOBAL" + ociDNSScopePriv = "PRIVATE" + ociDNSLoopback = "127.0.0.1" +) + +var ( + ociDNSIDRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,255}$`) + ociDNSDomainRe = regexp.MustCompile(`^[A-Za-z0-9_*.-]{1,255}$`) + ociDNSLabelRe = regexp.MustCompile(`^[A-Za-z0-9_*]([A-Za-z0-9_-]*[A-Za-z0-9_])?$`) +) + +// ociDNSQueryKeys are the only query parameters each listing forwards; everything else is dropped. +// +//nolint:gochecknoglobals // immutable allow-list +var ociDNSQueryKeys = map[string][]string{ + ociDNSZones: { + ociDNSKeyComp, ociDNSKeyScope, ociDNSKeyViewID, ociDNSKeyName, "nameContains", "zoneType", ociDNSKeyState, + ociDNSKeyLimit, ociDNSKeyPage, ociDNSKeySortBy, ociDNSKeySortOrd, + }, + ociDNSRecords: { + ociDNSKeyScope, ociDNSKeyViewID, ociDNSKeyDomain, "domainContains", ociDNSKeyRtype, "zoneVersion", + ociDNSKeyLimit, ociDNSKeyPage, ociDNSKeySortBy, ociDNSKeySortOrd, + }, + ociDNSViews: {ociDNSKeyComp, ociDNSKeyDisplay, ociDNSKeyLimit, ociDNSKeyPage}, + ociDNSResolvers: {ociDNSKeyComp, ociDNSKeyDisplay, ociDNSKeyLimit, ociDNSKeyPage}, + ociDNSEndpoints: {ociDNSKeyLimit, ociDNSKeyPage}, + ociDNSTsigKeys: {ociDNSKeyComp, ociDNSKeyName, ociDNSKeyLimit, ociDNSKeyPage}, + ociDNSPolicies: {ociDNSKeyComp, ociDNSKeyDisplay, ociDNSKeyLimit, ociDNSKeyPage}, + ociDNSZoneItem: {ociDNSKeyScope, ociDNSKeyViewID}, +} + +// ociDNSQueryTypes are the record types the Try it panel may ask for. +// +//nolint:gochecknoglobals // immutable allow-list +var ociDNSQueryTypes = []string{"A", "AAAA", "CNAME", "MX", "NS", "TXT", "SOA", "SRV", "PTR", "CAA"} + +type ociDNSCall struct { + body func(c *echo.Context) ([]byte, bool) + path string + coll string + cond bool +} + +// registerOCIDNSRoutes serves the OCI DNS page by dispatching in-process to the emulator's control plane without +// request signing. Only the zone, record, view, resolver, TSIG key and steering policy operations the page uses are +// reachable, each write rebuilt from fixed fields; TSIG secrets are removed from every answer, and the Try it query +// goes to the emulator's own DNS data plane on the loopback address only. +func (h *DashboardHandler) registerOCIDNSRoutes() { + var inner *echo.Echo + + ops := h.config.OCIDNSOps + if ops != nil { + inner = echo.New() + inner.Any("/*", ops.Trusted()) + } + + guard := func(next echo.HandlerFunc) echo.HandlerFunc { + return ociGuard(inner != nil, "OCI DNS", next) + } + + h.SubRouter.Add(http.MethodGet, ociDNSPrefix+"/compartments", guard(func(c *echo.Context) error { + comps := ops.Backend.Compartments() + if !slices.Contains(comps, idbackend.TenancyID) { + comps = slices.Insert(comps, 0, idbackend.TenancyID) + } + + return c.JSON(http.StatusOK, map[string]any{ociCompartmentsKey: comps}) + })) + + h.SubRouter.Add(http.MethodGet, ociDNSPrefix+"/dataplane", guard(func(c *echo.Context) error { + port := ops.DataPlanePort() + + return c.JSON(http.StatusOK, map[string]any{ociDNSKeyEnabled: port > 0, ociDNSKeyPort: port}) + })) + + h.SubRouter.Add(http.MethodPost, ociDNSPrefix+"/query", guard(func(c *echo.Context) error { + return ociDNSTryIt(c, ops.DataPlanePort()) + })) + + serve := guard(func(c *echo.Context) error { return ociDNSServe(c, inner) }) + + for _, m := range []string{http.MethodGet, http.MethodPost, http.MethodPatch, http.MethodDelete} { + h.SubRouter.Add(m, ociDNSPrefix+"/api/*", serve) + } +} + +// ociDNSServe forwards one allow-listed control-plane call. +func ociDNSServe(c *echo.Context, inner *echo.Echo) error { + r := c.Request() + + parts, ok := ociSplitPath(strings.TrimPrefix(r.URL.EscapedPath(), ociDNSPrefix+"/api")) + if !ok { + return ociInvalid(c, "unsupported OCI DNS proxy request") + } + + call, ok := ociDNSResolve(r.Method, parts) + if !ok { + return ociInvalid(c, "unsupported OCI DNS proxy request") + } + + var body []byte + + if call.body != nil { + if body, ok = call.body(c); !ok { + return ociInvalid(c, "invalid OCI DNS request") + } + } + + ifMatch := "" + if call.cond { + ifMatch = r.Header.Get("If-Match") + if len(ifMatch) > ociDNSMaxName { + return ociInvalid(c, "invalid If-Match header") + } + } + + query := ociArtQuery(r.URL.Query(), ociDNSQueryKeys[call.coll]) + r.Header = http.Header{} + + if ifMatch != "" { + r.Header.Set("If-Match", ifMatch) + } + + r.URL.RawQuery = query + + rec := ociArtForward(c, inner, call.path, body) + if rec.Code < http.StatusMultipleChoices { + rec.Body = bytes.NewBuffer(ociDNSScrub(rec.Body.Bytes())) + } + + return ociRelay(c, rec, "opc-next-page", "etag", "opc-request-id", "opc-total-items") +} + +// ociDNSResolve maps a proxy request to its emulator call; false means the request is outside the allow-list. +func ociDNSResolve(method string, parts []string) (ociDNSCall, bool) { + if len(parts) > ociDNSSubSegs || (len(parts) >= ociDNSItemSegs && !ociDNSIDOK(parts[1])) { + return ociDNSCall{}, false + } + + var ( + call ociDNSCall + ok bool + ) + + switch parts[0] { + case ociDNSZones: + call, ok = ociDNSZoneCall(method, parts) + case ociDNSViews, ociDNSTsigKeys, ociDNSPolicies: + call, ok = ociDNSCall{path: "/" + parts[0]}, method == http.MethodGet && len(parts) == 1 + case ociDNSResolvers: + call, ok = ociDNSResolverCall(method, parts) + } + + if !ok { + return ociDNSCall{}, false + } + + if call.coll == "" { + call.coll = parts[0] + } + + call.path = ociDNSAPIPrefix + call.path + + return call, true +} + +func ociDNSZoneCall(method string, parts []string) (ociDNSCall, bool) { + switch { + case len(parts) == 1 && method == http.MethodGet: + return ociDNSCall{path: "/zones"}, true + case len(parts) == 1 && method == http.MethodPost: + return ociDNSCall{path: "/zones", body: ociDNSZoneBody}, true + case len(parts) == ociDNSItemSegs && method == http.MethodGet: + return ociDNSCall{path: "/zones/" + parts[1], coll: ociDNSZoneItem}, true + case len(parts) == ociDNSItemSegs && method == http.MethodDelete: + return ociDNSCall{path: "/zones/" + parts[1], coll: ociDNSZoneItem, cond: true}, true + case len(parts) == ociDNSSubSegs && parts[2] == ociDNSRecords && method == http.MethodGet: + return ociDNSCall{path: "/zones/" + parts[1] + "/records", coll: ociDNSRecords}, true + case len(parts) == ociDNSSubSegs && parts[2] == ociDNSRecords && method == http.MethodPatch: + return ociDNSCall{ + path: "/zones/" + parts[1] + "/records", coll: ociDNSRecords, cond: true, body: ociDNSRecordsBody, + }, true + } + + return ociDNSCall{}, false +} + +func ociDNSResolverCall(method string, parts []string) (ociDNSCall, bool) { + switch { + case len(parts) == 1 && method == http.MethodGet: + return ociDNSCall{path: "/resolvers"}, true + case len(parts) == ociDNSSubSegs && parts[2] == ociDNSEndpoints && method == http.MethodGet: + return ociDNSCall{path: "/resolvers/" + parts[1] + "/endpoints", coll: ociDNSEndpoints}, true + } + + return ociDNSCall{}, false +} + +func ociDNSIDOK(seg string) bool { return ociDNSIDRe.MatchString(unescapedPath(seg)) } + +func ociDNSName(s string) bool { return s != "" && len(s) <= ociDNSMaxName } + +func ociDNSZoneBody(c *echo.Context) ([]byte, bool) { + var in struct { + Name string `json:"name"` + Compartment string `json:"compartmentId"` + ZoneType string `json:"zoneType"` + Scope string `json:"scope"` + ViewID string `json:"viewId"` + } + + if !ociDecodeBodyN(c, &in, ociDNSMaxBody) { + return nil, false + } + + scope := in.Scope + if scope == "" { + scope = ociDNSScopeGlob + } + + if !ociDNSName(in.Name) || !ociDNSName(in.Compartment) || (in.ZoneType != "" && in.ZoneType != "PRIMARY") || + (scope != ociDNSScopeGlob && scope != ociDNSScopePriv) || + (scope == ociDNSScopePriv) != (in.ViewID != "") || len(in.ViewID) > ociDNSMaxName { + return nil, false + } + + out := map[string]any{ + ociDNSKeyName: in.Name, ociDNSKeyComp: in.Compartment, "zoneType": "PRIMARY", ociDNSKeyScope: scope, + } + + if in.ViewID != "" { + out[ociDNSKeyViewID] = in.ViewID + } + + return ociArtMarshal(out) +} + +type ociDNSRecordIn struct { + TTL *int `json:"ttl"` + Operation string `json:"operation"` + Domain string `json:"domain"` + Rtype string `json:"rtype"` + Rdata string `json:"rdata"` +} + +func (in ociDNSRecordIn) build() (map[string]any, bool) { + if (in.Operation != ociDNSOpAdd && in.Operation != ociDNSOpRemove) || !ociDNSDomainRe.MatchString(in.Domain) || + !slices.Contains(ociDNSQueryTypes, in.Rtype) || in.Rdata == "" || len(in.Rdata) > ociDNSMaxRdata || + strings.ContainsFunc(in.Rdata, func(r rune) bool { return r < ' ' }) { + return nil, false + } + + out := map[string]any{ + "operation": in.Operation, ociDNSKeyDomain: in.Domain, ociDNSKeyRtype: in.Rtype, "rdata": in.Rdata, + } + + if in.TTL != nil { + if *in.TTL < 0 || *in.TTL > ociDNSMaxTTL { + return nil, false + } + + out["ttl"] = *in.TTL + } else if in.Operation == ociDNSOpAdd { + return nil, false + } + + return out, true +} + +func ociDNSRecordsBody(c *echo.Context) ([]byte, bool) { + var in struct { + Items []ociDNSRecordIn `json:"items"` + } + + if !ociDecodeBodyN(c, &in, ociDNSMaxBody) || len(in.Items) == 0 || len(in.Items) > ociDNSMaxItems { + return nil, false + } + + items := make([]map[string]any, 0, len(in.Items)) + + for _, it := range in.Items { + built, ok := it.build() + if !ok { + return nil, false + } + + items = append(items, built) + } + + return ociArtMarshal(map[string]any{"items": items}) +} + +// ociDNSScrub removes TSIG secrets from any answer. +func ociDNSScrub(raw []byte) []byte { + if len(bytes.TrimSpace(raw)) == 0 { + return raw + } + + var doc any + if json.Unmarshal(raw, &doc) != nil { + return []byte("{}") + } + + ociDNSStrip(doc) + + out, err := json.Marshal(doc) + if err != nil { + return []byte("{}") + } + + return out +} + +func ociDNSStrip(v any) { + switch t := v.(type) { + case []any: + for _, x := range t { + ociDNSStrip(x) + } + case map[string]any: + delete(t, ociDNSKeySecret) + + for _, x := range t { + ociDNSStrip(x) + } + } +} + +// ociDNSQName validates a query name: a dotted host name of at most 253 characters with valid labels. +func ociDNSQName(name string) (string, bool) { + n := strings.TrimSuffix(name, ".") + if n == "" || len(n) > ociDNSMaxFQDN { + return "", false + } + + for label := range strings.SplitSeq(n, ".") { + if len(label) > ociDNSMaxLabel || !ociDNSLabelRe.MatchString(label) { + return "", false + } + } + + return mdns.Fqdn(n), true +} + +type ociDNSRR struct { + Name string `json:"name"` + Type string `json:"type"` + Data string `json:"data"` + TTL uint32 `json:"ttl"` +} + +func ociDNSRRs(in []mdns.RR) []ociDNSRR { + out := make([]ociDNSRR, 0, len(in)) + + for _, rr := range in { + hdr := rr.Header() + out = append(out, ociDNSRR{ + Name: hdr.Name, Type: mdns.TypeToString[hdr.Rrtype], TTL: hdr.Ttl, + Data: strings.TrimSpace(strings.TrimPrefix(rr.String(), hdr.String())), + }) + } + + return out +} + +// ociDNSTryIt runs one query against the emulator's DNS data plane. The address is always the loopback host with +// the configured data plane port; nothing in the request selects where the query goes. +func ociDNSTryIt(c *echo.Context, port int) error { + if port <= 0 { + return cloudJSONFail(c, http.StatusServiceUnavailable, "DataPlaneDisabled", + "the OCI DNS data plane is not running (OCI_DNS_PORT_DNS is 0)") + } + + var in struct { + Name string `json:"name"` + Type string `json:"type"` + } + + if !ociDecodeBodyN(c, &in, ociDNSMaxBody) { + return ociInvalid(c, "invalid query request") + } + + fqdn, ok := ociDNSQName(in.Name) + if !ok || !slices.Contains(ociDNSQueryTypes, in.Type) { + return ociInvalid(c, "query name or type is not valid") + } + + msg := new(mdns.Msg) + msg.SetQuestion(fqdn, mdns.StringToType[in.Type]) + + ctx, cancel := context.WithTimeout(c.Request().Context(), ociDNSQueryWait) + defer cancel() + + addr := net.JoinHostPort(ociDNSLoopback, strconv.Itoa(port)) + + resp, _, err := (&mdns.Client{Net: "udp", Timeout: ociDNSQueryWait}).ExchangeContext(ctx, msg, addr) + if err == nil && resp.Truncated { + resp, _, err = (&mdns.Client{Net: "tcp", Timeout: ociDNSQueryWait}).ExchangeContext(ctx, msg, addr) + } + + if err != nil { + return cloudJSONFail(c, http.StatusBadGateway, "QueryFailed", "the DNS data plane did not answer") + } + + return c.JSON(http.StatusOK, map[string]any{ + "rcode": mdns.RcodeToString[resp.Rcode], "authoritative": resp.Authoritative, + "answer": ociDNSRRs(resp.Answer), "authority": ociDNSRRs(resp.Ns), "additional": ociDNSRRs(resp.Extra), + ociDNSKeyName: fqdn, ociDNSKeyType: in.Type, "server": addr, + }) +} diff --git a/dashboard/ocidns_proxy_test.go b/dashboard/ocidns_proxy_test.go new file mode 100644 index 0000000000..99e5bb0a33 --- /dev/null +++ b/dashboard/ocidns_proxy_test.go @@ -0,0 +1,409 @@ +package dashboard_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/ocidns" +) + +const ( + dnsComp = "ocid1.compartment.oc1..dnscomp" + dnsBase = "/dashboard/api/v1/oci/dns" + dnsAPI = dnsBase + "/api" + dnsSecret = "c3VwZXItc2VjcmV0LXRzaWctdmFsdWU=" +) + +type dnsFixture struct { + h *dashboard.DashboardHandler + ops *ocidns.Handler +} + +func newDNSDashboard(t *testing.T, dataPlane bool) dnsFixture { + t.Helper() + + ops := ocidns.NewHandler(ocidns.NewInMemoryBackend()) + ops.Port, ops.DNSPort, ops.DNSEphemeral = 0, 0, dataPlane + + require.NoError(t, ops.StartWorker(t.Context())) + t.Cleanup(func() { ops.Shutdown(context.Background()) }) + + return dnsFixture{h: dashboard.NewHandler(dashboard.Config{OCIDNSOps: ops}), ops: ops} +} + +func (f dnsFixture) direct(t *testing.T, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + e := echo.New() + e.Any("/*", f.ops.Trusted()) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + e.ServeHTTP(rec, req) + + return rec +} + +func (f dnsFixture) proxy(t *testing.T, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + return lbDo(t, f.h, method, path, body) +} + +func (f dnsFixture) zone(t *testing.T, name string) string { + t.Helper() + + rec := f.proxy(t, http.MethodPost, dnsAPI+"/zones", + `{"name":"`+name+`","compartmentId":"`+dnsComp+`","zoneType":"PRIMARY","scope":"GLOBAL"}`) + require.Less(t, rec.Code, http.StatusMultipleChoices, rec.Body.String()) + + var z struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &z)) + + return z.ID +} + +func (f dnsFixture) addRecord(t *testing.T, zoneID, domain, rtype, rdata string) { + t.Helper() + + rec := f.proxy(t, http.MethodPatch, dnsAPI+"/zones/"+zoneID+"/records", + `{"items":[{"operation":"ADD","domain":"`+domain+`","rtype":"`+rtype+`","rdata":"`+rdata+`","ttl":60}]}`) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) +} + +func TestOCIDNSProxyZoneAndRecordLifecycle(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + + rec := f.proxy(t, http.MethodPost, dnsAPI+"/zones", + `{"name":"example.com","compartmentId":"`+dnsComp+`","zoneType":"PRIMARY","scope":"GLOBAL", + "freeformTags":{"owner":"mallory"},"migrationSource":"DYNECT","externalMasters":[{"address":"10.9.9.9"}]}`) + require.Less(t, rec.Code, http.StatusMultipleChoices, rec.Body.String()) + assert.NotContains(t, rec.Body.String(), "mallory") + assert.NotContains(t, rec.Body.String(), "10.9.9.9") + + var zone struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &zone)) + + list := f.proxy(t, http.MethodGet, dnsAPI+"/zones?compartmentId="+dnsComp+"&scope=GLOBAL", "") + require.Equal(t, http.StatusOK, list.Code, list.Body.String()) + assert.Contains(t, list.Body.String(), `"name":"example.com"`) + + records := dnsAPI + "/zones/" + zone.ID + "/records" + f.addRecord(t, zone.ID, "www.example.com", "A", "192.0.2.1") + f.addRecord(t, zone.ID, "app.example.com", "CNAME", "www.example.com.") + + got := f.proxy(t, http.MethodGet, records, "") + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), `"rdata":"192.0.2.1"`) + assert.Contains(t, got.Body.String(), `"rtype":"CNAME"`) + require.NotEmpty(t, got.Header().Get("ETag"), "etag is relayed for If-Match") + + edit := f.proxy(t, http.MethodPatch, records, `{"items":[ + {"operation":"REMOVE","domain":"www.example.com","rtype":"A","rdata":"192.0.2.1","ttl":60}, + {"operation":"ADD","domain":"www.example.com","rtype":"A","rdata":"192.0.2.2","ttl":120}]}`) + require.Equal(t, http.StatusOK, edit.Code, edit.Body.String()) + + got = f.proxy(t, http.MethodGet, records+"?domain=www.example.com&rtype=A", "") + assert.Contains(t, got.Body.String(), `"rdata":"192.0.2.2"`) + assert.NotContains(t, got.Body.String(), `"rdata":"192.0.2.1"`) + + del := f.proxy(t, http.MethodPatch, records, + `{"items":[{"operation":"REMOVE","domain":"www.example.com","rtype":"A","rdata":"192.0.2.2"}]}`) + require.Equal(t, http.StatusOK, del.Code, del.Body.String()) + + gone := f.proxy(t, http.MethodDelete, dnsAPI+"/zones/"+zone.ID, "") + assert.Less(t, gone.Code, http.StatusMultipleChoices, gone.Body.String()) +} + +func TestOCIDNSProxyEtagConflict(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + id := f.zone(t, "conflict.example") + records := dnsAPI + "/zones/" + id + "/records" + + first := f.proxy(t, http.MethodGet, records, "") + etag := first.Header().Get("ETag") + require.NotEmpty(t, etag) + + body := `{"items":[{"operation":"ADD","domain":"a.conflict.example","rtype":"A","rdata":"192.0.2.1","ttl":60}]}` + + patch := func(ifMatch string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPatch, records, strings.NewReader(body)) + req.Header.Set("If-Match", ifMatch) + req.Header.Set("Authorization", "Bearer should-not-reach-emulator") + require.NoError(t, f.h.Handler()(echo.NewContext(req, rec))) + + return rec + } + + require.Equal(t, http.StatusOK, patch(etag).Code) + + stale := patch(etag) + assert.Equal(t, http.StatusPreconditionFailed, stale.Code, stale.Body.String()) + assert.Contains(t, stale.Body.String(), "NoEtagMatch") +} + +func TestOCIDNSProxyRejectsOutsideAllowList(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + id := f.zone(t, "reject.example") + rec := dnsAPI + "/zones/" + id + "/records" + + tests := []struct { + name string + method string + path string + body string + }{ + {"delete tsig key", http.MethodDelete, dnsAPI + "/tsigKeys/k1", ""}, + {"get one tsig key", http.MethodGet, dnsAPI + "/tsigKeys/k1", ""}, + {"update zone", http.MethodPut, dnsAPI + "/zones/" + id, `{}`}, + {"create resolver endpoint", http.MethodPost, dnsAPI + "/resolvers/r1/endpoints", `{}`}, + {"create view", http.MethodPost, dnsAPI + "/views", `{}`}, + {"zone content", http.MethodGet, dnsAPI + "/zones/" + id + "/content", ""}, + {"replace records", http.MethodPut, rec, `{"items":[]}`}, + {"domain records", http.MethodGet, rec + "/www.reject.example", ""}, + {"attachments", http.MethodGet, dnsAPI + "/steeringPolicyAttachments", ""}, + {"traversal", http.MethodGet, dnsAPI + "/zones/%2e%2e", ""}, + {"encoded slash", http.MethodGet, dnsAPI + "/zones/a%2Fb", ""}, + {"empty api", http.MethodGet, dnsAPI + "/", ""}, + {"unknown collection", http.MethodGet, dnsAPI + "/admin", ""}, + {"admin path", http.MethodPost, dnsAPI + "/_gopherstack/keys", `{}`}, + { + "secondary zone", http.MethodPost, dnsAPI + "/zones", + `{"name":"x.example","compartmentId":"` + dnsComp + `","zoneType":"SECONDARY"}`, + }, + { + "private zone without view", http.MethodPost, dnsAPI + "/zones", + `{"name":"x.example","compartmentId":"` + dnsComp + `","scope":"PRIVATE"}`, + }, + { + "global zone with view", http.MethodPost, dnsAPI + "/zones", + `{"name":"x.example","compartmentId":"` + dnsComp + `","scope":"GLOBAL","viewId":"v"}`, + }, + {"zone without name", http.MethodPost, dnsAPI + "/zones", `{"compartmentId":"` + dnsComp + `"}`}, + { + "unknown operation", http.MethodPatch, rec, + `{"items":[{"operation":"REQUIRE","domain":"a.reject.example","rtype":"A","rdata":"1.1.1.1","ttl":1}]}`, + }, + { + "bad record type", http.MethodPatch, rec, + `{"items":[{"operation":"ADD","domain":"a.reject.example","rtype":"ALIAS","rdata":"x","ttl":1}]}`, + }, + { + "add without ttl", http.MethodPatch, rec, + `{"items":[{"operation":"ADD","domain":"a.reject.example","rtype":"A","rdata":"1.1.1.1"}]}`, + }, + { + "bad domain", http.MethodPatch, rec, + `{"items":[{"operation":"ADD","domain":"a b","rtype":"A","rdata":"1.1.1.1","ttl":1}]}`, + }, + {"empty items", http.MethodPatch, rec, `{"items":[]}`}, + {"malformed body", http.MethodPatch, rec, `{`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := f.proxy(t, tt.method, tt.path, tt.body) + assert.Contains(t, []int{http.StatusBadRequest, http.StatusMethodNotAllowed}, got.Code, got.Body.String()) + }) + } +} + +func TestOCIDNSProxyTsigSecretStripped(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + + created := f.direct(t, http.MethodPost, "/20180115/tsigKeys", + `{"name":"transfer.example","algorithm":"hmac-sha256","secret":"`+dnsSecret+`","compartmentId":"`+dnsComp+`"}`) + require.Less(t, created.Code, http.StatusMultipleChoices, created.Body.String()) + require.Contains(t, created.Body.String(), dnsSecret, "fixture: the emulator returns the secret on create") + + var key struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(created.Body.Bytes(), &key)) + + direct := f.direct(t, http.MethodGet, "/20180115/tsigKeys/"+key.ID, "") + require.Contains(t, direct.Body.String(), dnsSecret, "fixture: the emulator returns the secret on get") + + got := f.proxy(t, http.MethodGet, dnsAPI+"/tsigKeys?compartmentId="+dnsComp, "") + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), `"name":"transfer.example"`) + assert.Contains(t, got.Body.String(), `"algorithm":"hmac-sha256"`) + assert.NotContains(t, got.Body.String(), dnsSecret) + assert.NotContains(t, got.Body.String(), `"secret"`) + + one := f.proxy(t, http.MethodGet, dnsAPI+"/tsigKeys/"+key.ID, "") + assert.Equal(t, http.StatusBadRequest, one.Code) +} + +func TestOCIDNSProxyReadOnlyLists(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + + view := f.direct(t, http.MethodPost, "/20180115/views", `{"compartmentId":"`+dnsComp+`","displayName":"corp-view"}`) + require.Less(t, view.Code, http.StatusMultipleChoices, view.Body.String()) + + policy := f.direct(t, http.MethodPost, "/20180115/steeringPolicies", + `{"compartmentId":"`+dnsComp+`","displayName":"failover","template":"FAILOVER","ttl":30, + "answers":[{"name":"a","rtype":"A","rdata":"192.0.2.9","pool":"p"}], + "rules":[{"ruleType":"FILTER", + "defaultAnswerData":[{"answerCondition":"answer.isDisabled != true","shouldKeep":true}]}]}`) + require.Less(t, policy.Code, http.StatusMultipleChoices, policy.Body.String()) + + tests := []struct { + name string + path string + want string + }{ + {"compartments", dnsBase + "/compartments", dnsComp}, + {"views", dnsAPI + "/views?compartmentId=" + dnsComp, `"displayName":"corp-view"`}, + {"policies", dnsAPI + "/steeringPolicies?compartmentId=" + dnsComp, `"displayName":"failover"`}, + {"resolvers", dnsAPI + "/resolvers?compartmentId=" + dnsComp, `[`}, + {"dataplane", dnsBase + "/dataplane", `"enabled":false`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := f.proxy(t, http.MethodGet, tt.path, "") + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), tt.want) + }) + } +} + +func TestOCIDNSProxyTryIt(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, true) + port := f.ops.DataPlanePort() + require.Positive(t, port) + + id := f.zone(t, "tryit.example") + f.addRecord(t, id, "www.tryit.example", "A", "192.0.2.77") + + dp := f.proxy(t, http.MethodGet, dnsBase+"/dataplane", "") + assert.Contains(t, dp.Body.String(), `"enabled":true`) + + query := func(body string) *httptest.ResponseRecorder { + return f.proxy(t, http.MethodPost, dnsBase+"/query", body) + } + + t.Run("answers from the data plane", func(t *testing.T) { + t.Parallel() + + got := query(`{"name":"www.tryit.example","type":"A"}`) + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + + var out map[string]any + + require.NoError(t, json.Unmarshal(got.Body.Bytes(), &out)) + assert.Equal(t, "NOERROR", out["rcode"]) + assert.Equal(t, true, out["authoritative"]) + + answer, ok := out["answer"].([]any) + require.True(t, ok) + require.Len(t, answer, 1) + + rr, ok := answer[0].(map[string]any) + require.True(t, ok) + assert.Equal(t, "A", rr["type"]) + assert.Equal(t, "192.0.2.77", rr["data"]) + assert.InDelta(t, 60, rr["ttl"], 0) + }) + + t.Run("loopback only", func(t *testing.T) { + t.Parallel() + + got := query(`{"name":"www.tryit.example.","type":"A","server":"8.8.8.8:53","address":"203.0.113.1","port":53}`) + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), `"server":"127.0.0.1:`) + assert.NotContains(t, got.Body.String(), "8.8.8.8") + assert.NotContains(t, got.Body.String(), "203.0.113.1") + }) + + t.Run("unknown name is nxdomain", func(t *testing.T) { + t.Parallel() + + got := query(`{"name":"missing.tryit.example","type":"A"}`) + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), `"rcode":"NXDOMAIN"`) + }) + + invalid := []struct{ name, body string }{ + {"space in name", `{"name":"bad name.example","type":"A"}`}, + {"empty name", `{"name":"","type":"A"}`}, + {"empty label", `{"name":"a..example","type":"A"}`}, + {"long label", `{"name":"` + strings.Repeat("a", 64) + `.example","type":"A"}`}, + {"long name", `{"name":"` + strings.Repeat("abcdefgh.", 30) + `example","type":"A"}`}, + {"zone transfer", `{"name":"tryit.example","type":"AXFR"}`}, + {"any", `{"name":"tryit.example","type":"ANY"}`}, + {"lowercase type", `{"name":"tryit.example","type":"a"}`}, + {"malformed", `{`}, + } + + for _, tt := range invalid { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := query(tt.body) + assert.Equal(t, http.StatusBadRequest, got.Code, got.Body.String()) + }) + } +} + +func TestOCIDNSProxyTryItWithoutDataPlane(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + + got := f.proxy(t, http.MethodPost, dnsBase+"/query", `{"name":"www.example.com","type":"A"}`) + assert.Equal(t, http.StatusServiceUnavailable, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), "DataPlaneDisabled") +} + +func TestOCIDNSProxyUnavailableWithoutService(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + for _, p := range []struct{ method, path string }{ + {http.MethodGet, dnsBase + "/compartments"}, + {http.MethodGet, dnsBase + "/dataplane"}, + {http.MethodPost, dnsBase + "/query"}, + {http.MethodGet, dnsAPI + "/zones?compartmentId=" + dnsComp}, + {http.MethodPatch, dnsAPI + "/zones/z1/records"}, + } { + got := lbDo(t, h, p.method, p.path, `{}`) + assert.Equal(t, http.StatusServiceUnavailable, got.Code, "%s %s", p.method, p.path) + } +} diff --git a/dashboard/ocifilestorage_proxy.go b/dashboard/ocifilestorage_proxy.go new file mode 100644 index 0000000000..4a1763d0ad --- /dev/null +++ b/dashboard/ocifilestorage_proxy.go @@ -0,0 +1,398 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/netip" + "regexp" + "slices" + "strings" + + "github.com/labstack/echo/v5" + + idbackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +const ( + ociFSPrefix = ociProxyPrefix + "/filestorage" + ociFSAPIPrefix = "/20171215" + ociFSMaxBody = 1 << 16 + ociFSMaxName = 255 + ociFSMaxPath = 1024 + ociFSMaxOptions = 32 + ociFSItemSegs = 2 + ociFSKeyComp = "compartmentId" + ociFSKeyAD = "availabilityDomain" + ociFSKeyState = "lifecycleState" + ociFSKeyName = "displayName" + ociFSKeyLimit = "limit" + ociFSKeyPage = "page" + ociFSKeySortBy = "sortBy" + ociFSKeySortOrd = "sortOrder" + ociFSKeyFS = "fileSystemId" + ociFSKeyExportSt = "exportSetId" + ociFSFileSystems = "fileSystems" + ociFSMountTarget = "mountTargets" + ociFSExportSets = "exportSets" + ociFSExports = "exports" + ociFSSnapshots = "snapshots" + ociFSPolicies = "filesystemSnapshotPolicies" + ociFSReplicas = "replications" + ociFSAccessRW = "READ_WRITE" + ociFSAccessRO = "READ_ONLY" + ociFSKeySubnet = "subnetId" + ociFSKeyAccess = "access" +) + +var ( + ociFSIDRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,255}$`) + ociFSSnapRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,255}$`) + ociFSLabelRe = regexp.MustCompile(`^[A-Za-z]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?$`) +) + +type ociFSColl struct { + create func(c *echo.Context) ([]byte, bool) + keys []string + remove bool +} + +// ociFSColls lists the collections the page reaches. A collection with a create builder accepts POST; remove allows +// DELETE of one item; every collection allows list and get. +// +//nolint:gochecknoglobals // immutable allow-list +var ociFSColls = map[string]ociFSColl{ + ociFSFileSystems: { + create: ociFSFileSystemBody, remove: true, + keys: []string{ + ociFSKeyComp, ociFSKeyAD, ociFSKeyState, ociFSKeyName, "parentFileSystemId", "sourceSnapshotId", + ociFSKeyLimit, ociFSKeyPage, ociFSKeySortBy, ociFSKeySortOrd, + }, + }, + ociFSMountTarget: { + create: ociFSMountTargetBody, remove: true, + keys: []string{ + ociFSKeyComp, ociFSKeyAD, ociFSKeyState, ociFSKeyName, ociFSKeyExportSt, + ociFSKeyLimit, ociFSKeyPage, ociFSKeySortBy, ociFSKeySortOrd, + }, + }, + ociFSExportSets: { + keys: []string{ + ociFSKeyComp, ociFSKeyAD, ociFSKeyState, ociFSKeyName, ociFSKeyLimit, ociFSKeyPage, + ociFSKeySortBy, ociFSKeySortOrd, + }, + }, + ociFSExports: { + create: ociFSExportBody, remove: true, + keys: []string{ + ociFSKeyComp, ociFSKeyExportSt, ociFSKeyFS, ociFSKeyState, ociFSKeyLimit, ociFSKeyPage, + ociFSKeySortBy, ociFSKeySortOrd, + }, + }, + ociFSSnapshots: { + create: ociFSSnapshotBody, remove: true, + keys: []string{ociFSKeyFS, ociFSKeyState, ociFSKeyLimit, ociFSKeyPage, ociFSKeySortBy, ociFSKeySortOrd}, + }, + ociFSPolicies: { + keys: []string{ + ociFSKeyComp, ociFSKeyAD, ociFSKeyState, ociFSKeyName, ociFSKeyLimit, ociFSKeyPage, + ociFSKeySortBy, ociFSKeySortOrd, + }, + }, + ociFSReplicas: { + keys: []string{ + ociFSKeyComp, ociFSKeyAD, ociFSKeyFS, ociFSKeyState, ociFSKeyName, ociFSKeyLimit, ociFSKeyPage, + ociFSKeySortBy, ociFSKeySortOrd, + }, + }, +} + +// registerOCIFileStorageRoutes serves the File Storage page by dispatching in-process to the emulator's control +// plane without request signing. Only the file system, mount target, export, snapshot, policy and replication +// operations the page uses are reachable, each write rebuilt from fixed fields. +func (h *DashboardHandler) registerOCIFileStorageRoutes() { + var inner *echo.Echo + + ops := h.config.OCIFileStorageOps + if ops != nil { + inner = echo.New() + inner.Any("/*", ops.Trusted()) + } + + guard := func(next echo.HandlerFunc) echo.HandlerFunc { + return ociGuard(inner != nil, "OCI File Storage", next) + } + + h.SubRouter.Add(http.MethodGet, ociFSPrefix+"/compartments", guard(func(c *echo.Context) error { + comps := ops.Backend.Compartments() + if !slices.Contains(comps, idbackend.TenancyID) { + comps = slices.Insert(comps, 0, idbackend.TenancyID) + } + + return c.JSON(http.StatusOK, map[string]any{ociCompartmentsKey: comps}) + })) + + h.SubRouter.Add(http.MethodGet, ociFSPrefix+"/nfs", guard(func(c *echo.Context) error { + return c.JSON(http.StatusOK, map[string]any{ + ociDNSKeyEnabled: ops.Backend.NFSEnabled(), "endpoints": ops.Backend.NFSEndpoints(), + }) + })) + + h.SubRouter.Add(http.MethodGet, ociFSPrefix+"/availabilitydomains", guard(h.ociCIAvailabilityDomains)) + h.SubRouter.Add(http.MethodGet, ociFSPrefix+"/subnets", guard(h.ociCISubnets)) + h.SubRouter.Add(http.MethodGet, ociFSPrefix+"/privateips/:id", guard(h.ociFSPrivateIP)) + + serve := guard(func(c *echo.Context) error { return ociFSServe(c, inner) }) + + for _, m := range []string{http.MethodGet, http.MethodPost, http.MethodDelete} { + h.SubRouter.Add(m, ociFSPrefix+"/api/*", serve) + } +} + +// ociFSPrivateIP reads the address of a mount target's private IP from OCI Core, reduced to what the page shows. +func (h *DashboardHandler) ociFSPrivateIP(c *echo.Context) error { + id := c.Param("id") + if !ociFSIDRe.MatchString(id) { + return ociInvalid(c, "invalid private IP id") + } + + out := map[string]any{} + + if ops := h.config.OCICoreOps; ops != nil { + inner := echo.New() + inner.Any("/*", ops.Trusted()) + + r := c.Request() + fwd := r.Clone(r.Context()) + fwd.URL.Path, fwd.URL.RawPath, fwd.URL.RawQuery = ociCICoreAPI+"/privateIps/"+id, "", "" + fwd.RequestURI, fwd.Method, fwd.Body, fwd.ContentLength = "", http.MethodGet, http.NoBody, 0 + fwd.Header = http.Header{} + + rec := httptest.NewRecorder() + inner.ServeHTTP(rec, fwd) + + var ip map[string]any + if rec.Code == http.StatusOK && json.Unmarshal(rec.Body.Bytes(), &ip) == nil { + out = map[string]any{ + "ipAddress": ip["ipAddress"], "hostnameLabel": ip["hostnameLabel"], ociFSKeySubnet: ip[ociFSKeySubnet], + } + } + } + + return c.JSON(http.StatusOK, out) +} + +// ociFSServe forwards one allow-listed control-plane call. +func ociFSServe(c *echo.Context, inner *echo.Echo) error { + r := c.Request() + + parts, ok := ociSplitPath(strings.TrimPrefix(r.URL.EscapedPath(), ociFSPrefix+"/api")) + if !ok { + return ociInvalid(c, "unsupported OCI File Storage proxy request") + } + + coll, build, ok := ociFSResolve(r.Method, parts) + if !ok { + return ociInvalid(c, "unsupported OCI File Storage proxy request") + } + + var body []byte + + if build != nil { + if body, ok = build(c); !ok { + return ociInvalid(c, "invalid OCI File Storage request") + } + } + + query := ociArtQuery(r.URL.Query(), coll.keys) + r.Header = http.Header{} + r.URL.RawQuery = "" + + if r.Method == http.MethodGet && len(parts) == 1 { + r.URL.RawQuery = query + } + + rec := ociArtForward(c, inner, ociFSAPIPrefix+"/"+strings.Join(parts, "/"), body) + + return ociRelay(c, rec, "opc-next-page", "etag", "opc-request-id") +} + +// ociFSResolve maps a proxy request to its collection and body builder; false means outside the allow-list. +func ociFSResolve(method string, parts []string) (ociFSColl, func(*echo.Context) ([]byte, bool), bool) { + if len(parts) > ociFSItemSegs || (len(parts) == ociFSItemSegs && !ociFSIDRe.MatchString(unescapedPath(parts[1]))) { + return ociFSColl{}, nil, false + } + + coll, known := ociFSColls[parts[0]] + if !known { + return ociFSColl{}, nil, false + } + + item := len(parts) == ociFSItemSegs + + switch { + case method == http.MethodGet: + return coll, nil, true + case method == http.MethodPost && !item && coll.create != nil: + return coll, coll.create, true + case method == http.MethodDelete && item && coll.remove: + return coll, nil, true + } + + return ociFSColl{}, nil, false +} + +func ociFSName(s string) bool { return s != "" && len(s) <= ociFSMaxName } + +func ociFSOptName(s string) bool { return len(s) <= ociFSMaxName } + +func ociFSFileSystemBody(c *echo.Context) ([]byte, bool) { + var in struct { + DisplayName string `json:"displayName"` + Compartment string `json:"compartmentId"` + AD string `json:"availabilityDomain"` + Source string `json:"sourceSnapshotId"` + } + + if !ociDecodeBodyN(c, &in, ociFSMaxBody) || !ociFSName(in.Compartment) || !ociFSName(in.AD) || + !ociFSOptName(in.DisplayName) || (in.Source != "" && !ociFSIDRe.MatchString(in.Source)) { + return nil, false + } + + out := map[string]any{ociFSKeyComp: in.Compartment, ociFSKeyAD: in.AD} + + if in.DisplayName != "" { + out[ociFSKeyName] = in.DisplayName + } + + if in.Source != "" { + out["sourceSnapshotId"] = in.Source + } + + return ociArtMarshal(out) +} + +func ociFSMountTargetBody(c *echo.Context) ([]byte, bool) { + var in struct { + DisplayName string `json:"displayName"` + Compartment string `json:"compartmentId"` + AD string `json:"availabilityDomain"` + SubnetID string `json:"subnetId"` + Hostname string `json:"hostnameLabel"` + IPAddress string `json:"ipAddress"` + } + + if !ociDecodeBodyN(c, &in, ociFSMaxBody) || !ociFSName(in.Compartment) || !ociFSName(in.AD) || + !ociFSIDRe.MatchString(in.SubnetID) || !ociFSOptName(in.DisplayName) || + (in.Hostname != "" && !ociFSLabelRe.MatchString(in.Hostname)) { + return nil, false + } + + out := map[string]any{ociFSKeyComp: in.Compartment, ociFSKeyAD: in.AD, ociFSKeySubnet: in.SubnetID} + + if in.DisplayName != "" { + out[ociFSKeyName] = in.DisplayName + } + + if in.Hostname != "" { + out["hostnameLabel"] = in.Hostname + } + + if in.IPAddress != "" { + if _, err := netip.ParseAddr(in.IPAddress); err != nil { + return nil, false + } + + out["ipAddress"] = in.IPAddress + } + + return ociArtMarshal(out) +} + +type ociFSOption struct { + Privileged *bool `json:"requirePrivilegedSourcePort"` + Source string `json:"source"` + Access string `json:"access"` + Squash string `json:"identitySquash"` +} + +func (o ociFSOption) build() (map[string]any, bool) { + if !ociFSSource(o.Source) || !slices.Contains([]string{ociFSAccessRW, ociFSAccessRO}, o.Access) || + !slices.Contains([]string{"NONE", "ROOT", "ALL"}, o.Squash) { + return nil, false + } + + out := map[string]any{"source": o.Source, ociFSKeyAccess: o.Access, "identitySquash": o.Squash} + + if o.Privileged != nil { + out["requirePrivilegedSourcePort"] = *o.Privileged + } + + return out, true +} + +// ociFSSource accepts one IPv4 or IPv6 address or CIDR block. +func ociFSSource(s string) bool { + if _, err := netip.ParsePrefix(s); err == nil { + return true + } + + _, err := netip.ParseAddr(s) + + return err == nil +} + +func ociFSExportBody(c *echo.Context) ([]byte, bool) { + var in struct { + ExportSet string `json:"exportSetId"` + FileSystem string `json:"fileSystemId"` + Path string `json:"path"` + Options []ociFSOption `json:"exportOptions"` + } + + if !ociDecodeBodyN(c, &in, ociFSMaxBody) || !ociFSIDRe.MatchString(in.ExportSet) || + !ociFSIDRe.MatchString(in.FileSystem) || !ociFSExportPath(in.Path) || len(in.Options) > ociFSMaxOptions { + return nil, false + } + + opts := make([]map[string]any, 0, len(in.Options)) + + for _, o := range in.Options { + built, ok := o.build() + if !ok { + return nil, false + } + + opts = append(opts, built) + } + + return ociArtMarshal(map[string]any{ + ociFSKeyExportSt: in.ExportSet, ociFSKeyFS: in.FileSystem, "path": in.Path, "exportOptions": opts, + }) +} + +func ociFSExportPath(p string) bool { + return len(p) > 0 && len(p) <= ociFSMaxPath && strings.HasPrefix(p, "/") && + !strings.ContainsFunc(p, func(r rune) bool { return r < ' ' }) +} + +func ociFSSnapshotBody(c *echo.Context) ([]byte, bool) { + var in struct { + FileSystem string `json:"fileSystemId"` + Name string `json:"name"` + } + + if !ociDecodeBodyN(c, &in, ociFSMaxBody) || !ociFSIDRe.MatchString(in.FileSystem) || + !ociFSSnapRe.MatchString(in.Name) { + return nil, false + } + + return ociArtMarshal(map[string]any{ociFSKeyFS: in.FileSystem, netPxNameKey: in.Name}) +} + +// registerStorageAndWorkflowRoutes registers the File Storage and Logic Apps pages. +func (h *DashboardHandler) registerStorageAndWorkflowRoutes() { + h.registerOCIFileStorageRoutes() + h.registerAzureLogicRoutes() + h.registerSearchAndWebRoutes() +} diff --git a/dashboard/ocifilestorage_proxy_test.go b/dashboard/ocifilestorage_proxy_test.go new file mode 100644 index 0000000000..db483b756f --- /dev/null +++ b/dashboard/ocifilestorage_proxy_test.go @@ -0,0 +1,357 @@ +package dashboard_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocifilestorage" +) + +const ( + fsComp = "ocid1.compartment.oc1..fscomp" + fsAD = "Uocm:IAD-AD-1" + fsBase = "/dashboard/api/v1/oci/filestorage" + fsAPI = fsBase + "/api" +) + +type fsFixture struct { + h *dashboard.DashboardHandler + ops *ocifilestorage.Handler + subnet string +} + +func idOf(t *testing.T, v any) string { + t.Helper() + + raw, err := json.Marshal(v) + require.NoError(t, err) + + var out struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(raw, &out)) + + return out.ID +} + +func newFSDashboard(t *testing.T, nfs bool) fsFixture { + t.Helper() + + backend := ocifilestorage.NewInMemoryBackend() + backend.SetTransition(0) + + core := ocicore.NewInMemoryBackend() + vcn, _, aerr := core.CreateVcn( + ocicore.CreateVcnIn{CompartmentID: fsComp, DisplayName: "vcn", CidrBlock: "10.0.0.0/16"}, + ) + require.Nil(t, aerr) + + sub, _, aerr := core.CreateSubnet(ocicore.CreateSubnetIn{ + CompartmentID: fsComp, VcnID: idOf(t, vcn), DisplayName: "files", CidrBlock: "10.0.1.0/24", + }) + require.Nil(t, aerr) + + backend.SetNetwork(ocifilestorage.Network{ + Subnet: func(id string) (ocifilestorage.SubnetInfo, bool) { + ref, live := core.LookupSubnet(id) + + return ocifilestorage.SubnetInfo{VcnID: ref.VcnID, CIDR: ref.CidrBlock, AD: ref.AvailabilityDomain}, live + }, + CreateNic: func(r ocifilestorage.NicRequest) (ocifilestorage.Nic, error) { + nic, err := core.CreateServiceNic(ocicore.ServiceNicIn{ + CompartmentID: r.CompartmentID, AD: r.AD, SubnetID: r.SubnetID, DisplayName: r.DisplayName, + HostnameLabel: r.Hostname, PrivateIP: r.IPAddress, + }) + if err != nil { + return ocifilestorage.Nic{}, err + } + + return ocifilestorage.Nic{VnicID: nic.VnicID, PrivateIPID: nic.PrivateIPID, IPAddress: nic.IPAddress}, nil + }, + DeleteNic: core.DeleteServiceNic, + }) + + if nfs { + plane, err := ocifilestorage.NewNFSPlane(ocifilestorage.NFSConfig{Dir: t.TempDir()}) + require.NoError(t, err) + t.Cleanup(plane.Close) + backend.SetDataPlane(plane) + } + + ops := ocifilestorage.NewHandler(backend) + + return fsFixture{ + h: dashboard.NewHandler(dashboard.Config{OCIFileStorageOps: ops, OCICoreOps: ocicore.NewHandler(core)}), + ops: ops, subnet: idOf(t, sub), + } +} + +func (f fsFixture) proxy(t *testing.T, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + return lbDo(t, f.h, method, path, body) +} + +func (f fsFixture) create(t *testing.T, coll, body string) map[string]any { + t.Helper() + + rec := f.proxy(t, http.MethodPost, fsAPI+"/"+coll, body) + require.Less(t, rec.Code, http.StatusMultipleChoices, rec.Body.String()) + + var out map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + + return out +} + +func (f fsFixture) fileSystem(t *testing.T, name string) string { + t.Helper() + + id, ok := f.create(t, "fileSystems", + `{"displayName":"`+name+`","compartmentId":"`+fsComp+`","availabilityDomain":"`+fsAD+`"}`)["id"].(string) + require.True(t, ok) + + return id +} + +func TestOCIFileStorageProxyLifecycle(t *testing.T) { + t.Parallel() + + f := newFSDashboard(t, false) + + rec := f.proxy(t, http.MethodPost, fsAPI+"/fileSystems", + `{"displayName":"data","compartmentId":"`+fsComp+`","availabilityDomain":"`+fsAD+`",`+ + `"kmsKeyId":"evil","freeformTags":{"owner":"mallory"},"locks":[{"type":"FULL"}]}`) + require.Less(t, rec.Code, http.StatusMultipleChoices, rec.Body.String()) + assert.NotContains(t, rec.Body.String(), "mallory") + assert.NotContains(t, rec.Body.String(), "evil") + + fsID := f.fileSystem(t, "second") + + list := f.proxy(t, http.MethodGet, + fsAPI+"/fileSystems?compartmentId="+fsComp+"&availabilityDomain="+fsAD+"&sneaky=1", "") + require.Equal(t, http.StatusOK, list.Code, list.Body.String()) + assert.Contains(t, list.Body.String(), `"displayName":"data"`) + assert.Contains(t, list.Body.String(), `"meteredBytes":0`) + + snap := f.create(t, "snapshots", `{"fileSystemId":"`+fsID+`","name":"nightly"}`) + snapID, ok := snap["id"].(string) + require.True(t, ok) + + snaps := f.proxy(t, http.MethodGet, fsAPI+"/snapshots?fileSystemId="+fsID, "") + require.Equal(t, http.StatusOK, snaps.Code, snaps.Body.String()) + assert.Contains(t, snaps.Body.String(), `"name":"nightly"`) + + clone := f.create(t, "fileSystems", `{"displayName":"copy","compartmentId":"`+fsComp+`",`+ + `"availabilityDomain":"`+fsAD+`","sourceSnapshotId":"`+snapID+`"}`) + assert.Contains(t, clone, "sourceDetails") + + mt := f.create(t, "mountTargets", `{"displayName":"mt","compartmentId":"`+fsComp+`",`+ + `"availabilityDomain":"`+fsAD+`","subnetId":"`+f.subnet+`"}`) + setID, ok := mt["exportSetId"].(string) + require.True(t, ok) + + exp := f.create(t, "exports", `{"exportSetId":"`+setID+`","fileSystemId":"`+fsID+`","path":"/data",`+ + `"exportOptions":[{"source":"10.0.0.0/16","access":"READ_ONLY","identitySquash":"ALL",`+ + `"requirePrivilegedSourcePort":false,"anonymousUid":0}]}`) + expID, ok := exp["id"].(string) + require.True(t, ok) + + ips, ok := mt["privateIpIds"].([]any) + require.True(t, ok) + require.Len(t, ips, 1) + + ip := f.proxy(t, http.MethodGet, fsBase+"/privateips/"+ips[0].(string), "") + require.Equal(t, http.StatusOK, ip.Code, ip.Body.String()) + assert.Contains(t, ip.Body.String(), `"ipAddress":"10.0.1.`) + assert.NotContains(t, ip.Body.String(), "vnicId") + + badIP := f.proxy(t, http.MethodGet, fsBase+"/privateips/a%20b", "") + assert.Equal(t, http.StatusBadRequest, badIP.Code) + + got := f.proxy(t, http.MethodGet, fsAPI+"/exports/"+expID, "") + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), `"access":"READ_ONLY"`) + assert.Contains(t, got.Body.String(), `"identitySquash":"ALL"`) + assert.NotContains(t, got.Body.String(), "anonymousUid\":0") + + for _, p := range []string{ + "exportSets?compartmentId=" + fsComp + "&availabilityDomain=" + fsAD, + "mountTargets?compartmentId=" + fsComp + "&availabilityDomain=" + fsAD, + "filesystemSnapshotPolicies?compartmentId=" + fsComp + "&availabilityDomain=" + fsAD, + "replications?compartmentId=" + fsComp + "&availabilityDomain=" + fsAD, + "exports?exportSetId=" + setID, + } { + r := f.proxy(t, http.MethodGet, fsAPI+"/"+p, "") + assert.Equal(t, http.StatusOK, r.Code, "%s: %s", p, r.Body.String()) + } + + for _, p := range []string{"exports/" + expID, "snapshots/" + snapID} { + r := f.proxy(t, http.MethodDelete, fsAPI+"/"+p, "") + assert.Less(t, r.Code, http.StatusMultipleChoices, "%s: %s", p, r.Body.String()) + } + + comps := f.proxy(t, http.MethodGet, fsBase+"/compartments", "") + require.Equal(t, http.StatusOK, comps.Code) + assert.Contains(t, comps.Body.String(), fsComp) +} + +func TestOCIFileStorageProxyRejectsOutsideAllowList(t *testing.T) { + t.Parallel() + + f := newFSDashboard(t, false) + id := f.fileSystem(t, "guarded") + + tests := []struct { + name string + method string + path string + body string + }{ + {"change compartment", http.MethodPost, fsAPI + "/fileSystems/" + id + "/actions/changeCompartment", `{}`}, + {"add lock", http.MethodPost, fsAPI + "/fileSystems/" + id + "/actions/addLock", `{}`}, + {"quota rules", http.MethodGet, fsAPI + "/fileSystems/" + id + "/quotaRules", ""}, + {"delete export set", http.MethodDelete, fsAPI + "/exportSets/x", ""}, + {"create export set", http.MethodPost, fsAPI + "/exportSets", `{}`}, + {"create policy", http.MethodPost, fsAPI + "/filesystemSnapshotPolicies", `{}`}, + {"delete policy", http.MethodDelete, fsAPI + "/filesystemSnapshotPolicies/p", ""}, + {"create replication", http.MethodPost, fsAPI + "/replications", `{}`}, + {"delete replication", http.MethodDelete, fsAPI + "/replications/r", ""}, + {"connectors", http.MethodGet, fsAPI + "/outboundConnectors", ""}, + {"post to item", http.MethodPost, fsAPI + "/fileSystems/" + id, `{}`}, + {"delete collection", http.MethodDelete, fsAPI + "/fileSystems", ""}, + {"traversal", http.MethodGet, fsAPI + "/fileSystems/%2e%2e", ""}, + {"encoded slash", http.MethodGet, fsAPI + "/fileSystems/a%2Fb", ""}, + {"admin keys", http.MethodGet, fsAPI + "/_gopherstack/oci/keys", ""}, + {"empty", http.MethodGet, fsAPI + "/", ""}, + {"deep path", http.MethodGet, fsAPI + "/fileSystems/" + id + "/snapshots", ""}, + {"fs missing ad", http.MethodPost, fsAPI + "/fileSystems", `{"compartmentId":"` + fsComp + `"}`}, + { + "fs bad source", http.MethodPost, fsAPI + "/fileSystems", + `{"compartmentId":"` + fsComp + `","availabilityDomain":"` + fsAD + `","sourceSnapshotId":"a/b"}`, + }, + { + "mt bad subnet", http.MethodPost, fsAPI + "/mountTargets", + `{"compartmentId":"` + fsComp + `","availabilityDomain":"` + fsAD + `","subnetId":"a b"}`, + }, + { + "mt bad hostname", http.MethodPost, fsAPI + "/mountTargets", + `{"compartmentId":"` + fsComp + `","availabilityDomain":"` + fsAD + `","subnetId":"` + f.subnet + + `","hostnameLabel":"-bad"}`, + }, + { + "mt bad ip", http.MethodPost, fsAPI + "/mountTargets", + `{"compartmentId":"` + fsComp + `","availabilityDomain":"` + fsAD + `","subnetId":"` + f.subnet + + `","ipAddress":"nope"}`, + }, + { + "export bad path", http.MethodPost, fsAPI + "/exports", + `{"exportSetId":"s","fileSystemId":"` + id + `","path":"data"}`, + }, + { + "export bad source", http.MethodPost, fsAPI + "/exports", + `{"exportSetId":"s","fileSystemId":"` + id + `","path":"/d","exportOptions":[` + + `{"source":"everyone","access":"READ_WRITE","identitySquash":"ROOT"}]}`, + }, + { + "export bad access", http.MethodPost, fsAPI + "/exports", + `{"exportSetId":"s","fileSystemId":"` + id + `","path":"/d","exportOptions":[` + + `{"source":"10.0.0.1","access":"ADMIN","identitySquash":"ROOT"}]}`, + }, + { + "export bad squash", http.MethodPost, fsAPI + "/exports", + `{"exportSetId":"s","fileSystemId":"` + id + `","path":"/d","exportOptions":[` + + `{"source":"10.0.0.1","access":"READ_WRITE","identitySquash":"WHATEVER"}]}`, + }, + {"snapshot bad name", http.MethodPost, fsAPI + "/snapshots", `{"fileSystemId":"` + id + `","name":"a/b"}`}, + {"snapshot no fs", http.MethodPost, fsAPI + "/snapshots", `{"name":"x"}`}, + {"malformed", http.MethodPost, fsAPI + "/snapshots", `{`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := f.proxy(t, tt.method, tt.path, tt.body) + assert.Equal(t, http.StatusBadRequest, got.Code, "%s %s: %s", tt.method, tt.path, got.Body.String()) + }) + } +} + +func TestOCIFileStorageProxyNoUpdates(t *testing.T) { + t.Parallel() + + f := newFSDashboard(t, false) + id := f.fileSystem(t, "frozen") + + for _, p := range []string{"/fileSystems/" + id, "/mountTargets/m", "/exports/e"} { + got := f.proxy(t, http.MethodPut, fsAPI+p, `{"displayName":"renamed"}`) + assert.GreaterOrEqual(t, got.Code, http.StatusBadRequest, p) + } + + got := f.proxy(t, http.MethodGet, fsAPI+"/fileSystems/"+id, "") + require.Equal(t, http.StatusOK, got.Code) + assert.Contains(t, got.Body.String(), `"displayName":"frozen"`) +} + +func TestOCIFileStorageProxyNFSEndpoint(t *testing.T) { + t.Parallel() + + stub := newFSDashboard(t, false) + off := stub.proxy(t, http.MethodGet, fsBase+"/nfs", "") + require.Equal(t, http.StatusOK, off.Code) + assert.Contains(t, off.Body.String(), `"enabled":false`) + + f := newFSDashboard(t, true) + fsID := f.fileSystem(t, "nfs") + mt := f.create(t, "mountTargets", `{"compartmentId":"`+fsComp+`","availabilityDomain":"`+fsAD+ + `","subnetId":"`+f.subnet+`"}`) + setID, ok := mt["exportSetId"].(string) + require.True(t, ok) + + f.create(t, "exports", `{"exportSetId":"`+setID+`","fileSystemId":"`+fsID+`","path":"/nfs",`+ + `"exportOptions":[{"source":"127.0.0.1","access":"READ_WRITE","identitySquash":"NONE"}]}`) + + on := f.proxy(t, http.MethodGet, fsBase+"/nfs", "") + require.Equal(t, http.StatusOK, on.Code) + + var out struct { + Endpoints map[string]string `json:"endpoints"` + Enabled bool `json:"enabled"` + } + + require.NoError(t, json.Unmarshal(on.Body.Bytes(), &out)) + assert.True(t, out.Enabled) + require.Len(t, out.Endpoints, 1) + + for _, addr := range out.Endpoints { + assert.True(t, strings.HasPrefix(addr, "127.0.0.1:"), addr) + } +} + +func TestOCIFileStorageProxyUnavailableWithoutService(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + for _, p := range []struct{ method, path string }{ + {http.MethodGet, fsBase + "/compartments"}, + {http.MethodGet, fsBase + "/nfs"}, + {http.MethodGet, fsBase + "/availabilitydomains"}, + {http.MethodGet, fsAPI + "/fileSystems?compartmentId=" + fsComp}, + {http.MethodPost, fsAPI + "/snapshots"}, + {http.MethodDelete, fsAPI + "/exports/e1"}, + } { + got := lbDo(t, h, p.method, p.path, `{}`) + assert.Equal(t, http.StatusServiceUnavailable, got.Code, "%s %s", p.method, p.path) + } +} diff --git a/dashboard/provider.go b/dashboard/provider.go index 1769675206..60171390d3 100644 --- a/dashboard/provider.go +++ b/dashboard/provider.go @@ -19,14 +19,19 @@ import ( "github.com/blackbirdworks/gopherstack/services/azureappconfig" "github.com/blackbirdworks/gopherstack/services/azurearm" "github.com/blackbirdworks/gopherstack/services/azureblob" + "github.com/blackbirdworks/gopherstack/services/azurecontainerregistry" "github.com/blackbirdworks/gopherstack/services/azureeventgrid" "github.com/blackbirdworks/gopherstack/services/azureeventhubs" "github.com/blackbirdworks/gopherstack/services/azurefile" "github.com/blackbirdworks/gopherstack/services/azurefunctions" + "github.com/blackbirdworks/gopherstack/services/azuregraph" "github.com/blackbirdworks/gopherstack/services/azurekeyvault" "github.com/blackbirdworks/gopherstack/services/azuremonitor" "github.com/blackbirdworks/gopherstack/services/azurequeue" + "github.com/blackbirdworks/gopherstack/services/azureredis" + "github.com/blackbirdworks/gopherstack/services/azuresearch" "github.com/blackbirdworks/gopherstack/services/azureservicebus" + "github.com/blackbirdworks/gopherstack/services/azuresql" "github.com/blackbirdworks/gopherstack/services/azuretable" "github.com/blackbirdworks/gopherstack/services/backup" "github.com/blackbirdworks/gopherstack/services/cloudformation" @@ -75,12 +80,16 @@ import ( "github.com/blackbirdworks/gopherstack/services/kinesis" "github.com/blackbirdworks/gopherstack/services/kms" "github.com/blackbirdworks/gopherstack/services/lambda" + "github.com/blackbirdworks/gopherstack/services/lambdaweb" "github.com/blackbirdworks/gopherstack/services/ociapigateway" "github.com/blackbirdworks/gopherstack/services/ociartifacts" "github.com/blackbirdworks/gopherstack/services/ocicontainerengine" "github.com/blackbirdworks/gopherstack/services/ocicontainerinstances" "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocidatabase" + "github.com/blackbirdworks/gopherstack/services/ocidns" "github.com/blackbirdworks/gopherstack/services/ocievents" + "github.com/blackbirdworks/gopherstack/services/ocifilestorage" "github.com/blackbirdworks/gopherstack/services/ocifunctions" "github.com/blackbirdworks/gopherstack/services/ociidentity" "github.com/blackbirdworks/gopherstack/services/ociloadbalancer" @@ -164,6 +173,10 @@ func (p *Provider) Init(ctx *service.AppContext) (service.Registerable, error) { conf.AzureARMOps, _ = getHandler[*azurearm.Handler](ap.GetAzureARMHandler()) } + if gp, hasGraph := ctx.Config.(interface{ GetAzureGraphHandler() service.Registerable }); hasGraph { + conf.AzureGraphOps, _ = getHandler[*azuregraph.Handler](gp.GetAzureGraphHandler()) + } + if bp, hasBlob := ctx.Config.(interface{ GetAzureBlobHandler() service.Registerable }); hasBlob { conf.AzureBlobOps, _ = getHandler[*azureblob.Handler](bp.GetAzureBlobHandler()) } @@ -331,6 +344,26 @@ func linkCloudExtras(conf *Config, cfg any) { conf.AzureAppConfigOps, _ = getHandler[*azureappconfig.Handler](p.GetAzureAppConfigHandler()) } + if p, ok := cfg.(interface{ GetAzureRedisHandler() service.Registerable }); ok { + conf.AzureRedisOps, _ = getHandler[*azureredis.Handler](p.GetAzureRedisHandler()) + } + + if p, ok := cfg.(interface{ GetAzureSQLHandler() service.Registerable }); ok { + conf.AzureSQLOps, _ = getHandler[*azuresql.Handler](p.GetAzureSQLHandler()) + } + + if p, ok := cfg.(interface{ GetAzureACRHandler() service.Registerable }); ok { + conf.AzureACROps, _ = getHandler[*azurecontainerregistry.Handler](p.GetAzureACRHandler()) + } + + if p, ok := cfg.(interface{ GetAzureSearchHandler() service.Registerable }); ok { + conf.AzureSearchOps, _ = getHandler[*azuresearch.Handler](p.GetAzureSearchHandler()) + } + + if p, ok := cfg.(interface{ GetLambdaWebHandler() service.Registerable }); ok { + conf.LambdaWebOps, _ = getHandler[*lambdaweb.Handler](p.GetLambdaWebHandler()) + } + if p, ok := cfg.(interface{ GetGCPSQLHandler() service.Registerable }); ok { conf.GCPSQLOps, _ = getHandler[*gcpsql.Handler](p.GetGCPSQLHandler()) } @@ -484,6 +517,18 @@ func linkOCIObservabilityExtras(conf *Config, cfg any) { p.GetOCIContainerInstancesHandler()) } + if p, ok := cfg.(interface{ GetOCIDNSHandler() service.Registerable }); ok { + conf.OCIDNSOps, _ = getHandler[*ocidns.Handler](p.GetOCIDNSHandler()) + } + + if p, ok := cfg.(interface{ GetOCIDatabaseHandler() service.Registerable }); ok { + conf.OCIDatabaseOps, _ = getHandler[*ocidatabase.Handler](p.GetOCIDatabaseHandler()) + } + + if p, ok := cfg.(interface{ GetOCIFileStorageHandler() service.Registerable }); ok { + conf.OCIFileStorageOps, _ = getHandler[*ocifilestorage.Handler](p.GetOCIFileStorageHandler()) + } + if p, ok := cfg.(interface{ GetOCIEventsHandler() service.Registerable }); ok { conf.OCIEventsOps, _ = getHandler[*ocievents.Handler](p.GetOCIEventsHandler()) } diff --git a/dashboard/static/icons/azure-acr.svg b/dashboard/static/icons/azure-acr.svg new file mode 100644 index 0000000000..f429045490 --- /dev/null +++ b/dashboard/static/icons/azure-acr.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/azure-entra.svg b/dashboard/static/icons/azure-entra.svg new file mode 100644 index 0000000000..284a7498ef --- /dev/null +++ b/dashboard/static/icons/azure-entra.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/azure-logic.svg b/dashboard/static/icons/azure-logic.svg new file mode 100644 index 0000000000..49e06ce64f --- /dev/null +++ b/dashboard/static/icons/azure-logic.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/azure-network.svg b/dashboard/static/icons/azure-network.svg new file mode 100644 index 0000000000..a916a74353 --- /dev/null +++ b/dashboard/static/icons/azure-network.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/azure-redis.svg b/dashboard/static/icons/azure-redis.svg new file mode 100644 index 0000000000..e7fd8a5dcf --- /dev/null +++ b/dashboard/static/icons/azure-redis.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/azure-search.svg b/dashboard/static/icons/azure-search.svg new file mode 100644 index 0000000000..f61c2c0c88 --- /dev/null +++ b/dashboard/static/icons/azure-search.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/azure-sql.svg b/dashboard/static/icons/azure-sql.svg new file mode 100644 index 0000000000..539c9863d4 --- /dev/null +++ b/dashboard/static/icons/azure-sql.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/lambdaweb.svg b/dashboard/static/icons/lambdaweb.svg new file mode 100644 index 0000000000..f1fb688577 --- /dev/null +++ b/dashboard/static/icons/lambdaweb.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/oci-database.svg b/dashboard/static/icons/oci-database.svg new file mode 100644 index 0000000000..129550c538 --- /dev/null +++ b/dashboard/static/icons/oci-database.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/oci-dns.svg b/dashboard/static/icons/oci-dns.svg new file mode 100644 index 0000000000..f1069aa133 --- /dev/null +++ b/dashboard/static/icons/oci-dns.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/oci-filestorage.svg b/dashboard/static/icons/oci-filestorage.svg new file mode 100644 index 0000000000..4e105dce8f --- /dev/null +++ b/dashboard/static/icons/oci-filestorage.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/system_ports_test.go b/dashboard/system_ports_test.go index 9b7792b38f..28a039bd66 100644 --- a/dashboard/system_ports_test.go +++ b/dashboard/system_ports_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/dashboard" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" ) func TestCollectAllocatedPorts(t *testing.T) { @@ -36,8 +36,7 @@ func TestCollectAllocatedPorts(t *testing.T) { t.Cleanup(func() { _ = ln.Close() }) } - alloc, err := portalloc.New(1, 2) - require.NoError(t, err) + alloc := portalloctest.New(t, 1) require.NoError(t, alloc.Reserve(fixedPort, "gcpfake")) require.NoError(t, alloc.Reserve(0, "disabled")) diff --git a/dashboard/ui.go b/dashboard/ui.go index 21a764e3a4..6f43c87f5f 100644 --- a/dashboard/ui.go +++ b/dashboard/ui.go @@ -47,14 +47,19 @@ import ( azureappconfigbackend "github.com/blackbirdworks/gopherstack/services/azureappconfig" azurearmbackend "github.com/blackbirdworks/gopherstack/services/azurearm" azureblobbackend "github.com/blackbirdworks/gopherstack/services/azureblob" + azureacrbackend "github.com/blackbirdworks/gopherstack/services/azurecontainerregistry" azureeventgridbackend "github.com/blackbirdworks/gopherstack/services/azureeventgrid" azureeventhubsbackend "github.com/blackbirdworks/gopherstack/services/azureeventhubs" azurefilebackend "github.com/blackbirdworks/gopherstack/services/azurefile" azurefunctionsbackend "github.com/blackbirdworks/gopherstack/services/azurefunctions" + azuregraphbackend "github.com/blackbirdworks/gopherstack/services/azuregraph" azurekeyvaultbackend "github.com/blackbirdworks/gopherstack/services/azurekeyvault" azuremonitorbackend "github.com/blackbirdworks/gopherstack/services/azuremonitor" azurequeuebackend "github.com/blackbirdworks/gopherstack/services/azurequeue" + azureredisbackend "github.com/blackbirdworks/gopherstack/services/azureredis" + azuresearchbackend "github.com/blackbirdworks/gopherstack/services/azuresearch" azureservicebusbackend "github.com/blackbirdworks/gopherstack/services/azureservicebus" + azuresqlbackend "github.com/blackbirdworks/gopherstack/services/azuresql" azuretablebackend "github.com/blackbirdworks/gopherstack/services/azuretable" backupbackend "github.com/blackbirdworks/gopherstack/services/backup" batchbackend "github.com/blackbirdworks/gopherstack/services/batch" @@ -140,6 +145,7 @@ import ( kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" lakeformationbackend "github.com/blackbirdworks/gopherstack/services/lakeformation" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" + lambdawebbackend "github.com/blackbirdworks/gopherstack/services/lambdaweb" managedblockchainbackend "github.com/blackbirdworks/gopherstack/services/managedblockchain" mediaconvertbackend "github.com/blackbirdworks/gopherstack/services/mediaconvert" mediastorebackend "github.com/blackbirdworks/gopherstack/services/mediastore" @@ -153,7 +159,10 @@ import ( ocicontainerenginebackend "github.com/blackbirdworks/gopherstack/services/ocicontainerengine" ocicontainerinstancesbackend "github.com/blackbirdworks/gopherstack/services/ocicontainerinstances" ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + ocidatabasebackend "github.com/blackbirdworks/gopherstack/services/ocidatabase" + ocidnsbackend "github.com/blackbirdworks/gopherstack/services/ocidns" ocieventsbackend "github.com/blackbirdworks/gopherstack/services/ocievents" + ocifilestoragebackend "github.com/blackbirdworks/gopherstack/services/ocifilestorage" ocifunctionsbackend "github.com/blackbirdworks/gopherstack/services/ocifunctions" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" ociloadbalancerbackend "github.com/blackbirdworks/gopherstack/services/ociloadbalancer" @@ -554,6 +563,8 @@ type Config struct { AppSyncOps *appsyncbackend.Handler // AzureARMOps is dispatched in-process by the /dashboard/api/v1/azure/arm proxy. AzureARMOps *azurearmbackend.Handler + // AzureGraphOps is dispatched in-process by the /dashboard/api/v1/azure/graph proxy. + AzureGraphOps *azuregraphbackend.Handler // GCPStorageOps is dispatched in-process by the /dashboard/api/v1/gcp/storage proxy. GCPStorageOps *gcpstoragebackend.Handler // GCPBigQueryOps is dispatched in-process by the /dashboard/api/v1/gcp/bigquery proxy. @@ -630,6 +641,12 @@ type Config struct { OCIArtifactsOps *ociartifactsbackend.Handler // OCIContainerInstancesOps is dispatched in-process by its proxy. OCIContainerInstancesOps *ocicontainerinstancesbackend.Handler + // OCIDNSOps is dispatched in-process by its proxy. + OCIDNSOps *ocidnsbackend.Handler + // OCIDatabaseOps is dispatched in-process by its proxy. + OCIDatabaseOps *ocidatabasebackend.Handler + // OCIFileStorageOps is dispatched in-process by its proxy. + OCIFileStorageOps *ocifilestoragebackend.Handler // AzureFileOps is dispatched in-process by the /dashboard/api/v1/azure/file proxy. AzureFileOps *azurefilebackend.Handler // AzureFunctionsOps is called in-process by the /dashboard/api/v1/azure/functions proxy. @@ -650,6 +667,16 @@ type Config struct { AzureEventGridOps *azureeventgridbackend.Handler // AzureAppConfigOps is called in-process by the /dashboard/api/v1/azure/appconfig proxy. AzureAppConfigOps *azureappconfigbackend.Handler + // AzureRedisOps is called in-process by the /dashboard/api/v1/azure/redis proxy. + AzureRedisOps *azureredisbackend.Handler + // AzureSQLOps is called in-process by the /dashboard/api/v1/azure/sqlquery proxy. + AzureSQLOps *azuresqlbackend.Handler + // AzureACROps is called in-process by the /dashboard/api/v1/azure/acr proxy. + AzureACROps *azureacrbackend.Handler + // AzureSearchOps is dispatched in-process by the /dashboard/api/v1/azure/search* proxies. + AzureSearchOps *azuresearchbackend.Handler + // LambdaWebOps is dispatched in-process by the /dashboard/api/v1/lambdaweb proxy. + LambdaWebOps *lambdawebbackend.Handler // GCPSQLOps is dispatched in-process by the /dashboard/api/v1/gcp/sql proxy. GCPSQLOps *gcpsqlbackend.Handler // GCPLoggingOps is called in-process by the /dashboard/api/v1/gcp/logging proxy. @@ -913,6 +940,9 @@ func (h *DashboardHandler) setupSubRouter() { h.registerAzureEventHubsRoutes() h.registerAzureEventGridRoutes() h.registerAzureAppConfigRoutes() + h.registerAzureRedisRoutes() + h.registerAzureSQLRoutes() + h.registerAzureACRRoutes() h.registerGCPSQLRoutes() h.registerGCPLoggingRoutes() h.registerSTSMetricsRoute() diff --git a/desktop/go.mod b/desktop/go.mod index 7d2eccae26..7efc48f553 100644 --- a/desktop/go.mod +++ b/desktop/go.mod @@ -8,6 +8,8 @@ require ( ) require ( + cel.dev/cel-go v0.33.2 // indirect + cel.dev/expr v0.25.2 // indirect cloud.google.com/go/apigateway v1.17.0 // indirect cloud.google.com/go/artifactregistry v1.27.0 // indirect cloud.google.com/go/auth v0.24.1 // indirect @@ -39,18 +41,20 @@ require ( connectrpc.com/connect v1.21.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect github.com/Microsoft/go-winio v0.6.3 // indirect + github.com/ProtonMail/go-crypto v1.4.1 // indirect github.com/adrg/xdg v0.5.3 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect github.com/alecthomas/kong v1.16.1 // indirect github.com/alicebob/miniredis/v2 v2.40.0 // indirect github.com/andybalholm/brotli v1.2.6 // indirect - github.com/aws/aws-sdk-go-v2 v1.47.3 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect github.com/aws/aws-sdk-go-v2/config v1.33.9 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect github.com/aws/aws-sdk-go-v2/service/acm v1.50.4 // indirect github.com/aws/aws-sdk-go-v2/service/amplify v1.48.4 // indirect @@ -106,16 +110,19 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect github.com/aws/aws-sdk-go-v2/service/transcribe v1.66.4 // indirect github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4 // indirect - github.com/aws/smithy-go v1.28.5 // indirect + github.com/aws/smithy-go v1.28.8 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/cedar-policy/cedar-go v1.8.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudflare/circl v1.6.4 // indirect github.com/coder/websocket v1.8.14 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/distribution/v3 v3.1.2 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2/v2 v2.8.4 // indirect @@ -126,21 +133,42 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dop251/goja v0.0.0-20261012212121-0c308c5ed11b // indirect github.com/dustin/go-humanize v1.1.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.4 // indirect + github.com/go-git/go-billy/v5 v5.9.2 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-mysql-org/go-mysql v1.16.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/jsonpointer v1.0.1 // indirect + github.com/go-openapi/jsonreference v1.0.1 // indirect + github.com/go-openapi/swag v0.29.2 // indirect + github.com/go-openapi/swag/cmdutils v0.29.2 // indirect + github.com/go-openapi/swag/conv v0.29.2 // indirect + github.com/go-openapi/swag/fileutils v0.29.2 // indirect + github.com/go-openapi/swag/jsonutils v0.29.2 // indirect + github.com/go-openapi/swag/loading v0.29.2 // indirect + github.com/go-openapi/swag/mangling v0.29.2 // indirect + github.com/go-openapi/swag/netutils v0.29.2 // indirect + github.com/go-openapi/swag/pools v0.29.2 // indirect + github.com/go-openapi/swag/stringutils v0.29.2 // indirect + github.com/go-openapi/swag/typeutils v0.29.2 // indirect + github.com/go-openapi/swag/yamlutils v0.29.2 // indirect github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect github.com/go-sql-driver/mysql v1.10.1 // indirect github.com/go-stomp/stomp/v3 v3.1.5 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/go-webauthn/webauthn v0.18.2 // indirect + github.com/go-webauthn/x v0.3.1 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/godbus/dbus/v5 v5.2.3 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/snappy v1.0.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/google/jsonschema-go v0.4.3 // indirect github.com/google/pprof v0.0.0-20261012231050-2af12c3f4d87 // indirect github.com/google/s2a-go v0.1.11 // indirect @@ -149,7 +177,7 @@ require ( github.com/googleapis/gax-go/v2 v2.27.0 // indirect github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/mux v1.8.1 // indirect - github.com/gorilla/websocket v1.5.3 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect github.com/hashicorp/golang-lru/arc/v2 v2.0.7 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect @@ -158,6 +186,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.11.0 // indirect github.com/jackc/puddle/v2 v2.2.3 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.20.3 // indirect github.com/kr/fs v0.1.0 // indirect github.com/labstack/echo/v5 v5.4.1 // indirect @@ -168,9 +197,11 @@ require ( github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/moby/api v1.56.1 // indirect github.com/moby/moby/client v0.6.2 // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/mochi-mqtt/server/v2 v2.7.9 // indirect github.com/modelcontextprotocol/go-sdk v1.8.0 // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -178,6 +209,7 @@ require ( github.com/parquet-go/bitpack v1.1.0 // indirect github.com/parquet-go/jsonlite v1.5.5 // indirect github.com/parquet-go/parquet-go v0.32.0 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.33 // indirect github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 // indirect github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 // indirect @@ -189,6 +221,7 @@ require ( github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect github.com/rabbitmq/amqp091-go v1.15.0 // indirect + github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect github.com/recolabs/gnata v0.6.1 // indirect github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 // indirect github.com/redis/go-redis/extra/redisotel/v9 v9.23.0 // indirect @@ -198,14 +231,21 @@ require ( github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect github.com/shopspring/decimal v1.5.0 // indirect + github.com/sijms/go-ora/v2 v2.9.0 // indirect github.com/sirupsen/logrus v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/tidwall/gjson v1.20.0 // indirect github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.2 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/twmb/franz-go v1.22.1 // indirect github.com/twmb/franz-go/pkg/kmsg v1.15.0 // indirect github.com/twpayne/go-geom v1.7.0 // indirect + github.com/ulikunitz/xz v0.5.17 // indirect github.com/vektah/gqlparser/v2 v2.5.62 // indirect + github.com/willscott/go-nfs v0.0.4 // indirect + github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect @@ -240,12 +280,15 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.58.0 // indirect golang.org/x/exp v0.0.0-20261011100332-faecfb24eb49 // indirect + golang.org/x/mod v0.42.0 // indirect golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sync v0.24.0 // indirect golang.org/x/sys v0.49.0 // indirect + golang.org/x/term v0.47.0 // indirect golang.org/x/text v0.43.0 // indirect golang.org/x/time v0.16.0 // indirect google.golang.org/api v0.304.0 // indirect @@ -254,12 +297,25 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20261012220534-4341edf3f941 // indirect google.golang.org/grpc v1.84.0 // indirect google.golang.org/protobuf v1.36.12 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.37.2 // indirect + k8s.io/apimachinery v0.37.2 // indirect + k8s.io/client-go v0.37.2 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect + k8s.io/streaming v0.37.2 // indirect + k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect modernc.org/libc v1.78.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect modernc.org/sqlite v1.60.1 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect software.sslmate.com/src/go-pkcs12 v0.7.4 // indirect ) diff --git a/desktop/go.sum b/desktop/go.sum index be0874c61d..058d7ca646 100644 --- a/desktop/go.sum +++ b/desktop/go.sum @@ -1,3 +1,5 @@ +cel.dev/cel-go v0.33.2 h1:eAcEdBguc6AFrGbPOyE0g168Uey4kKFW9J6qsBLSeMU= +cel.dev/cel-go v0.33.2/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= @@ -74,8 +76,8 @@ firebase.google.com/go/v4 v4.22.0 h1:+7ClMN69jOITseDXC7q0OKjMkRl3ZEJwZaLB900mvYs firebase.google.com/go/v4 v4.22.0/go.mod h1:tXyH/bqgOst15PmgHUSFaLX9woyDwclTtz/cRX+NocQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3 h1:qgnWTiWs1G7AqcKZegFvkJD1bmXJcaDU3uC0ELSgZ5U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3/go.mod h1:XtxD48QPInqoj+GMlcRlXHEYsoOxFnP6w2JgIZ9rDS4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.15.0 h1:w3gDUJlJ0BfBsHH02DCBDwbkmMSXJqBj6nBOCo9iocY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.15.0/go.mod h1:7scl8mG/IFLVjJ9ef/ltdND2T1z0CkgVSaQoUxLvmeU= github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig/v2 v2.2.0 h1:80ijeACessBdODMsTjR0zf7t1Uvhc+hCqcXc0BcW9ZA= github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig/v2 v2.2.0/go.mod h1:RPzAQTYyoren3gG5K0QSiWTSUhGCB47LY//1MGdODv0= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.5.0 h1:wtCn7MemMD9eo4/NdpJ6S/MFD2BV2CDwoEfvl5th2vM= @@ -116,6 +118,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventgrid/armeventgrid/v2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventgrid/armeventgrid/v2 v2.3.0/go.mod h1:NOM/LtCJfoU69VkmxPxV6PMxDm/MsOvmsao/5V5Rhgs= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/logic/armlogic v1.2.0 h1:EMNgS+pCj2/2LL7+nWG8zPf9sp4u8icP5FNwoBhyc8M= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/logic/armlogic v1.2.0/go.mod h1:TsM36SmGxYC24DiOTR9wPuBj5HYphihMC6xlnX536bE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.12.0 h1:qmeVFr8V9dDzhLZDDgdqyOEl+XLhhMpm7xrOajxqKuk= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.12.0/go.mod h1:x/aYmuTilxu4yQMFj2r3ZE3iiO2wxYFFmh6dRg2IWiQ= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 h1:L7G3dExHBgUxsO3qpTGhk/P2dgnYyW48yn7AO33Tbek= @@ -136,6 +140,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armlocks v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armlocks v1.2.0/go.mod h1:GE1wqa9Ny9eZ8wHtHqbCE7mMsFfVbdEY0itmzYV8JEg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armpolicy v1.0.0 h1:FKYUd0TZfjjWtdk7LR+6rI64yNY1gMe9rNV40Y2Fh4o= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armpolicy v1.0.0/go.mod h1:w987ZDGBg74HZu5sAqaBEhDTlvlJlxPggIhBiDviYhE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/search/armsearch v1.4.0 h1:zBdabY8pMSMLPb1XJnFSEdJi9Bd0h+VMjh1uU8B6Yp8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/search/armsearch v1.4.0/go.mod h1:Y2Q3nB3UfSnG9nALOpPAjflXPM3jL/n2ZmYIu2Occ9g= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/sql/armsql v1.2.0 h1:S087deZ0kP1RUg4pU7w9U9xpUedTCbOtz+mnd0+hrkQ= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/sql/armsql v1.2.0/go.mod h1:B4cEyXrWBmbfMDAPnpJ1di7MAt5DKP57jPEObAvZChg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= @@ -160,8 +166,8 @@ github.com/Azure/go-amqp v1.7.0 h1:9VlH/LEWr386XWWJRNON0eslFqSClYBXP4HewvIqkDQ= github.com/Azure/go-amqp v1.7.0/go.mod h1:pCJaHsvRlmmFUpxyQbh2qPkUFqYJeRBTqJSHKJadvPg= github.com/Azure/go-ansiterm v0.0.0-20261008234906-119c4d577770 h1:n3LUIJHwvkpfvfkt0dD7at4iYfQD24ML2jflWi3r2ew= github.com/Azure/go-ansiterm v0.0.0-20261008234906-119c4d577770/go.mod h1:DMNF6Kxv+EcuUvVuNkvC3f+BjEEmc/Fn9aU+qiD8zpU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= -github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 h1:0O6j18nQDoIff6/mtosT5LNU2WehEwp+LqfX6699oqY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1/go.mod h1:xdYAf5bjkOpsd3auA8riiJW4vneBubt9caavL7626jQ= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0 h1:BzsL0qE7LvtTEtXG7Dt5NS1EP0CQwI21HZfj9aGghhw= @@ -178,6 +184,8 @@ github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/Microsoft/go-winio v0.6.3 h1:M/52tsoyTfwaxajLnXZm7PqkbMGRH8iSWP+8fcg9MQ4= github.com/Microsoft/go-winio v0.6.3/go.mod h1:ma0QpxizD4fyyzJ1C7MIblymTD2Nxz5Mho/3CepwIZg= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= @@ -194,6 +202,8 @@ github.com/andybalholm/brotli v1.2.6 h1:ftYnfj6usCp+UGV5kSJ3+chpMQgU+gJf/AxsUQ52 github.com/andybalholm/brotli v1.2.6/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antlr/antlr4 v0.0.0-20181218183524-be58ebffde8e h1:yxMh4HIdsSh2EqxUESWvzszYMNzOugRyYCeohfwNULM= github.com/antlr/antlr4 v0.0.0-20181218183524-be58ebffde8e/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= @@ -202,24 +212,28 @@ github.com/apache/tinkerpop/gremlin-go/v3 v3.8.2 h1:kE0xuL6UncFQgPYwmQfBT440wNOM github.com/apache/tinkerpop/gremlin-go/v3 v3.8.2/go.mod h1:HkUXCwiMMjv3oBUqwrbqRLSkbHh9csEC3aW7Ntw/uFc= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-dax-go v1.2.15 h1:30rH3+QgjpjemrVg0NGIG5FnB1izJZ7jUZuBb1Fy8ak= github.com/aws/aws-dax-go v1.2.15/go.mod h1:4f/qGLBQlPYd+fmAfG4n4oSvN19JdKNYYmsr90/MPso= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= -github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2 v1.47.7 h1:jBmtzRtGWVAzbfc4RBWo1dTte0ZZH8p9U42YJIw2+Bw= +github.com/aws/aws-sdk-go-v2 v1.47.7/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ= github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/dsql/auth v1.2.8 h1:udoNKmbYngh4RpmURFjIaduwPjS/pTHkySWCTkS9Qf0= +github.com/aws/aws-sdk-go-v2/feature/dsql/auth v1.2.8/go.mod h1:kAkwdYfRHYr653KcQh9vLui/BP4CFLrYvdD9npWb5Dc= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10 h1:DKtPO+uoekvZX1tIFGVlWypYGSYkHPb7rliUI1wZtEw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10/go.mod h1:uAi0RDdZZz8IRpXERptWALeCgQG9Ye/HiMLxkxlt8IU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10 h1:r8p95YELmfmuV9VvAaFcLesVBZduCCCccxpzQKkNThw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10/go.mod h1:W3M52Mqai5TjgzyL/dEG+UwE+Xz9PykyjXelvbAzWIA= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.57.3 h1:m7EQZA6WxCbiExtCYq1wJLvK/dTQgQc1KvEOBnF9GlY= @@ -444,6 +458,8 @@ github.com/aws/aws-sdk-go-v2/service/lakeformation v1.55.3 h1:E9E99pv5u2Uc0VkgXp github.com/aws/aws-sdk-go-v2/service/lakeformation v1.55.3/go.mod h1:IlKDS05ABuNxYQ/UspDTEoKVOmkpXne18fVJFMTOKeQ= github.com/aws/aws-sdk-go-v2/service/lambda v1.112.2 h1:QcDoECg2br9JOaQ6C0fR3Mkcj9b/+dFSRuxZCLS3tME= github.com/aws/aws-sdk-go-v2/service/lambda v1.112.2/go.mod h1:GZ1UAuKbyxBdaRNbhKbAobG2VcWrEg5YmgWstJA3sl8= +github.com/aws/aws-sdk-go-v2/service/lambdaweb v1.2.5 h1:0y77BcwEEKEuwUSDuJsxpUC0UYxK5iTjkwDVAFkxGe0= +github.com/aws/aws-sdk-go-v2/service/lambdaweb v1.2.5/go.mod h1:yLxwAiadnny6r4QkMjWn0oChHryB6xbSIpMdGkr7doA= github.com/aws/aws-sdk-go-v2/service/lightsail v1.66.4 h1:M0KgyGQkqxe/MHR5Lj1PZGgwnJOnfN5bCDh470sIJDg= github.com/aws/aws-sdk-go-v2/service/lightsail v1.66.4/go.mod h1:cYtZI/MoJuzdSBYXg4enE8RkNzKRkoMr0qATq+ThsIw= github.com/aws/aws-sdk-go-v2/service/macie2 v1.59.3 h1:hbejt97Z2rKhxcjlpLu85REd1/w9Z8omGQkXl68FEI4= @@ -476,6 +492,8 @@ github.com/aws/aws-sdk-go-v2/service/mwaa v1.48.3 h1:y5NMX0onYGKLbL2sXgSsMiRiQNg github.com/aws/aws-sdk-go-v2/service/mwaa v1.48.3/go.mod h1:1VKiGS0MW4kgvZ9iT0SYDDqcbe/h3gcx/olyQT5M30w= github.com/aws/aws-sdk-go-v2/service/neptune v1.53.3 h1:ykzA1zMCGNBrvG3rTU/0CJ+mdG3oDVWd13srP4/u9ec= github.com/aws/aws-sdk-go-v2/service/neptune v1.53.3/go.mod h1:XLgBl5jRJlB9CejUkggkmr+DO5zkI2MyvyP7nXLuWsI= +github.com/aws/aws-sdk-go-v2/service/neptunedata v1.27.3 h1:eBcQCfO3BRhnEHGgyUILmkmGW1uCKyl3KT/QbKoUlsA= +github.com/aws/aws-sdk-go-v2/service/neptunedata v1.27.3/go.mod h1:jZw7kiGWC8nU0D2B5AEZMpTITmS5Je13gMH7aZFeP+o= github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.76.1 h1:5vjcmM/0scMQ7T2Gf9QxAqUehUbLeyl6viZZj+bYdKw= github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.76.1/go.mod h1:UwfhRPgauP3bzV9eY3M94mpRATlNTzzlYtPTGjd85Zk= github.com/aws/aws-sdk-go-v2/service/networkmanager v1.50.3 h1:7CgYmJqLDr1Y+QohY1LGkvbKddehyr5PtHjQ0KtuxWM= @@ -632,8 +650,8 @@ github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4 h1:s3r96tGUdWFhIolHNwR9+ github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4/go.mod h1:rupBg6/2hKMz1uCa7sd/RSMmLOXJQWgMzYIOB+Qi2EA= github.com/aws/aws-sdk-go-v2/service/xray v1.45.3 h1:GDr+18TEmx5ivPIaYqCWjVtPGSbcVZQkgX3PzhfzkKE= github.com/aws/aws-sdk-go-v2/service/xray v1.45.3/go.mod h1:My/sZif0SANOKoH+IiyDW9AlWR2ITZQny7u7pTBYBMQ= -github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= -github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ= +github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= @@ -642,6 +660,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM= +github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= github.com/cedar-policy/cedar-go v1.8.0 h1:9gcU7EHXwHC2RMdpph68yTAkdB3behTTssC+kt4GoS8= github.com/cedar-policy/cedar-go v1.8.0/go.mod h1:h5+3CVW1oI5LXVskJG+my9TFCYI5yjh/+Ul3EJie6MI= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -650,6 +670,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U= +github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= @@ -666,9 +688,12 @@ github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/distribution/distribution/v3 v3.1.2 h1:/Bv2YIqqSR00HiO49FT93hovy6r9IlrzwUDsJjCN5qg= @@ -695,6 +720,8 @@ github.com/ebitengine/purego v0.11.1 h1:2zpWRSQNVKN4eKsKO9eM1ILDgWfYMY9GwqRmK6Xe github.com/ebitengine/purego v0.11.1/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE= github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= @@ -702,8 +729,12 @@ github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMD github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= -github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/fxamacker/cbor/v2 v2.9.4 h1:xwjVlxEMR3S605oUlgBjKLTTeGFciYPGYCtF/35LKGo= +github.com/fxamacker/cbor/v2 v2.9.4/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE= +github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= +github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4= +github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-json-experiment/json v0.0.0-20251027170946-4849db3c2f7e h1:Lf/gRkoycfOBPa42vU2bbgPurFong6zXeFtPoxholzU= github.com/go-json-experiment/json v0.0.0-20251027170946-4849db3c2f7e/go.mod h1:uNVvRXArCGbZ508SxYYTC5v1JWoz2voff5pm25jU1Ok= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -715,6 +746,40 @@ github.com/go-mysql-org/go-mysql v1.16.0 h1:odv4Ygtc1WHJv3uUF2aoJdE1RS7tA0sD3ET9 github.com/go-mysql-org/go-mysql v1.16.0/go.mod h1:VjBTZTTDKL8OMXUAhNbg3VHaVVq9HOXJEBLpAKBFIfE= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/jsonpointer v1.0.1 h1:2KxywRmNwJkT/FMBa3iRNHEaAxSJvjqoufQZy3au1Mg= +github.com/go-openapi/jsonpointer v1.0.1/go.mod h1:wI7ZYsFmbIi9nBXOZqgDaS/bqOchRGZjqxFli7FBYxY= +github.com/go-openapi/jsonreference v1.0.1 h1:4zJ7AmYDKNmD3aSpfPnFNCFA5E80/xMHUNKgydaLh38= +github.com/go-openapi/jsonreference v1.0.1/go.mod h1:dYplQXa6p5lXprLcJ8LE2iU7vNpXsAHDQ5ZAgL+Qx3A= +github.com/go-openapi/swag v0.29.2 h1:9n8frkcsuQRA0INU31VbxNxt1zBaHxrHYGEf5BUjFi4= +github.com/go-openapi/swag v0.29.2/go.mod h1:RkCiX1gCVXWgLOavDXE3ALJrQ/VO2ziiIzOUs3z10xo= +github.com/go-openapi/swag/cmdutils v0.29.2 h1:cXEzX/nWCODon251f+1HgL75/R4XXML5IZxgLCWMoro= +github.com/go-openapi/swag/cmdutils v0.29.2/go.mod h1:XziaSVzYqkDvpWXzASfWajw2YvJrLscq610O2XiyqJw= +github.com/go-openapi/swag/conv v0.29.2 h1:8c9shoB8l0QRSR6ymq1llHdBWDqpIgJfjTGHx3Vuhm0= +github.com/go-openapi/swag/conv v0.29.2/go.mod h1:AZS0YigTNf8qNtD6WqJz/N4RUNmpr76SyjFGkq4+p6Q= +github.com/go-openapi/swag/fileutils v0.29.2 h1:mdUL+Vw5ah1fO1AvFCoHeIyv7YZsCz2MN3KNqd/lLVI= +github.com/go-openapi/swag/fileutils v0.29.2/go.mod h1:7QKmmodjAebaq61lGVJa8ldLdGeMF4DSgYv8tXOGGo0= +github.com/go-openapi/swag/jsonutils v0.29.2 h1:uZNSD2/rJDYAfvsLYkykTzXuyxM3QpDKV9jhRHrMu6k= +github.com/go-openapi/swag/jsonutils v0.29.2/go.mod h1:ONTdNvj3Y+IXRzfa17E+Rgt2ns/qiSOYjIo2K7v2yDs= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.2 h1:w+Fd6EBOMGlC74GYGHqCLGyb3Bpams8TtNrgM/SG4jo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.2/go.mod h1:HC2egPORFzbj/gf05Zrfgz8Mgplx3oW7hgGi7pjZVhM= +github.com/go-openapi/swag/loading v0.29.2 h1:QU1ry24e6r6Shoxe380Nx0X5iKkpRO+hJffsyJ2dVlY= +github.com/go-openapi/swag/loading v0.29.2/go.mod h1:DqilDjuiJKETecsS4TRopWG1acUDsfqN39ZmQhPu6uI= +github.com/go-openapi/swag/mangling v0.29.2 h1:ltdo3K0tTP4P6zWhVVWDU9D37/YYBuMJRdZxiS2Dtog= +github.com/go-openapi/swag/mangling v0.29.2/go.mod h1:RjDi4TnItAenS3cgCXSpIl1kKbjG7il6pnmQVl/8bp8= +github.com/go-openapi/swag/netutils v0.29.2 h1:dBli6jyUa93sDS/XYWShJzf4+nnVfFKfvY9SLUaUzB8= +github.com/go-openapi/swag/netutils v0.29.2/go.mod h1:1pM6Xg/Um8E1eps3umWl8Hme3ByWXOtHsvFC0CULHUE= +github.com/go-openapi/swag/pools v0.29.2 h1:PlGoDRF8WtSyXkedZZkpW3f9wDhFf3u8KtaBcmgmh8c= +github.com/go-openapi/swag/pools v0.29.2/go.mod h1:V3lhxVT4qDYRqc2MRSSbLsTYIYV/2HlHafhyEkmzLIc= +github.com/go-openapi/swag/stringutils v0.29.2 h1:lcnBxwAaysT3bMUVBfn9V/PkfVqkurpqufKU6rTUMGk= +github.com/go-openapi/swag/stringutils v0.29.2/go.mod h1:i9cdh7sGaa0nm3CqIvH5IM5h2QClk1wns2ktKeILvRI= +github.com/go-openapi/swag/typeutils v0.29.2 h1:O7aVvkTs3pXwikgtrPLenigEMdQFgONKRooQA9pgf2I= +github.com/go-openapi/swag/typeutils v0.29.2/go.mod h1:7+GDG+uz9Ke+eVt4rClZg7wklSDp8hT/mKNh/pC26TE= +github.com/go-openapi/swag/yamlutils v0.29.2 h1:IFKFFeDnuIwzfsuWRQU+rI8iL3g4XyAYjV/RlnlxPLM= +github.com/go-openapi/swag/yamlutils v0.29.2/go.mod h1:7MGqtcrK73sxQ4ceiyIf8qiXS/s09JLMdR5esK5euYQ= +github.com/go-openapi/testify/enable/yaml/v2 v2.7.0 h1:wPW6YRgx3+SID1yUy/Xwa17L8kFEaEKod2VRbJDZNUs= +github.com/go-openapi/testify/enable/yaml/v2 v2.7.0/go.mod h1:mI1M88etYbc3PhgHsWQK2kwvNwW5aGFqMPbmib+SGIs= +github.com/go-openapi/testify/v2 v2.7.0 h1:bycOreEj6wfBvijg3YFogZ/sFjTCDmQnwSodSzHa3X8= +github.com/go-openapi/testify/v2 v2.7.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q= github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= @@ -723,6 +788,10 @@ github.com/go-stomp/stomp/v3 v3.1.5 h1:Pikz1OSusmSKUm5mRKYfXQZaDatfZ+EnBBA1JJ2xE github.com/go-stomp/stomp/v3 v3.1.5/go.mod h1:ztzZej6T2W4Y6FlD+Tb5n7HQP3/O5UNQiuC169pIp10= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-webauthn/webauthn v0.18.2 h1:0BeftmEHU7i3Dv0VFwBtidy/ba37Vcdjvqst9EYu8Sk= +github.com/go-webauthn/webauthn v0.18.2/go.mod h1:hEXaOuLxvZ3zG9miZe3ehlyeVso9AtklXG+kTn36k+A= +github.com/go-webauthn/x v0.3.1 h1:1ff37z3XfmTTomkhlURgGizLIDyOvPgTt2t9nlzKLRo= +github.com/go-webauthn/x v0.3.1/go.mod h1:ZInxAynYXfBPvvm5gzKZ7geBlL23K71xASMgohHl/Rg= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= @@ -750,12 +819,21 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba h1:qJEJcuLzH5KDR0gKc0zcktin6KSAwL7+jWKBYceddTc= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba/go.mod h1:EFYHy8/1y2KfgTAsx7Luu7NGhoxtuVHnNo8jE7FikKc= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/pprof v0.0.0-20261012231050-2af12c3f4d87 h1:P18JvwDzBYAeC8SIfnee/YJzKk1dysuN6rUwmF6CxZY= github.com/google/pprof v0.0.0-20261012231050-2af12c3f4d87/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM= +github.com/google/rpmpack v0.7.1 h1:YdWh1IpzOjBz60Wvdw0TU0A5NWP+JTVHA5poDqwMO2o= +github.com/google/rpmpack v0.7.1/go.mod h1:h1JL16sUTWCLI/c39ox1rDaTBo3BXUQGjczVJyK4toU= github.com/google/s2a-go v0.1.11 h1:7v1V1TraNGP63WptsAkNqr2w8jgm4vM6A6qx6Lsw9iM= github.com/google/s2a-go v0.1.11/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -768,8 +846,8 @@ github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyE github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 h1:Bd7KaOxzULLxtZ/K5s1aLbWhR0+5RToO65TXHsf3bqQ= github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0/go.mod h1:nN7ts3dFXKtCZWc//yfkpcQNKJABg16/uDVAZpLDalo= github.com/hashicorp/golang-lru/arc/v2 v2.0.7 h1:QxkVTxwColcduO+LP7eJO56r2hFiG8zEbfAAzRv52KQ= @@ -792,10 +870,14 @@ github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -833,6 +915,8 @@ github.com/moby/moby/client v0.6.2 h1:ekSPlMccrZGcw6Scb8T9D39qE0mB+XupU/yS0zNw9N github.com/moby/moby/client v0.6.2/go.mod h1:DGcnO7qmPJrqobx/ggQgQ/AUk+2wgVdQv7uxB0/mEsc= github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0= @@ -845,18 +929,26 @@ github.com/mochi-mqtt/server/v2 v2.7.9 h1:y0g4vrSLAag7T07l2oCzOa/+nKVLoazKEWAArw github.com/mochi-mqtt/server/v2 v2.7.9/go.mod h1:lZD3j35AVNqJL5cezlnSkuG05c0FCHSsfAKSPBOSbqc= github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtwfBpOTAI+Urx4= github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= github.com/nicksnyder/go-i18n/v2 v2.5.0 h1:3wH1gpaekcgGuwzWdSu7JwJhH9Tk87k1ezt0i1p2/Is= github.com/nicksnyder/go-i18n/v2 v2.5.0/go.mod h1:DrhgsSDZxoAfvVrBVLXoxZn/pN5TXqaDbq7ju94viiQ= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opensearch-project/opensearch-go/v4 v4.8.1 h1:9r9x+pt5QG49EZfG0bqQgCexFcQkeMkVGM54dpX0b9Q= +github.com/opensearch-project/opensearch-go/v4 v4.8.1/go.mod h1:OEBTtFm2ZeJ8G/ZbE3ITbKuKlT+9ujruJF3Vj5LBMqo= github.com/oracle/oci-go-sdk/v65 v65.126.2 h1:7lv9XuPcX9DzW9z+y/W2ENub71yyOP7DM8+BPCh8hgg= github.com/oracle/oci-go-sdk/v65 v65.126.2/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w= github.com/parquet-go/bitpack v1.1.0 h1:b80hfKZwIJ3Ku684xJSEj/9G0tywQPKOtmTV8si30mo= @@ -865,6 +957,8 @@ github.com/parquet-go/jsonlite v1.5.5 h1:TIqJ+b+mYoTvvEt1/ICS4fBacQO0xWwAbvtcrKj github.com/parquet-go/jsonlite v1.5.5/go.mod h1:hN57TVzxtr04kaax5XwNgCCsh1KE7BT6dvY6X/zNH90= github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM= github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4/v4 v4.1.33 h1:GjG1TJ1V4IzKP8L96muuuDNpTwd7D+l2ccXrjAbe014= github.com/pierrec/lz4/v4 v4.1.33/go.mod h1:7SE9MC2STkNtL4PIwGhjmyVwvILaGI9/COYQNBhKM/c= github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 h1:cLgCk5mwDG9lDH+dPK8TmEliTjyGJwwKN0qevWAl8IY= @@ -880,6 +974,8 @@ github.com/pkg/sftp v1.13.11/go.mod h1:uNkH9roSXglNJqM+glJJi+TQXQUm0fXFWqCFmT8hs github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 h1:XA01Vk/wv9YikCi1V51yRzIHPMT5of9+cMpZoZvDn/M= github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0= @@ -894,6 +990,8 @@ github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0o github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/rabbitmq/amqp091-go v1.15.0 h1:LEQL4/yp48/Wigt6A6XOu18RQRo8ZHtB5I/KZJn+gkw= github.com/rabbitmq/amqp091-go v1.15.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/recolabs/gnata v0.6.1 h1:2YCxrmUK64UFm1fPBFYe5jFHS6Q8EKJC+34QCujeU44= github.com/recolabs/gnata v0.6.1/go.mod h1:+u5DmoYMyHl+9Uuo9KQXLhbKHip+M5N8S/QYAPSG0XU= github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 h1:YOQCtePRR/cBrxtZB5u2ueYei3zGJm1n0KB2tvth0k0= @@ -916,16 +1014,22 @@ github.com/shirou/gopsutil/v4 v4.26.9 h1:CaBo/hBFqvlJoLvAQEKdHdf5GD97/MN2ACfyv63 github.com/shirou/gopsutil/v4 v4.26.9/go.mod h1:nKH+8wX2zxr/mDbsR+AAy6Qd2z4ZOQ7wfQaEp+yrYmg= github.com/shopspring/decimal v1.5.0 h1:HxSWk31RmJv0EyY4oLiGQB9u7WPludUwPBdvQZnYFPc= github.com/shopspring/decimal v1.5.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sijms/go-ora/v2 v2.9.0 h1:+iQbUeTeCOFMb5BsOMgUhV8KWyrv9yjKpcK4x7+MFrg= +github.com/sijms/go-ora/v2 v2.9.0/go.mod h1:QgFInVi3ZWyqAiJwzBQA+nbKYKH77tdp1PYoCqhR2dU= github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg= github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -942,6 +1046,8 @@ github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.2 h1:dz1jrRuE7or/74V490B4/GP1pZm5WKlt2bgCP5A83w8= github.com/tidwall/pretty v1.2.2/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= @@ -952,10 +1058,18 @@ github.com/twmb/franz-go/pkg/kmsg v1.15.0 h1:mILdOQm4JkWojbSYVQ7KJNJ1qMdYNsLrdNO github.com/twmb/franz-go/pkg/kmsg v1.15.0/go.mod h1:sGkGxKexpRLBThFHJLjjywcyfkK09qB/g+X+wFWZQ/A= github.com/twpayne/go-geom v1.7.0 h1:zMHTHNlenoHAc2ZSiHFh7SMoG0Wdw8gF0orBnrBhfjE= github.com/twpayne/go-geom v1.7.0/go.mod h1:y9Q2YcKKaC2PHXwtRvwEunOPx5Oa+U8aAxPmFvscROk= +github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= +github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= github.com/vektah/gqlparser/v2 v2.5.62 h1:GJP/emj1wIWKs0U3SHcAgzknWgylTqQ1qH2gNL+wSWc= github.com/vektah/gqlparser/v2 v2.5.62/go.mod h1:JNK+plRwKdXLsF/qPFPe5tE0z4s1WeroD9S5LR8um/Q= github.com/wailsapp/wails/v3 v3.0.0 h1:lBe89kFu23bVz1dc1H9v0srkHQgWsy7BAKpH4sLQuDw= github.com/wailsapp/wails/v3 v3.0.0/go.mod h1:U3p3/QfSer2XxXM3XWhwis9O7LvvuNKN/AcGoEFD+K8= +github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI= +github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg= +github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 h1:U0DnHRZFzoIV1oFEZczg5XyPut9yxk9jjtax/9Bxr/o= +github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= @@ -1039,6 +1153,8 @@ go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= @@ -1128,6 +1244,10 @@ google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1135,6 +1255,20 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +k8s.io/api v0.37.2 h1:PnxqnUFy93jr2GcpWDkKOm8wfy25kT9wa572MuQS8oc= +k8s.io/api v0.37.2/go.mod h1:+NVRLft99kx+Jg9v34GmHw867Dapim89TSyCsn2hK+M= +k8s.io/apimachinery v0.37.2 h1:G4t9RGoPDbQML38hz/kQQiltttgd3egg2ODCR8aIc2o= +k8s.io/apimachinery v0.37.2/go.mod h1:qaAvSNvgU2x07wRM95MSHSJTPQ/ob/LeNAGbeDgqQJg= +k8s.io/client-go v0.37.2 h1:sgFtyRZukGkOrsu+QgS7LYnbFFJ5ut3MchdBNduD//E= +k8s.io/client-go v0.37.2/go.mod h1:bo3LFC8YbCtKFPX21Yj8puMsoJLAv/+SvuOODma0s54= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= +k8s.io/streaming v0.37.2 h1:xmzp9nbqIbjvsRcqrBYa+K5XPQ6Jy/Dm3zZ8mN916AI= +k8s.io/streaming v0.37.2/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= +k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs= +k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= modernc.org/cc/v4 v4.30.1 h1:ZXInYH2CD24GPWSwOFPwB6JrnFt4/6d7ceW1Ukd30Bk= modernc.org/cc/v4 v4.30.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.36.2 h1:j6d5HuXtBxF00vEcI5RE0O6EIrG6RyRpgMiGV2IK9ys= @@ -1165,5 +1299,13 @@ modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= software.sslmate.com/src/go-pkcs12 v0.7.4 h1:OvJ7g/OB9RuVAOzahlFLns6HeIcP+SJotsDqmLfhd8w= software.sslmate.com/src/go-pkcs12 v0.7.4/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/docker-compose.yml b/docker-compose.yml index 139bd57ca6..549602dbb8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,10 @@ services: start_period: 5s volumes: - gopherstack-data:/data + # Docker-backed engines (Lambda, ECS, ...): the image runs as UID 65532, so also mount the socket and + # join its group, e.g. `DOCKER_GID=$(stat -c %g /var/run/docker.sock) docker compose up`. + # - /var/run/docker.sock:/var/run/docker.sock + # group_add: ["${DOCKER_GID}"] volumes: gopherstack-data: diff --git a/docs/architecture/elasticache.md b/docs/architecture/elasticache.md index fb65b8c48c..d22e49b291 100644 --- a/docs/architecture/elasticache.md +++ b/docs/architecture/elasticache.md @@ -46,15 +46,22 @@ gopherstack --elasticache-engine stub ## `docker` -A real Redis container is started via the Docker daemon when a cluster is created, and stopped when the cluster is deleted. +Real Redis (or Valkey) containers run the data plane, started through the container runtime (Docker or Podman): -**When to use:** When you need the full Redis stack (Lua scripts, modules, AUTH, TLS) that miniredis does not support. +- Standalone cache clusters get one container. +- Replication groups without cluster mode get one primary plus replicas wired with `REPLICAOF`; `PrimaryEndpoint` targets the primary and `ReaderEndpoint` the first replica. +- Cluster-mode groups launch `NumNodeGroups x (1 + replicas)` containers with `cluster-enabled yes`, form a real cluster (`CLUSTER MEET`/`ADDSLOTSRANGE`/`REPLICATE`) over the 16384-slot map (`NodeGroupConfiguration.Slots` is honoured), and announce `127.0.0.1` (or `ELASTICACHE_HOST`) as the hostname so `MOVED` redirects reach the host. Every node has its own port in `NodeGroupMembers[].ReadEndpoint`. +- `TransitEncryptionEnabled` serves TLS only (certificates from the emulator CA, `InMemoryBackend.DockerCAPEM()`); mode `preferred` serves plaintext. +- `AuthToken` protects the `default` user; RBAC users and user groups become `ACL SETUSER` accounts on every node, re-synced on user, user group and replication group changes. +- `CreateSnapshot` reads each node group primary's RDB; `SnapshotName` seeds the new primaries from it. + +**When to use:** When you need the full Redis stack (Lua, modules, ACLs, TLS, cluster slots) that miniredis does not support. ```bash gopherstack --elasticache-engine docker ``` -Requires Docker to be running. The image `redis:7-alpine` is used by default. Port allocation follows the same range as `embedded` mode. +Requires a container runtime. Images are `redis:7.x-alpine` (`valkey/valkey:8.x-alpine` for Valkey). Ports come from the port allocator range. ## Configuration reference diff --git a/docs/docker.md b/docs/docker.md index aa7b9818d0..e549775205 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -80,6 +80,12 @@ environment: - ELASTICACHE_ENGINE=embedded ``` +## Running as non-root + +The image is built on `gcr.io/distroless/static:nonroot` and runs as `65532:65532` by default. The binary is `/usr/local/bin/gopherstack` (the working directory, so `./gopherstack health` keeps working), `/data` is owned by 65532 and `/tmp` is `1777`. The image used to be `FROM scratch` and ran as root with the binary at `/root/gopherstack`; that path is gone. + +To run as root again (for example to keep an older root-owned volume without a `chown`), pass `--user 0:0` / `user: "0:0"`. + ## Volume-backed persistence State snapshots are written to `/data` inside the container. Mount a named volume or host path to keep state across container restarts: @@ -90,6 +96,18 @@ volumes: - ./local-data:/data # host path ``` +A new named volume inherits the ownership of `/data` (65532) and just works. A host path keeps the host's ownership, so give the container user write access, using either of: + +```bash +mkdir -p local-data && sudo chown 65532:65532 local-data +docker run --rm -p 8000:8000 -e PERSIST=true -v "$PWD/local-data:/data" ghcr.io/blackbirdworks/gopherstack:latest + +# or run as your own user +docker run --rm --user "$(id -u):$(id -g)" -p 8000:8000 -e PERSIST=true -v "$PWD/local-data:/data" ghcr.io/blackbirdworks/gopherstack:latest +``` + +Without write access Gopherstack logs `mkdir /data/...: permission denied`. A named volume created by an older, root-running image is still owned by root; fix it once with `docker run --rm -v gopherstack-data:/data alpine chown -R 65532:65532 /data`. + ## Multi-service application example Combine Gopherstack with your application in a single Compose file: @@ -165,10 +183,19 @@ docker run --rm -p 8000:8000 gopherstack:local Lambda image-based functions require the Docker daemon. When running Gopherstack itself inside Docker, you need to mount the host Docker socket and set `LAMBDA_DOCKER_HOST` to the host IP: +The container user (65532) is not in the socket's group, so add that group with `--group-add`/`group_add`: + +```bash +docker run --rm -p 8000:8000 -v /var/run/docker.sock:/var/run/docker.sock \ + --group-add "$(stat -c %g /var/run/docker.sock)" ghcr.io/blackbirdworks/gopherstack:latest +``` + ```yaml services: gopherstack: image: ghcr.io/blackbirdworks/gopherstack:latest + group_add: + - "${DOCKER_GID}" # export DOCKER_GID=$(stat -c %g /var/run/docker.sock) environment: - LAMBDA_DOCKER_HOST=host-gateway # or your host IP volumes: @@ -178,6 +205,8 @@ services: - "8000:8000" ``` +On Docker Desktop the socket is typically `root:root` inside the VM; use `--group-add 0` or `--user 0:0`. Without the group, engine-backed services (Lambda, ECS, ...) cannot reach the daemon and stay metadata-only. + See [Lambda architecture](architecture/lambda.md) for details. ## Registry credentials and mirrors diff --git a/docs/engines.md b/docs/engines.md index d57b1e0c7d..040285a0e8 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -16,7 +16,7 @@ If Podman is the reachable runtime and `CONTAINER_RUNTIME` is unset, it is set t A per-service flag (`--rds-engine`, ...), its env var (`RDS_ENGINE`, ...) or a non-default value always wins over `--engines`, so `GOPHERSTACK_ENGINES=auto RDS_ENGINE=stub` runs everything real except RDS. The ECS task runtime is -`--ecs-runtime` / `GOPHERSTACK_ECS_RUNTIME` (`none` or `docker`); Timestream for InfluxDB follows +`--ecs-runtime` / `GOPHERSTACK_ECS_RUNTIME` (`none` or `docker`; with `CONTAINER_RUNTIME=kubernetes` each task runs as a Pod, see [kubernetes.md](kubernetes.md)); Timestream for InfluxDB follows `TIMESTREAM_INFLUXDB_ENGINE`. Resolved modes are not written to persisted config. @@ -46,7 +46,7 @@ Adopting services: Cloud SQL (gcpsql), EC2 docker instances, ECS docker tasks an - ECS: a restored RUNNING task claims one container per container definition (`taskArn|containerName`). All must exist, run, and use the definition's image; then exit, health and log watching resume (logs from adoption time on). Otherwise every claimed container is removed and the task is STOPPED ("Task containers were lost when the - emulator restarted"). Containers behind security-group relays are never adopted. + emulator restarted"). Containers behind security-group relays and awsvpc tasks (pause container and task network) are never adopted. - Batch: containers carry the identity and service labels but no resource key. A job lost to a crash is failed ("Container execution was interrupted") and its container removed at the next startup. diff --git a/docs/kubernetes.md b/docs/kubernetes.md new file mode 100644 index 0000000000..0598c145d8 --- /dev/null +++ b/docs/kubernetes.md @@ -0,0 +1,127 @@ +# Kubernetes (Helm chart) + +`charts/gopherstack` deploys Gopherstack into a cluster. It renders a Deployment (one replica, `Recreate`), a Service, a ConfigMap of environment settings, and optionally a PVC, an Ingress and a Docker-in-Docker sidecar. + +## Install + +From a checkout: + +```bash +helm install gopherstack ./charts/gopherstack +kubectl port-forward svc/gopherstack-gopherstack 8000:8000 +AWS_ENDPOINT_URL=http://localhost:8000 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test \ + aws s3 mb s3://demo +``` + +The image is `ghcr.io/blackbirdworks/gopherstack`, tagged with the chart `appVersion` unless `image.tag` is set. In-cluster clients use `http://-gopherstack..svc.cluster.local:8000`. + +## Common values + +| Value | Default | Effect | +|---|---|---| +| `config.region`, `config.accountId`, `config.logLevel` | `us-east-1`, `000000000000`, `info` | `REGION`, `ACCOUNT_ID`, `LOG_LEVEL` | +| `config.services` / `config.disableServices` | `[]` | `GOPHERSTACK_SERVICES` / `GOPHERSTACK_DISABLE_SERVICES` | +| `config.engines` | `stub` | `GOPHERSTACK_ENGINES` (`stub`, `auto`, `docker`) | +| `config.enforceIam` | `false` | `GOPHERSTACK_ENFORCE_IAM` | +| `config.extra` | `{}` | any other env var, `KEY: value` | +| `persistence.enabled` | `false` | PVC mounted at `/data`, sets `PERSIST=true` and `GOPHERSTACK_DATA_DIR` | +| `service.type` / `service.port` | `ClusterIP` / `8000` | main endpoint (`PORT` follows `service.port`) | +| `service.extraPorts` | `[]` | extra named container and Service ports (below) | +| `ingress.*` | disabled | standard Ingress for the main port | +| `probes.*` | `/_gopherstack/health` | liveness, readiness and startup probes | +| `container.runtime` | `docker` | `kubernetes` runs Lambda functions and ECS tasks as Pods through the Kubernetes API (below) | +| `container.kubernetes.*` | see `values.yaml` | namespace, fetch image and code port for the Pod executor | +| `dind.enabled` | `false` | privileged Docker-in-Docker sidecar (below) | +| `resources`, `nodeSelector`, `tolerations`, `affinity` | empty | scheduling | + +`values.schema.json` validates the values at install time. + +## Persistence + +With `persistence.enabled=true` Gopherstack snapshots to the PVC on shutdown and restores on start, so state survives pod restarts. Without it, state is in memory. Use `persistence.existingClaim` to bring your own PVC. + +## Per-cloud ports + +AWS services share the main port. Azure, GCP and OCI emulators listen on dedicated ports, listed in `fixedServicePorts` in `internal/app/cli.go`. Expose the ones you need: + +```yaml +service: + extraPorts: + - name: azure-blob # AZURE_BLOB_PORT, default 10000 + port: 10000 + - name: cosmos # COSMOSDB_PORT, default 8081 + port: 8081 +``` + +Names must be at most 15 characters. A few defaults: Azure Blob 10000, Queue 10001, Table 10002, Service Bus 10003, Cosmos 8081, GCP Pub/Sub 8085, GCP Storage 4443, IoT MQTT 1883. If you change a port with `config.extra` (for example `AZURE_BLOB_PORT`), change the matching `extraPorts` entry. + +## Docker-backed engines + +Engines that start real containers (RDS, EKS, Lambda containers, ECS tasks, EC2 docker provider, ...) need a Docker daemon, which a pod does not have. By default the chart runs with `config.engines=stub`, so those services answer from metadata only. Mounting the node's Docker socket is not supported by the chart. + +To use them anyway, enable the sidecar: + +```yaml +dind: + enabled: true +config: + engines: docker +``` + +This adds a `docker:dind` container that runs **privileged** and exposes `tcp://127.0.0.1:2375` (no TLS, pod-local) as `DOCKER_HOST`. It needs a cluster that permits privileged pods and is not covered by the install test. + +## Kubernetes pod executor + +`container.runtime=kubernetes` (sets `CONTAINER_RUNTIME=kubernetes`) runs Lambda function containers as Pods in the release namespace (or `container.kubernetes.namespace`), so no privileged sidecar is needed. Docker stays the default. The chart then adds: + +- a namespace-scoped `Role` and `RoleBinding`: `pods` create/get/list/watch/delete and `pods/log` get. No cluster-wide rules; `secrets` (create/patch/delete) and `pods/exec` (create) only with `container.kubernetes.ecs.enabled` / `.exec`; +- the ServiceAccount token mount (off otherwise); +- `POD_IP`, `LAMBDA_DOCKER_HOST` and `CONTAINER_K8S_ADVERTISE_HOST` set to the Gopherstack Pod IP, plus the `fn-code` container port. + +How it works: + +- Function Pods get no ServiceAccount token. They reach the Lambda Runtime API at `:` (ports come from `PORT_RANGE_START`...; each function runtime listens on its own port), so a NetworkPolicy must allow function Pods to reach the Gopherstack Pod on those ports and on `container.kubernetes.codePort`. A Service port is not used because the ports are dynamic. +- There are no bind mounts. The function zip and layers are held in memory and an initContainer (`container.kubernetes.fetchImage`, needs `sh`, `wget`, `unzip`) downloads them from Gopherstack on the code port into read-only `emptyDir` volumes at `/var/task` and `/opt`. Each Pod gets a random bearer token (in the initContainer env only, never logged) that is valid for that Pod's archives alone and is revoked as soon as the Pod is running or removed. The transfer is plain HTTP inside the cluster; the token is visible to whoever can read Pods in the namespace, so restrict `get pods` there. Zips with absolute or `..` entries are rejected. +- Memory limit and request equal the function memory size. `OCIRuntime` (Lambda isolation) maps to `runtimeClassName`. +- Pods are labelled `gopherstack.managed=true`; the existing startup reconcile lists them and removes dead leftovers of earlier runs. + +Not supported by the Pod executor yet: EC2 and other engines (they still need docker), EFS file systems, Lambda hot reload, extensions in layers, `LAMBDA_DOCKER_FLAGS`, host DNS/CA injection, Lambda `Exec`, and container stop/start (a Pod is created or deleted). + +### ECS tasks as Pods + +With `CONTAINER_RUNTIME=kubernetes` and ECS containers enabled (`config.engines=docker` or `GOPHERSTACK_ECS_RUNTIME=docker`) every ECS task is **one Pod**; its containers share the Pod's network namespace, as under `awsvpc`. The Pod is named `ecs-`. Chart values `container.kubernetes.ecs.enabled` (secrets create/patch/delete) and `container.kubernetes.ecs.exec` (`pods/exec` create) add only the grants the feature needs; both default to off. + +| ECS | Pod | +|---|---| +| `entryPoint`, `command`, `environment`, `workingDirectory`, `user` (numeric or `root`), `readonlyRootFilesystem`, `linuxParameters.capabilities` | `command`, `args`, `env`, `workingDir`, `runAsUser`/`runAsGroup`, `readOnlyRootFilesystem`, `capabilities` | +| `secrets`, `environmentFiles` | resolved server-side into a Kubernetes `Secret` named after the Pod and referenced with `secretKeyRef`. The Pod owns the Secret (`ownerReferences`), and it is deleted with the task. Values are never in the Pod spec, logs or annotations | +| container `memory` / `memoryReservation` | memory limit / request | +| container `cpu` (units, 1024 = 1 vCPU) | cpu request. Task `cpu` and `memory` cap every container, as ECS caps the task | +| `healthCheck` (`CMD`, `CMD-SHELL`) | exec `readinessProbe` (interval, timeout, retries, startPeriod). Health is UNKNOWN until the first success, then HEALTHY/UNHEALTHY; a container that never becomes ready stays UNKNOWN | +| `stopTimeout` | `terminationGracePeriodSeconds` (the largest container value, default 30) | +| `essential` | the task stops when an essential container exits (`EssentialContainerExited`), exit codes are recorded per container; a non-essential exit leaves the task running | +| `dependsOn` `SUCCESS` | the target runs as an init container, in dependency order. It must be non-essential; a non-zero exit fails the task (`TaskFailedToStart`) | +| `dependsOn` `START` | accepted: containers are listed in dependency order but a Pod starts them together, so startup order is not guaranteed | +| `volumesFrom`, `mountPoints`, `linuxParameters.tmpfs`, `sharedMemorySize` | the source container's mounts, `emptyDir` volumes (memory-backed for tmpfs and `/dev/shm`) | +| task `volumes` with no host path, task-scoped `dockerVolumeConfiguration` | `emptyDir`, shared by the task's containers and gone with the Pod | +| `portMappings` | `containerPort` entries. Reachability is `:`; `hostPort` is ignored. The task's `privateIPv4Address` becomes the Pod IP, so ECS service load balancer registration (ELBv2) and Cloud Map use it | +| `logConfiguration` `awslogs` | each container's Pod log stream is forwarded to CloudWatch Logs. The stream name is `//`, or `/` without a prefix | +| ECS Exec (`ExecuteCommand`) | `pods/exec` with a TTY, through the SSM session data channel (needs `container.kubernetes.ecs.exec`) | +| task state | PROVISIONING, PENDING and ACTIVATING are driven by the backend; RUNNING once the Pod runs, STOPPED with `stopCode`, `stoppedReason`, exit codes and `OutOfMemoryError` reasons from the Pod's container statuses. A Pod deleted outside Gopherstack stops the task (exit 137). Image pull failures report `CannotPullContainerError` | + +Refused with `TaskFailedToStart` (a clear `stoppedReason`) instead of being approximated: host-path volumes, EFS, `configuredAtLaunch`, shared `dockerVolumeConfiguration`, `networkMode` `host`/`none`, `pidMode`/`ipcMode` `host`, `privileged`, `disableNetworking`, `devices`, container `restartPolicy`, `repositoryCredentials`, FireLens, and `dependsOn` `COMPLETE` and `HEALTHY`. `ulimits` and `ephemeralStorage` are ignored. Security-group enforcement, task adoption after a Gopherstack restart (leftover Pods are removed by the startup reconcile) and task networking policies are not implemented for Pods. + +Running Gopherstack outside the cluster: set `CONTAINER_RUNTIME=kubernetes`, `CONTAINER_K8S_KUBECONFIG=` and `CONTAINER_K8S_ADVERTISE_HOST=` (also `LAMBDA_DOCKER_HOST` to the same value). + +## Security + +The container drops all capabilities, forbids privilege escalation, uses the `RuntimeDefault` seccomp profile and a read-only root filesystem (`/tmp` and `/data` are writable volumes). It runs as the image's non-root user (UID/GID 65532, `runAsNonRoot: true`); `fsGroup` 65532 makes the PVC writable. The release image is distroless `static:nonroot`, so its root directory is world-searchable under containerd as well. To run docker-backed engines from the cluster, enable the `dind` sidecar rather than mounting a host socket. It mounts no service account token and creates no RBAC unless `container.runtime=kubernetes` (see above). + +## Tests + +```bash +task helm-test # lint + golden templates (docker, pinned alpine/helm image) +task helm-install-test # installs into a real k3s cluster started with pkgs/k3sengine +``` + +Both run `go test ./test/helm/` with `GOPHERSTACK_TEST_DOCKER=1`. The install test loads a locally built image (`bin/gopherstack-linux` via `Dockerfile.test`, or `GOPHERSTACK_HELM_IMAGE`), installs the chart with persistence on a NodePort, checks `/_gopherstack/health`, creates and lists an S3 bucket, restarts the pod to prove persistence, then uninstalls. Refresh goldens with `GOPHERSTACK_UPDATE_GOLDEN=1`. diff --git a/examples/apigw-websocket-chat/docker-compose.yml b/examples/apigw-websocket-chat/docker-compose.yml index 57e31609fb..88539f36b9 100644 --- a/examples/apigw-websocket-chat/docker-compose.yml +++ b/examples/apigw-websocket-chat/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/examples/cognito-api-auth/docker-compose.yml b/examples/cognito-api-auth/docker-compose.yml index 57e31609fb..88539f36b9 100644 --- a/examples/cognito-api-auth/docker-compose.yml +++ b/examples/cognito-api-auth/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/examples/ddb-lambda-chain/docker-compose.yml b/examples/ddb-lambda-chain/docker-compose.yml index 457d22c557..ff0134c487 100644 --- a/examples/ddb-lambda-chain/docker-compose.yml +++ b/examples/ddb-lambda-chain/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/examples/ec2-docker/docker-compose.yml b/examples/ec2-docker/docker-compose.yml index a995c31863..117e34dfb0 100644 --- a/examples/ec2-docker/docker-compose.yml +++ b/examples/ec2-docker/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" environment: # Run the EC2 service with the docker-backed compute provider so that # RunInstances actually spawns a container (sshd, ec2-user) instead of diff --git a/examples/kinesis-lambda-aggregator/docker-compose.yml b/examples/kinesis-lambda-aggregator/docker-compose.yml index 57e31609fb..88539f36b9 100644 --- a/examples/kinesis-lambda-aggregator/docker-compose.yml +++ b/examples/kinesis-lambda-aggregator/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/examples/s3-lambda-processor/docker-compose.yml b/examples/s3-lambda-processor/docker-compose.yml index 57e31609fb..88539f36b9 100644 --- a/examples/s3-lambda-processor/docker-compose.yml +++ b/examples/s3-lambda-processor/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/examples/stepfunctions-order-workflow/docker-compose.yml b/examples/stepfunctions-order-workflow/docker-compose.yml index 57e31609fb..88539f36b9 100644 --- a/examples/stepfunctions-order-workflow/docker-compose.yml +++ b/examples/stepfunctions-order-workflow/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/go.mod b/go.mod index 1f17120ca1..406efcb7be 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/blackbirdworks/gopherstack go 1.27.3 require ( + cel.dev/cel-go v0.33.2 cloud.google.com/go/apigateway v1.17.0 cloud.google.com/go/artifactregistry v1.27.0 cloud.google.com/go/batch v1.24.1 @@ -34,6 +35,7 @@ require ( connectrpc.com/connect v1.21.0 firebase.google.com/go/v4 v4.22.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.15.0 github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig/v2 v2.2.0 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.5.0 github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1 @@ -53,6 +55,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v9 v9.7.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventgrid/armeventgrid/v2 v2.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/logic/armlogic v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.12.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/mysql/armmysqlflexibleservers/v2 v2.0.0 @@ -63,6 +66,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/redis/armredis/v3 v3.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armlocks v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armpolicy v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/search/armsearch v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/sql/armsql v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.5.0 @@ -71,16 +75,18 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 github.com/Azure/azure-sdk-for-go/sdk/storage/azqueue v1.0.1 + github.com/ProtonMail/go-crypto v1.4.1 github.com/alecthomas/kong v1.16.1 github.com/alicebob/miniredis/v2 v2.40.0 github.com/andybalholm/brotli v1.2.6 github.com/apache/tinkerpop/gremlin-go/v3 v3.8.2 github.com/aws/aws-dax-go v1.2.15 github.com/aws/aws-sdk-go v1.55.8 - github.com/aws/aws-sdk-go-v2 v1.47.3 + github.com/aws/aws-sdk-go-v2 v1.47.7 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 github.com/aws/aws-sdk-go-v2/config v1.33.9 github.com/aws/aws-sdk-go-v2/credentials v1.20.9 + github.com/aws/aws-sdk-go-v2/feature/dsql/auth v1.2.8 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.57.3 github.com/aws/aws-sdk-go-v2/service/account v1.42.2 @@ -188,6 +194,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/kms v1.61.4 github.com/aws/aws-sdk-go-v2/service/lakeformation v1.55.3 github.com/aws/aws-sdk-go-v2/service/lambda v1.112.2 + github.com/aws/aws-sdk-go-v2/service/lambdaweb v1.2.5 github.com/aws/aws-sdk-go-v2/service/lightsail v1.66.4 github.com/aws/aws-sdk-go-v2/service/macie2 v1.59.3 github.com/aws/aws-sdk-go-v2/service/managedblockchain v1.40.3 @@ -204,6 +211,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/mq v1.45.3 github.com/aws/aws-sdk-go-v2/service/mwaa v1.48.3 github.com/aws/aws-sdk-go-v2/service/neptune v1.53.3 + github.com/aws/aws-sdk-go-v2/service/neptunedata v1.27.3 github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.76.1 github.com/aws/aws-sdk-go-v2/service/networkmanager v1.50.3 github.com/aws/aws-sdk-go-v2/service/networkmonitor v1.21.3 @@ -282,21 +290,25 @@ require ( github.com/aws/aws-sdk-go-v2/service/workmail v1.45.3 github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4 github.com/aws/aws-sdk-go-v2/service/xray v1.45.3 - github.com/aws/smithy-go v1.28.5 + github.com/aws/smithy-go v1.28.8 github.com/blevesearch/snowballstem v0.9.0 github.com/cedar-policy/cedar-go v1.8.0 github.com/containerd/errdefs v1.0.0 github.com/distribution/distribution/v3 v3.1.2 github.com/dop251/goja v0.0.0-20261012212121-0c308c5ed11b github.com/eclipse/paho.mqtt.golang v1.5.1 + github.com/fxamacker/cbor/v2 v2.9.4 + github.com/go-git/go-billy/v5 v5.9.2 github.com/go-mysql-org/go-mysql v1.16.0 github.com/go-sql-driver/mysql v1.10.1 github.com/go-stomp/stomp/v3 v3.1.5 + github.com/go-webauthn/webauthn v0.18.2 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/go-cmp v0.7.0 + github.com/google/rpmpack v0.7.1 github.com/google/uuid v1.6.0 github.com/googleapis/gax-go/v2 v2.27.0 - github.com/gorilla/websocket v1.5.3 + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/iskorotkov/avro/v2 v2.35.0 github.com/jackc/pgx/v5 v5.11.0 github.com/klauspost/compress v1.20.3 @@ -309,6 +321,7 @@ require ( github.com/modelcontextprotocol/go-sdk v1.8.0 github.com/mxschmitt/playwright-go v0.6201.1 github.com/opencontainers/image-spec v1.1.1 + github.com/opensearch-project/opensearch-go/v4 v4.8.1 github.com/oracle/oci-go-sdk/v65 v65.126.2 github.com/parquet-go/parquet-go v0.32.0 github.com/pkg/sftp v1.13.11 @@ -317,11 +330,15 @@ require ( github.com/rabbitmq/amqp091-go v1.15.0 github.com/recolabs/gnata v0.6.1 github.com/redis/go-redis/v9 v9.23.0 + github.com/sijms/go-ora/v2 v2.9.0 github.com/stretchr/testify v1.12.1 github.com/testcontainers/testcontainers-go v0.44.0 github.com/twmb/franz-go v1.22.1 github.com/twmb/franz-go/pkg/kmsg v1.15.0 + github.com/ulikunitz/xz v0.5.17 github.com/vektah/gqlparser/v2 v2.5.62 + github.com/willscott/go-nfs v0.0.4 + github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 go.mongodb.org/mongo-driver/v2 v2.9.2 go.uber.org/goleak v1.3.0 golang.org/x/crypto v0.58.0 @@ -336,6 +353,9 @@ require ( google.golang.org/grpc v1.84.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.37.2 + k8s.io/apimachinery v0.37.2 + k8s.io/client-go v0.37.2 modernc.org/sqlite v1.60.1 software.sslmate.com/src/go-pkcs12 v0.7.4 ) @@ -353,6 +373,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect github.com/Azure/go-amqp v1.7.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20261008234906-119c4d577770 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 // indirect github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect @@ -361,10 +382,11 @@ require ( github.com/Microsoft/go-winio v0.6.3 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect github.com/antlr/antlr4 v0.0.0-20181218183524-be58ebffde8e // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apache/arrow-go/v18 v18.7.0 // indirect github.com/apache/arrow/go/v15 v15.0.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect @@ -373,15 +395,19 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitfield/gotestdox v0.2.2 // indirect + github.com/cavaliergopher/cpio v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudflare/circl v1.6.4 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.2.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2/v2 v2.8.4 // indirect @@ -393,6 +419,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.1.0 // indirect github.com/ebitengine/purego v0.11.1 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/fatih/color v1.19.0 // indirect @@ -402,9 +429,24 @@ require ( github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/jsonpointer v1.0.1 // indirect + github.com/go-openapi/jsonreference v1.0.1 // indirect + github.com/go-openapi/swag v0.29.2 // indirect + github.com/go-openapi/swag/cmdutils v0.29.2 // indirect + github.com/go-openapi/swag/conv v0.29.2 // indirect + github.com/go-openapi/swag/fileutils v0.29.2 // indirect + github.com/go-openapi/swag/jsonutils v0.29.2 // indirect + github.com/go-openapi/swag/loading v0.29.2 // indirect + github.com/go-openapi/swag/mangling v0.29.2 // indirect + github.com/go-openapi/swag/netutils v0.29.2 // indirect + github.com/go-openapi/swag/pools v0.29.2 // indirect + github.com/go-openapi/swag/stringutils v0.29.2 // indirect + github.com/go-openapi/swag/typeutils v0.29.2 // indirect + github.com/go-openapi/swag/yamlutils v0.29.2 // indirect github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/go-webauthn/x v0.3.1 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/gofrs/flock v0.13.1 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect @@ -414,6 +456,8 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/google/jsonschema-go v0.4.3 // indirect github.com/google/pprof v0.0.0-20261012231050-2af12c3f4d87 // indirect github.com/google/s2a-go v0.1.11 // indirect @@ -428,8 +472,11 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 // indirect github.com/magiconair/properties v1.18.12 // indirect github.com/mattn/go-colorable v0.1.15 // indirect @@ -437,11 +484,13 @@ require ( github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.3.3 // indirect github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/user v0.4.1 // indirect github.com/moby/sys/userns v0.2.1 // indirect github.com/moby/term v0.5.2 // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.1.0 // indirect github.com/nicksnyder/go-i18n/v2 v2.5.0 // indirect @@ -449,15 +498,18 @@ require ( github.com/palkan/mulint v1.1.0 // indirect github.com/parquet-go/bitpack v1.1.0 // indirect github.com/parquet-go/jsonlite v1.5.5 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.33 // indirect github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 // indirect github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 // indirect github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 // indirect github.com/prometheus/common v0.72.1 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect + github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 // indirect github.com/redis/go-redis/extra/redisotel/v9 v9.23.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect @@ -468,13 +520,16 @@ require ( github.com/shopspring/decimal v1.5.0 // indirect github.com/sirupsen/logrus v1.10.2 // indirect github.com/sony/gobreaker/v2 v2.4.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/tidwall/gjson v1.20.0 // indirect github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.2 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect github.com/twpayne/go-geom v1.7.0 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect @@ -524,11 +579,21 @@ require ( golang.org/x/vuln v1.7.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/appengine/v2 v2.0.6 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gotest.tools/gotestsum v1.13.0 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect + k8s.io/streaming v0.37.2 // indirect + k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect modernc.org/libc v1.78.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) tool ( diff --git a/go.sum b/go.sum index 2e68fcd70b..b6948c95d7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cel.dev/cel-go v0.33.2 h1:eAcEdBguc6AFrGbPOyE0g168Uey4kKFW9J6qsBLSeMU= +cel.dev/cel-go v0.33.2/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -83,6 +85,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3 h1:qgnWTiWs1G7AqcKZegFvkJD1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3/go.mod h1:XtxD48QPInqoj+GMlcRlXHEYsoOxFnP6w2JgIZ9rDS4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.15.0 h1:w3gDUJlJ0BfBsHH02DCBDwbkmMSXJqBj6nBOCo9iocY= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.15.0/go.mod h1:7scl8mG/IFLVjJ9ef/ltdND2T1z0CkgVSaQoUxLvmeU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig/v2 v2.2.0 h1:80ijeACessBdODMsTjR0zf7t1Uvhc+hCqcXc0BcW9ZA= github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig/v2 v2.2.0/go.mod h1:RPzAQTYyoren3gG5K0QSiWTSUhGCB47LY//1MGdODv0= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.5.0 h1:wtCn7MemMD9eo4/NdpJ6S/MFD2BV2CDwoEfvl5th2vM= @@ -129,6 +133,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.2.0 h1:+lnL github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.2.0/go.mod h1:tStOHrivWUrcBolspvKV70Us1ckESYGYSHdG4LX8zyY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/logic/armlogic v1.2.0 h1:EMNgS+pCj2/2LL7+nWG8zPf9sp4u8icP5FNwoBhyc8M= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/logic/armlogic v1.2.0/go.mod h1:TsM36SmGxYC24DiOTR9wPuBj5HYphihMC6xlnX536bE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.12.0 h1:qmeVFr8V9dDzhLZDDgdqyOEl+XLhhMpm7xrOajxqKuk= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.12.0/go.mod h1:x/aYmuTilxu4yQMFj2r3ZE3iiO2wxYFFmh6dRg2IWiQ= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.3.0 h1:L7G3dExHBgUxsO3qpTGhk/P2dgnYyW48yn7AO33Tbek= @@ -155,6 +161,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1. github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1 h1:guyQA4b8XB2sbJZXzUnOF9mn0WDBv/ZT7me9wTipKtE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1/go.mod h1:8h8yhzh9o+0HeSIhUxYny+rEQajScrfIpNktvgYG3Q8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/search/armsearch v1.4.0 h1:zBdabY8pMSMLPb1XJnFSEdJi9Bd0h+VMjh1uU8B6Yp8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/search/armsearch v1.4.0/go.mod h1:Y2Q3nB3UfSnG9nALOpPAjflXPM3jL/n2ZmYIu2Occ9g= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/sql/armsql v1.2.0 h1:S087deZ0kP1RUg4pU7w9U9xpUedTCbOtz+mnd0+hrkQ= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/sql/armsql v1.2.0/go.mod h1:B4cEyXrWBmbfMDAPnpJ1di7MAt5DKP57jPEObAvZChg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= @@ -179,6 +187,8 @@ github.com/Azure/go-amqp v1.7.0 h1:9VlH/LEWr386XWWJRNON0eslFqSClYBXP4HewvIqkDQ= github.com/Azure/go-amqp v1.7.0/go.mod h1:pCJaHsvRlmmFUpxyQbh2qPkUFqYJeRBTqJSHKJadvPg= github.com/Azure/go-ansiterm v0.0.0-20261008234906-119c4d577770 h1:n3LUIJHwvkpfvfkt0dD7at4iYfQD24ML2jflWi3r2ew= github.com/Azure/go-ansiterm v0.0.0-20261008234906-119c4d577770/go.mod h1:DMNF6Kxv+EcuUvVuNkvC3f+BjEEmc/Fn9aU+qiD8zpU= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 h1:0O6j18nQDoIff6/mtosT5LNU2WehEwp+LqfX6699oqY= github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1/go.mod h1:xdYAf5bjkOpsd3auA8riiJW4vneBubt9caavL7626jQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -202,6 +212,8 @@ github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/Microsoft/go-winio v0.6.3 h1:M/52tsoyTfwaxajLnXZm7PqkbMGRH8iSWP+8fcg9MQ4= github.com/Microsoft/go-winio v0.6.3/go.mod h1:ma0QpxizD4fyyzJ1C7MIblymTD2Nxz5Mho/3CepwIZg= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= @@ -216,6 +228,8 @@ github.com/andybalholm/brotli v1.2.6 h1:ftYnfj6usCp+UGV5kSJ3+chpMQgU+gJf/AxsUQ52 github.com/andybalholm/brotli v1.2.6/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antlr/antlr4 v0.0.0-20181218183524-be58ebffde8e h1:yxMh4HIdsSh2EqxUESWvzszYMNzOugRyYCeohfwNULM= github.com/antlr/antlr4 v0.0.0-20181218183524-be58ebffde8e/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= @@ -226,24 +240,28 @@ github.com/apache/tinkerpop/gremlin-go/v3 v3.8.2 h1:kE0xuL6UncFQgPYwmQfBT440wNOM github.com/apache/tinkerpop/gremlin-go/v3 v3.8.2/go.mod h1:HkUXCwiMMjv3oBUqwrbqRLSkbHh9csEC3aW7Ntw/uFc= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-dax-go v1.2.15 h1:30rH3+QgjpjemrVg0NGIG5FnB1izJZ7jUZuBb1Fy8ak= github.com/aws/aws-dax-go v1.2.15/go.mod h1:4f/qGLBQlPYd+fmAfG4n4oSvN19JdKNYYmsr90/MPso= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= -github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2 v1.47.7 h1:jBmtzRtGWVAzbfc4RBWo1dTte0ZZH8p9U42YJIw2+Bw= +github.com/aws/aws-sdk-go-v2 v1.47.7/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ= github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/dsql/auth v1.2.8 h1:udoNKmbYngh4RpmURFjIaduwPjS/pTHkySWCTkS9Qf0= +github.com/aws/aws-sdk-go-v2/feature/dsql/auth v1.2.8/go.mod h1:kAkwdYfRHYr653KcQh9vLui/BP4CFLrYvdD9npWb5Dc= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10 h1:DKtPO+uoekvZX1tIFGVlWypYGSYkHPb7rliUI1wZtEw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10/go.mod h1:uAi0RDdZZz8IRpXERptWALeCgQG9Ye/HiMLxkxlt8IU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10 h1:r8p95YELmfmuV9VvAaFcLesVBZduCCCccxpzQKkNThw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10/go.mod h1:W3M52Mqai5TjgzyL/dEG+UwE+Xz9PykyjXelvbAzWIA= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.57.3 h1:m7EQZA6WxCbiExtCYq1wJLvK/dTQgQc1KvEOBnF9GlY= @@ -468,6 +486,8 @@ github.com/aws/aws-sdk-go-v2/service/lakeformation v1.55.3 h1:E9E99pv5u2Uc0VkgXp github.com/aws/aws-sdk-go-v2/service/lakeformation v1.55.3/go.mod h1:IlKDS05ABuNxYQ/UspDTEoKVOmkpXne18fVJFMTOKeQ= github.com/aws/aws-sdk-go-v2/service/lambda v1.112.2 h1:QcDoECg2br9JOaQ6C0fR3Mkcj9b/+dFSRuxZCLS3tME= github.com/aws/aws-sdk-go-v2/service/lambda v1.112.2/go.mod h1:GZ1UAuKbyxBdaRNbhKbAobG2VcWrEg5YmgWstJA3sl8= +github.com/aws/aws-sdk-go-v2/service/lambdaweb v1.2.5 h1:0y77BcwEEKEuwUSDuJsxpUC0UYxK5iTjkwDVAFkxGe0= +github.com/aws/aws-sdk-go-v2/service/lambdaweb v1.2.5/go.mod h1:yLxwAiadnny6r4QkMjWn0oChHryB6xbSIpMdGkr7doA= github.com/aws/aws-sdk-go-v2/service/lightsail v1.66.4 h1:M0KgyGQkqxe/MHR5Lj1PZGgwnJOnfN5bCDh470sIJDg= github.com/aws/aws-sdk-go-v2/service/lightsail v1.66.4/go.mod h1:cYtZI/MoJuzdSBYXg4enE8RkNzKRkoMr0qATq+ThsIw= github.com/aws/aws-sdk-go-v2/service/macie2 v1.59.3 h1:hbejt97Z2rKhxcjlpLu85REd1/w9Z8omGQkXl68FEI4= @@ -500,6 +520,8 @@ github.com/aws/aws-sdk-go-v2/service/mwaa v1.48.3 h1:y5NMX0onYGKLbL2sXgSsMiRiQNg github.com/aws/aws-sdk-go-v2/service/mwaa v1.48.3/go.mod h1:1VKiGS0MW4kgvZ9iT0SYDDqcbe/h3gcx/olyQT5M30w= github.com/aws/aws-sdk-go-v2/service/neptune v1.53.3 h1:ykzA1zMCGNBrvG3rTU/0CJ+mdG3oDVWd13srP4/u9ec= github.com/aws/aws-sdk-go-v2/service/neptune v1.53.3/go.mod h1:XLgBl5jRJlB9CejUkggkmr+DO5zkI2MyvyP7nXLuWsI= +github.com/aws/aws-sdk-go-v2/service/neptunedata v1.27.3 h1:eBcQCfO3BRhnEHGgyUILmkmGW1uCKyl3KT/QbKoUlsA= +github.com/aws/aws-sdk-go-v2/service/neptunedata v1.27.3/go.mod h1:jZw7kiGWC8nU0D2B5AEZMpTITmS5Je13gMH7aZFeP+o= github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.76.1 h1:5vjcmM/0scMQ7T2Gf9QxAqUehUbLeyl6viZZj+bYdKw= github.com/aws/aws-sdk-go-v2/service/networkfirewall v1.76.1/go.mod h1:UwfhRPgauP3bzV9eY3M94mpRATlNTzzlYtPTGjd85Zk= github.com/aws/aws-sdk-go-v2/service/networkmanager v1.50.3 h1:7CgYmJqLDr1Y+QohY1LGkvbKddehyr5PtHjQ0KtuxWM= @@ -656,8 +678,8 @@ github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4 h1:s3r96tGUdWFhIolHNwR9+ github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4/go.mod h1:rupBg6/2hKMz1uCa7sd/RSMmLOXJQWgMzYIOB+Qi2EA= github.com/aws/aws-sdk-go-v2/service/xray v1.45.3 h1:GDr+18TEmx5ivPIaYqCWjVtPGSbcVZQkgX3PzhfzkKE= github.com/aws/aws-sdk-go-v2/service/xray v1.45.3/go.mod h1:My/sZif0SANOKoH+IiyDW9AlWR2ITZQny7u7pTBYBMQ= -github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= -github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ= +github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE= @@ -668,6 +690,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM= +github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= github.com/cedar-policy/cedar-go v1.8.0 h1:9gcU7EHXwHC2RMdpph68yTAkdB3behTTssC+kt4GoS8= github.com/cedar-policy/cedar-go v1.8.0/go.mod h1:h5+3CVW1oI5LXVskJG+my9TFCYI5yjh/+Ul3EJie6MI= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -678,6 +702,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U= +github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= @@ -697,6 +723,8 @@ github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GK github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -731,6 +759,8 @@ github.com/ebitengine/purego v0.11.1 h1:2zpWRSQNVKN4eKsKO9eM1ILDgWfYMY9GwqRmK6Xe github.com/ebitengine/purego v0.11.1/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= github.com/eclipse/paho.mqtt.golang v1.5.1 h1:/VSOv3oDLlpqR2Epjn1Q7b2bSTplJIeV2ISgCl2W7nE= github.com/eclipse/paho.mqtt.golang v1.5.1/go.mod h1:1/yJCneuyOoCOzKSsOTUc0AJfpsItBGWvYpBLimhArU= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -751,6 +781,10 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.4 h1:xwjVlxEMR3S605oUlgBjKLTTeGFciYPGYCtF/35LKGo= +github.com/fxamacker/cbor/v2 v2.9.4/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE= +github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4= github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -763,6 +797,40 @@ github.com/go-mysql-org/go-mysql v1.16.0/go.mod h1:VjBTZTTDKL8OMXUAhNbg3VHaVVq9H github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/jsonpointer v1.0.1 h1:2KxywRmNwJkT/FMBa3iRNHEaAxSJvjqoufQZy3au1Mg= +github.com/go-openapi/jsonpointer v1.0.1/go.mod h1:wI7ZYsFmbIi9nBXOZqgDaS/bqOchRGZjqxFli7FBYxY= +github.com/go-openapi/jsonreference v1.0.1 h1:4zJ7AmYDKNmD3aSpfPnFNCFA5E80/xMHUNKgydaLh38= +github.com/go-openapi/jsonreference v1.0.1/go.mod h1:dYplQXa6p5lXprLcJ8LE2iU7vNpXsAHDQ5ZAgL+Qx3A= +github.com/go-openapi/swag v0.29.2 h1:9n8frkcsuQRA0INU31VbxNxt1zBaHxrHYGEf5BUjFi4= +github.com/go-openapi/swag v0.29.2/go.mod h1:RkCiX1gCVXWgLOavDXE3ALJrQ/VO2ziiIzOUs3z10xo= +github.com/go-openapi/swag/cmdutils v0.29.2 h1:cXEzX/nWCODon251f+1HgL75/R4XXML5IZxgLCWMoro= +github.com/go-openapi/swag/cmdutils v0.29.2/go.mod h1:XziaSVzYqkDvpWXzASfWajw2YvJrLscq610O2XiyqJw= +github.com/go-openapi/swag/conv v0.29.2 h1:8c9shoB8l0QRSR6ymq1llHdBWDqpIgJfjTGHx3Vuhm0= +github.com/go-openapi/swag/conv v0.29.2/go.mod h1:AZS0YigTNf8qNtD6WqJz/N4RUNmpr76SyjFGkq4+p6Q= +github.com/go-openapi/swag/fileutils v0.29.2 h1:mdUL+Vw5ah1fO1AvFCoHeIyv7YZsCz2MN3KNqd/lLVI= +github.com/go-openapi/swag/fileutils v0.29.2/go.mod h1:7QKmmodjAebaq61lGVJa8ldLdGeMF4DSgYv8tXOGGo0= +github.com/go-openapi/swag/jsonutils v0.29.2 h1:uZNSD2/rJDYAfvsLYkykTzXuyxM3QpDKV9jhRHrMu6k= +github.com/go-openapi/swag/jsonutils v0.29.2/go.mod h1:ONTdNvj3Y+IXRzfa17E+Rgt2ns/qiSOYjIo2K7v2yDs= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.2 h1:w+Fd6EBOMGlC74GYGHqCLGyb3Bpams8TtNrgM/SG4jo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.2/go.mod h1:HC2egPORFzbj/gf05Zrfgz8Mgplx3oW7hgGi7pjZVhM= +github.com/go-openapi/swag/loading v0.29.2 h1:QU1ry24e6r6Shoxe380Nx0X5iKkpRO+hJffsyJ2dVlY= +github.com/go-openapi/swag/loading v0.29.2/go.mod h1:DqilDjuiJKETecsS4TRopWG1acUDsfqN39ZmQhPu6uI= +github.com/go-openapi/swag/mangling v0.29.2 h1:ltdo3K0tTP4P6zWhVVWDU9D37/YYBuMJRdZxiS2Dtog= +github.com/go-openapi/swag/mangling v0.29.2/go.mod h1:RjDi4TnItAenS3cgCXSpIl1kKbjG7il6pnmQVl/8bp8= +github.com/go-openapi/swag/netutils v0.29.2 h1:dBli6jyUa93sDS/XYWShJzf4+nnVfFKfvY9SLUaUzB8= +github.com/go-openapi/swag/netutils v0.29.2/go.mod h1:1pM6Xg/Um8E1eps3umWl8Hme3ByWXOtHsvFC0CULHUE= +github.com/go-openapi/swag/pools v0.29.2 h1:PlGoDRF8WtSyXkedZZkpW3f9wDhFf3u8KtaBcmgmh8c= +github.com/go-openapi/swag/pools v0.29.2/go.mod h1:V3lhxVT4qDYRqc2MRSSbLsTYIYV/2HlHafhyEkmzLIc= +github.com/go-openapi/swag/stringutils v0.29.2 h1:lcnBxwAaysT3bMUVBfn9V/PkfVqkurpqufKU6rTUMGk= +github.com/go-openapi/swag/stringutils v0.29.2/go.mod h1:i9cdh7sGaa0nm3CqIvH5IM5h2QClk1wns2ktKeILvRI= +github.com/go-openapi/swag/typeutils v0.29.2 h1:O7aVvkTs3pXwikgtrPLenigEMdQFgONKRooQA9pgf2I= +github.com/go-openapi/swag/typeutils v0.29.2/go.mod h1:7+GDG+uz9Ke+eVt4rClZg7wklSDp8hT/mKNh/pC26TE= +github.com/go-openapi/swag/yamlutils v0.29.2 h1:IFKFFeDnuIwzfsuWRQU+rI8iL3g4XyAYjV/RlnlxPLM= +github.com/go-openapi/swag/yamlutils v0.29.2/go.mod h1:7MGqtcrK73sxQ4ceiyIf8qiXS/s09JLMdR5esK5euYQ= +github.com/go-openapi/testify/enable/yaml/v2 v2.7.0 h1:wPW6YRgx3+SID1yUy/Xwa17L8kFEaEKod2VRbJDZNUs= +github.com/go-openapi/testify/enable/yaml/v2 v2.7.0/go.mod h1:mI1M88etYbc3PhgHsWQK2kwvNwW5aGFqMPbmib+SGIs= +github.com/go-openapi/testify/v2 v2.7.0 h1:bycOreEj6wfBvijg3YFogZ/sFjTCDmQnwSodSzHa3X8= +github.com/go-openapi/testify/v2 v2.7.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q= github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= @@ -773,6 +841,10 @@ github.com/go-stomp/stomp/v3 v3.1.5 h1:Pikz1OSusmSKUm5mRKYfXQZaDatfZ+EnBBA1JJ2xE github.com/go-stomp/stomp/v3 v3.1.5/go.mod h1:ztzZej6T2W4Y6FlD+Tb5n7HQP3/O5UNQiuC169pIp10= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-webauthn/webauthn v0.18.2 h1:0BeftmEHU7i3Dv0VFwBtidy/ba37Vcdjvqst9EYu8Sk= +github.com/go-webauthn/webauthn v0.18.2/go.mod h1:hEXaOuLxvZ3zG9miZe3ehlyeVso9AtklXG+kTn36k+A= +github.com/go-webauthn/x v0.3.1 h1:1ff37z3XfmTTomkhlURgGizLIDyOvPgTt2t9nlzKLRo= +github.com/go-webauthn/x v0.3.1/go.mod h1:ZInxAynYXfBPvvm5gzKZ7geBlL23K71xASMgohHl/Rg= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= @@ -816,6 +888,8 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU= github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -827,6 +901,11 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba h1:qJEJcuLzH5KDR0gKc0zcktin6KSAwL7+jWKBYceddTc= +github.com/google/go-tpm-tools v0.3.13-0.20230620182252-4639ecce2aba/go.mod h1:EFYHy8/1y2KfgTAsx7Luu7NGhoxtuVHnNo8jE7FikKc= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= @@ -835,6 +914,8 @@ github.com/google/pprof v0.0.0-20261012231050-2af12c3f4d87 h1:P18JvwDzBYAeC8SIfn github.com/google/pprof v0.0.0-20261012231050-2af12c3f4d87/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/rpmpack v0.7.1 h1:YdWh1IpzOjBz60Wvdw0TU0A5NWP+JTVHA5poDqwMO2o= +github.com/google/rpmpack v0.7.1/go.mod h1:h1JL16sUTWCLI/c39ox1rDaTBo3BXUQGjczVJyK4toU= github.com/google/s2a-go v0.1.11 h1:7v1V1TraNGP63WptsAkNqr2w8jgm4vM6A6qx6Lsw9iM= github.com/google/s2a-go v0.1.11/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= @@ -850,8 +931,8 @@ github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyE github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 h1:Bd7KaOxzULLxtZ/K5s1aLbWhR0+5RToO65TXHsf3bqQ= github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0/go.mod h1:nN7ts3dFXKtCZWc//yfkpcQNKJABg16/uDVAZpLDalo= github.com/hashicorp/golang-lru/arc/v2 v2.0.7 h1:QxkVTxwColcduO+LP7eJO56r2hFiG8zEbfAAzRv52KQ= @@ -878,10 +959,16 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -917,6 +1004,8 @@ github.com/moby/moby/client v0.6.2 h1:ekSPlMccrZGcw6Scb8T9D39qE0mB+XupU/yS0zNw9N github.com/moby/moby/client v0.6.2/go.mod h1:DGcnO7qmPJrqobx/ggQgQ/AUk+2wgVdQv7uxB0/mEsc= github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/sys/mount v0.3.5 h1:eS3fsZTjHaBihwjp4/+5Z3jxqLXYsbwxqpVSfFv3M00= github.com/moby/sys/mount v0.3.5/go.mod h1:WUQDO+/uCiCIkIztx8SrwIDVn2dtMFRBebRhpDFT71M= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= @@ -933,8 +1022,12 @@ github.com/mochi-mqtt/server/v2 v2.7.9 h1:y0g4vrSLAag7T07l2oCzOa/+nKVLoazKEWAArw github.com/mochi-mqtt/server/v2 v2.7.9/go.mod h1:lZD3j35AVNqJL5cezlnSkuG05c0FCHSsfAKSPBOSbqc= github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtwfBpOTAI+Urx4= github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxschmitt/playwright-go v0.6201.1 h1:KBBDopE+IIdjAp2FIk6zhLcKCCSSx/T6rGjKZmfo/Bs= @@ -943,10 +1036,14 @@ github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= github.com/nicksnyder/go-i18n/v2 v2.5.0 h1:3wH1gpaekcgGuwzWdSu7JwJhH9Tk87k1ezt0i1p2/Is= github.com/nicksnyder/go-i18n/v2 v2.5.0/go.mod h1:DrhgsSDZxoAfvVrBVLXoxZn/pN5TXqaDbq7ju94viiQ= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opensearch-project/opensearch-go/v4 v4.8.1 h1:9r9x+pt5QG49EZfG0bqQgCexFcQkeMkVGM54dpX0b9Q= +github.com/opensearch-project/opensearch-go/v4 v4.8.1/go.mod h1:OEBTtFm2ZeJ8G/ZbE3ITbKuKlT+9ujruJF3Vj5LBMqo= github.com/oracle/oci-go-sdk/v65 v65.126.2 h1:7lv9XuPcX9DzW9z+y/W2ENub71yyOP7DM8+BPCh8hgg= github.com/oracle/oci-go-sdk/v65 v65.126.2/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w= github.com/palkan/mulint v1.1.0 h1:W0eSO5N53R4TEBXyFpwpWHeKPT7YWMXPyGAdj79isbk= @@ -957,6 +1054,8 @@ github.com/parquet-go/jsonlite v1.5.5 h1:TIqJ+b+mYoTvvEt1/ICS4fBacQO0xWwAbvtcrKj github.com/parquet-go/jsonlite v1.5.5/go.mod h1:hN57TVzxtr04kaax5XwNgCCsh1KE7BT6dvY6X/zNH90= github.com/parquet-go/parquet-go v0.32.0 h1:NWDqTUHfrCS4cJP/Fj2HlxvqsrVedWG3sayMkf+znzM= github.com/parquet-go/parquet-go v0.32.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4/v4 v4.1.33 h1:GjG1TJ1V4IzKP8L96muuuDNpTwd7D+l2ccXrjAbe014= github.com/pierrec/lz4/v4 v4.1.33/go.mod h1:7SE9MC2STkNtL4PIwGhjmyVwvILaGI9/COYQNBhKM/c= github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 h1:cLgCk5mwDG9lDH+dPK8TmEliTjyGJwwKN0qevWAl8IY= @@ -989,6 +1088,8 @@ github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0o github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/rabbitmq/amqp091-go v1.15.0 h1:LEQL4/yp48/Wigt6A6XOu18RQRo8ZHtB5I/KZJn+gkw= github.com/rabbitmq/amqp091-go v1.15.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/recolabs/gnata v0.6.1 h1:2YCxrmUK64UFm1fPBFYe5jFHS6Q8EKJC+34QCujeU44= github.com/recolabs/gnata v0.6.1/go.mod h1:+u5DmoYMyHl+9Uuo9KQXLhbKHip+M5N8S/QYAPSG0XU= github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 h1:YOQCtePRR/cBrxtZB5u2ueYei3zGJm1n0KB2tvth0k0= @@ -1011,10 +1112,14 @@ github.com/shirou/gopsutil/v4 v4.26.9 h1:CaBo/hBFqvlJoLvAQEKdHdf5GD97/MN2ACfyv63 github.com/shirou/gopsutil/v4 v4.26.9/go.mod h1:nKH+8wX2zxr/mDbsR+AAy6Qd2z4ZOQ7wfQaEp+yrYmg= github.com/shopspring/decimal v1.5.0 h1:HxSWk31RmJv0EyY4oLiGQB9u7WPludUwPBdvQZnYFPc= github.com/shopspring/decimal v1.5.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sijms/go-ora/v2 v2.9.0 h1:+iQbUeTeCOFMb5BsOMgUhV8KWyrv9yjKpcK4x7+MFrg= +github.com/sijms/go-ora/v2 v2.9.0/go.mod h1:QgFInVi3ZWyqAiJwzBQA+nbKYKH77tdp1PYoCqhR2dU= github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg= github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -1040,6 +1145,10 @@ github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.2 h1:dz1jrRuE7or/74V490B4/GP1pZm5WKlt2bgCP5A83w8= github.com/tidwall/pretty v1.2.2/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= @@ -1050,8 +1159,18 @@ github.com/twmb/franz-go/pkg/kmsg v1.15.0 h1:mILdOQm4JkWojbSYVQ7KJNJ1qMdYNsLrdNO github.com/twmb/franz-go/pkg/kmsg v1.15.0/go.mod h1:sGkGxKexpRLBThFHJLjjywcyfkK09qB/g+X+wFWZQ/A= github.com/twpayne/go-geom v1.7.0 h1:zMHTHNlenoHAc2ZSiHFh7SMoG0Wdw8gF0orBnrBhfjE= github.com/twpayne/go-geom v1.7.0/go.mod h1:y9Q2YcKKaC2PHXwtRvwEunOPx5Oa+U8aAxPmFvscROk= +github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= +github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= github.com/vektah/gqlparser/v2 v2.5.62 h1:GJP/emj1wIWKs0U3SHcAgzknWgylTqQ1qH2gNL+wSWc= github.com/vektah/gqlparser/v2 v2.5.62/go.mod h1:JNK+plRwKdXLsF/qPFPe5tE0z4s1WeroD9S5LR8um/Q= +github.com/wI2L/jsondiff v0.7.1 h1:Fg9+yj+1/x3UtPBJhR91TKEzRkrEEWcAcLbg9dzEaNM= +github.com/wI2L/jsondiff v0.7.1/go.mod h1:yAt2W7U6Jd4HK0RA8DGSGk0zDtfEtOUUJVnH/xICpjo= +github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI= +github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg= +github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 h1:U0DnHRZFzoIV1oFEZczg5XyPut9yxk9jjtax/9Bxr/o= +github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= @@ -1137,6 +1256,8 @@ go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= @@ -1274,6 +1395,10 @@ google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1288,6 +1413,20 @@ gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.37.2 h1:PnxqnUFy93jr2GcpWDkKOm8wfy25kT9wa572MuQS8oc= +k8s.io/api v0.37.2/go.mod h1:+NVRLft99kx+Jg9v34GmHw867Dapim89TSyCsn2hK+M= +k8s.io/apimachinery v0.37.2 h1:G4t9RGoPDbQML38hz/kQQiltttgd3egg2ODCR8aIc2o= +k8s.io/apimachinery v0.37.2/go.mod h1:qaAvSNvgU2x07wRM95MSHSJTPQ/ob/LeNAGbeDgqQJg= +k8s.io/client-go v0.37.2 h1:sgFtyRZukGkOrsu+QgS7LYnbFFJ5ut3MchdBNduD//E= +k8s.io/client-go v0.37.2/go.mod h1:bo3LFC8YbCtKFPX21Yj8puMsoJLAv/+SvuOODma0s54= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= +k8s.io/streaming v0.37.2 h1:xmzp9nbqIbjvsRcqrBYa+K5XPQ6Jy/Dm3zZ8mN916AI= +k8s.io/streaming v0.37.2/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= +k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs= +k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= modernc.org/cc/v4 v4.30.1 h1:ZXInYH2CD24GPWSwOFPwB6JrnFt4/6d7ceW1Ukd30Bk= modernc.org/cc/v4 v4.30.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.36.2 h1:j6d5HuXtBxF00vEcI5RE0O6EIrG6RyRpgMiGV2IK9ys= @@ -1320,5 +1459,13 @@ pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= software.sslmate.com/src/go-pkcs12 v0.7.4 h1:OvJ7g/OB9RuVAOzahlFLns6HeIcP+SJotsDqmLfhd8w= software.sslmate.com/src/go-pkcs12 v0.7.4/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/internal/app/agentcore_wiring.go b/internal/app/agentcore_wiring.go index da46c31875..e5dacbe334 100644 --- a/internal/app/agentcore_wiring.go +++ b/internal/app/agentcore_wiring.go @@ -13,13 +13,17 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/service" agentcoredataplane "github.com/blackbirdworks/gopherstack/services/bedrockagentcore" agentcorebackend "github.com/blackbirdworks/gopherstack/services/bedrockagentcorecontrol" + cognitoidpbackend "github.com/blackbirdworks/gopherstack/services/cognitoidp" ecrbackend "github.com/blackbirdworks/gopherstack/services/ecr" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" s3backend "github.com/blackbirdworks/gopherstack/services/s3" secretsmanagerbackend "github.com/blackbirdworks/gopherstack/services/secretsmanager" ) -var errAgentCoreNoSecrets = errors.New("secrets manager is not wired") +var ( + errAgentCoreNoSecrets = errors.New("secrets manager is not wired") + errAgentCoreNoLambda = errors.New("lambda is not wired") +) const ( maxAgentCoreObjectBytes = 64 << 20 @@ -181,3 +185,44 @@ func wireAgentCore(byName map[string]service.Registerable) { dp.Backend.SetControl(h.Backend.DataPlane()) } } + +var _ agentcorebackend.LambdaInvoker = agentCoreRefs{} + +// InvokeLambda runs the function synchronously with the gateway client context. +func (r agentCoreRefs) InvokeLambda( + ctx context.Context, functionARN, clientContext string, payload []byte, +) ([]byte, string, error) { + parts := strings.Split(functionARN, ":") + if r.lambda == nil || len(parts) < arnResourceFields { + return nil, "", errAgentCoreNoLambda + } + + qi, ok := r.lambda.BackendFor(parts[3]).(lambdabackend.QualifierInvoker) + if !ok { + return nil, "", errAgentCoreNoLambda + } + + ctx = awsmeta.Set(ctx, &awsmeta.Metadata{Account: parts[4], Region: parts[3], Partition: awsmeta.DefaultPartition}) + + out, _, fnErr, _, err := qi.InvokeFunctionWithQualifier( + ctx, parts[6], "", clientContext, "", lambdabackend.InvocationTypeRequestResponse, payload) + if err != nil { + return nil, "", fmt.Errorf("invoke %s: %w", functionARN, err) + } + + return out, fnErr, nil +} + +// wireAgentCoreAuth lets gateway authorizers verify Cognito-issued JWTs and AWS_IAM signatures. +func wireAgentCoreAuth(byName map[string]service.Registerable, sigV4Secret string) { + h, ok := byName["BedrockAgentCoreControl"].(*agentcorebackend.Handler) + if !ok { + return + } + + if cognito, found := byName["CognitoIDP"].(*cognitoidpbackend.Handler); found { + h.Backend.SetJWKSProvider(cognito) + } + + h.Backend.SetSigV4Secret(sigV4Secret) +} diff --git a/internal/app/amp_wiring.go b/internal/app/amp_wiring.go index 8337da2f6e..ece3161322 100644 --- a/internal/app/amp_wiring.go +++ b/internal/app/amp_wiring.go @@ -2,11 +2,16 @@ package app import ( "context" + "errors" "fmt" + "time" "github.com/blackbirdworks/gopherstack/pkgs/service" ampbackend "github.com/blackbirdworks/gopherstack/services/amp" + cwlogsbackend "github.com/blackbirdworks/gopherstack/services/cloudwatchlogs" eksbackend "github.com/blackbirdworks/gopherstack/services/eks" + iambackend "github.com/blackbirdworks/gopherstack/services/iam" + kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" snsbackend "github.com/blackbirdworks/gopherstack/services/sns" ) @@ -78,5 +83,83 @@ func wireAMPScraperSources(byName map[string]service.Registerable) { if aok { wireAMPAlertNotifications(ampH, byName) + wireAMPKeysAndLogs(ampH, byName) + } +} + +// ampKeys answers KMS key lookups for CreateWorkspace kmsKeyArn. +type ampKeys struct{ kms kmsbackend.StorageBackend } + +func (k ampKeys) KMSKeyState(ctx context.Context, keyARN string) (bool, bool) { + out, err := k.kms.DescribeKey(ctx, &kmsbackend.DescribeKeyInput{KeyID: keyARN}) + if err != nil { + return false, false + } + + return true, out.KeyMetadata.Enabled && out.KeyMetadata.KeyState == kmsbackend.KeyStateEnabled +} + +// ampLogs checks log groups and delivers workspace log lines in the in-process CloudWatch Logs. +type ampLogs struct{ logs cwlogsbackend.StorageBackend } + +func (l ampLogs) LogGroupExists(ctx context.Context, region, group string) bool { + groups, _, err := l.logs.DescribeLogGroups(cwlogsbackend.WithRegion(ctx, region), group, "", "", 0) + if err != nil { + return false + } + + for _, g := range groups { + if g.LogGroupName == group { + return true + } + } + + return false +} + +func (l ampLogs) PutLogLine(ctx context.Context, region, group, stream, message string, at time.Time) error { + rctx := cwlogsbackend.WithRegion(ctx, region) + + if _, err := l.logs.CreateLogStream(rctx, group, stream); err != nil && + !errors.Is(err, cwlogsbackend.ErrLogStreamAlreadyExist) { + return fmt.Errorf("amp log stream: %w", err) + } + + _, err := l.logs.PutLogEvents(rctx, group, stream, "", + []cwlogsbackend.InputLogEvent{{Message: message, Timestamp: at.UnixMilli()}}) + if err != nil { + return fmt.Errorf("amp log events: %w", err) + } + + return nil +} + +func wireAMPKeysAndLogs(ampH *ampbackend.Handler, byName map[string]service.Registerable) { + if kmsH, ok := byName["KMS"].(*kmsbackend.Handler); ok { + ampH.Backend.SetKeyChecker(ampKeys{kms: kmsH.Backend}) + } + + if logsH, ok := byName["CloudWatchLogs"].(*cwlogsbackend.Handler); ok { + ampH.Backend.SetLogSink(ampLogs{logs: logsH.Backend}) + } +} + +// ampPolicyEvaluator runs a workspace resource policy through the IAM evaluator for the caller account. +func ampPolicyEvaluator(doc, action, resource, callerAccount string) (bool, bool) { + res := iambackend.EvaluatePolicies([]string{doc}, action, resource, + iambackend.ConditionContext{PrincipalAccount: callerAccount}) + + return res == iambackend.EvalAllow, res == iambackend.EvalExplicitDeny +} + +// wireAMPDataPlaneAuth makes the Prometheus endpoints require aps-scoped SigV4 when --validate-sigv4 is on. +func wireAMPDataPlaneAuth(appCtx *service.AppContext, byName map[string]service.Registerable) { + cli, ok := appCtx.Config.(*CLI) + if !ok || !cli.ValidateSigV4 { + return + } + + if ampH, aok := byName["AMP"].(*ampbackend.Handler); aok { + ampH.Backend.SetDataPlaneAuth(&backend.DataPlaneAuth{Secret: cli.SigV4Secret, Evaluate: ampPolicyEvaluator}) } } diff --git a/internal/app/amp_wiring_test.go b/internal/app/amp_wiring_test.go index aff35fc0e3..8bc60316d3 100644 --- a/internal/app/amp_wiring_test.go +++ b/internal/app/amp_wiring_test.go @@ -9,6 +9,8 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/service" ampbackend "github.com/blackbirdworks/gopherstack/services/amp" + cwlogsbackend "github.com/blackbirdworks/gopherstack/services/cloudwatchlogs" + kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" snsbackend "github.com/blackbirdworks/gopherstack/services/sns" ) @@ -88,3 +90,84 @@ func TestAMPAlertManagerDeliversToSNS(t *testing.T) { }) } } + +func TestAMPKeysAndLogsWiring(t *testing.T) { + t.Parallel() + + ctx := t.Context() + kmsBk := kmsbackend.NewInMemoryBackend() + logsBk := cwlogsbackend.NewInMemoryBackend() + + live, err := kmsBk.CreateKey(ctx, &kmsbackend.CreateKeyInput{}) + require.NoError(t, err) + + off, err := kmsBk.CreateKey(ctx, &kmsbackend.CreateKeyInput{}) + require.NoError(t, err) + require.NoError(t, kmsBk.DisableKey(ctx, &kmsbackend.DisableKeyInput{KeyID: off.KeyMetadata.KeyID})) + + _, err = logsBk.CreateLogGroup(ctx, "/amp", "", "") + require.NoError(t, err) + + keys := ampKeys{kms: kmsBk} + logs := ampLogs{logs: logsBk} + + tests := []struct { + name string + key string + wantExists bool + wantEnabled bool + }{ + {name: "enabled", key: live.KeyMetadata.Arn, wantExists: true, wantEnabled: true}, + {name: "disabled", key: off.KeyMetadata.Arn, wantExists: true}, + {name: "unknown", key: "arn:aws:kms:us-east-1:000000000000:key/00000000-0000-0000-0000-000000000000"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + exists, enabled := keys.KMSKeyState(t.Context(), tt.key) + assert.Equal(t, tt.wantExists, exists) + assert.Equal(t, tt.wantEnabled, enabled) + }) + } + + assert.True(t, logs.LogGroupExists(ctx, "us-east-1", "/amp")) + assert.False(t, logs.LogGroupExists(ctx, "us-east-1", "/other")) + + require.NoError(t, logs.PutLogLine(ctx, "us-east-1", "/amp", "ruler", "one", time.Now())) + require.NoError(t, logs.PutLogLine(ctx, "us-east-1", "/amp", "ruler", "two", time.Now())) +} + +func TestAMPPolicyEvaluator(t *testing.T) { + t.Parallel() + + const ( + arn = "arn:aws:aps:us-east-1:123456789012:workspace/ws-1" + allow = `{"Version":"2012-10-17","Statement":[` + + `{"Effect":"Allow","Principal":"*","Action":"aps:RemoteWrite","Resource":"*"}]}` + deny = `{"Version":"2012-10-17","Statement":[` + + `{"Effect":"Deny","Principal":"*","Action":"aps:*","Resource":"*"}]}` + ) + + tests := []struct { + name string + doc, action string + allowed bool + denied bool + }{ + {name: "allow", doc: allow, action: "aps:RemoteWrite", allowed: true}, + {name: "other action", doc: allow, action: "aps:QueryMetrics"}, + {name: "deny", doc: deny, action: "aps:QueryMetrics", denied: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + allowed, denied := ampPolicyEvaluator(tt.doc, tt.action, arn, "123456789012") + assert.Equal(t, tt.allowed, allowed) + assert.Equal(t, tt.denied, denied) + }) + } +} diff --git a/internal/app/appsync_sigv4_wiring_test.go b/internal/app/appsync_sigv4_wiring_test.go index dd9aec6ea8..082bae6202 100644 --- a/internal/app/appsync_sigv4_wiring_test.go +++ b/internal/app/appsync_sigv4_wiring_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" appsyncbackend "github.com/blackbirdworks/gopherstack/services/appsync" ) @@ -55,8 +55,7 @@ func TestInitializeServices_AppSyncSigV4SecretWiring(t *testing.T) { t.Parallel() cli := &CLI{AccountID: "000000000000", Region: "us-east-1", SigV4Secret: "custom-secret"} - portAlloc, err := portalloc.New(19200, 19300) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) appCtx := &service.AppContext{ Logger: slog.Default(), diff --git a/internal/app/athena_engine.go b/internal/app/athena_engine.go index f6e3a8e924..1d92b10023 100644 --- a/internal/app/athena_engine.go +++ b/internal/app/athena_engine.go @@ -180,3 +180,13 @@ func (a *athenaGlueAdapter) CommitTableMetadata(db, name, expected, next string) return err } + +// SetTableColumns replaces a table's Glue columns after an Iceberg schema change. +func (a *athenaGlueAdapter) SetTableColumns(db, name string, cols []athenabackend.Column) error { + t, err := a.glueTable(db, name) + if err != nil { + return err + } + + return a.backend.SetTableColumns(t.DatabaseName, t.Name, glueColumns(cols)) +} diff --git a/internal/app/azureaca_wiring.go b/internal/app/azureaca_wiring.go new file mode 100644 index 0000000000..2c011708d3 --- /dev/null +++ b/internal/app/azureaca_wiring.go @@ -0,0 +1,65 @@ +package app + +import ( + "errors" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/azurearm" + "github.com/blackbirdworks/gopherstack/services/azurequeue" + "github.com/blackbirdworks/gopherstack/services/azureservicebus" +) + +var errMeterUnavailable = errors.New("queue service is not running") + +// wireAzureARMQueueMeters lets Container Apps KEDA scale rules and event jobs read the Service Bus and Storage +// queue depths of the in-process services. +func wireAzureARMQueueMeters(byName map[string]service.Registerable) { + armH, ok := byName["AzureARM"].(*azurearm.Handler) + if !ok { + return + } + + m := &azureACAQueueMeters{} + + if h, hOk := byName["AzureServiceBus"].(*azureservicebus.Handler); hOk { + m.serviceBus = h.Backend + } + + if h, hOk := byName["AzureQueue"].(*azurequeue.Handler); hOk { + m.queues = h.Backend + } + + armH.Registry.SetACAQueueMeters(m) +} + +// azureACAQueueMeters adapts the Service Bus and Storage queue backends to azurearm.ACAQueueMeters. +type azureACAQueueMeters struct { + serviceBus azureservicebus.StorageBackend + queues azurequeue.StorageBackend +} + +func (m *azureACAQueueMeters) ServiceBusMessages(queue, topic, subscription string) (int, error) { + if m.serviceBus == nil { + return 0, errMeterUnavailable + } + + if queue != "" { + info, err := m.serviceBus.GetQueueInfo(queue) + + return info.ActiveMessageCount, err + } + + info, err := m.serviceBus.GetSubscriptionInfo(topic, subscription) + + return info.ActiveMessageCount, err +} + +func (m *azureACAQueueMeters) StorageQueueMessages(queue string) (int, error) { + if m.queues == nil { + return 0, errMeterUnavailable + } + + props, err := m.queues.GetQueueProperties(queue) + + return props.ApproximateMessageCount, err +} diff --git a/internal/app/azureappconfig_wiring_test.go b/internal/app/azureappconfig_wiring_test.go index 79bca22fde..3fe99cccf6 100644 --- a/internal/app/azureappconfig_wiring_test.go +++ b/internal/app/azureappconfig_wiring_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/services/azureappconfig" "github.com/blackbirdworks/gopherstack/services/azurearm" ) @@ -29,12 +28,11 @@ func TestReserveFixedServicePorts_AzureAppConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(10000, 10100) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.want, tt.port) - cli := CLI{AzureAppConfig: azureappconfig.Settings{Port: tt.port}} + cli := CLI{AzureAppConfig: azureappconfig.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.want, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.want, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/azureappservice_slots_sdk_test.go b/internal/app/azureappservice_slots_sdk_test.go index ca8b98ce6e..988acf4ece 100644 --- a/internal/app/azureappservice_slots_sdk_test.go +++ b/internal/app/azureappservice_slots_sdk_test.go @@ -596,14 +596,17 @@ func TestAzureAppServiceARMSDK_SlotLimits(t *testing.T) { s := newFnSDKStack(t) plan := s.createPlanSKU(t, "limitplan", tt.kind, tt.sku, tt.tier) + props := &armappservice.SiteProperties{ + ServerFarmID: new(*plan.ID), + SiteConfig: &armappservice.SiteConfig{LinuxFxVersion: new(limitFx(tt.siteKind))}, + } + if tt.tier == "FlexConsumption" { + props.SiteConfig = &armappservice.SiteConfig{} + props.FunctionAppConfig = flexSDKConfig() + } + poller, err := s.factory.NewWebAppsClient().BeginCreateOrUpdate(t.Context(), webSDKRG, "limited", - armappservice.Site{ - Location: new("local"), Kind: new(tt.siteKind), - Properties: &armappservice.SiteProperties{ - ServerFarmID: new(*plan.ID), - SiteConfig: &armappservice.SiteConfig{LinuxFxVersion: new(limitFx(tt.siteKind))}, - }, - }, nil) + armappservice.Site{Location: new("local"), Kind: new(tt.siteKind), Properties: props}, nil) require.NoError(t, err) _, err = poller.PollUntilDone(t.Context(), pollOpts()) require.NoError(t, err) diff --git a/internal/app/azurearm_port_reservation_test.go b/internal/app/azurearm_port_reservation_test.go index 11b1356723..e029c2d1bb 100644 --- a/internal/app/azurearm_port_reservation_test.go +++ b/internal/app/azurearm_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" azurearmbackend "github.com/blackbirdworks/gopherstack/services/azurearm" ) @@ -26,18 +24,16 @@ func TestReserveFixedServicePorts_AzureARM(t *testing.T) { tests := []struct { name string armPort int - rangeStart int - rangeEnd int wantBlockedFromPool bool }{ { - name: "default azurearm port collides with default pool range", - armPort: azurearmbackend.DefaultPort, rangeStart: 10000, rangeEnd: 10100, + name: "default azurearm port collides with default pool range", + armPort: azurearmbackend.DefaultPort, wantBlockedFromPool: true, }, { - name: "custom azurearm port outside a custom pool range", - armPort: 9998, rangeStart: 10000, rangeEnd: 10100, + name: "custom azurearm port outside a custom pool range", + armPort: 9998, wantBlockedFromPool: false, }, } @@ -46,13 +42,12 @@ func TestReserveFixedServicePorts_AzureARM(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantBlockedFromPool, tt.armPort) - cli := CLI{AzureARM: azurearmbackend.Settings{Port: tt.armPort}} + cli := CLI{AzureARM: azurearmbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(tt.armPort), tt.name) + assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(port), tt.name) }) } } diff --git a/internal/app/azureblob_port_reservation_test.go b/internal/app/azureblob_port_reservation_test.go index 9dfd087541..f82cbb993e 100644 --- a/internal/app/azureblob_port_reservation_test.go +++ b/internal/app/azureblob_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" azureblobbackend "github.com/blackbirdworks/gopherstack/services/azureblob" ) @@ -24,18 +22,16 @@ func TestReserveFixedServicePorts(t *testing.T) { tests := []struct { name string azurePort int - rangeStart int - rangeEnd int wantBlockedFromPool bool }{ { - name: "default azure port collides with default pool range", - azurePort: azureblobbackend.DefaultPort, rangeStart: 10000, rangeEnd: 10100, + name: "default azure port collides with default pool range", + azurePort: azureblobbackend.DefaultPort, wantBlockedFromPool: true, }, { - name: "custom azure port outside a custom pool range", - azurePort: 9999, rangeStart: 10000, rangeEnd: 10100, + name: "custom azure port outside a custom pool range", + azurePort: 9999, wantBlockedFromPool: false, }, } @@ -44,13 +40,12 @@ func TestReserveFixedServicePorts(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantBlockedFromPool, tt.azurePort) - cli := CLI{AzureBlob: azureblobbackend.Settings{Port: tt.azurePort}} + cli := CLI{AzureBlob: azureblobbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(tt.azurePort), tt.name) + assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(port), tt.name) }) } } diff --git a/internal/app/azureeventgrid_wiring.go b/internal/app/azureeventgrid_wiring.go index abc83479ce..9dab14ed7d 100644 --- a/internal/app/azureeventgrid_wiring.go +++ b/internal/app/azureeventgrid_wiring.go @@ -22,8 +22,10 @@ func wireAzureARMAndEventGrid(byName map[string]service.Registerable) { byName["AzureARM"], byName["AzureServiceBus"], byName["AzureKeyVault"], byName["CosmosDB"], byName["AzureEventHubs"], ) + wireAzureARMQueueMeters(byName) wireAzureEventGrid(byName) wireAzureAppConfig(byName) + wireAzureSearch(byName) wireAzureContainerRegistry(byName) wireAzureFunctions(byName) wireAzureFile(byName) diff --git a/internal/app/azureeventgrid_wiring_test.go b/internal/app/azureeventgrid_wiring_test.go index 6b456b3bb1..bbe58ec283 100644 --- a/internal/app/azureeventgrid_wiring_test.go +++ b/internal/app/azureeventgrid_wiring_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" "github.com/blackbirdworks/gopherstack/services/azureblob" "github.com/blackbirdworks/gopherstack/services/azureeventgrid" @@ -36,12 +35,11 @@ func TestReserveFixedServicePorts_AzureEventGrid(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(10000, 10100) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.want, tt.port) - cli := CLI{AzureEventGrid: azureeventgrid.Settings{Port: tt.port}} + cli := CLI{AzureEventGrid: azureeventgrid.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.want, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.want, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/azureeventhubs_wiring_test.go b/internal/app/azureeventhubs_wiring_test.go index 126b7e2a60..1150ccc5f8 100644 --- a/internal/app/azureeventhubs_wiring_test.go +++ b/internal/app/azureeventhubs_wiring_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/services/azurearm" "github.com/blackbirdworks/gopherstack/services/azureeventhubs" ) @@ -29,12 +28,11 @@ func TestReserveFixedServicePorts_AzureEventHubs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(10000, 10100) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.want, tt.port) - cli := CLI{AzureEventHubs: azureeventhubs.Settings{Port: tt.port}} + cli := CLI{AzureEventHubs: azureeventhubs.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.want, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.want, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/azurefile_port_reservation_test.go b/internal/app/azurefile_port_reservation_test.go index 75552effb4..a90f82e5c3 100644 --- a/internal/app/azurefile_port_reservation_test.go +++ b/internal/app/azurefile_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" azurefilebackend "github.com/blackbirdworks/gopherstack/services/azurefile" ) @@ -18,18 +16,16 @@ func TestReserveFixedServicePorts_AzureFile(t *testing.T) { tests := []struct { name string azurePort int - rangeStart int - rangeEnd int wantBlockedFromPool bool }{ { - name: "default azure files port collides with default pool range", - azurePort: azurefilebackend.DefaultPort, rangeStart: 10000, rangeEnd: 10100, + name: "default azure files port collides with default pool range", + azurePort: azurefilebackend.DefaultPort, wantBlockedFromPool: true, }, { - name: "custom azure files port outside a custom pool range", - azurePort: 9998, rangeStart: 10000, rangeEnd: 10100, + name: "custom azure files port outside a custom pool range", + azurePort: 9998, wantBlockedFromPool: false, }, } @@ -38,13 +34,12 @@ func TestReserveFixedServicePorts_AzureFile(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantBlockedFromPool, tt.azurePort) - cli := CLI{AzureFile: azurefilebackend.Settings{Port: tt.azurePort}} + cli := CLI{AzureFile: azurefilebackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(tt.azurePort), tt.name) + assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(port), tt.name) }) } } diff --git a/internal/app/azurefunctions_admin_sdk_test.go b/internal/app/azurefunctions_admin_sdk_test.go new file mode 100644 index 0000000000..eabc429b60 --- /dev/null +++ b/internal/app/azurefunctions_admin_sdk_test.go @@ -0,0 +1,285 @@ +package app + +import ( + "archive/zip" + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v6" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func flexSDKConfig() *armappservice.FunctionAppConfig { + return &armappservice.FunctionAppConfig{ + Deployment: &armappservice.FunctionsDeployment{Storage: &armappservice.FunctionsDeploymentStorage{ + Type: new(armappservice.FunctionsDeploymentStorageTypeBlobContainer), + Value: new("https://acct.blob.core.windows.net/deploy"), + Authentication: &armappservice.FunctionsDeploymentStorageAuthentication{ + Type: new(armappservice.AuthenticationTypeSystemAssignedIdentity), + }, + }}, + Runtime: &armappservice.FunctionsRuntime{ + Name: new(armappservice.RuntimeNameNode), + Version: new("22"), + }, + ScaleAndConcurrency: &armappservice.FunctionsScaleAndConcurrency{InstanceMemoryMB: new(int32(2048))}, + } +} + +func (s *fnSDKStack) createFlexPlan(t *testing.T, name string) *armappservice.Plan { + t.Helper() + + poller, err := s.factory.NewPlansClient().BeginCreateOrUpdate(t.Context(), fnSDKRG, name, armappservice.Plan{ + Location: new("local"), Kind: new("functionapp"), + SKU: &armappservice.SKUDescription{Name: new("FC1"), Tier: new("FlexConsumption")}, + Properties: &armappservice.PlanProperties{Reserved: new(true)}, + }, nil) + require.NoError(t, err) + + res, err := poller.PollUntilDone(t.Context(), pollOpts()) + require.NoError(t, err) + + return &res.Plan +} + +func (s *fnSDKStack) putSiteWith( + t *testing.T, name, planID string, mutate func(*armappservice.SiteProperties), +) error { + t.Helper() + + props := &armappservice.SiteProperties{ServerFarmID: new(planID), SiteConfig: &armappservice.SiteConfig{}} + mutate(props) + + poller, err := s.factory.NewWebAppsClient().BeginCreateOrUpdate(t.Context(), fnSDKRG, name, armappservice.Site{ + Location: new("local"), Kind: new("functionapp,linux"), Properties: props, + }, nil) + if err != nil { + return err + } + + _, err = poller.PollUntilDone(t.Context(), pollOpts()) + + return err +} + +func TestAzureFunctionsARMSDK_Flex(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(*armappservice.FunctionAppConfig) + name string + wantCode int + }{ + {func(*armappservice.FunctionAppConfig) {}, "valid", 0}, + {func(c *armappservice.FunctionAppConfig) { c.Runtime.Name = new(armappservice.RuntimeName("cobol")) }, + "unknown runtime", http.StatusBadRequest}, + {func(c *armappservice.FunctionAppConfig) { c.Runtime.Version = new("14") }, + "unsupported node version", http.StatusBadRequest}, + {func(c *armappservice.FunctionAppConfig) { c.ScaleAndConcurrency.InstanceMemoryMB = new(int32(1024)) }, + "bad memory size", http.StatusBadRequest}, + {func(c *armappservice.FunctionAppConfig) { + c.ScaleAndConcurrency.MaximumInstanceCount = new(int32(1001)) + }, + "too many instances", http.StatusBadRequest}, + {func(c *armappservice.FunctionAppConfig) { c.Deployment.Storage.Value = new("not a url") }, + "bad storage url", http.StatusBadRequest}, + {func(c *armappservice.FunctionAppConfig) { + c.Deployment.Storage.Authentication.Type = new(armappservice.AuthenticationTypeUserAssignedIdentity) + }, "identity auth without identity", http.StatusBadRequest}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + s := newFnSDKStack(t) + plan := s.createFlexPlan(t, "flexplan") + cfg := flexSDKConfig() + tt.mutate(cfg) + + err := s.putSiteWith(t, "flexapp", *plan.ID, func(p *armappservice.SiteProperties) { + p.FunctionAppConfig = cfg + }) + + if tt.wantCode != 0 { + require.Error(t, err) + assert.Equal(t, tt.wantCode, responseCode(err)) + + return + } + + require.NoError(t, err) + + got, getErr := s.factory.NewWebAppsClient().Get(t.Context(), fnSDKRG, "flexapp", nil) + require.NoError(t, getErr) + + fc := got.Properties.FunctionAppConfig + require.NotNil(t, fc) + assert.Equal(t, "node", string(*fc.Runtime.Name)) + assert.Equal(t, int32(100), *fc.ScaleAndConcurrency.MaximumInstanceCount, "default instance count") + assert.Equal(t, int32(2048), *fc.ScaleAndConcurrency.InstanceMemoryMB) + assert.Equal(t, "https://acct.blob.core.windows.net/deploy", *fc.Deployment.Storage.Value) + + info, ok := s.backend.App("flexapp") + require.True(t, ok) + assert.Equal( + t, + "mcr.microsoft.com/azure-functions/node:4-node22", + info.Image, + "runtime.version picks the image", + ) + }) + } + + t.Run("config needs a flex plan", func(t *testing.T) { + t.Parallel() + + s := newFnSDKStack(t) + plan := s.createPlan(t, "dynplan") + + err := s.putSiteWith(t, "dynapp", *plan.ID, func(p *armappservice.SiteProperties) { + p.FunctionAppConfig = flexSDKConfig() + }) + require.Error(t, err) + assert.Equal(t, http.StatusBadRequest, responseCode(err)) + }) + + t.Run("flex plan needs config", func(t *testing.T) { + t.Parallel() + + s := newFnSDKStack(t) + plan := s.createFlexPlan(t, "flexplan") + + err := s.putSiteWith(t, "flexapp", *plan.ID, func(*armappservice.SiteProperties) {}) + require.Error(t, err) + assert.Equal(t, http.StatusBadRequest, responseCode(err)) + }) + + t.Run("publish without a type", func(t *testing.T) { + t.Parallel() + + s := newFnSDKStack(t) + plan := s.createFlexPlan(t, "flexplan") + require.NoError(t, s.putSiteWith(t, "flexapp", *plan.ID, func(p *armappservice.SiteProperties) { + p.FunctionAppConfig = flexSDKConfig() + })) + + poller, err := s.factory.NewWebAppsClient().BeginListPublishingCredentials(t.Context(), fnSDKRG, "flexapp", nil) + require.NoError(t, err) + + cred, err := poller.PollUntilDone(t.Context(), pollOpts()) + require.NoError(t, err) + + user, pass := *cred.Properties.PublishingUserName, *cred.Properties.PublishingPassword + host := "flexapp.scm." + fnSDKDomain + + code, _ := s.call(t, http.MethodPost, host, "/api/publish?RemoteBuild=false", fnZip(t), user, pass) + assert.Equal(t, http.StatusOK, code) + + code, _ = s.call(t, http.MethodPost, host, "/api/publish?RemoteBuild=true", fnZip(t), user, pass) + assert.Equal(t, http.StatusBadRequest, code) + }) +} + +func TestAzureFunctionsARMSDK_Management(t *testing.T) { + t.Parallel() + + s := newFnSDKStack(t) + plan := s.createPlan(t, "mgmtplan") + apps := s.factory.NewWebAppsClient() + + deploy := func(t *testing.T, name string) { + t.Helper() + + s.createSite(t, name, *plan.ID, map[string]string{"FUNCTIONS_WORKER_RUNTIME": "node"}) + _, err := s.backend.Deploy(name, fnZip(t), "t", "m") + require.NoError(t, err) + } + + t.Run("delete function", func(t *testing.T) { + t.Parallel() + + deploy(t, "delapp") + + _, err := apps.DeleteFunction(t.Context(), fnSDKRG, "delapp", "hello", nil) + require.NoError(t, err) + + zr, err := zip.NewReader(bytes.NewReader(s.backend.Package("delapp")), int64(len(s.backend.Package("delapp")))) + require.NoError(t, err) + + for _, f := range zr.File { + assert.False(t, strings.HasPrefix(f.Name, "hello/"), f.Name) + } + + assert.NotEmpty(t, zr.File, "host.json stays") + + _, err = apps.DeleteFunction(t.Context(), fnSDKRG, "delapp", "ghost", nil) + require.Error(t, err) + assert.Equal(t, http.StatusNotFound, responseCode(err)) + + _, err = apps.DeleteFunction(t.Context(), fnSDKRG, "delapp", "tick", nil) + require.Error(t, err) + assert.Equal(t, http.StatusBadRequest, responseCode(err), "tick has no function.json") + }) + + t.Run("admin token", func(t *testing.T) { + t.Parallel() + + deploy(t, "tokapp") + + res, err := apps.GetFunctionsAdminToken(t.Context(), fnSDKRG, "tokapp", nil) + require.NoError(t, err) + require.NotNil(t, res.Value) + + parts := strings.Split(*res.Value, ".") + require.Len(t, parts, 3) + + claims, err := base64.RawURLEncoding.DecodeString(parts[1]) + require.NoError(t, err) + + var c map[string]any + require.NoError(t, json.Unmarshal(claims, &c)) + assert.Equal(t, "https://tokapp.azurewebsites.net/azurefunctions", c["aud"]) + assert.Greater(t, c["exp"], c["iat"]) + + sig, err := base64.RawURLEncoding.DecodeString(parts[2]) + require.NoError(t, err) + + key := s.rt.lastEnv("tokapp")["WEBSITE_AUTH_ENCRYPTION_KEY"] + assert.Empty(t, key, "no host started yet") + + _, err = s.backend.Functions(t.Context(), "tokapp") + require.NoError(t, err) + + raw := s.rt.lastEnv("tokapp")["WEBSITE_AUTH_ENCRYPTION_KEY"] + require.NotEmpty(t, raw) + + keyBytes, err := hex.DecodeString(raw) + require.NoError(t, err) + + mac := hmac.New(sha256.New, keyBytes) + _, _ = mac.Write([]byte(parts[0] + "." + parts[1])) + assert.True(t, hmac.Equal(sig, mac.Sum(nil)), "the host's signing key validates the token") + }) + + t.Run("sync triggers", func(t *testing.T) { + t.Parallel() + + deploy(t, "syncapp") + + _, err := apps.SyncFunctionTriggers(t.Context(), fnSDKRG, "syncapp", nil) + require.NoError(t, err) + + _, err = apps.SyncFunctionTriggers(t.Context(), fnSDKRG, "ghostapp", nil) + require.Error(t, err) + assert.Equal(t, http.StatusNotFound, responseCode(err)) + }) +} diff --git a/internal/app/azurefunctions_wiring.go b/internal/app/azurefunctions_wiring.go index ce108a4c1a..0a0cd63d56 100644 --- a/internal/app/azurefunctions_wiring.go +++ b/internal/app/azurefunctions_wiring.go @@ -8,7 +8,10 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/service" "github.com/blackbirdworks/gopherstack/services/azurearm" + "github.com/blackbirdworks/gopherstack/services/azureblob" "github.com/blackbirdworks/gopherstack/services/azurefunctions" + "github.com/blackbirdworks/gopherstack/services/azurequeue" + "github.com/blackbirdworks/gopherstack/services/azuretable" ) // GetAzureFunctionsSettings returns Azure Functions settings (azurefunctions.ConfigProvider). @@ -16,16 +19,37 @@ func (c *CLI) GetAzureFunctionsSettings() azurefunctions.Settings { return c.Azu // wireAzureFunctions gives ARM's Microsoft.Web provider the Azure Functions data plane. func wireAzureFunctions(byName map[string]service.Registerable) { - armH, ok := byName["AzureARM"].(*azurearm.Handler) - if !ok { + h, hOk := byName["AzureFunctions"].(*azurefunctions.Handler) + if !hOk { return } - if h, hOk := byName["AzureFunctions"].(*azurefunctions.Handler); hOk { + h.Backend.SetNetwork(azureFunctionsNetwork(byName)) + + if armH, ok := byName["AzureARM"].(*azurearm.Handler); ok { armH.Registry.SetFunctionApps(&azureFunctionAppsAdapter{backend: h.Backend}) } } +// azureFunctionsNetwork reports the storage listener ports function host containers connect to. +func azureFunctionsNetwork(byName map[string]service.Registerable) azurefunctions.Network { + var n azurefunctions.Network + + if h, ok := byName["AzureBlob"].(*azureblob.Handler); ok { + n.BlobPort = h.Port + } + + if h, ok := byName["AzureQueue"].(*azurequeue.Handler); ok { + n.QueuePort = h.Port + } + + if h, ok := byName["AzureTable"].(*azuretable.Handler); ok { + n.TablePort = h.Port + } + + return n +} + // azureFunctionAppsAdapter adapts services/azurefunctions's backend to services/azurearm.FunctionApps. type azureFunctionAppsAdapter struct { backend *azurefunctions.InMemoryBackend @@ -39,6 +63,10 @@ func mapFunctionsError(err error) error { return azurearm.ErrWebSiteNotFound case errors.Is(err, azurefunctions.ErrFunctionNotFound): return azurearm.ErrWebFunctionNotFound + case errors.Is(err, azurefunctions.ErrFunctionInCode): + return azurearm.ErrWebFunctionInCode + case errors.Is(err, azurefunctions.ErrFunctionAppOnly): + return azurearm.ErrWebNotFunctionApp case errors.Is(err, azurefunctions.ErrKeyNotFound): return azurearm.ErrWebKeyNotFound case errors.Is(err, azurefunctions.ErrInvalidKeyType): @@ -60,6 +88,7 @@ func (a *azureFunctionAppsAdapter) PutApp( ) (azurearm.FunctionAppInfo, error) { appCfg := azurefunctions.AppConfig{ Kind: cfg.Kind, LinuxFx: cfg.LinuxFx, AppCommandLine: cfg.AppCommandLine, HealthCheckPath: cfg.HealthCheckPath, + Flex: cfg.Flex, } if err := azurefunctions.ValidateConfig(appCfg); err != nil && (!isSlotName(name) || cfg.LinuxFx != "") { @@ -171,3 +200,17 @@ func (a *azureFunctionAppsAdapter) SwapContent(first, second string) error { func (a *azureFunctionAppsAdapter) WarmUp(ctx context.Context, name string) error { return mapFunctionsError(a.backend.WarmUp(ctx, name)) } + +func (a *azureFunctionAppsAdapter) DeleteFunction(ctx context.Context, name, fn string) error { + return mapFunctionsError(a.backend.DeleteFunction(ctx, name, fn)) +} + +func (a *azureFunctionAppsAdapter) AdminToken(name string) (string, error) { + t, err := a.backend.AdminToken(name) + + return t, mapFunctionsError(err) +} + +func (a *azureFunctionAppsAdapter) SyncTriggers(ctx context.Context, name string) error { + return mapFunctionsError(a.backend.SyncTriggers(ctx, name)) +} diff --git a/internal/app/azuregraph_credential_sdk_test.go b/internal/app/azuregraph_credential_sdk_test.go new file mode 100644 index 0000000000..a8b2c74e86 --- /dev/null +++ b/internal/app/azuregraph_credential_sdk_test.go @@ -0,0 +1,231 @@ +package app + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "math/big" + "net/http" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/golang-jwt/jwt/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/aadauth" + "github.com/blackbirdworks/gopherstack/services/azuregraph" +) + +const graphScope = "https://graph.microsoft.com/.default" + +func (s *graphStack) clientOptions() azcore.ClientOptions { + return azcore.ClientOptions{ + Cloud: cloud.Configuration{ActiveDirectoryAuthorityHost: s.arm.URL}, + Transport: s.arm.Client(), + } +} + +func (s *graphStack) bootstrapToken(t *testing.T) string { + t.Helper() + + status, boot := s.token(t, "", "") + require.Equal(t, http.StatusOK, status) + + tok, _ := boot["access_token"].(string) + + return tok +} + +// registerApp creates an application with a service principal through Graph and returns both IDs. +func (s *graphStack) registerApp(t *testing.T, admin, name string) (string, string, string) { + t.Helper() + + status, app := s.call(t, admin, http.MethodPost, "/v1.0/applications", `{"displayName":"`+name+`"}`) + require.Equal(t, http.StatusCreated, status) + + appID, _ := app["appId"].(string) + objectID, _ := app["id"].(string) + + status, sp := s.call(t, admin, http.MethodPost, "/v1.0/servicePrincipals", `{"appId":"`+appID+`"}`) + require.Equal(t, http.StatusCreated, status) + + spID, _ := sp["id"].(string) + + return appID, objectID, spID +} + +func selfSignedCert(t *testing.T) (*x509.Certificate, *rsa.PrivateKey) { + t.Helper() + + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "sdk-test"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + + cert, err := x509.ParseCertificate(der) + require.NoError(t, err) + + return cert, key +} + +func tokenClaims(t *testing.T, token string) jwt.MapClaims { + t.Helper() + + claims := jwt.MapClaims{} + _, _, err := jwt.NewParser().ParseUnverified(token, claims) + require.NoError(t, err) + + return claims +} + +func TestAzureIdentityClientCertificateCredential(t *testing.T) { + t.Parallel() + + s := newGraphStack(t) + admin := s.bootstrapToken(t) + appID, objectID, spID := s.registerApp(t, admin, "cert-daemon") + cert, key := selfSignedCert(t) + + keyBody, err := json.Marshal(map[string]any{"keyCredential": map[string]any{ + "type": "AsymmetricX509Cert", "usage": "Verify", "key": cert.Raw, + }}) + require.NoError(t, err) + + status, added := s.call(t, admin, http.MethodPost, "/v1.0/applications/"+objectID+"/addKey", string(keyBody)) + require.Equal(t, http.StatusOK, status, added) + + newCred := func(c *x509.Certificate, k *rsa.PrivateKey) *azidentity.ClientCertificateCredential { + cred, cerr := azidentity.NewClientCertificateCredential(azuregraph.DefaultTenantID, appID, + []*x509.Certificate{c}, k, &azidentity.ClientCertificateCredentialOptions{ + ClientOptions: s.clientOptions(), DisableInstanceDiscovery: true, + }) + require.NoError(t, cerr) + + return cred + } + + t.Run("registered certificate obtains a token usable on graph", func(t *testing.T) { + t.Parallel() + + tok, gerr := newCred( + cert, + key, + ).GetToken(context.Background(), policy.TokenRequestOptions{Scopes: []string{graphScope}}) + require.NoError(t, gerr) + + claims := tokenClaims(t, tok.Token) + assert.Equal(t, spID, claims["oid"]) + assert.Equal(t, appID, claims["appid"]) + + code, got := s.call(t, tok.Token, http.MethodGet, "/v1.0/servicePrincipals(appId='"+appID+"')", "") + require.Equal(t, http.StatusOK, code) + assert.Equal(t, spID, got["id"]) + }) + + t.Run("unregistered certificate is rejected", func(t *testing.T) { + t.Parallel() + + otherCert, otherKey := selfSignedCert(t) + _, gerr := newCred( + otherCert, + otherKey, + ).GetToken(context.Background(), policy.TokenRequestOptions{Scopes: []string{graphScope}}) + require.Error(t, gerr) + assert.Contains(t, gerr.Error(), "AADSTS700027") + }) +} + +func TestAzureIdentityClientAssertionCredentialFederated(t *testing.T) { + t.Parallel() + + s := newGraphStack(t) + admin := s.bootstrapToken(t) + appID, objectID, spID := s.registerApp(t, admin, "federated-daemon") + + const ( + issuer = "https://issuer.example.test" + subject = "workload-1" + audience = "api://AzureADTokenExchange" + ) + + status, fic := s.call(t, admin, http.MethodPost, "/v1.0/applications/"+objectID+"/federatedIdentityCredentials", + `{"name":"workload","issuer":"`+issuer+`","subject":"`+subject+`","audiences":["`+audience+`"]}`) + require.Equal(t, http.StatusCreated, status, fic) + + emulatorAssertion := func(sub string) func(context.Context) (string, error) { + return func(context.Context) (string, error) { + return s.armH.Issuer.IssueClientCredentialsToken(aadauth.ClientCredentialsClaims{ + Issuer: issuer, Audience: audience, TenantID: azuregraph.DefaultTenantID, ClientID: sub, ObjectID: sub, + }, 0) + } + } + + foreign, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + forged := func(context.Context) (string, error) { + now := time.Now() + tok := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iss": issuer, "sub": subject, "aud": audience, "nbf": now.Unix(), "exp": now.Add(time.Hour).Unix(), + }) + tok.Header["kid"] = s.armH.Issuer.KeyID() + + return tok.SignedString(foreign) + } + + tests := []struct { + assertion func(context.Context) (string, error) + name string + wantErr string + }{ + {emulatorAssertion(subject), "assertion from the emulator issuer", ""}, + {emulatorAssertion("other-workload"), "subject without a credential", "AADSTS70021"}, + {forged, "forged signature under the emulator key id", "AADSTS700027"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cred, cerr := azidentity.NewClientAssertionCredential( + azuregraph.DefaultTenantID, + appID, + tt.assertion, + &azidentity.ClientAssertionCredentialOptions{ + ClientOptions: s.clientOptions(), + DisableInstanceDiscovery: true, + }, + ) + require.NoError(t, cerr) + + tok, gerr := cred.GetToken(context.Background(), policy.TokenRequestOptions{Scopes: []string{graphScope}}) + if tt.wantErr != "" { + require.Error(t, gerr) + assert.Contains(t, gerr.Error(), tt.wantErr) + + return + } + + require.NoError(t, gerr) + assert.Equal(t, spID, tokenClaims(t, tok.Token)["oid"]) + + code, body := s.call(t, tok.Token, http.MethodGet, "/v1.0/applications/"+objectID, "") + assert.Equal(t, http.StatusOK, code, body) + }) + } +} diff --git a/internal/app/azuregraph_port_reservation_test.go b/internal/app/azuregraph_port_reservation_test.go index e8771725f0..7a179ed6c1 100644 --- a/internal/app/azuregraph_port_reservation_test.go +++ b/internal/app/azuregraph_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" azuregraphbackend "github.com/blackbirdworks/gopherstack/services/azuregraph" ) @@ -27,13 +25,12 @@ func TestReserveFixedServicePorts_AzureGraph(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(10000, 10100) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantBlockedFromPool, tt.port) - cli := CLI{AzureGraph: azuregraphbackend.Settings{Port: tt.port}} + cli := CLI{AzureGraph: azuregraphbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/azuregraph_wiring.go b/internal/app/azuregraph_wiring.go index 2c5d7049ed..37a4ebe463 100644 --- a/internal/app/azuregraph_wiring.go +++ b/internal/app/azuregraph_wiring.go @@ -1,6 +1,7 @@ package app import ( + "crypto" "errors" "github.com/blackbirdworks/gopherstack/pkgs/service" @@ -31,6 +32,14 @@ func wireAzureGraph(byName map[string]service.Registerable) { graphH.Issuer = armH.Issuer } + if armH.Issuer != nil { + graphH.Backend.SetIssuerKeys(func(kid string) (crypto.PublicKey, bool) { + k, found := armH.Issuer.PublicKeyForKID(kid) + + return k, found + }) + } + armH.SetClientAuthenticator(&graphClientAuth{backend: graphH.Backend}) armH.SetPrincipalDirectory(graphH.Backend) } @@ -43,11 +52,21 @@ type graphClientAuth struct { func (a *graphClientAuth) AuthenticateClient(clientID, secret string) (string, bool, error) { oid, known, err := a.backend.AuthenticateClient(clientID, secret) + return oid, known, armError(err) +} + +func (a *graphClientAuth) AuthenticateAssertion(clientID, assertion string, audiences []string) (string, bool, error) { + oid, known, err := a.backend.AuthenticateAssertion(clientID, assertion, audiences) + + return oid, known, armError(err) +} + +func armError(err error) error { if ce, ok := errors.AsType[*azuregraph.ClientAuthError](err); ok { - return "", known, &azurearm.ClientAuthError{ + return &azurearm.ClientAuthError{ Name: ce.Name, Description: ce.Description, Status: ce.Status, Code: ce.Code, } } - return oid, known, err + return err } diff --git a/internal/app/azuregraph_wiring_test.go b/internal/app/azuregraph_wiring_test.go index 30ff349c3c..ffc5b4e670 100644 --- a/internal/app/azuregraph_wiring_test.go +++ b/internal/app/azuregraph_wiring_test.go @@ -28,8 +28,10 @@ func (graphARMConfig) GetAzureARMSettings() azurearm.Settings { } type graphStack struct { - arm *httptest.Server - graph *httptest.Server + arm *httptest.Server + graph *httptest.Server + armH *azurearm.Handler + graphH *azuregraph.Handler } func newGraphStack(t *testing.T) *graphStack { @@ -51,7 +53,7 @@ func newGraphStack(t *testing.T) *graphStack { ae.Any("/*", armH.Handler()) ge.Any("/*", graphH.Handler()) - s := &graphStack{arm: httptest.NewTLSServer(ae), graph: httptest.NewTLSServer(ge)} + s := &graphStack{arm: httptest.NewTLSServer(ae), graph: httptest.NewTLSServer(ge), armH: armH, graphH: graphH} t.Cleanup(s.arm.Close) t.Cleanup(s.graph.Close) diff --git a/internal/app/azuremonitor_port_reservation_test.go b/internal/app/azuremonitor_port_reservation_test.go index c18421b520..038bc69677 100644 --- a/internal/app/azuremonitor_port_reservation_test.go +++ b/internal/app/azuremonitor_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" azuremonitorbackend "github.com/blackbirdworks/gopherstack/services/azuremonitor" ) @@ -18,17 +16,14 @@ func TestReserveFixedServicePorts_AzureMonitor(t *testing.T) { tests := []struct { name string port int - rangeStart int - rangeEnd int wantBlockedFromPool bool }{ { name: "default port collides with default pool range", port: azuremonitorbackend.DefaultPort, - rangeStart: 10000, rangeEnd: 10100, wantBlockedFromPool: true, + wantBlockedFromPool: true, }, { - name: "custom port outside a custom pool range", port: 9997, - rangeStart: 10000, rangeEnd: 10100, wantBlockedFromPool: false, + name: "custom port outside a custom pool range", port: 9997, wantBlockedFromPool: false, }, } @@ -36,13 +31,12 @@ func TestReserveFixedServicePorts_AzureMonitor(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantBlockedFromPool, tt.port) - cli := CLI{AzureMonitor: azuremonitorbackend.Settings{Port: tt.port}} + cli := CLI{AzureMonitor: azuremonitorbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(tt.port), tt.name) + assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(port), tt.name) }) } } diff --git a/internal/app/azuresearch_wiring.go b/internal/app/azuresearch_wiring.go new file mode 100644 index 0000000000..6f09058893 --- /dev/null +++ b/internal/app/azuresearch_wiring.go @@ -0,0 +1,156 @@ +package app + +import ( + "errors" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/azurearm" + "github.com/blackbirdworks/gopherstack/services/azureblob" + "github.com/blackbirdworks/gopherstack/services/azuresearch" + "github.com/blackbirdworks/gopherstack/services/cosmosdb" +) + +// wireAzureSearch gives ARM's Microsoft.Search provider the Azure AI Search data plane and gives the +// data plane the in-process Blob Storage and Cosmos DB emulators as indexer sources. +func wireAzureSearch(byName map[string]service.Registerable) { + searchH, ok := byName["AzureSearch"].(*azuresearch.Handler) + if !ok { + return + } + + if armH, armOk := byName["AzureARM"].(*azurearm.Handler); armOk { + armH.Registry.SetSearch(&azureSearchAdapter{b: searchH.Backend, port: searchH.Port}) + } + + if blobH, blobOk := byName["AzureBlob"].(*azureblob.Handler); blobOk { + searchH.Backend.SetBlobSource(azureSearchBlobs{b: blobH.Backend}) + } + + if cosmosH, cosmosOk := byName["CosmosDB"].(*cosmosdb.Handler); cosmosOk { + searchH.Backend.SetCosmosSource(azureSearchCosmos{b: cosmosH.Backend}) + } +} + +// azureSearchBlobs exposes the Blob Storage emulator to the search indexers. +type azureSearchBlobs struct { + b azureblob.StorageBackend +} + +func (s azureSearchBlobs) ListBlobs(container string) ([]azuresearch.BlobEntry, error) { + var out []azuresearch.BlobEntry + + marker := "" + + for { + page, err := s.b.ListBlobsPage(container, azureblob.ListBlobsOptions{Marker: marker}) + if err != nil { + return nil, err + } + + for _, e := range page.Entries { + if e.IsPrefix { + continue + } + + out = append(out, azuresearch.BlobEntry{ + Name: e.Blob.Name, ContentType: e.Blob.ContentType, ETag: e.Blob.ETag, MD5: e.Blob.ContentMD5, + Size: e.Blob.ContentLength, LastModified: e.Blob.LastModified, Metadata: e.Blob.Metadata, + }) + } + + if page.NextMarker == "" { + return out, nil + } + + marker = page.NextMarker + } +} + +func (s azureSearchBlobs) ReadBlob(container, name string) ([]byte, error) { + _, data, err := s.b.GetBlobAt(container, name, "") + + return data, err +} + +// azureSearchCosmos exposes the Cosmos DB emulator to the search indexers. +type azureSearchCosmos struct { + b cosmosdb.StorageBackend +} + +func (s azureSearchCosmos) ListDocuments(database, container string) ([]azuresearch.CosmosDoc, error) { + docs, err := s.b.ListDocuments(database, container) + if err != nil { + return nil, err + } + + out := make([]azuresearch.CosmosDoc, 0, len(docs)) + for _, d := range docs { + out = append(out, azuresearch.CosmosDoc{ID: d.ID, ETag: d.ETag, Timestamp: d.Timestamp, Body: d.Body}) + } + + return out, nil +} + +// azureSearchAdapter adapts services/azuresearch's backend to services/azurearm.SearchPlane. +type azureSearchAdapter struct { + b *azuresearch.InMemoryBackend + port int +} + +func (a *azureSearchAdapter) Port() int { return a.port } + +var _ azurearm.SearchPlane = (*azureSearchAdapter)(nil) + +func (a *azureSearchAdapter) PutService(name string, cfg azurearm.SearchServiceConfig) (time.Time, error) { + return a.b.PutService(name, azuresearch.ServiceConfig(cfg)) +} + +func (a *azureSearchAdapter) DeleteService(name string) error { + return ignoreMissing(a.b.DeleteService(name), azuresearch.ErrServiceNotFound) +} + +func searchKeysOf(k azuresearch.AdminKeys, err error) (azurearm.SearchAdminKeys, error) { + return azurearm.SearchAdminKeys{Primary: k.Primary, Secondary: k.Secondary}, err +} + +func (a *azureSearchAdapter) AdminKeys(name string) (azurearm.SearchAdminKeys, error) { + return searchKeysOf(a.b.Keys(name)) +} + +func (a *azureSearchAdapter) RegenerateAdminKey(name, kind string) (azurearm.SearchAdminKeys, error) { + return searchKeysOf(a.b.RegenerateAdminKey(name, kind)) +} + +func (a *azureSearchAdapter) QueryKeys(name string) ([]azurearm.SearchQueryKey, error) { + keys, err := a.b.ListQueryKeys(name) + if err != nil { + return nil, err + } + + out := make([]azurearm.SearchQueryKey, 0, len(keys)) + for _, k := range keys { + out = append(out, azurearm.SearchQueryKey(k)) + } + + return out, nil +} + +func (a *azureSearchAdapter) CreateQueryKey(name, keyName string) (azurearm.SearchQueryKey, error) { + k, err := a.b.CreateQueryKey(name, keyName) + if errors.Is(err, azuresearch.ErrQueryKeyNameTaken) { + return azurearm.SearchQueryKey{}, azurearm.ErrSearchQueryKeyNameTaken + } + + return azurearm.SearchQueryKey(k), err +} + +func (a *azureSearchAdapter) DeleteQueryKey(name, key string) error { + if err := a.b.DeleteQueryKey(name, key); errors.Is(err, azuresearch.ErrQueryKeyNotFound) { + return azurearm.ErrSearchQueryKeyNotFound + } else if err != nil { + return err + } + + return nil +} diff --git a/internal/app/azuresearch_wiring_test.go b/internal/app/azuresearch_wiring_test.go new file mode 100644 index 0000000000..7ebeaea1e1 --- /dev/null +++ b/internal/app/azuresearch_wiring_test.go @@ -0,0 +1,301 @@ +package app + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/testclient" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/search/armsearch" + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/azurearm" + "github.com/blackbirdworks/gopherstack/services/azureblob" + "github.com/blackbirdworks/gopherstack/services/azuresearch" + "github.com/blackbirdworks/gopherstack/services/cosmosdb" +) + +type searchSDKCred struct{} + +func (searchSDKCred) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: "emulator", ExpiresOn: time.Now().Add(time.Hour)}, nil +} + +func TestReserveFixedServicePorts_AzureSearch(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + port int + want bool + }{ + {name: "default port inside pool", port: azuresearch.DefaultPort, want: true}, + {name: "port outside pool", port: 9995, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + alloc, port := fixedPortPool(t, tt.want, tt.port) + + cli := CLI{AzureSearch: azuresearch.Settings{Port: port}} + reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) + assert.Equal(t, tt.want, alloc.IsAllocated(port)) + }) + } +} + +func TestAzureSearchDefaultPortsAgree(t *testing.T) { + t.Parallel() + + assert.Equal(t, azuresearch.DefaultPort, azurearm.DefaultSearchPort) +} + +type searchE2E struct { + factory *armsearch.ClientFactory + blobs *azureblob.InMemoryBackend + cosmos *cosmosdb.InMemoryBackend + search *azuresearch.Handler + data string + arm string +} + +func newSearchE2E(t *testing.T) *searchE2E { + t.Helper() + + armSvc, err := (&azurearm.Provider{}).Init(&service.AppContext{}) + require.NoError(t, err) + + armH, ok := armSvc.(*azurearm.Handler) + require.True(t, ok) + + t.Cleanup(func() { armH.Shutdown(context.Background()) }) + + clock := &struct{ now time.Time }{now: time.Date(2026, 11, 1, 9, 0, 0, 0, time.UTC)} + searchH := azuresearch.NewHandler( + azuresearch.NewInMemoryBackend( + azuresearch.WithClock(func() time.Time { return clock.now }), + ), + azuresearch.DefaultSettings(), + ) + blobs := azureblob.NewInMemoryBackend() + cosmos := cosmosdb.NewInMemoryBackend() + + wireAzureSearch(map[string]service.Registerable{ + "AzureARM": armH, + "AzureSearch": searchH, + "AzureBlob": azureblob.NewHandler(blobs), + "CosmosDB": cosmosdb.NewHandler(cosmos), + }) + + armE := echo.New() + armE.Any("/*", armH.Handler()) + + armSrv := httptest.NewTLSServer(armE) + t.Cleanup(armSrv.Close) + + dataE := echo.New() + dataE.Any("/*", searchH.Handler()) + + dataSrv := httptest.NewServer(dataE) + t.Cleanup(dataSrv.Close) + + f, err := armsearch.NewClientFactory("00000000-0000-0000-0000-000000000000", searchSDKCred{}, &arm.ClientOptions{ + Cloud: cloud.Configuration{Services: map[cloud.ServiceName]cloud.ServiceConfiguration{ + cloud.ResourceManager: {Endpoint: armSrv.URL, Audience: "https://management.core.windows.net/"}, + }}, + Transport: armSrv.Client(), Retry: policy.RetryOptions{MaxRetries: -1}, + }) + require.NoError(t, err) + + return &searchE2E{factory: f, blobs: blobs, cosmos: cosmos, search: searchH, data: dataSrv.URL, arm: armSrv.URL} +} + +func (e *searchE2E) call(t *testing.T, method, path, key string, body any) (int, map[string]any) { + t.Helper() + + var rd io.Reader + + if body != nil { + raw, err := json.Marshal(body) + require.NoError(t, err) + + rd = bytes.NewReader(raw) + } + + req, err := http.NewRequestWithContext(t.Context(), method, e.data+path, rd) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Api-Key", key) + + res, err := testclient.New(t).Do(req) + require.NoError(t, err) + + defer res.Body.Close() + + raw, err := io.ReadAll(res.Body) + require.NoError(t, err) + + var out map[string]any + + _ = json.Unmarshal(raw, &out) + + return res.StatusCode, out +} + +func TestAzureSearchARMKeysUnlockDataPlane(t *testing.T) { + t.Parallel() + + e := newSearchE2E(t) + poller, err := e.factory.NewServicesClient().BeginCreateOrUpdate(t.Context(), "rg", "e2esearch", armsearch.Service{ + Location: new("westus"), SKU: &armsearch.SKU{Name: new(armsearch.SKUNameBasic)}, + }, nil, nil) + require.NoError(t, err) + + svc, err := poller.PollUntilDone(t.Context(), nil) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(*svc.Properties.Endpoint, "https://e2esearch.search.")) + + keys, err := e.factory.NewAdminKeysClient().Get(t.Context(), "rg", "e2esearch", nil, nil) + require.NoError(t, err) + + idx := map[string]any{"name": "things", "fields": []any{ + map[string]any{"name": "id", "type": "Edm.String", "key": true}, + map[string]any{"name": "body", "type": "Edm.String", "searchable": true}, + }} + + status, _ := e.call(t, http.MethodPost, "/indexes?api-version=2024-07-01", *keys.PrimaryKey, idx) + assert.Equal(t, http.StatusCreated, status, "the ARM-issued admin key authenticates the data plane") + + status, _ = e.call(t, http.MethodPost, "/indexes?api-version=2024-07-01", "not-a-key", idx) + assert.Equal(t, http.StatusForbidden, status) + + qk, err := e.factory.NewQueryKeysClient().Create(t.Context(), "rg", "e2esearch", "reader", nil, nil) + require.NoError(t, err) + + status, _ = e.call(t, http.MethodGet, "/indexes('things')/docs?search=*&api-version=2024-07-01", *qk.Key, nil) + assert.Equal(t, http.StatusOK, status, "an ARM-created query key can query") + + status, _ = e.call(t, http.MethodPost, "/indexes?api-version=2024-07-01", *qk.Key, idx) + assert.Equal(t, http.StatusForbidden, status, "a query key cannot manage indexes") + + regen, err := e.factory.NewAdminKeysClient(). + Regenerate(t.Context(), "rg", "e2esearch", armsearch.AdminKeyKindPrimary, nil, nil) + require.NoError(t, err) + + status, _ = e.call(t, http.MethodGet, "/indexes?api-version=2024-07-01", *keys.PrimaryKey, nil) + assert.Equal(t, http.StatusForbidden, status, "regenerating revokes the old key") + + status, _ = e.call(t, http.MethodGet, "/indexes?api-version=2024-07-01", *regen.PrimaryKey, nil) + assert.Equal(t, http.StatusOK, status) + + _, err = e.factory.NewServicesClient().Delete(t.Context(), "rg", "e2esearch", nil, nil) + require.NoError(t, err) + + status, _ = e.call(t, http.MethodGet, "/indexes?api-version=2024-07-01", *regen.PrimaryKey, nil) + assert.Equal(t, http.StatusForbidden, status, "deleting the ARM resource deletes the data-plane service") +} + +func TestAzureSearchIndexerFromInProcessEmulators(t *testing.T) { + t.Parallel() + + e := newSearchE2E(t) + poller, err := e.factory.NewServicesClient().BeginCreateOrUpdate(t.Context(), "rg", "idxsvc", armsearch.Service{ + Location: new("westus"), SKU: &armsearch.SKU{Name: new(armsearch.SKUNameBasic)}, + }, nil, nil) + require.NoError(t, err) + + _, err = poller.PollUntilDone(t.Context(), nil) + require.NoError(t, err) + + keys, err := e.factory.NewAdminKeysClient().Get(t.Context(), "rg", "idxsvc", nil, nil) + require.NoError(t, err) + + key := *keys.PrimaryKey + + require.NoError(t, e.blobs.CreateContainer("docs")) + _, err = e.blobs.PutBlob("docs", "readme.txt", []byte("emulated search indexing"), "text/plain") + require.NoError(t, err) + + _, err = e.cosmos.CreateDatabase("db") + require.NoError(t, err) + _, err = e.cosmos.CreateContainer("db", cosmosdb.ContainerSpec{ID: "items", PartitionKeyPath: "/pk"}) + require.NoError(t, err) + _, err = e.cosmos.CreateDocument( + "db", + "items", + map[string]any{"id": "c1", "pk": "p", "content": "cosmos record"}, + false, + ) + require.NoError(t, err) + + for _, step := range []struct { + body map[string]any + path string + want int + }{ + {path: "/indexes", body: map[string]any{"name": "docs", "fields": []any{ + map[string]any{"name": "id", "type": "Edm.String", "key": true}, + map[string]any{"name": "content", "type": "Edm.String", "searchable": true}, + }}, want: http.StatusCreated}, + {path: "/datasources", body: map[string]any{ + "name": "blobs", "type": "azureblob", "container": map[string]any{"name": "docs"}, + "credentials": map[string]any{"connectionString": "DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=a2V5;"}, //nolint:lll // test fixture + }, want: http.StatusCreated}, + {path: "/datasources", body: map[string]any{ + "name": "cosmos", "type": "cosmosdb", "container": map[string]any{"name": "items"}, + "credentials": map[string]any{"connectionString": "AccountEndpoint=https://x;AccountKey=a2V5;Database=db"}, + }, want: http.StatusCreated}, + {path: "/indexers", body: map[string]any{ + "name": "from-blob", "dataSourceName": "blobs", "targetIndexName": "docs", + "fieldMappings": []any{map[string]any{ + "sourceFieldName": "metadata_storage_path", "targetFieldName": "id", "mappingFunction": map[string]any{"name": "base64Encode"}, //nolint:lll // test fixture + }}, + }, want: http.StatusCreated}, + {path: "/indexers", body: map[string]any{"name": "from-cosmos", "dataSourceName": "cosmos", "targetIndexName": "docs"}, want: http.StatusCreated}, //nolint:lll // test fixture + } { + status, body := e.call(t, http.MethodPost, step.path+"?api-version=2024-07-01", key, step.body) + require.Equal(t, step.want, status, "%s: %v", step.path, body) + } + + for _, name := range []string{"from-blob", "from-cosmos"} { + status, _ := e.call(t, http.MethodPost, "/indexers('"+name+"')/search.run?api-version=2024-07-01", key, nil) + require.Equal(t, http.StatusAccepted, status) + } + + status, body := e.call( + t, + http.MethodGet, + "/indexes('docs')/docs?search=*&$count=true&api-version=2024-07-01", + key, + nil, + ) + require.Equal(t, http.StatusOK, status) + assert.EqualValues(t, 2, body["@odata.count"], "one document from Blob Storage and one from Cosmos DB") + + status, body = e.call(t, http.MethodGet, "/indexes('docs')/docs?search=emulated&api-version=2024-07-01", key, nil) + require.Equal(t, http.StatusOK, status) + assert.Len(t, body["value"], 1) + + status, body = e.call(t, http.MethodGet, "/indexers('from-blob')/search.status?api-version=2024-07-01", key, nil) + require.Equal(t, http.StatusOK, status) + + last, _ := body["lastResult"].(map[string]any) + assert.Equal(t, "success", last["status"]) +} diff --git a/internal/app/azureservicebus_port_reservation_test.go b/internal/app/azureservicebus_port_reservation_test.go index 0a9c39fd4a..9781c75d68 100644 --- a/internal/app/azureservicebus_port_reservation_test.go +++ b/internal/app/azureservicebus_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" azureservicebusbackend "github.com/blackbirdworks/gopherstack/services/azureservicebus" ) @@ -28,18 +26,16 @@ func TestReserveFixedServicePorts_AzureServiceBus(t *testing.T) { tests := []struct { name string azurePort int - rangeStart int - rangeEnd int wantBlockedFromPool bool }{ { - name: "default azure service bus port collides with default pool range", - azurePort: azureservicebusbackend.DefaultPort, rangeStart: 10000, rangeEnd: 10100, + name: "default azure service bus port collides with default pool range", + azurePort: azureservicebusbackend.DefaultPort, wantBlockedFromPool: true, }, { - name: "custom azure service bus port outside a custom pool range", - azurePort: 9997, rangeStart: 10000, rangeEnd: 10100, + name: "custom azure service bus port outside a custom pool range", + azurePort: 9997, wantBlockedFromPool: false, }, } @@ -48,13 +44,12 @@ func TestReserveFixedServicePorts_AzureServiceBus(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantBlockedFromPool, tt.azurePort) - cli := CLI{AzureServiceBus: azureservicebusbackend.Settings{Port: tt.azurePort}} + cli := CLI{AzureServiceBus: azureservicebusbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(tt.azurePort), tt.name) + assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(port), tt.name) }) } } diff --git a/internal/app/azuretable_port_reservation_test.go b/internal/app/azuretable_port_reservation_test.go index 337c2fbfb8..e5cc7298cb 100644 --- a/internal/app/azuretable_port_reservation_test.go +++ b/internal/app/azuretable_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" azuretablebackend "github.com/blackbirdworks/gopherstack/services/azuretable" ) @@ -27,18 +25,16 @@ func TestReserveFixedServicePorts_AzureTable(t *testing.T) { tests := []struct { name string azurePort int - rangeStart int - rangeEnd int wantBlockedFromPool bool }{ { - name: "default azure table port collides with default pool range", - azurePort: azuretablebackend.DefaultPort, rangeStart: 10000, rangeEnd: 10100, + name: "default azure table port collides with default pool range", + azurePort: azuretablebackend.DefaultPort, wantBlockedFromPool: true, }, { - name: "custom azure table port outside a custom pool range", - azurePort: 9998, rangeStart: 10000, rangeEnd: 10100, + name: "custom azure table port outside a custom pool range", + azurePort: 9998, wantBlockedFromPool: false, }, } @@ -47,13 +43,12 @@ func TestReserveFixedServicePorts_AzureTable(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantBlockedFromPool, tt.azurePort) - cli := CLI{AzureTable: azuretablebackend.Settings{Port: tt.azurePort}} + cli := CLI{AzureTable: azuretablebackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(tt.azurePort), tt.name) + assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(port), tt.name) }) } } diff --git a/internal/app/cli.go b/internal/app/cli.go index acc58566b0..beb6fee914 100644 --- a/internal/app/cli.go +++ b/internal/app/cli.go @@ -104,6 +104,7 @@ import ( azuremonitorbackend "github.com/blackbirdworks/gopherstack/services/azuremonitor" azurequeuebackend "github.com/blackbirdworks/gopherstack/services/azurequeue" azureredisbackend "github.com/blackbirdworks/gopherstack/services/azureredis" + azuresearchbackend "github.com/blackbirdworks/gopherstack/services/azuresearch" azureservicebusbackend "github.com/blackbirdworks/gopherstack/services/azureservicebus" azuresqlbackend "github.com/blackbirdworks/gopherstack/services/azuresql" azurestoragevhostbackend "github.com/blackbirdworks/gopherstack/services/azurestoragevhost" @@ -222,6 +223,7 @@ import ( kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" lakeformationbackend "github.com/blackbirdworks/gopherstack/services/lakeformation" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" + lambdawebbackend "github.com/blackbirdworks/gopherstack/services/lambdaweb" lightsailbackend "github.com/blackbirdworks/gopherstack/services/lightsail" macie2backend "github.com/blackbirdworks/gopherstack/services/macie2" managedblockchainbackend "github.com/blackbirdworks/gopherstack/services/managedblockchain" @@ -247,7 +249,10 @@ import ( ocioke "github.com/blackbirdworks/gopherstack/services/ocicontainerengine" ocicontainerinstances "github.com/blackbirdworks/gopherstack/services/ocicontainerinstances" ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + ocidatabase "github.com/blackbirdworks/gopherstack/services/ocidatabase" + ocidns "github.com/blackbirdworks/gopherstack/services/ocidns" ocievents "github.com/blackbirdworks/gopherstack/services/ocievents" + ocifilestorage "github.com/blackbirdworks/gopherstack/services/ocifilestorage" ocifunctionsbackend "github.com/blackbirdworks/gopherstack/services/ocifunctions" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" ociloadbalancer "github.com/blackbirdworks/gopherstack/services/ociloadbalancer" @@ -451,6 +456,7 @@ type CLI struct { cloudtrailHandler service.Registerable azureARMHandler service.Registerable azureBlobHandler service.Registerable + azureGraphHandler service.Registerable gcpStorageHandler service.Registerable gcpPubSubHandler service.Registerable gcpFirestoreHandler service.Registerable @@ -473,6 +479,11 @@ type CLI struct { azureEventHubsHandler service.Registerable azureEventGridHandler service.Registerable azureAppConfigHandler service.Registerable + azureSearchHandler service.Registerable + azureRedisHandler service.Registerable + azureSQLHandler service.Registerable + azureACRHandler service.Registerable + lambdaWebHandler service.Registerable azureFunctionsHandler service.Registerable azureFileHandler service.Registerable gcpWorkflowsHandler service.Registerable @@ -499,6 +510,9 @@ type CLI struct { ociLoadBalancerHandler service.Registerable ociArtifactsHandler service.Registerable ociContainerInstancesHandler service.Registerable + ociDNSHandler service.Registerable + ociDatabaseHandler service.Registerable + ociFileStorageHandler service.Registerable azureMonitorHandler service.Registerable ociVaultHandler service.Registerable ociStreamingHandler service.Registerable @@ -651,6 +665,9 @@ type CLI struct { AzureGraph azuregraphbackend.Settings `embed:"" prefix:"azure-graph-"` CodeBuild codebuildbackend.Settings `embed:"" prefix:"codebuild-"` Glue gluebackend.Settings `embed:"" prefix:"glue-"` + DSQL dsqlbackend.Settings `embed:"" prefix:"dsql-"` + LambdaWeb lambdawebbackend.Settings `embed:"" prefix:"lambdaweb-"` + Transcribe transcribebackend.Settings `embed:"" prefix:"transcribe-"` GCPIdentityToolkit gcpidtkbackend.Settings `embed:"" prefix:"gcp-identitytoolkit-"` AzureKeyVault azurekeyvaultbackend.Settings `embed:"" prefix:"azure-keyvault-"` S3 s3backend.Settings `embed:"" prefix:"s3-"` @@ -665,6 +682,8 @@ type CLI struct { OCIContainerEngine ocioke.Settings `embed:"" prefix:"oci-oke-"` OCIContainerInstances ocicontainerinstances.Settings `embed:""` OCIMySQL ocimysqlbackend.Settings `embed:"" prefix:"oci-mysql-"` + OCIDatabase ocidatabase.Settings `embed:"" prefix:"oci-database-"` + OCIFileStorage ocifilestorage.Settings `embed:"" prefix:"oci-filestorage-"` AgentCore agentcorebackend.Settings `embed:"" prefix:"agentcore-"` GCPCloudBuild gcpcloudbuildbackend.Settings `embed:"" prefix:"gcp-cloudbuild-"` GCPBatch gcpbatchbackend.Settings `embed:"" prefix:"gcp-batch-"` @@ -714,6 +733,7 @@ type CLI struct { GCPCompute gcpcomputebackend.Settings `embed:"" prefix:"gcp-compute-"` GCPContainer gcpcontainerbackend.Settings `embed:"" prefix:"gcp-container-"` GCPAPIGateway gcpapigatewaybackend.Settings `embed:"" prefix:"gcp-apigateway-"` + AzureSearch azuresearchbackend.Settings `embed:"" prefix:"azure-search-"` PortRangeStart int ` name:"port-range-start" env:"PORT_RANGE_START" default:"10000" help:"Start of the port range for resource endpoints."` //nolint:lll // config struct tags are intentionally verbose PortRangeEnd int ` name:"port-range-end" env:"PORT_RANGE_END" default:"10100" help:"End (exclusive) of the port range for resource endpoints."` //nolint:lll // config struct tags are intentionally verbose EC2DockerSSHPortMin int ` name:"ec2-docker-ssh-port-min" env:"EC2_DOCKER_SSH_PORT_MIN" default:"0" help:"Lower bound of the host TCP port range used to map EC2-docker SSH (0 = let Docker pick)."` //nolint:lll // config struct tags are intentionally verbose @@ -754,6 +774,7 @@ type CLI struct { OCIEvents ocievents.Settings `embed:"" prefix:"oci-events-"` OCILoadBalancer ociloadbalancer.Settings `embed:"" prefix:"oci-loadbalancer-"` OCIArtifacts ociartifacts.Settings `embed:"" prefix:"oci-artifacts-"` + OCIDNS ocidns.Settings `embed:"" prefix:"oci-dns-"` } // GetGlobalConfig returns the centralised account ID and region (config.Provider). @@ -999,6 +1020,11 @@ func (c *CLI) GetOCIMySQLEngine() string { return c.engineMode("oci-mysql-engine", c.OCIMySQL.Engine) } +// GetOCIDatabaseEngine returns the OCI Database engine mode (ocidatabase.EngineConfigProvider). +func (c *CLI) GetOCIDatabaseEngine() string { + return c.engineMode("oci-database-engine", c.OCIDatabase.Engine) +} + // GetOCINotificationsSettings returns OCI Notifications settings (ocinotifications.ConfigProvider). func (c *CLI) GetOCINotificationsSettings() ocinotificationsbackend.Settings { return c.OCINotifications @@ -1025,6 +1051,9 @@ func (c *CLI) GetAzureAppConfigSettings() azureappconfigbackend.Settings { return c.AzureAppConfig } +// GetAzureSearchSettings returns Azure AI Search settings (azuresearch.ConfigProvider). +func (c *CLI) GetAzureSearchSettings() azuresearchbackend.Settings { return c.AzureSearch } + // GetAzureRedisSettings returns Azure Cache for Redis settings (azureredis.ConfigProvider). func (c *CLI) GetAzureRedisSettings() azureredisbackend.Settings { return c.AzureRedis } @@ -1505,6 +1534,30 @@ func (c *CLI) GetGlueSettings() gluebackend.Settings { return st } +// GetDSQLSettings returns DSQL settings (dsql.ConfigProvider). +func (c *CLI) GetDSQLSettings() dsqlbackend.Settings { + st := c.DSQL + st.Engine = c.engineMode("dsql-engine", st.Engine) + + return st +} + +// GetLambdaWebSettings returns Lambda Web Functions settings (lambdaweb.ConfigProvider). +func (c *CLI) GetLambdaWebSettings() lambdawebbackend.Settings { + st := c.LambdaWeb + st.Engine = c.engineMode("lambdaweb-engine", st.Engine) + + return st +} + +// GetTranscribeSettings returns Transcribe settings (transcribe.ConfigProvider). +func (c *CLI) GetTranscribeSettings() transcribebackend.Settings { + st := c.Transcribe + st.Engine = c.engineMode("transcribe-engine", st.Engine) + + return st +} + // GetEC2Settings returns EC2 settings (ec2.ConfigProvider). func (c *CLI) GetEC2Settings() ec2backend.Settings { return c.EC2 @@ -2770,6 +2823,7 @@ func fixedServicePorts(cli CLI) []fixedServicePort { {"AzureEventHubsAMQPTLS", "azureeventhubs-amqp-tls", cli.AzureEventHubs.AMQPTLSPort}, {"AzureEventGrid", "azureeventgrid", cli.AzureEventGrid.Port}, {"AzureAppConfig", "azureappconfig", cli.AzureAppConfig.Port}, + {"AzureSearch", "azuresearch", cli.AzureSearch.Port}, {"AzureContainerRegistry", "azurecontainerregistry", cli.AzureACR.Port}, {"AzureFunctions", "azurefunctions", cli.AzureFunctions.Port}, {"AzureARM", "azurearm", cli.AzureARM.Port}, @@ -2793,6 +2847,10 @@ func fixedServicePorts(cli CLI) []fixedServicePort { {"OCILoadBalancer", "ociloadbalancer", cli.OCILoadBalancer.Port}, {"OCIArtifacts", "ociartifacts", cli.OCIArtifacts.Port}, {"OCIContainerInstances", "ocicontainerinstances", cli.OCIContainerInstances.Port}, + {"OCIDNS", "ocidns", cli.OCIDNS.Port}, + {"OCIDNSServer", "ocidns-server", cli.OCIDNS.DNSPort}, + {"OCIDatabase", "ocidatabase", cli.OCIDatabase.Port}, + {"OCIFileStorage", "ocifilestorage", cli.OCIFileStorage.Port}, } } @@ -3166,6 +3224,7 @@ func buildEchoServer( registerSSOEndpoints(e, services) registerEC2NetworkEndpoints(e, services) registerEKSEndpoints(e, services) + registerTranscribeEndpoints(e, services) registerPricingEndpoints(e, services) e.POST("/_gopherstack/snapshot", buildSnapshotHandler(persistManager)) e.POST("/_gopherstack/load", buildLoadHandler(persistManager)) @@ -3528,6 +3587,9 @@ func storeCLIRecentHandlers(cli *CLI, byName map[string]service.Registerable) { cli.ociLoadBalancerHandler = byName["OCILoadBalancer"] cli.ociArtifactsHandler = byName["OCIArtifacts"] cli.ociContainerInstancesHandler = byName["OCIContainerInstances"] + cli.ociDNSHandler = byName["OCIDNS"] + cli.ociDatabaseHandler = byName["OCIDatabase"] + cli.ociFileStorageHandler = byName["OCIFileStorage"] storeAdditionalCLIHandlers(cli, byName) } @@ -3796,6 +3858,7 @@ func wireCrossServiceDependencies( wireRDSAutoPauseMetrics(appCtx.JanitorCtx, byName) wireStorageAndSecretsIntegrations(byName) wireAppSyncAndStreamsIntegrations(byName, sigV4SecretOf(appCtx)) + wireAMPDataPlaneAuth(appCtx, byName) wireSchedulerAndPipesIntegrations(byName) wireGovernanceIntegrations(byName, services) wireOCIStreamingVault(byName) @@ -3810,6 +3873,10 @@ func wireCrossServiceDependencies( wireOCILoadBalancer(byName) wireOCIArtifacts(byName) wireOCIContainerInstances(byName) + wireOCIDNS(byName) + wireOCIDatabase(byName) + wireOCIFileStorage(byName) + wireOCICoreVault(byName) } // wireOCIStreamingVault lets OCI Streaming validate customEncryptionKey references against OCI Vault keys. @@ -3820,6 +3887,26 @@ func wireOCIStreamingVault(byName map[string]service.Registerable) { if sok && vok { streamingH.Backend.SetKeyChecker(vaultH.Backend.KeyState) } + + if osH, ook := byName["OCIObjectStorage"].(*ociobjectstoragebackend.Handler); ook && vok { + osH.Backend.SetKeyChecker(vaultH.Backend.KeyState) + vaultH.Backend.SetBlobStore(osH.Backend) + } + + if fnH, fok := byName["OCIFunctions"].(*ocifunctionsbackend.Handler); fok && vok { + vaultH.Backend.SetFunctionInvoker(func(ctx context.Context, fnID string, body []byte) error { + out, err := fnH.Backend.Invoke(ctx, fnID, ocifunctionsbackend.InvokeIn{Body: body}) + if err != nil { + return err + } + + if out.Status >= http.StatusMultipleChoices { + return fmt.Errorf("%w: status %d", ocivaultbackend.ErrFunctionFailed, out.Status) + } + + return nil + }) + } } // wireOCIQueueNotifications lets OCI Queue validate customEncryptionKeyId against OCI Vault keys and lets OCI @@ -4985,6 +5072,7 @@ func wireAppSyncAndStreamsIntegrations(byName map[string]service.Registerable, s wireAppSyncSigV4(byName["AppSync"], sigV4Secret) wireLambdaSigV4(byName["Lambda"], sigV4Secret) + wireAgentCoreAuth(byName, sigV4Secret) // Wire DynamoDB Streams → DynamoDB backend so streams share the same in-memory data. wireDynamoDBStreams(byName["DynamoDB"], byName["DynamoDBStreams"]) @@ -5269,6 +5357,7 @@ func getLatestServiceProviders() []service.Provider { // Extracted from getServiceProviders to satisfy the funlen limit. func getNewestServiceProviders() []service.Provider { return append([]service.Provider{ + &lambdawebbackend.Provider{}, &mwaabackend.Provider{}, &neptunebackend.Provider{}, &signinbackend.Provider{}, @@ -5314,6 +5403,9 @@ func getLeadingRecentServiceProviders() []service.Provider { &ociloadbalancer.Provider{}, &ociartifacts.Provider{}, &ocicontainerinstances.Provider{}, + &ocidns.Provider{}, + &ocidatabase.Provider{}, + &ocifilestorage.Provider{}, &gcppubsubbackend.Provider{}, &gcpsecretmanagerbackend.Provider{}, &gcpkmsbackend.Provider{}, @@ -5417,6 +5509,7 @@ func getMostRecentServiceProviders() []service.Provider { &lightsailbackend.Provider{}, &oambackend.Provider{}, &servicequotasbackend.Provider{}, + &azuresearchbackend.Provider{}, }...) } @@ -8027,6 +8120,22 @@ func (a *codepipelineCodeBuildAdapter) BuildStatus(ctx context.Context, buildID return builds[0].BuildStatus, nil } +func (a *codepipelineCodeBuildAdapter) BuildExportedVariables( + ctx context.Context, buildID string, +) (map[string]string, error) { + builds, _ := a.handler.BackendFor(awsmeta.Region(ctx)).BatchGetBuilds([]string{buildID}) + if len(builds) == 0 { + return nil, fmt.Errorf("%w: build %s not found", codebuildbackend.ErrNotFound, buildID) + } + + vars := make(map[string]string, len(builds[0].ExportedEnvironmentVariables)) + for _, v := range builds[0].ExportedEnvironmentVariables { + vars[v.Name] = v.Value + } + + return vars, nil +} + // wireCodePipelineS3 gives CodePipeline S3 for Source/S3 and Deploy/S3 actions and the // artifact store. s3.InMemoryBackend satisfies codepipeline.ArtifactS3 directly. func wireCodePipelineS3(codepipelineReg, s3Reg service.Registerable) { @@ -13168,6 +13277,8 @@ func setupChaosAndRegistry( wireFISFaultStore(cli.fisHandler, faultStore) wireDynamoDBFaultStore(services, faultStore) + wireSimulatorSCPs(cli.IAMEnforceSCPs, services) + registry, err := setupRegistry( e, log, @@ -13193,6 +13304,7 @@ func setupChaosAndRegistry( wireCloudControlHandlers(e, services) wireServiceRoleAuthorizer(services, cli.EnforceIAM) wireRDSIAMAuth(e, services, cli.EnforceIAM) + wireDSQLAuth(e, services, cli.EnforceIAM) wireNeptuneIAMAuth(services, cli.EnforceIAM) wireDocDBCA(e, services) wireKafkaAuth(e, services, cli.EnforceIAM) @@ -13324,6 +13436,8 @@ func setupRegistry( Recorder: iamRecorder, Audit: iamAudit, SCPs: scps, + ResourceOwners: resourceOwnersFor(scps != nil, services), + RCPExemptions: rcpExemptionsFor(scps != nil, services), } registry.Use(service.Middleware(iambackend.EnforcementMiddleware(iamBackend, ecfg))) @@ -13405,6 +13519,10 @@ func extractStorageResourcePolicyProvider(svc service.Registerable) iambackend.R if b, ok := h.Backend.(sqsPolicyBackend); ok { return &sqsPolicyAdapter{backend: b} } + case *iambackend.Handler: + if b, ok := h.Backend.(iambackend.RoleGetter); ok { + return iambackend.RoleTrustPolicyProvider{Roles: b} + } case *kmsbackend.Handler: if b, ok := h.Backend.(kmsPolicyBackend); ok { return &kmsPolicyAdapter{backend: b} @@ -13633,6 +13751,7 @@ func (a *sqsPolicyAdapter) GetResourcePolicy( out, err := a.backend.GetQueueAttributes(&sqsbackend.GetQueueAttributesInput{ QueueURL: queueURL, AttributeNames: []string{"Policy"}, + Region: accountID + "@" + parts[3], }) if err != nil { return "", err diff --git a/internal/app/cloud_dashboard_handlers.go b/internal/app/cloud_dashboard_handlers.go index 30be537570..0cb0e56122 100644 --- a/internal/app/cloud_dashboard_handlers.go +++ b/internal/app/cloud_dashboard_handlers.go @@ -17,6 +17,31 @@ func (c *CLI) GetAzureEventGridHandler() service.Registerable { return c.azureEv //nolint:ireturn // architecturally required to return interface func (c *CLI) GetAzureAppConfigHandler() service.Registerable { return c.azureAppConfigHandler } +// GetAzureSearchHandler returns the Azure AI Search handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetAzureSearchHandler() service.Registerable { return c.azureSearchHandler } + +// GetAzureRedisHandler returns the Azure Cache for Redis handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetAzureRedisHandler() service.Registerable { return c.azureRedisHandler } + +// GetAzureSQLHandler returns the Azure SQL handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetAzureSQLHandler() service.Registerable { return c.azureSQLHandler } + +// GetAzureACRHandler returns the Azure Container Registry handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetAzureACRHandler() service.Registerable { return c.azureACRHandler } + +// GetLambdaWebHandler returns the Lambda Web Functions handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetLambdaWebHandler() service.Registerable { return c.lambdaWebHandler } + // GetGCPSQLHandler returns the Cloud SQL handler for the dashboard proxy. // //nolint:ireturn // architecturally required to return interface @@ -52,6 +77,12 @@ func storeConsoleHandlers(cli *CLI, byName map[string]service.Registerable) { cli.gcpCloudBuildHandler = byName["GCPCloudBuild"] cli.gcpRedisHandler = byName["GCPRedis"] cli.gcpMonitoringHandler = byName["GCPMonitoring"] + cli.azureGraphHandler = byName["AzureGraph"] + cli.azureSearchHandler = byName["AzureSearch"] + cli.azureRedisHandler = byName["AzureRedis"] + cli.azureSQLHandler = byName["AzureSQL"] + cli.azureACRHandler = byName["AzureContainerRegistry"] + cli.lambdaWebHandler = byName["LambdaWeb"] } // GetGCPWorkflowsHandler returns the GCPWorkflows handler for the dashboard proxy. @@ -193,3 +224,8 @@ func (c *CLI) GetGCPResourceManagerHandler() service.Registerable { return c.gcp // //nolint:ireturn // architecturally required to return interface func (c *CLI) GetGCPBatchHandler() service.Registerable { return c.gcpBatchHandler } + +// GetAzureGraphHandler returns the Microsoft Graph handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetAzureGraphHandler() service.Registerable { return c.azureGraphHandler } diff --git a/internal/app/cloudformation_settings.go b/internal/app/cloudformation_settings.go index bad6d4f0b7..beb6567b35 100644 --- a/internal/app/cloudformation_settings.go +++ b/internal/app/cloudformation_settings.go @@ -1,9 +1,21 @@ package app -// CloudFormationSettings configures CloudFormation template validation. +import "time" + +// DefaultCloudFormationProvisioningDelay is how long each stack step stays *_IN_PROGRESS. +const DefaultCloudFormationProvisioningDelay = 300 * time.Millisecond + +// CloudFormationSettings configures CloudFormation template validation and provisioning. type CloudFormationSettings struct { + ProvisioningDelay time.Duration `name:"provisioning-delay" env:"GOPHERSTACK_CFN_PROVISIONING_DELAY" default:"300ms" help:"How long each stack step stays in *_IN_PROGRESS before completing, like real CloudFormation. 0 completes stacks instantly."` //nolint:lll // config struct tags are intentionally verbose + ValidateProperties bool `name:"validate-properties" env:"GOPHERSTACK_CLOUDFORMATION_VALIDATE_PROPERTIES" default:"true" negatable:"" help:"Reject unknown, wrong-typed and missing required resource properties at stack operations, like CloudFormation. Disable to accept any property."` //nolint:lll // config struct tags are intentionally verbose } // GetCloudFormationValidateProperties reports whether resource property validation is on. func (c *CLI) GetCloudFormationValidateProperties() bool { return c.CloudFormation.ValidateProperties } + +// GetCloudFormationProvisioningDelay returns the per-step *_IN_PROGRESS duration. +func (c *CLI) GetCloudFormationProvisioningDelay() time.Duration { + return c.CloudFormation.ProvisioningDelay +} diff --git a/internal/app/cloudformation_settings_test.go b/internal/app/cloudformation_settings_test.go index bfdcd43163..13b942e4c1 100644 --- a/internal/app/cloudformation_settings_test.go +++ b/internal/app/cloudformation_settings_test.go @@ -2,6 +2,7 @@ package app import ( "testing" + "time" "github.com/alecthomas/kong" "github.com/stretchr/testify/assert" @@ -37,3 +38,45 @@ func TestCloudFormationValidatePropertiesFlag(t *testing.T) { }) } } + +func TestCloudFormationProvisioningDelayFlag(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + args []string + want time.Duration + }{ + {name: "default", want: DefaultCloudFormationProvisioningDelay}, + {name: "instant", args: []string{"--cloudformation-provisioning-delay=0"}}, + {name: "custom", args: []string{"--cloudformation-provisioning-delay=2s"}, want: 2 * time.Second}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var root rootCLI + + parser, err := kong.New(&root, kong.Name("gopherstack"), kong.Writers(nil, nil)) + require.NoError(t, err) + _, err = parser.Parse(tt.args) + require.NoError(t, err) + + assert.Equal(t, tt.want, root.Serve.GetCloudFormationProvisioningDelay()) + }) + } +} + +func TestCloudFormationProvisioningDelayEnv(t *testing.T) { + t.Setenv("GOPHERSTACK_CFN_PROVISIONING_DELAY", "750ms") + + var root rootCLI + + parser, err := kong.New(&root, kong.Name("gopherstack"), kong.Writers(nil, nil)) + require.NoError(t, err) + _, err = parser.Parse(nil) + require.NoError(t, err) + + assert.Equal(t, 750*time.Millisecond, root.Serve.GetCloudFormationProvisioningDelay()) +} diff --git a/internal/app/codebuild_engine.go b/internal/app/codebuild_engine.go index f21493b076..bc6040c8ae 100644 --- a/internal/app/codebuild_engine.go +++ b/internal/app/codebuild_engine.go @@ -5,6 +5,7 @@ import ( "context" "errors" "io" + "time" "github.com/aws/aws-sdk-go-v2/aws" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" @@ -16,6 +17,7 @@ import ( s3backend "github.com/blackbirdworks/gopherstack/services/s3" secretsmanagerbackend "github.com/blackbirdworks/gopherstack/services/secretsmanager" ssmbackend "github.com/blackbirdworks/gopherstack/services/ssm" + stsbackend "github.com/blackbirdworks/gopherstack/services/sts" ) // wireCodeBuildEngine gives docker-engine CodeBuild access to S3, CloudWatch Logs, CodeCommit, SSM and Secrets Manager. @@ -51,6 +53,54 @@ func wireCodeBuildEngine(byName map[string]service.Registerable) { cbH.SetEngineServices(svc) } +// codebuildCredentials issues service-role credentials from STS. +type codebuildCredentials struct { + byName map[string]service.Registerable +} + +func (c *codebuildCredentials) IssueBuildCredentials( + _ context.Context, roleArn, sessionName string, +) (codebuildbackend.BuildCredentials, error) { + stsH, ok := c.byName["STS"].(*stsbackend.Handler) + if !ok { + return codebuildbackend.BuildCredentials{}, errCodeBuildBackendMissing + } + + stsBk, ok := stsH.Backend.(*stsbackend.InMemoryBackend) + if !ok { + return codebuildbackend.BuildCredentials{}, errCodeBuildBackendMissing + } + + out, err := stsBk.AssumeRoleForService(codebuildbackend.ServicePrincipal(), roleArn, sessionName) + if err != nil { + return codebuildbackend.BuildCredentials{}, err + } + + creds := out.AssumeRoleResult.Credentials + + exp, err := time.Parse(time.RFC3339, creds.Expiration) + if err != nil { + return codebuildbackend.BuildCredentials{}, err + } + + return codebuildbackend.BuildCredentials{ + AccessKeyID: creds.AccessKeyID, SecretAccessKey: creds.SecretAccessKey, + SessionToken: creds.SessionToken, Expires: exp, + }, nil +} + +// wireCodeBuildRuntimeEnv gives docker-engine builds the emulator endpoint and service-role credentials. +func wireCodeBuildRuntimeEnv(cli *CLI, byName map[string]service.Registerable) { + cbH, ok := byName["CodeBuild"].(*codebuildbackend.Handler) + if !ok || !cbH.Backend.EngineEnabled() || cli == nil || cli.Port == "" { + return + } + + cbH.Backend.SetEngineRuntime( + lambdaEndpointURL(cli.Lambda.DockerHost, cli.Port), &codebuildCredentials{byName: byName}, + ) +} + var errCodeBuildBackendMissing = errors.New("backend is not available") type codebuildS3 struct{ backend s3backend.StorageBackend } diff --git a/internal/app/codebuild_engine_test.go b/internal/app/codebuild_engine_test.go new file mode 100644 index 0000000000..9afb96ecb5 --- /dev/null +++ b/internal/app/codebuild_engine_test.go @@ -0,0 +1,50 @@ +package app + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + stsbackend "github.com/blackbirdworks/gopherstack/services/sts" +) + +func TestCodeBuildCredentialsIssuedFromSTS(t *testing.T) { + t.Parallel() + + tests := []struct { + byName map[string]service.Registerable + name string + wantErr bool + }{ + { + name: "sts wired", + byName: map[string]service.Registerable{ + "STS": stsbackend.NewHandler(stsbackend.NewInMemoryBackend()), + }, + }, + {name: "sts missing", byName: map[string]service.Registerable{}, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + c := &codebuildCredentials{byName: tt.byName} + creds, err := c.IssueBuildCredentials(t.Context(), "arn:aws:iam::000000000000:role/svc", "AWSCodeBuild-x") + + if tt.wantErr { + require.Error(t, err) + + return + } + + require.NoError(t, err) + assert.NotEmpty(t, creds.AccessKeyID) + assert.NotEmpty(t, creds.SecretAccessKey) + assert.NotEmpty(t, creds.SessionToken) + assert.False(t, creds.Expires.IsZero()) + }) + } +} diff --git a/internal/app/cognito_delivery.go b/internal/app/cognito_delivery.go index 0fb3c048ae..65ce4d8a46 100644 --- a/internal/app/cognito_delivery.go +++ b/internal/app/cognito_delivery.go @@ -145,16 +145,20 @@ func wireCognitoMessageDelivery(cognitoReg, sesReg, snsReg service.Registerable, cognitoH.Backend.SetMessageSender(sender) } -// cognitoKMSEncrypter encrypts custom-sender codes through the KMS emulator. -type cognitoKMSEncrypter struct{ kms kmsbackend.StorageBackend } - -func (e cognitoKMSEncrypter) EncryptCode(ctx context.Context, keyID string, plaintext []byte) ([]byte, error) { - out, err := e.kms.Encrypt(ctx, &kmsbackend.EncryptInput{KeyID: keyID, Plaintext: plaintext}) +// cognitoKMSDataKeys generates the data keys custom-sender codes are encrypted under. +type cognitoKMSDataKeys struct{ kms kmsbackend.StorageBackend } + +func (e cognitoKMSDataKeys) GenerateDataKey( + ctx context.Context, keyID string, encCtx map[string]string, +) (*cognitoidpbackend.DataKey, error) { + out, err := e.kms.GenerateDataKey(ctx, &kmsbackend.GenerateDataKeyInput{ + KeyID: keyID, KeySpec: "AES_256", EncryptionContext: encCtx, + }) if err != nil { - return nil, fmt.Errorf("kms encrypt: %w", err) + return nil, fmt.Errorf("kms generate data key: %w", err) } - return out.CiphertextBlob, nil + return &cognitoidpbackend.DataKey{KeyARN: out.KeyID, Plaintext: out.Plaintext, Ciphertext: out.CiphertextBlob}, nil } // wireCognitoCodeEncryption lets CustomEmailSender/CustomSMSSender triggers receive KMS-encrypted codes. @@ -166,5 +170,5 @@ func wireCognitoCodeEncryption(cognitoReg, kmsReg service.Registerable) { return } - cognitoH.Backend.SetCodeEncrypter(cognitoKMSEncrypter{kms: kmsH.Backend}) + cognitoH.Backend.SetDataKeyGenerator(cognitoKMSDataKeys{kms: kmsH.Backend}) } diff --git a/internal/app/cognito_kms_datakey_test.go b/internal/app/cognito_kms_datakey_test.go new file mode 100644 index 0000000000..4f0ec4a003 --- /dev/null +++ b/internal/app/cognito_kms_datakey_test.go @@ -0,0 +1,55 @@ +package app + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" +) + +func TestCognitoKMSDataKeys(t *testing.T) { + t.Parallel() + + tests := []struct { + decrypt map[string]string + name string + wantErr bool + }{ + {name: "same context decrypts", decrypt: map[string]string{"userpool-id": "pool-1"}}, + {name: "other context rejected", decrypt: map[string]string{"userpool-id": "pool-2"}, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kms := kmsbackend.NewInMemoryBackend() + + created, err := kms.CreateKey(ctx, &kmsbackend.CreateKeyInput{}) + require.NoError(t, err) + + dk, err := cognitoKMSDataKeys{kms: kms}.GenerateDataKey( + ctx, created.KeyMetadata.KeyID, map[string]string{"userpool-id": "pool-1"}, + ) + require.NoError(t, err) + assert.Equal(t, created.KeyMetadata.Arn, dk.KeyARN) + assert.Len(t, dk.Plaintext, 32) + + out, err := kms.Decrypt(ctx, &kmsbackend.DecryptInput{ + CiphertextBlob: dk.Ciphertext, EncryptionContext: tt.decrypt, + }) + if tt.wantErr { + require.Error(t, err) + + return + } + + require.NoError(t, err) + assert.Equal(t, dk.Plaintext, out.Plaintext) + }) + } +} diff --git a/internal/app/connect_wiring.go b/internal/app/connect_wiring.go index 8f84d5b75f..a7f1508941 100644 --- a/internal/app/connect_wiring.go +++ b/internal/app/connect_wiring.go @@ -2,6 +2,7 @@ package app import ( "context" + "io" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" @@ -11,8 +12,20 @@ import ( ebbackend "github.com/blackbirdworks/gopherstack/services/eventbridge" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" s3backend "github.com/blackbirdworks/gopherstack/services/s3" + sesbackend "github.com/blackbirdworks/gopherstack/services/ses" ) +// connectMailer delivers Connect outbound email through SES. +type connectMailer struct{ ses *sesbackend.InMemoryBackend } + +func (m connectMailer) SendEmail(_ context.Context, msg connectbackend.EmailMessage) error { + _, err := m.ses.SendEmail(sesbackend.SendEmailInput{ + From: msg.From, To: msg.To, Cc: msg.Cc, Subject: msg.Subject, BodyHTML: msg.HTML, BodyText: msg.Text, + }) + + return err +} + // connectResolvers checks Lambda functions and S3 objects named by Connect configuration. type connectResolvers struct { lambda lambdabackend.StorageBackend @@ -37,6 +50,17 @@ func (r connectResolvers) ObjectExists(ctx context.Context, bucket, key string) return err == nil } +func (r connectResolvers) ReadObject(ctx context.Context, bucket, key string) ([]byte, error) { + out, err := r.s3.GetObject(ctx, &awss3.GetObjectInput{Bucket: &bucket, Key: &key}) + if err != nil { + return nil, err + } + + defer out.Body.Close() + + return io.ReadAll(out.Body) +} + // connectIntegrations registers Connect integration associations with AppIntegrations. type connectIntegrations struct { ai *appintegrationsbackend.InMemoryBackend @@ -109,6 +133,12 @@ func wireConnectIntegrations(byName map[string]service.Registerable) { cn.Backend.SetIntegrations(connectIntegrations{ai: ai.Backend}) } + if sesH, ok := byName["SES"].(*sesbackend.Handler); ok { + if bk, bkOK := sesH.Backend.(*sesbackend.InMemoryBackend); bkOK { + cn.Backend.SetMailer(connectMailer{ses: bk}) + } + } + lambdaH, lok := byName["Lambda"].(*lambdabackend.Handler) s3H, sok := byName["S3"].(*s3backend.S3Handler) diff --git a/internal/app/connect_wiring_test.go b/internal/app/connect_wiring_test.go index b756c2b618..779f6f4699 100644 --- a/internal/app/connect_wiring_test.go +++ b/internal/app/connect_wiring_test.go @@ -12,6 +12,7 @@ import ( connecttypes "github.com/aws/aws-sdk-go-v2/service/connect/types" "github.com/aws/aws-sdk-go-v2/service/eventbridge" "github.com/aws/aws-sdk-go-v2/service/s3" + sessdk "github.com/aws/aws-sdk-go-v2/service/ses" "github.com/aws/smithy-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,6 +23,7 @@ type connectWiringEnv struct { integrations *aisdk.Client events *eventbridge.Client objects *s3.Client + mail *sessdk.Client instance *connectsdk.CreateInstanceOutput } @@ -42,6 +44,7 @@ func TestInitializeServices_ConnectWiring(t *testing.T) { integrations: aisdk.NewFromConfig(cfg), events: eventbridge.NewFromConfig(cfg), objects: s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true }), + mail: sessdk.NewFromConfig(cfg), } tests := []struct { @@ -59,6 +62,7 @@ func TestInitializeServices_ConnectWiring(t *testing.T) { assert.Equal(t, "InvalidRequestException", connectErrCode(err)) }}, {name: "prompts read S3 objects", run: connectPromptCase}, + {name: "outbound email is delivered through SES", run: connectOutboundEmailCase}, {name: "storage config needs its bucket", run: func(t *testing.T, e *connectWiringEnv) { t.Helper() @@ -186,3 +190,61 @@ func connectPromptCase(t *testing.T, e *connectWiringEnv) { _, err = e.connect.CreatePrompt(t.Context(), prompt) require.NoError(t, err) } + +func connectOutboundEmailCase(t *testing.T, e *connectWiringEnv) { + t.Helper() + + _, err := e.mail.VerifyEmailIdentity(t.Context(), &sessdk.VerifyEmailIdentityInput{ + EmailAddress: aws.String("support@wired.example.com"), + }) + require.NoError(t, err) + + _, err = e.connect.CreateEmailAddress(t.Context(), &connectsdk.CreateEmailAddressInput{ + InstanceId: e.instance.Id, EmailAddress: aws.String("support@wired.example.com"), + }) + require.NoError(t, err) + + rp, err := e.connect.ListRoutingProfiles( + t.Context(), + &connectsdk.ListRoutingProfilesInput{InstanceId: e.instance.Id}, + ) + require.NoError(t, err) + + sp, err := e.connect.ListSecurityProfiles( + t.Context(), &connectsdk.ListSecurityProfilesInput{InstanceId: e.instance.Id}) + require.NoError(t, err) + + agent, err := e.connect.CreateUser(t.Context(), &connectsdk.CreateUserInput{ + InstanceId: e.instance.Id, Username: aws.String("mailer"), Password: aws.String("Passw0rd!Passw0rd"), + RoutingProfileId: rp.RoutingProfileSummaryList[0].Id, + SecurityProfileIds: []string{aws.ToString(sp.SecurityProfileSummaryList[0].Id)}, + PhoneConfig: &connecttypes.UserPhoneConfig{PhoneType: connecttypes.PhoneTypeSoftPhone}, + }) + require.NoError(t, err) + + userID := agent.UserId + + contact, err := e.connect.CreateContact(t.Context(), &connectsdk.CreateContactInput{ + InstanceId: e.instance.Id, Channel: connecttypes.ChannelEmail, + InitiationMethod: connecttypes.ContactInitiationMethodOutbound, + UserInfo: &connecttypes.UserInfo{UserId: userID}, + }) + require.NoError(t, err) + + _, err = e.connect.StartOutboundEmailContact(t.Context(), &connectsdk.StartOutboundEmailContactInput{ + InstanceId: e.instance.Id, ContactId: contact.ContactId, + FromEmailAddress: &connecttypes.EmailAddressInfo{EmailAddress: aws.String("support@wired.example.com")}, + DestinationEmailAddress: &connecttypes.EmailAddressInfo{EmailAddress: aws.String("cust@example.org")}, + EmailMessage: &connecttypes.OutboundEmailContent{ + MessageSourceType: connecttypes.OutboundMessageSourceTypeRaw, + RawMessage: &connecttypes.OutboundRawMessage{ + Subject: aws.String("hi"), Body: aws.String("body"), ContentType: aws.String("text/plain"), + }, + }, + }) + require.NoError(t, err) + + quota, err := e.mail.GetSendQuota(t.Context(), &sessdk.GetSendQuotaInput{}) + require.NoError(t, err) + assert.InDelta(t, 1.0, quota.SentLast24Hours, 0.01) +} diff --git a/internal/app/cosmosdb_port_reservation_test.go b/internal/app/cosmosdb_port_reservation_test.go index d5f82f4db0..d0689a46ea 100644 --- a/internal/app/cosmosdb_port_reservation_test.go +++ b/internal/app/cosmosdb_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" cosmosdbbackend "github.com/blackbirdworks/gopherstack/services/cosmosdb" ) @@ -35,18 +33,16 @@ func TestReserveFixedServicePorts_CosmosDB(t *testing.T) { tests := []struct { name string cosmosPort int - rangeStart int - rangeEnd int wantBlockedFromPool bool }{ { - name: "default cosmosdb port falls outside the default pool range", - cosmosPort: cosmosdbbackend.DefaultPort, rangeStart: 10000, rangeEnd: 10100, + name: "default cosmosdb port falls outside the default pool range", + cosmosPort: cosmosdbbackend.DefaultPort, wantBlockedFromPool: false, }, { - name: "default cosmosdb port falls inside a custom pool range", - cosmosPort: cosmosdbbackend.DefaultPort, rangeStart: 8000, rangeEnd: 8100, + name: "default cosmosdb port falls inside a custom pool range", + cosmosPort: cosmosdbbackend.DefaultPort, wantBlockedFromPool: true, }, } @@ -55,13 +51,12 @@ func TestReserveFixedServicePorts_CosmosDB(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantBlockedFromPool, tt.cosmosPort) - cli := CLI{CosmosDB: cosmosdbbackend.Settings{Port: tt.cosmosPort}} + cli := CLI{CosmosDB: cosmosdbbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(tt.cosmosPort), tt.name) + assert.Equal(t, tt.wantBlockedFromPool, alloc.IsAllocated(port), tt.name) }) } } diff --git a/internal/app/cross_service_region_test.go b/internal/app/cross_service_region_test.go index 0acfa03936..522c8e874b 100644 --- a/internal/app/cross_service_region_test.go +++ b/internal/app/cross_service_region_test.go @@ -10,7 +10,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/awsmeta" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" cwbackend "github.com/blackbirdworks/gopherstack/services/cloudwatch" cwlogsbackend "github.com/blackbirdworks/gopherstack/services/cloudwatchlogs" @@ -30,8 +30,7 @@ func crossRegionServices(t *testing.T) map[string]service.Registerable { t.Helper() cli := &CLI{AccountID: crossAcct, Region: crossHome} - portAlloc, err := portalloc.New(19600, 19700) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) cli.faultStore = chaos.NewFaultStore() diff --git a/internal/app/dms_rules_docker_test.go b/internal/app/dms_rules_docker_test.go new file mode 100644 index 0000000000..da6f1b627f --- /dev/null +++ b/internal/app/dms_rules_docker_test.go @@ -0,0 +1,324 @@ +package app + +import ( + "fmt" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice" + dmstypes "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice/types" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const filterItemsRule = `{"rule-type":"selection","rule-id":"1","rule-name":"1","rule-action":"include",` + + `"object-locator":{"schema-name":"appdb","table-name":"items"},"filters":[` + + `{"filter-type":"source","column-name":"id","filter-conditions":[{"filter-operator":"ste","value":"5"}]},` + + `{"filter-type":"source","column-name":"qty","filter-conditions":[{"filter-operator":"gte","value":"10"}]}]}` + +func columnRule(id, action, column, extra string) string { + return fmt.Sprintf(`{"rule-type":"transformation","rule-id":"%s","rule-name":"%s","rule-target":"column",`+ + `"rule-action":"%s","object-locator":{"schema-name":"appdb","table-name":"items","column-name":"%s"}%s}`, + id, id, action, column, extra) +} + +func TestDMSDockerFiltersAndColumnRules(t *testing.T) { + t.Parallel() + requireDMSDocker(t, "mysql:8.4", "postgres:16") + + h := newDMSHarness(t) + srcEP, tgtEP := h.createDB(t, "src", "mysql", "8.4"), h.createDB(t, "tgt", "postgres", "16") + src, tgt := openDB(t, "mysql", srcEP), openDB(t, "postgres", tgtEP) + + mustExec(t, src, + "CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(40), qty INT)", + "CREATE TABLE logs (id INT PRIMARY KEY)", + "INSERT INTO items VALUES (1,'alpha',10),(2,'beta',5),(3,'gamma',30),(6,'delta',60)", + "INSERT INTO logs VALUES (1)") + + ri := h.instance(t) + srcARN := h.endpoint(t, "src-ep", dmstypes.ReplicationEndpointTypeValueSource, "mysql", &srcEP, nil) + tgtARN := h.endpoint(t, "tgt-ep", dmstypes.ReplicationEndpointTypeValueTarget, "postgres", &tgtEP, nil) + + mapping := mappingRules( + filterItemsRule, + `{"rule-type":"selection","rule-id":"2","rule-name":"2","rule-action":"include",`+ + `"object-locator":{"schema-name":"appdb","table-name":"log_"}}`, + columnRule("3", "add-column", "label", + `,"value":"label","expression":"upper($name)||'-'||$id","data-type":{"type":"string","length":30}`), + columnRule("4", "change-data-type", "qty", `,"data-type":{"type":"string","length":"12"}`), + ) + + taskARN := h.task(t, "rules", srcARN, tgtARN, ri, "full-load-and-cdc", mapping) + h.awaitStatus(t, taskARN, "running") + + const q = "SELECT id::text, name, qty, label FROM appdb.items ORDER BY id" + + awaitRows(t, tgt, q, [][]string{{"1", "alpha", "10", "ALPHA-1"}, {"3", "gamma", "30", "GAMMA-3"}}) + awaitRows(t, tgt, "SELECT id::text FROM appdb.logs", [][]string{{"1"}}) + + types := queryRows(t, tgt, "SELECT column_name, data_type, COALESCE(character_maximum_length::text,'') "+ + "FROM information_schema.columns WHERE table_schema='appdb' AND table_name='items' ORDER BY ordinal_position") + assert.Equal(t, [][]string{ + {"id", "integer", ""}, {"name", "character varying", "40"}, {"qty", "character varying", "12"}, + {"label", "character varying", "30"}, + }, types) + + mustExec(t, src, + "INSERT INTO items VALUES (4,'epsilon',40),(5,'low',1),(9,'high-id',90)", + "UPDATE items SET name='ALPHA2' WHERE id=1", + "UPDATE items SET qty=1 WHERE id=3", + "UPDATE items SET qty=70 WHERE id=2") + awaitRows(t, tgt, q, [][]string{ + {"1", "ALPHA2", "10", "ALPHA2-1"}, {"2", "beta", "70", "BETA-2"}, {"4", "epsilon", "40", "EPSILON-4"}, + }) + + h.stop(t, taskARN) +} + +func TestDMSDockerS3SourceCSVAndCDC(t *testing.T) { + t.Parallel() + requireDMSDocker(t, "postgres:16") + + h := newDMSHarness(t) + tgtEP := h.createDB(t, "tgt", "postgres", "16") + tgt := openDB(t, "postgres", tgtEP) + + s3c := s3.NewFromConfig(h.cfg, func(o *s3.Options) { o.UsePathStyle = true }) + _, err := s3c.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String("dms-in")}) + require.NoError(t, err) + + put := func(key, body string) { + _, perr := s3c.PutObject(t.Context(), &s3.PutObjectInput{ + Bucket: aws.String("dms-in"), Key: aws.String(key), Body: strings.NewReader(body), + }) + require.NoError(t, perr) + } + + put("src/hr/employee/LOAD001.csv", "101,Smith,2014-06-04 10:00:00.123,12.50\n102,Jones,2015-10-08 00:00:00.000,7\n") + + const definition = `{"TableCount":"1","Tables":[{"TableName":"employee","TablePath":"hr/employee/",` + + `"TableOwner":"hr","TableColumns":[` + + `{"ColumnName":"Id","ColumnType":"INT8","ColumnNullable":"false","ColumnIsPk":"true"},` + + `{"ColumnName":"LastName","ColumnType":"STRING","ColumnLength":"20","ColumnNullable":"true"},` + + `{"ColumnName":"HireDate","ColumnType":"DATETIME","ColumnScale":"3","ColumnNullable":"true"},` + + `{"ColumnName":"Rate","ColumnType":"NUMERIC","ColumnPrecision":"5","ColumnScale":"2","ColumnNullable":"true"}],` + + `"TableColumnsTotal":"4"}]}` + + ri := h.instance(t) + srcARN := h.endpoint(t, "s3-src", dmstypes.ReplicationEndpointTypeValueSource, "s3", nil, + func(in *databasemigrationservice.CreateEndpointInput) { + in.ExternalTableDefinition = aws.String(definition) + in.S3Settings = &dmstypes.S3Settings{ + BucketName: aws.String("dms-in"), BucketFolder: aws.String("src"), CdcPath: aws.String("changes"), + ServiceAccessRoleArn: aws.String(dmsRoleARN), + } + }) + tgtARN := h.endpoint(t, "tgt-ep", dmstypes.ReplicationEndpointTypeValueTarget, "postgres", &tgtEP, nil) + + conn, err := h.dms.TestConnection(t.Context(), &databasemigrationservice.TestConnectionInput{ + ReplicationInstanceArn: aws.String(ri), EndpointArn: aws.String(srcARN), + }) + require.NoError(t, err) + assert.Equal(t, "successful", aws.ToString(conn.Connection.Status)) + + taskARN := h.task(t, "s3-to-pg", srcARN, tgtARN, ri, "full-load-and-cdc", `{"rules":[`+ + `{"rule-type":"selection","rule-id":"1","rule-name":"1","rule-action":"include",`+ + `"object-locator":{"schema-name":"hr","table-name":"employee"}}]}`) + h.awaitStatus(t, taskARN, "running") + + const q = `SELECT "Id"::text, "LastName", to_char("HireDate",'YYYY-MM-DD HH24:MI:SS.MS'), "Rate"::text ` + + `FROM hr.employee ORDER BY "Id"` + + awaitRows(t, tgt, q, [][]string{ + {"101", "Smith", "2014-06-04 10:00:00.123", "12.50"}, {"102", "Jones", "2015-10-08 00:00:00.000", "7.00"}, + }) + + put("src/changes/CDC00001.csv", + "INSERT,employee,hr,103,Brown,2017-03-13 00:00:00.000,9.99\n"+ + "U,employee,hr,101,Smith-Updated,2014-06-04 10:00:00.123,13.00\n"+ + "D,employee,hr,102,Jones,2015-10-08 00:00:00.000,7.00\n") + awaitRows(t, tgt, q, [][]string{ + { + "101", + "Smith-Updated", + "2014-06-04 10:00:00.123", + "13.00", + }, {"103", "Brown", "2017-03-13 00:00:00.000", "9.99"}, + }) + + require.Eventually(t, func() bool { + s := h.tableStats(t, taskARN)["hr.employee"] + + return s.FullLoadRows == 2 && s.Inserts == 1 && s.Updates == 1 && s.Deletes == 1 + }, dmsDockerWait, dmsPoll) + + h.stop(t, taskARN) +} + +func TestDMSDockerEndpointFromSecretsManager(t *testing.T) { + t.Parallel() + requireDMSDocker(t, "mysql:8.4", "postgres:16") + + h := newDMSHarness(t) + srcEP, tgtEP := h.createDB(t, "src", "mysql", "8.4"), h.createDB(t, "tgt", "postgres", "16") + src, tgt := openDB(t, "mysql", srcEP), openDB(t, "postgres", tgtEP) + + mustExec( + t, + src, + "CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(40))", + "INSERT INTO items VALUES (1,'alpha')", + ) + + sm := secretsmanager.NewFromConfig(h.cfg) + secret := func(name, password string) string { + out, err := sm.CreateSecret(t.Context(), &secretsmanager.CreateSecretInput{ + Name: aws.String(name), SecretString: aws.String(fmt.Sprintf( + `{"username":"master","password":%q,"host":%q,"port":"%d"}`, password, srcEP.host, srcEP.port)), + }) + require.NoError(t, err) + + return aws.ToString(out.ARN) + } + + secretEndpoint := func(id, secretID string) string { + return h.endpoint(t, id, dmstypes.ReplicationEndpointTypeValueSource, "mysql", nil, + func(in *databasemigrationservice.CreateEndpointInput) { + in.MySQLSettings = &dmstypes.MySQLSettings{ + DatabaseName: aws.String("appdb"), SecretsManagerSecretId: aws.String(secretID), + SecretsManagerAccessRoleArn: aws.String(dmsRoleARN), + } + }) + } + + ri := h.instance(t) + goodARN := secretEndpoint("good-ep", secret("dms/good", dmsMasterPass)) + badARN := secretEndpoint("bad-ep", secret("dms/bad", "wrong-password-xyz")) + tgtARN := h.endpoint(t, "tgt-ep", dmstypes.ReplicationEndpointTypeValueTarget, "postgres", &tgtEP, nil) + + test := func(arn string) *dmstypes.Connection { + out, err := h.dms.TestConnection(t.Context(), &databasemigrationservice.TestConnectionInput{ + ReplicationInstanceArn: aws.String(ri), EndpointArn: aws.String(arn), + }) + require.NoError(t, err) + + return out.Connection + } + + assert.Equal(t, "successful", aws.ToString(test(goodARN).Status)) + + bad := test(badARN) + assert.Equal(t, "failed", aws.ToString(bad.Status)) + assert.NotContains(t, aws.ToString(bad.LastFailureMessage), "wrong-password-xyz") + assert.NotContains(t, aws.ToString(bad.LastFailureMessage), dmsMasterPass) + + taskARN := h.task(t, "secret-task", goodARN, tgtARN, ri, "full-load", tableMapping("items")) + h.awaitStatus(t, taskARN, "stopped") + assert.Equal(t, [][]string{{"1", "alpha"}}, queryRows(t, tgt, "SELECT id::text, name FROM appdb.items")) +} + +func TestDMSDockerServerlessReplication(t *testing.T) { + t.Parallel() + requireDMSDocker(t, "mysql:8.4", "postgres:16") + + h := newDMSHarness(t) + srcEP, tgtEP := h.createDB(t, "src", "mysql", "8.4"), h.createDB(t, "tgt", "postgres", "16") + src, tgt := openDB(t, "mysql", srcEP), openDB(t, "postgres", tgtEP) + + mustExec(t, src, "CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(40))", + "INSERT INTO items VALUES (1,'alpha'),(2,'beta')") + + srcARN := h.endpoint(t, "src-ep", dmstypes.ReplicationEndpointTypeValueSource, "mysql", &srcEP, nil) + tgtARN := h.endpoint(t, "tgt-ep", dmstypes.ReplicationEndpointTypeValueTarget, "postgres", &tgtEP, nil) + + cfg, err := h.dms.CreateReplicationConfig(t.Context(), &databasemigrationservice.CreateReplicationConfigInput{ + ReplicationConfigIdentifier: aws.String( + "serverless", + ), ReplicationType: dmstypes.MigrationTypeValueFullLoadAndCdc, + SourceEndpointArn: aws.String(srcARN), TargetEndpointArn: aws.String(tgtARN), + TableMappings: aws.String(tableMapping("items")), + ComputeConfig: &dmstypes.ComputeConfig{MinCapacityUnits: aws.Int32(2), MaxCapacityUnits: aws.Int32(8)}, + }) + require.NoError(t, err) + + cfgARN := aws.ToString(cfg.ReplicationConfig.ReplicationConfigArn) + replication := func() dmstypes.Replication { + out, derr := h.dms.DescribeReplications(t.Context(), &databasemigrationservice.DescribeReplicationsInput{ + Filters: []dmstypes.Filter{{Name: aws.String("replication-config-arn"), Values: []string{cfgARN}}}, + }) + require.NoError(t, derr) + require.Len(t, out.Replications, 1) + + return out.Replications[0] + } + await := func(status string) dmstypes.Replication { + var r dmstypes.Replication + + require.Eventually(t, func() bool { + r = replication() + if aws.ToString(r.Status) == "failed" { + require.FailNow(t, "replication failed: "+strings.Join(r.FailureMessages, "; ")) + } + + return aws.ToString(r.Status) == status + }, dmsDockerWait, dmsPoll) + + return r + } + start := func(kind string) { + _, serr := h.dms.StartReplication(t.Context(), &databasemigrationservice.StartReplicationInput{ + ReplicationConfigArn: aws.String(cfgARN), StartReplicationType: aws.String(kind), + }) + require.NoError(t, serr) + } + + start("start-replication") + await("running") + + const q = "SELECT id::text, name FROM appdb.items ORDER BY id" + + awaitRows(t, tgt, q, [][]string{{"1", "alpha"}, {"2", "beta"}}) + + mustExec(t, src, "INSERT INTO items VALUES (3,'gamma')", "UPDATE items SET name='BETA' WHERE id=2") + awaitRows(t, tgt, q, [][]string{{"1", "alpha"}, {"2", "BETA"}, {"3", "gamma"}}) + + require.Eventually(t, func() bool { + stats, serr := h.dms.DescribeReplicationTableStatistics(t.Context(), + &databasemigrationservice.DescribeReplicationTableStatisticsInput{ReplicationConfigArn: aws.String(cfgARN)}) + if serr != nil || len(stats.ReplicationTableStatistics) != 1 { + return false + } + + s := stats.ReplicationTableStatistics[0] + + return s.FullLoadRows == 2 && s.Inserts == 1 && s.Updates == 1 + }, dmsDockerWait, dmsPoll) + + running := replication() + require.NotNil(t, running.ReplicationStats) + assert.EqualValues(t, 1, running.ReplicationStats.TablesLoaded) + assert.EqualValues(t, 100, running.ReplicationStats.FullLoadProgressPercent) + require.NotNil(t, running.ProvisionData) + assert.EqualValues(t, 2, running.ProvisionData.ProvisionedCapacityUnits) + + _, err = h.dms.StopReplication(t.Context(), &databasemigrationservice.StopReplicationInput{ + ReplicationConfigArn: aws.String(cfgARN), + }) + require.NoError(t, err) + assert.Equal(t, "Stop Reason NORMAL", aws.ToString(await("stopped").StopReason)) + + mustExec(t, src, "INSERT INTO items VALUES (4,'delta')") + start("resume-processing") + await("running") + awaitRows(t, tgt, q, [][]string{{"1", "alpha"}, {"2", "BETA"}, {"3", "gamma"}, {"4", "delta"}}) + + _, err = h.dms.StopReplication(t.Context(), &databasemigrationservice.StopReplicationInput{ + ReplicationConfigArn: aws.String(cfgARN), + }) + require.NoError(t, err) + await("stopped") +} diff --git a/internal/app/dms_stores.go b/internal/app/dms_stores.go index 5a9ded327e..c62d77f559 100644 --- a/internal/app/dms_stores.go +++ b/internal/app/dms_stores.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dynamodb" @@ -17,6 +18,7 @@ import ( ddbbackend "github.com/blackbirdworks/gopherstack/services/dynamodb" kinesisbackend "github.com/blackbirdworks/gopherstack/services/kinesis" s3backend "github.com/blackbirdworks/gopherstack/services/s3" + secretsmanagerbackend "github.com/blackbirdworks/gopherstack/services/secretsmanager" ) // wireRDSIntegrations connects the RDS Data API and DMS to the services they read and write. @@ -53,6 +55,25 @@ func wireDMSStores(byName map[string]service.Registerable) { } dmsH.Backend.SetStores(stores) + + if h, hok := byName["SecretsManager"].(*secretsmanagerbackend.Handler); hok { + dmsH.Backend.SetSecrets(dmsSecrets{bk: h.Backend}) + } +} + +// dmsSecrets resolves endpoint SecretsManagerSecretId values against the in-process Secrets Manager. +type dmsSecrets struct { + bk secretsmanagerbackend.StorageBackend +} + +func (d dmsSecrets) SecretString(ctx context.Context, region, secretID string) (string, error) { + out, err := d.bk.GetSecretValue(secretsmanagerbackend.WithRegion(ctx, region), + &secretsmanagerbackend.GetSecretValueInput{SecretID: secretID}) + if err != nil { + return "", err + } + + return out.SecretString, nil } type dmsObjects struct{ bk *s3backend.InMemoryBackend } @@ -65,6 +86,42 @@ func (o *dmsObjects) PutObject(ctx context.Context, bucket, key string, body []b return err } +func (o *dmsObjects) ListObjects(ctx context.Context, bucket, prefix string) ([]string, error) { + var ( + keys []string + token *string + ) + + for { + out, err := o.bk.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), Prefix: aws.String(prefix), ContinuationToken: token, + }) + if err != nil { + return nil, err + } + + for _, obj := range out.Contents { + keys = append(keys, aws.ToString(obj.Key)) + } + + if !aws.ToBool(out.IsTruncated) || out.NextContinuationToken == nil { + return keys, nil + } + + token = out.NextContinuationToken + } +} + +func (o *dmsObjects) GetObject(ctx context.Context, bucket, key string) ([]byte, error) { + out, err := o.bk.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)}) + if err != nil { + return nil, err + } + defer out.Body.Close() + + return io.ReadAll(out.Body) +} + func (o *dmsObjects) CheckBucket(ctx context.Context, bucket string) error { _, err := o.bk.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)}) diff --git a/internal/app/dsql_wiring.go b/internal/app/dsql_wiring.go new file mode 100644 index 0000000000..2a96a602d0 --- /dev/null +++ b/internal/app/dsql_wiring.go @@ -0,0 +1,102 @@ +package app + +import ( + "net/http" + "path/filepath" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/dsql" + iambackend "github.com/blackbirdworks/gopherstack/services/iam" + stsbackend "github.com/blackbirdworks/gopherstack/services/sts" +) + +const ( + dsqlCAPath = "/_gopherstack/dsql/ca.pem" + dsqlClusterPath = "/_gopherstack/dsql/clusters/:id" +) + +// GetDSQLCADir persists the docker-mode DSQL CA under the data dir when --persist is on (dsql.CADirConfig). +func (c *CLI) GetDSQLCADir() string { + if !c.Persist { + return "" + } + + return filepath.Join(c.resolvedDataDir(), "dsql-ca") +} + +// wireDSQLAuth connects DSQL IAM-token authentication to IAM and STS and serves the data-plane CA and each +// cluster's connection address. +func wireDSQLAuth(e *echo.Echo, services []service.Registerable, enforceIAM bool) { + var ( + dsqlH *dsql.Handler + iamH *iambackend.Handler + stsH *stsbackend.Handler + ) + + for _, svc := range services { + switch h := svc.(type) { + case *dsql.Handler: + dsqlH = h + case *iambackend.Handler: + iamH = h + case *stsbackend.Handler: + stsH = h + } + } + + if dsqlH == nil { + return + } + + bk, ok := dsqlH.Backend.(*dsql.InMemoryBackend) + if !ok { + return + } + + bk.SetIAMAuth(func() dsql.IAMAuthConfig { return dsqlIAMAuthConfig(iamH, stsH, enforceIAM) }) + + e.GET(dsqlCAPath, func(c *echo.Context) error { + pem := bk.EngineCAPEM() + if len(pem) == 0 { + return c.String(http.StatusNotFound, "DSQL docker engine is not enabled") + } + + return c.Blob(http.StatusOK, "application/x-pem-file", pem) + }) + + e.GET(dsqlClusterPath, func(c *echo.Context) error { + id := c.Param("id") + + host, port, found := bk.ConnectAddr(id) + if !found { + return c.String(http.StatusNotFound, "cluster has no running data plane") + } + + cl, err := bk.GetCluster(id) + if err != nil { + return c.String(http.StatusNotFound, "cluster not found") + } + + return c.JSON(http.StatusOK, map[string]any{"host": host, "port": port, "endpoint": cl.Endpoint}) + }) +} + +func dsqlIAMAuthConfig(iamH *iambackend.Handler, stsH *stsbackend.Handler, enforceIAM bool) dsql.IAMAuthConfig { + cfg := dsql.IAMAuthConfig{Enforce: enforceIAM} + + if iamH != nil { + if ids, ok := iamH.Backend.(dsql.IAMAuthIdentityProvider); ok { + cfg.Identities = ids + } + } + + if stsH != nil { + if ss, ok := stsH.Backend.(dsql.IAMAuthSessionProvider); ok { + cfg.Sessions = ss + } + } + + return cfg +} diff --git a/internal/app/dsql_wiring_test.go b/internal/app/dsql_wiring_test.go new file mode 100644 index 0000000000..8cd7822e4f --- /dev/null +++ b/internal/app/dsql_wiring_test.go @@ -0,0 +1,123 @@ +package app + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/dsql" + iambackend "github.com/blackbirdworks/gopherstack/services/iam" + stsbackend "github.com/blackbirdworks/gopherstack/services/sts" +) + +type dsqlFakeInstance struct{} + +func (dsqlFakeInstance) Addr() (string, int) { return "127.0.0.1", 41234 } + +func (dsqlFakeInstance) Close(context.Context) {} + +type dsqlFakeLauncher struct{} + +func (dsqlFakeLauncher) Launch(context.Context, dsql.ClusterRef) (dsql.Instance, error) { + return dsqlFakeInstance{}, nil +} + +func TestWireDSQLAuth(t *testing.T) { + t.Parallel() + + b := dsql.NewInMemoryBackend() + b.EnableEngine(dsqlFakeLauncher{}) + t.Cleanup(b.Close) + + c, err := b.CreateCluster("123456789012", "us-east-1", dsql.CreateClusterInput{}) + require.NoError(t, err) + require.Eventually(t, func() bool { + _, _, ok := b.ConnectAddr(c.Identifier) + + return ok + }, 10*time.Second, time.Millisecond) + + e := echo.New() + services := []service.Registerable{ + dsql.NewHandler(b), iambackend.NewHandler(iambackend.NewInMemoryBackend()), + stsbackend.NewHandler(stsbackend.NewInMemoryBackend()), + } + wireDSQLAuth(e, services, true) + + tests := []struct { + name string + path string + code int + }{ + {name: "cluster address", path: "/_gopherstack/dsql/clusters/" + c.Identifier, code: http.StatusOK}, + {name: "unknown cluster", path: "/_gopherstack/dsql/clusters/nope", code: http.StatusNotFound}, + {name: "no ca without docker engine", path: dsqlCAPath, code: http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, tt.path, nil)) + assert.Equal(t, tt.code, rec.Code) + + if tt.code != http.StatusOK { + return + } + + var got struct { + Host string `json:"host"` + Endpoint string `json:"endpoint"` + Port int `json:"port"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, "127.0.0.1", got.Host) + assert.Equal(t, 41234, got.Port) + assert.Equal(t, c.Endpoint, got.Endpoint) + }) + } +} + +func TestDSQLIAMAuthConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + withIAM bool + enforce bool + }{ + {name: "iam and sts present", withIAM: true, enforce: true}, + {name: "services absent"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var ( + iamH *iambackend.Handler + stsH *stsbackend.Handler + ) + + if tt.withIAM { + iamH = iambackend.NewHandler(iambackend.NewInMemoryBackend()) + stsH = stsbackend.NewHandler(stsbackend.NewInMemoryBackend()) + } + + cfg := dsqlIAMAuthConfig(iamH, stsH, tt.enforce) + assert.Equal(t, tt.enforce, cfg.Enforce) + assert.Equal(t, tt.withIAM, cfg.Identities != nil) + assert.Equal(t, tt.withIAM, cfg.Sessions != nil) + }) + } +} diff --git a/internal/app/ec2_ecs_region_wiring_test.go b/internal/app/ec2_ecs_region_wiring_test.go index c46f7df877..a03a8d7f9e 100644 --- a/internal/app/ec2_ecs_region_wiring_test.go +++ b/internal/app/ec2_ecs_region_wiring_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" cwbackend "github.com/blackbirdworks/gopherstack/services/cloudwatch" cwlogsbackend "github.com/blackbirdworks/gopherstack/services/cloudwatchlogs" @@ -46,8 +46,7 @@ func TestInitializeServices_EC2ECSRegionWiring(t *testing.T) { t.Parallel() cli := &CLI{AccountID: "000000000000", Region: "us-east-1"} - portAlloc, err := portalloc.New(19500, 19600) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) cli.faultStore = chaos.NewFaultStore() diff --git a/internal/app/ecs_runtime.go b/internal/app/ecs_runtime.go index 9f06a0c3e4..28d9d1c8e3 100644 --- a/internal/app/ecs_runtime.go +++ b/internal/app/ecs_runtime.go @@ -3,7 +3,9 @@ package app import ( "context" "errors" + "fmt" "io" + "os" "sync" "time" @@ -11,7 +13,9 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/labstack/echo/v5" + "github.com/blackbirdworks/gopherstack/pkgs/container" "github.com/blackbirdworks/gopherstack/pkgs/service" + ec2backend "github.com/blackbirdworks/gopherstack/services/ec2" ecsbackend "github.com/blackbirdworks/gopherstack/services/ecs" efsbackend "github.com/blackbirdworks/gopherstack/services/efs" ebbackend "github.com/blackbirdworks/gopherstack/services/eventbridge" @@ -26,6 +30,7 @@ var ( errECSSecretsUnavailable = errors.New("secrets backend unavailable") errECSSSMUnavailable = errors.New("ssm backend unavailable") errECSS3Unavailable = errors.New("s3 backend unavailable") + errECSEC2Unavailable = errors.New("ec2 backend unavailable") ) // ecsSecrets adapts Secrets Manager and SSM to ecs.SecretsResolver, resolved at call time. @@ -187,6 +192,7 @@ func wireECSRuntime(cli *CLI, byName map[string]service.Registerable) { bk.SetRoleCredentialIssuer(&ecsTaskCredentials{byName: byName}) bk.SetSecretsResolver(&ecsSecrets{byName: byName}) bk.SetEFSResolver(&ecsEFS{byName: byName}) + bk.SetEBSProvisioner(&ecsEBS{byName: byName}) bk.SetEnvFileStore(&ecsEnvFiles{byName: byName}) bk.SetExecSessionStarter(&ecsExecSessions{byName: byName}) @@ -233,7 +239,7 @@ func (a *ecsExecSessions) StartExecSession( return } - if shells, err := ssmbackend.NewDockerShellOpener(); err == nil { + if shells, err := newECSShellOpener(); err == nil { bk.SetShellOpener(shells) } }) @@ -276,3 +282,93 @@ func (a *ecsEFS) ResolveVolume( HostDir: m.HostDir, UID: m.UID, GID: m.GID, SecondaryGIDs: m.SecondaryGIDs, HasPosixUser: m.HasPosixUser, }, nil } + +// newECSShellOpener opens ECS Exec sessions through pods/exec on the kubernetes runtime, else the docker daemon. +func newECSShellOpener() (ssmbackend.ShellOpener, error) { + if os.Getenv("CONTAINER_RUNTIME") != string(container.RuntimeKubernetes) { + return ssmbackend.NewDockerShellOpener() + } + + rt, err := container.NewRuntime(container.Config{Runtime: container.RuntimeKubernetes}) + if err != nil { + return nil, fmt.Errorf("kubernetes runtime for ECS Exec: %w", err) + } + + k, ok := rt.(*container.KubernetesRuntime) + if !ok { + return nil, fmt.Errorf("%w: %T", container.ErrUnknownRuntime, rt) + } + + return ssmbackend.NewKubernetesShellOpener(k), nil +} + +// ecsEBS adapts the EC2 backend to ecs.EBSProvisioner, resolved at call time. +type ecsEBS struct { + byName map[string]service.Registerable +} + +func (a *ecsEBS) backend(region string) (*ec2backend.InMemoryBackend, error) { + h, ok := a.byName["EC2"].(*ec2backend.Handler) + if !ok { + return nil, errECSEC2Unavailable + } + + bk, ok := h.BackendFor(region).(*ec2backend.InMemoryBackend) + if !ok { + return nil, errECSEC2Unavailable + } + + return bk, nil +} + +func (a *ecsEBS) CreateTaskVolume(spec ecsbackend.EBSVolumeSpec) (string, error) { + bk, err := a.backend(spec.Region) + if err != nil { + return "", err + } + + vol, err := bk.CreateVolume("", spec.VolumeType, spec.SizeInGiB, spec.SnapshotID, spec.VolumeInitializationRate) + if err != nil { + return "", err + } + + fail := func(err error) (string, error) { + _ = bk.DeleteVolume(vol.ID) + + return "", err + } + + if spec.Encrypted != nil && *spec.Encrypted { + if err = bk.SetVolumeEncryption(vol.ID, true, spec.KMSKeyID); err != nil { + return fail(err) + } + } + + if spec.Iops > 0 || spec.Throughput > 0 { + if err = bk.SetVolumePerformance(vol.ID, spec.Iops, spec.Throughput); err != nil { + return fail(err) + } + } + + if len(spec.Tags) > 0 { + tags := make(map[string]string, len(spec.Tags)) + for _, t := range spec.Tags { + tags[t.Key] = t.Value + } + + if err = bk.CreateTags([]string{vol.ID}, tags); err != nil { + return fail(err) + } + } + + return vol.ID, nil +} + +func (a *ecsEBS) DeleteTaskVolume(region, volumeID string) error { + bk, err := a.backend(region) + if err != nil { + return err + } + + return bk.DeleteVolume(volumeID) +} diff --git a/internal/app/efs_subnet_addresses_test.go b/internal/app/efs_subnet_addresses_test.go index af1441e798..95a7a1a051 100644 --- a/internal/app/efs_subnet_addresses_test.go +++ b/internal/app/efs_subnet_addresses_test.go @@ -39,7 +39,7 @@ func TestInitializeServices_EFSNoFreeAddresses(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - services, err := initializeServicesT(t, newTestAppContext(t, 19800, 19900)) + services, err := initializeServicesT(t, newTestAppContext(t)) require.NoError(t, err) byName := serviceByName(services) diff --git a/internal/app/elbv2_region_subnet_test.go b/internal/app/elbv2_region_subnet_test.go index f6e999d572..e52b559da0 100644 --- a/internal/app/elbv2_region_subnet_test.go +++ b/internal/app/elbv2_region_subnet_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" elbv2backend "github.com/blackbirdworks/gopherstack/services/elbv2" ) @@ -17,8 +17,7 @@ func TestELBv2DefaultSubnetZoneFollowsLoadBalancerRegion(t *testing.T) { t.Parallel() cli := &CLI{AccountID: "000000000000", Region: "us-east-1"} - portAlloc, err := portalloc.New(19700, 19800) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) cli.faultStore = chaos.NewFaultStore() diff --git a/internal/app/engines.go b/internal/app/engines.go index 85d0bb7837..4a6847cf58 100644 --- a/internal/app/engines.go +++ b/internal/app/engines.go @@ -67,6 +67,10 @@ func engineEntries() []engineEntry { "oci-oke-engine", "OKE_ENGINE", enginesStub, }, {func(c *CLI) *string { return &c.OCIMySQL.Engine }, "oci-mysql-engine", "OCI_MYSQL_ENGINE", enginesStub}, + { + func(c *CLI) *string { return &c.OCIDatabase.Engine }, + "oci-database-engine", "OCI_DATABASE_ENGINE", enginesStub, + }, {func(c *CLI) *string { return &c.RDSEngine }, "rds-engine", "RDS_ENGINE", enginesStub}, {func(c *CLI) *string { return &c.DMSEngine }, "dms-engine", "DMS_ENGINE", enginesStub}, {func(c *CLI) *string { return &c.DocDBEngine }, "docdb-engine", "DOCDB_ENGINE", enginesStub}, @@ -80,6 +84,9 @@ func engineEntries() []engineEntry { {func(c *CLI) *string { return &c.SageMakerEngine }, "sagemaker-engine", "SAGEMAKER_ENGINE", enginesStub}, {func(c *CLI) *string { return &c.CodeBuild.Engine }, "codebuild-engine", "CODEBUILD_ENGINE", enginesStub}, {func(c *CLI) *string { return &c.Glue.Engine }, "glue-engine", "GLUE_ENGINE", enginesStub}, + {func(c *CLI) *string { return &c.DSQL.Engine }, "dsql-engine", "DSQL_ENGINE", enginesStub}, + {func(c *CLI) *string { return &c.LambdaWeb.Engine }, "lambdaweb-engine", "LAMBDAWEB_ENGINE", enginesStub}, + {func(c *CLI) *string { return &c.Transcribe.Engine }, "transcribe-engine", "TRANSCRIBE_ENGINE", enginesStub}, } } diff --git a/internal/app/error_wire_guards_test.go b/internal/app/error_wire_guards_test.go index 1f551b32b4..33c03703bc 100644 --- a/internal/app/error_wire_guards_test.go +++ b/internal/app/error_wire_guards_test.go @@ -339,6 +339,7 @@ func errorProbeUnreachable() map[string]string { return map[string]string{ "APIGatewayManagementAPI": "path-routed REST (@connections)", "AzureAppConfig": "non-AWS", + "AzureSearch": "non-AWS", "AzureContainerRegistry": "non-AWS", "AzureFunctions": "non-AWS", "AzureARM": "non-AWS", diff --git a/internal/app/fis_target_resolution_test.go b/internal/app/fis_target_resolution_test.go index e352dba002..57d5742fbb 100644 --- a/internal/app/fis_target_resolution_test.go +++ b/internal/app/fis_target_resolution_test.go @@ -70,7 +70,7 @@ func TestInitializeServices_FISTargetResolution(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - services, err := initializeServicesT(t, newTestAppContext(t, 19700, 19800)) + services, err := initializeServicesT(t, newTestAppContext(t)) require.NoError(t, err) byName := serviceByName(services) diff --git a/internal/app/fixed_ports_report_test.go b/internal/app/fixed_ports_report_test.go index e33ee60b3d..e0272efa59 100644 --- a/internal/app/fixed_ports_report_test.go +++ b/internal/app/fixed_ports_report_test.go @@ -8,19 +8,18 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" ) func TestFixedServicePorts_AllReported(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int + name string + span int }{ - {"default pool", 10000, 10100}, - {"pool containing fixed ports", 4000, 9100}, + {"small pool", 10}, + {"large pool", 100}, } for _, tt := range tests { @@ -36,8 +35,7 @@ func TestFixedServicePorts_AllReported(t *testing.T) { cli := root.Serve - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc := portalloctest.New(t, tt.span) reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) diff --git a/internal/app/gcpapigateway_port_reservation_test.go b/internal/app/gcpapigateway_port_reservation_test.go index f2ff2ecdd3..c56a36476f 100644 --- a/internal/app/gcpapigateway_port_reservation_test.go +++ b/internal/app/gcpapigateway_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpapigatewaybackend "github.com/blackbirdworks/gopherstack/services/gcpapigateway" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPAPIGateway(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpapigatewaybackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpapigatewaybackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpapigatewaybackend.DefaultPort, true}, + {"port outside pool range", gcpapigatewaybackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPAPIGateway: gcpapigatewaybackend.Settings{Port: tt.port}} + cli := CLI{GCPAPIGateway: gcpapigatewaybackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpbatch_port_reservation_test.go b/internal/app/gcpbatch_port_reservation_test.go index 8f140bdbab..b7989759ee 100644 --- a/internal/app/gcpbatch_port_reservation_test.go +++ b/internal/app/gcpbatch_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpbatchbackend "github.com/blackbirdworks/gopherstack/services/gcpbatch" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPBatch(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpbatchbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpbatchbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpbatchbackend.DefaultPort, true}, + {"port outside pool range", gcpbatchbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPBatch: gcpbatchbackend.Settings{Port: tt.port}} + cli := CLI{GCPBatch: gcpbatchbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpbigquery_port_reservation_test.go b/internal/app/gcpbigquery_port_reservation_test.go index b2e049b0d6..17f456054b 100644 --- a/internal/app/gcpbigquery_port_reservation_test.go +++ b/internal/app/gcpbigquery_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpbigquerybackend "github.com/blackbirdworks/gopherstack/services/gcpbigquery" ) @@ -15,26 +13,23 @@ func TestReserveFixedServicePorts_GCPBigQuery(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 9000, 9100, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, gcpbigquerybackend.DefaultPort) - cli := CLI{GCPBigQuery: gcpbigquerybackend.Settings{Port: gcpbigquerybackend.DefaultPort}} + cli := CLI{GCPBigQuery: gcpbigquerybackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(gcpbigquerybackend.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpbigtable_port_reservation_test.go b/internal/app/gcpbigtable_port_reservation_test.go index f103b4ac2a..2da5b35584 100644 --- a/internal/app/gcpbigtable_port_reservation_test.go +++ b/internal/app/gcpbigtable_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpbigtablebackend "github.com/blackbirdworks/gopherstack/services/gcpbigtable" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPBigtable(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpbigtablebackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpbigtablebackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpbigtablebackend.DefaultPort, true}, + {"port outside pool range", gcpbigtablebackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPBigtable: gcpbigtablebackend.Settings{Port: tt.port}} + cli := CLI{GCPBigtable: gcpbigtablebackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpcloudbuild_docker_results_test.go b/internal/app/gcpcloudbuild_docker_results_test.go new file mode 100644 index 0000000000..35d6d55f0f --- /dev/null +++ b/internal/app/gcpcloudbuild_docker_results_test.go @@ -0,0 +1,61 @@ +package app + +import ( + "testing" + + "cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + gcpstoragebackend "github.com/blackbirdworks/gopherstack/services/gcpstorage" +) + +func TestGCPCloudBuild_Docker_ManifestSourceAndStepResults(t *testing.T) { + t.Parallel() + + e := newBuildEnv(t) + + const project = "cb-docker-results" + + _, err := e.gcs.CreateBucket(project, gcpstoragebackend.Patch{"name": []byte(`"cb-m"`)}) + require.NoError(t, err) + + for name, body := range map[string]string{ + "blob": "FROM scratch\n", + "manifest.json": `{"dir/Dockerfile":{"sourceUrl":"gs://cb-m/blob"}}`, + } { + _, err = e.gcs.PutObject("cb-m", name, []byte(body), gcpstoragebackend.Patch{}, gcpstoragebackend.Preconds{}) + require.NoError(t, err) + } + + started := e.create(t, project, &cloudbuildpb.Build{ + Source: &cloudbuildpb.Source{Source: &cloudbuildpb.Source_StorageSourceManifest{ + StorageSourceManifest: &cloudbuildpb.StorageSourceManifest{Bucket: "cb-m", Object: "manifest.json"}, + }}, + Substitutions: map[string]string{"_WHO": "world"}, + Options: &cloudbuildpb.BuildOptions{ + AutomapSubstitutions: true, + SourceProvenanceHash: []cloudbuildpb.Hash_HashType{cloudbuildpb.Hash_SHA256, cloudbuildpb.Hash_MD5}, + }, + Steps: []*cloudbuildpb.BuildStep{ + shStep("a", `printf '%s' "$$_WHO:$$PROJECT_ID" > $$BUILDER_OUTPUT/output; wc -c < dir/Dockerfile`), + }, + }) + + got := e.finished(t, started) + require.Equal(t, cloudbuildpb.Build_SUCCESS, got.GetStatus(), got.GetStatusDetail()) + + require.Len(t, got.GetResults().GetBuildStepOutputs(), 1) + assert.Equal(t, "world:"+project, string(got.GetResults().GetBuildStepOutputs()[0])) + assert.Contains(t, e.logLines(t, project), `Step #0 - "a": 13`) + + require.Len(t, got.GetResults().GetBuildStepImages(), 1) + assert.Regexp(t, `^sha256:[0-9a-f]{64}$`, got.GetResults().GetBuildStepImages()[0]) + + assert.Equal(t, "manifest.json", got.GetSourceProvenance().GetResolvedStorageSourceManifest().GetObject()) + assert.NotZero(t, got.GetSourceProvenance().GetResolvedStorageSourceManifest().GetGeneration()) + assert.Len(t, got.GetSourceProvenance().GetFileHashes(), 1) + + assert.Contains(t, got.GetTiming(), "FETCHSOURCE") + assert.NotNil(t, got.GetSteps()[0].GetPullTiming()) +} diff --git a/internal/app/gcpcloudbuild_wiring.go b/internal/app/gcpcloudbuild_wiring.go index 30d91c63f0..f7327fad4e 100644 --- a/internal/app/gcpcloudbuild_wiring.go +++ b/internal/app/gcpcloudbuild_wiring.go @@ -6,11 +6,13 @@ import ( "net/http" "strconv" + "cloud.google.com/go/kms/apiv1/kmspb" "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb" "github.com/blackbirdworks/gopherstack/pkgs/service" gcparbackend "github.com/blackbirdworks/gopherstack/services/gcpartifactregistry" gcpcloudbuildbackend "github.com/blackbirdworks/gopherstack/services/gcpcloudbuild" + gcpkmsbackend "github.com/blackbirdworks/gopherstack/services/gcpkms" gcploggingbackend "github.com/blackbirdworks/gopherstack/services/gcplogging" gcppubsubbackend "github.com/blackbirdworks/gopherstack/services/gcppubsub" gcpsecretmanagerbackend "github.com/blackbirdworks/gopherstack/services/gcpsecretmanager" @@ -38,6 +40,29 @@ func (s cloudBuildStore) EnsureBucket(project, bucket string) error { return nil } +// ReadObjectGeneration reads an object at a generation (0 is the latest) and reports the generation read. +func (s cloudBuildStore) ReadObjectGeneration(bucket, object string, gen int64) ([]byte, int64, error) { + o, err := s.b.GetObject(bucket, object, gen, gcpstoragebackend.Preconds{}) + if err != nil { + return nil, 0, fmt.Errorf("read gs://%s/%s: %w", bucket, object, err) + } + + return o.Data, o.Generation, nil +} + +type cloudBuildKMS struct { + b *gcpkmsbackend.InMemoryBackend +} + +func (k cloudBuildKMS) Decrypt(keyName string, ciphertext []byte) ([]byte, error) { + res, err := k.b.Decrypt(&kmspb.DecryptRequest{Name: keyName, Ciphertext: ciphertext}) + if err != nil { + return nil, fmt.Errorf("kms decrypt: %w", err) + } + + return res.GetPlaintext(), nil +} + // cloudBuildPublisher publishes build status messages to the project's cloud-builds topic, creating it on demand. type cloudBuildPublisher struct { b *gcppubsubbackend.InMemoryBackend @@ -88,8 +113,13 @@ func cloudBuildServices(byName map[string]service.Registerable) gcpcloudbuildbac svc.Secrets = cloudBuildSecrets{b: sm.Backend} } + if kms, ok := byName["GCPKMS"].(*gcpkmsbackend.Handler); ok { + svc.KMS = cloudBuildKMS{b: kms.Backend} + } + if ar, ok := byName["GCPArtifactRegistry"].(*gcparbackend.Handler); ok { svc.RegistryAddr = "localhost:" + strconv.Itoa(ar.Port) + svc.Packages = ar.Backend } return svc @@ -99,6 +129,7 @@ func cloudBuildServices(byName map[string]service.Registerable) gcpcloudbuildbac func wireGCPStorageConsumers(byName map[string]service.Registerable) { wireGCPRedis(byName) wireGCPCloudBuild(byName) + wireGCPArtifactRegistry(byName) wireGCPEventarc(byName) wireGCPSecretRotation(byName) wireGCPWorkflows(byName) @@ -106,6 +137,7 @@ func wireGCPStorageConsumers(byName map[string]service.Registerable) { wireGCPContainer(byName) wireGCPIdentityToolkit(byName) wireGCPBatch(byName) + wireGCPStorageSecurity(byName) } // wireGCPCloudBuild connects Cloud Build to Cloud Storage, Pub/Sub, Cloud Logging, Secret Manager and the @@ -133,3 +165,32 @@ func wireGCPSecretRotation(byName map[string]service.Registerable) { sm.Backend.SetPublisher(logSinkPublisher{b: ps.Backend}) } } + +// wireGCPArtifactRegistry connects Artifact Registry to the GCS emulator (imports and exports) and to Secret +// Manager (remote repository credentials). +func wireGCPArtifactRegistry(byName map[string]service.Registerable) { + ar, okAR := byName["GCPArtifactRegistry"].(*gcparbackend.Handler) + gcs, okGCS := byName["GCPStorage"].(*gcpstoragebackend.Handler) + + if okAR && okGCS { + ar.Backend.SetObjectStore(gcsObjectStore{b: gcs.Backend}) + } + + if sm, okSM := byName["GCPSecretManager"].(*gcpsecretmanagerbackend.Handler); okAR && okSM { + ar.Backend.SetSecretResolver(arSecrets{b: sm.Backend}) + } +} + +// arSecrets resolves Artifact Registry upstream_credentials password secret versions. +type arSecrets struct { + b *gcpsecretmanagerbackend.InMemoryBackend +} + +func (s arSecrets) AccessSecretVersion(name string) ([]byte, error) { + resp, err := s.b.AccessSecretVersion(name) + if err != nil { + return nil, err + } + + return resp.GetPayload().GetData(), nil +} diff --git a/internal/app/gcpcloudbuild_wiring_test.go b/internal/app/gcpcloudbuild_wiring_test.go index fab4a6714c..ba1f376791 100644 --- a/internal/app/gcpcloudbuild_wiring_test.go +++ b/internal/app/gcpcloudbuild_wiring_test.go @@ -5,15 +5,16 @@ import ( "testing" "cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb" + "cloud.google.com/go/kms/apiv1/kmspb" "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb" "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" gcparbackend "github.com/blackbirdworks/gopherstack/services/gcpartifactregistry" gcpcloudbuildbackend "github.com/blackbirdworks/gopherstack/services/gcpcloudbuild" + gcpkmsbackend "github.com/blackbirdworks/gopherstack/services/gcpkms" gcppubsubbackend "github.com/blackbirdworks/gopherstack/services/gcppubsub" gcpsecretmanagerbackend "github.com/blackbirdworks/gopherstack/services/gcpsecretmanager" gcpstoragebackend "github.com/blackbirdworks/gopherstack/services/gcpstorage" @@ -121,26 +122,23 @@ func TestReserveFixedServicePorts_GCPCloudBuild(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, gcpcloudbuildbackend.DefaultPort) - cli := CLI{GCPCloudBuild: gcpcloudbuildbackend.Settings{Port: gcpcloudbuildbackend.DefaultPort}} + cli := CLI{GCPCloudBuild: gcpcloudbuildbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(gcpcloudbuildbackend.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } @@ -182,3 +180,59 @@ func TestWireGCPCloudBuild_PubsubTrigger(t *testing.T) { require.Len(t, builds, 1) assert.Equal(t, trigger.GetId(), builds[0].GetBuildTriggerId()) } + +func TestCloudBuildKMS(t *testing.T) { + t.Parallel() + + kms := gcpkmsbackend.NewInMemoryBackend() + const ring = "projects/p/locations/global" + + _, err := kms.CreateKeyRing(&kmspb.CreateKeyRingRequest{Parent: ring, KeyRingId: "r"}) + require.NoError(t, err) + + const key = ring + "/keyRings/r/cryptoKeys/k" + + _, err = kms.CreateCryptoKey(&kmspb.CreateCryptoKeyRequest{ + Parent: ring + "/keyRings/r", CryptoKeyId: "k", SkipInitialVersionCreation: false, + CryptoKey: &kmspb.CryptoKey{Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT}, + }) + require.NoError(t, err) + + enc, err := kms.Encrypt(&kmspb.EncryptRequest{Name: key, Plaintext: []byte("s3cret")}) + require.NoError(t, err) + + dec := cloudBuildKMS{b: kms} + + got, err := dec.Decrypt(key, enc.GetCiphertext()) + require.NoError(t, err) + assert.Equal(t, []byte("s3cret"), got) + + _, err = dec.Decrypt(key, []byte("garbage")) + require.Error(t, err) + + svc := cloudBuildServices(map[string]service.Registerable{"GCPKMS": gcpkmsbackend.NewHandler(kms)}) + assert.NotNil(t, svc.KMS) +} + +func TestCloudBuildStoreGenerations(t *testing.T) { + t.Parallel() + + store := cloudBuildStore{gcsObjectStore{b: gcpstoragebackend.NewInMemoryBackend()}} + require.NoError(t, store.EnsureBucket("p", "gen-bucket")) + require.NoError(t, store.WriteObject("gen-bucket", "o", []byte("v1"), "")) + + data, gen, err := store.ReadObjectGeneration("gen-bucket", "o", 0) + require.NoError(t, err) + assert.Equal(t, []byte("v1"), data) + assert.Positive(t, gen) + + require.NoError(t, store.WriteObject("gen-bucket", "o", []byte("v2"), "")) + + latest, gen2, err := store.ReadObjectGeneration("gen-bucket", "o", 0) + require.NoError(t, err) + assert.Equal(t, []byte("v2"), latest) + assert.Greater(t, gen2, gen) + + _, _, err = store.ReadObjectGeneration("gen-bucket", "o", gen+999) + require.Error(t, err) +} diff --git a/internal/app/gcpcloudtasks_port_reservation_test.go b/internal/app/gcpcloudtasks_port_reservation_test.go index cd553228cc..7fe72a7944 100644 --- a/internal/app/gcpcloudtasks_port_reservation_test.go +++ b/internal/app/gcpcloudtasks_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpcloudtasksbackend "github.com/blackbirdworks/gopherstack/services/gcpcloudtasks" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPCloudTasks(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpcloudtasksbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpcloudtasksbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpcloudtasksbackend.DefaultPort, true}, + {"port outside pool range", gcpcloudtasksbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPCloudTasks: gcpcloudtasksbackend.Settings{Port: tt.port}} + cli := CLI{GCPCloudTasks: gcpcloudtasksbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpcompute_port_reservation_test.go b/internal/app/gcpcompute_port_reservation_test.go index 8548db5397..994932cdf2 100644 --- a/internal/app/gcpcompute_port_reservation_test.go +++ b/internal/app/gcpcompute_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpcomputebackend "github.com/blackbirdworks/gopherstack/services/gcpcompute" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPCompute(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpcomputebackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpcomputebackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpcomputebackend.DefaultPort, true}, + {"port outside pool range", gcpcomputebackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPCompute: gcpcomputebackend.Settings{Port: tt.port}} + cli := CLI{GCPCompute: gcpcomputebackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpcontainer_port_reservation_test.go b/internal/app/gcpcontainer_port_reservation_test.go index 28434b06ff..07387e5757 100644 --- a/internal/app/gcpcontainer_port_reservation_test.go +++ b/internal/app/gcpcontainer_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpcontainerbackend "github.com/blackbirdworks/gopherstack/services/gcpcontainer" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPContainer(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpcontainerbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpcontainerbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpcontainerbackend.DefaultPort, true}, + {"port outside pool range", gcpcontainerbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPContainer: gcpcontainerbackend.Settings{Port: tt.port}} + cli := CLI{GCPContainer: gcpcontainerbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpdatastore_port_reservation_test.go b/internal/app/gcpdatastore_port_reservation_test.go index 61e5370e16..15c6526bca 100644 --- a/internal/app/gcpdatastore_port_reservation_test.go +++ b/internal/app/gcpdatastore_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpdatastorebackend "github.com/blackbirdworks/gopherstack/services/gcpdatastore" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPDatastore(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpdatastorebackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpdatastorebackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpdatastorebackend.DefaultPort, true}, + {"port outside pool range", gcpdatastorebackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPDatastore: gcpdatastorebackend.Settings{Port: tt.port}} + cli := CLI{GCPDatastore: gcpdatastorebackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpdns_port_reservation_test.go b/internal/app/gcpdns_port_reservation_test.go index 963b4cad95..5818614a38 100644 --- a/internal/app/gcpdns_port_reservation_test.go +++ b/internal/app/gcpdns_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpdnsbackend "github.com/blackbirdworks/gopherstack/services/gcpdns" ) @@ -15,29 +13,26 @@ func TestReserveFixedServicePorts_GCPDNS(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - dnsPort int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + dnsPort int + wantTaken bool }{ - {"ports inside pool range", gcpdnsbackend.DefaultPort, gcpdnsbackend.DefaultDNSPort, 8000, 9000, true}, - {"ports outside pool range", gcpdnsbackend.DefaultPort, gcpdnsbackend.DefaultDNSPort, 10000, 10100, false}, + {"ports inside pool range", gcpdnsbackend.DefaultPort, gcpdnsbackend.DefaultDNSPort, true}, + {"ports outside pool range", gcpdnsbackend.DefaultPort, gcpdnsbackend.DefaultDNSPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, ports := fixedPortPools(t, tt.wantTaken, tt.port, tt.dnsPort) - cli := CLI{GCPDNS: gcpdnsbackend.Settings{Port: tt.port, DNSPort: tt.dnsPort}} + cli := CLI{GCPDNS: gcpdnsbackend.Settings{Port: ports[0], DNSPort: ports[1]}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.dnsPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ports[0])) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ports[1])) }) } } diff --git a/internal/app/gcpenforce_wiring_test.go b/internal/app/gcpenforce_wiring_test.go index 505d087570..aba321cf37 100644 --- a/internal/app/gcpenforce_wiring_test.go +++ b/internal/app/gcpenforce_wiring_test.go @@ -21,7 +21,7 @@ import ( "google.golang.org/grpc/status" "github.com/blackbirdworks/gopherstack/pkgs/gcpenforce" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" "github.com/blackbirdworks/gopherstack/pkgs/testclient" gcpcomputebackend "github.com/blackbirdworks/gopherstack/services/gcpcompute" @@ -56,8 +56,7 @@ func startEnforceEnv(t *testing.T, enf gcpenforce.Settings) *enforceEnv { Domain: "x.test", DeleteRetention: gcprmbackend.DefaultDeleteRetention, }, } - portAlloc, err := portalloc.New(19700, 19800) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) services, err := initializeServicesT(t, &service.AppContext{ Logger: slog.Default(), Config: cli, JanitorCtx: t.Context(), PortAlloc: portAlloc, diff --git a/internal/app/gcpeventarc_port_reservation_test.go b/internal/app/gcpeventarc_port_reservation_test.go index f41e7c5d3f..736d79e8ff 100644 --- a/internal/app/gcpeventarc_port_reservation_test.go +++ b/internal/app/gcpeventarc_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpeventarcbackend "github.com/blackbirdworks/gopherstack/services/gcpeventarc" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPEventarc(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpeventarcbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpeventarcbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpeventarcbackend.DefaultPort, true}, + {"port outside pool range", gcpeventarcbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPEventarc: gcpeventarcbackend.Settings{Port: tt.port}} + cli := CLI{GCPEventarc: gcpeventarcbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpeventarc_wiring.go b/internal/app/gcpeventarc_wiring.go index 127b11edb7..1b5bc12f17 100644 --- a/internal/app/gcpeventarc_wiring.go +++ b/internal/app/gcpeventarc_wiring.go @@ -8,10 +8,15 @@ import ( "cloud.google.com/go/run/apiv2/runpb" "github.com/blackbirdworks/gopherstack/pkgs/service" + gcpbigquerybackend "github.com/blackbirdworks/gopherstack/services/gcpbigquery" gcpeventarcbackend "github.com/blackbirdworks/gopherstack/services/gcpeventarc" + gcpfirestorebackend "github.com/blackbirdworks/gopherstack/services/gcpfirestore" gcpfunctionsbackend "github.com/blackbirdworks/gopherstack/services/gcpfunctions" + gcploggingbackend "github.com/blackbirdworks/gopherstack/services/gcplogging" gcppubsubbackend "github.com/blackbirdworks/gopherstack/services/gcppubsub" + gcpresourcemanagerbackend "github.com/blackbirdworks/gopherstack/services/gcpresourcemanager" gcprunbackend "github.com/blackbirdworks/gopherstack/services/gcprun" + gcpsecretmanagerbackend "github.com/blackbirdworks/gopherstack/services/gcpsecretmanager" gcpstoragebackend "github.com/blackbirdworks/gopherstack/services/gcpstorage" gcpworkflowsbackend "github.com/blackbirdworks/gopherstack/services/gcpworkflows" ) @@ -84,6 +89,16 @@ func wireGCPEventarc(byName map[string]service.Registerable) { svc.Targets = targets ea.Backend.SetServices(svc) + wireGCPAudit(byName, ea) + + if fs, isFS := byName["GCPFirestore"].(*gcpfirestorebackend.Handler); isFS { + fs.Backend.ObserveDocuments(func(ev gcpfirestorebackend.DocumentEvent) { + ea.Backend.HandleDocumentEvent(gcpeventarcbackend.DocumentEvent{ + Kind: string(ev.Kind), Old: ev.Old, New: ev.New, UpdateMask: ev.UpdateMask, Time: ev.Time, + }) + }) + } + if gcs, isGCS := byName["GCPStorage"].(*gcpstoragebackend.Handler); isGCS { gcs.Backend.ObserveObjects(func(ev gcpstoragebackend.ObjectEvent) { ea.Backend.HandleStorageEvent(gcpeventarcbackend.StorageEvent{ @@ -92,3 +107,34 @@ func wireGCPEventarc(byName map[string]service.Registerable) { }) } } + +// wireGCPAudit points the audit emitters of the emulated services at the Logging emulator, and feeds the audit +// log entries it stores to Eventarc. +func wireGCPAudit(byName map[string]service.Registerable, ea *gcpeventarcbackend.Handler) { + lg, ok := byName["GCPLogging"].(*gcploggingbackend.Handler) + if !ok { + return + } + + lg.Backend.ObserveEntries(ea.Backend.HandleLogEntry) + + if h, isH := byName["GCPStorage"].(*gcpstoragebackend.Handler); isH { + h.Backend.Audit.SetSink(lg.Backend) + } + + if h, isH := byName["GCPBigQuery"].(*gcpbigquerybackend.Handler); isH { + h.Backend.Audit.SetSink(lg.Backend) + } + + if h, isH := byName["GCPFirestore"].(*gcpfirestorebackend.Handler); isH { + h.Backend.Audit.SetSink(lg.Backend) + } + + if h, isH := byName["GCPSecretManager"].(*gcpsecretmanagerbackend.Handler); isH { + h.Backend.Audit.SetSink(lg.Backend) + } + + if h, isH := byName["GCPResourceManager"].(*gcpresourcemanagerbackend.Handler); isH { + h.Backend.Audit.SetSink(lg.Backend) + } +} diff --git a/internal/app/gcpeventarc_wiring_test.go b/internal/app/gcpeventarc_wiring_test.go index 07bdebc0c9..a14594e284 100644 --- a/internal/app/gcpeventarc_wiring_test.go +++ b/internal/app/gcpeventarc_wiring_test.go @@ -14,6 +14,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/service" gcpeventarcbackend "github.com/blackbirdworks/gopherstack/services/gcpeventarc" + gcploggingbackend "github.com/blackbirdworks/gopherstack/services/gcplogging" gcppubsubbackend "github.com/blackbirdworks/gopherstack/services/gcppubsub" gcpstoragebackend "github.com/blackbirdworks/gopherstack/services/gcpstorage" ) @@ -111,6 +112,7 @@ func TestWireGCPEventarcStorageToHTTPEndpoint(t *testing.T) { ea := gcpeventarcbackend.NewHandler(gcpeventarcbackend.NewInMemoryBackend()) ea.Port = 0 + ea.Backend.SetOperationDelay(0) ps := gcppubsubbackend.NewHandler(gcppubsubbackend.NewInMemoryBackend()) ps.Port = 0 gcs := gcpstoragebackend.NewHandler(gcpstoragebackend.NewInMemoryBackend()) @@ -154,3 +156,54 @@ func TestWireGCPEventarcStorageToHTTPEndpoint(t *testing.T) { assert.Equal(t, "google.cloud.storage.object.v1.finalized", r.Header.Get("Ce-Type")) assert.Equal(t, "objects/f.txt", r.Header.Get("Ce-Subject")) } + +func TestWireGCPEventarcAuditLogToHTTPEndpoint(t *testing.T) { + t.Parallel() + + got := make(chan *http.Request, 1) + dest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got <- r + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(dest.Close) + + ea := gcpeventarcbackend.NewHandler(gcpeventarcbackend.NewInMemoryBackend()) + ea.Port = 0 + ea.Backend.SetOperationDelay(0) + ps := gcppubsubbackend.NewHandler(gcppubsubbackend.NewInMemoryBackend()) + ps.Port = 0 + gcs := gcpstoragebackend.NewHandler(gcpstoragebackend.NewInMemoryBackend()) + lg := gcploggingbackend.NewHandler(gcploggingbackend.NewInMemoryBackend()) + + wireGCPEventarc(map[string]service.Registerable{ + "GCPEventarc": ea, "GCPPubSub": ps, "GCPStorage": gcs, "GCPLogging": lg, + }) + + require.NoError(t, ps.StartWorker(t.Context())) + t.Cleanup(func() { ps.Shutdown(context.Background()) }) + require.NoError(t, ea.StartWorker(t.Context())) + t.Cleanup(func() { ea.Shutdown(context.Background()) }) + + _, err := ea.Backend.CreateTrigger(&eventarcpb.CreateTriggerRequest{ + Parent: "projects/p/locations/us-central1", TriggerId: "audit-trigger", + Trigger: &eventarcpb.Trigger{ + EventFilters: []*eventarcpb.EventFilter{ + {Attribute: "type", Value: "google.cloud.audit.log.v1.written"}, + {Attribute: "serviceName", Value: "storage.googleapis.com"}, + {Attribute: "methodName", Value: "storage.buckets.create"}, + }, + Destination: &eventarcpb.Destination{Descriptor_: &eventarcpb.Destination_HttpEndpoint{ + HttpEndpoint: &eventarcpb.HttpEndpoint{Uri: dest.URL + "/audit"}, + }}, + }, + }) + require.NoError(t, err) + + _, err = gcs.Backend.CreateBucket("p", gcpstoragebackend.Patch{"name": []byte(`"audited"`)}) + require.NoError(t, err) + + r := <-got + assert.Equal(t, "/audit", r.URL.Path) + assert.Equal(t, "google.cloud.audit.log.v1.written", r.Header.Get("Ce-Type")) + assert.Equal(t, "storage.googleapis.com/projects/_/buckets/audited", r.Header.Get("Ce-Subject")) +} diff --git a/internal/app/gcpfirestore_port_reservation_test.go b/internal/app/gcpfirestore_port_reservation_test.go index 89ef5b089c..17d3f7612f 100644 --- a/internal/app/gcpfirestore_port_reservation_test.go +++ b/internal/app/gcpfirestore_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpfirestorebackend "github.com/blackbirdworks/gopherstack/services/gcpfirestore" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPFirestore(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpfirestorebackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpfirestorebackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpfirestorebackend.DefaultPort, true}, + {"port outside pool range", gcpfirestorebackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPFirestore: gcpfirestorebackend.Settings{Port: tt.port}} + cli := CLI{GCPFirestore: gcpfirestorebackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpfunctions_port_reservation_test.go b/internal/app/gcpfunctions_port_reservation_test.go index 18f8ce78bf..78f53f9462 100644 --- a/internal/app/gcpfunctions_port_reservation_test.go +++ b/internal/app/gcpfunctions_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpfunctionsbackend "github.com/blackbirdworks/gopherstack/services/gcpfunctions" ) @@ -15,31 +13,33 @@ func TestReserveFixedServicePorts_GCPFunctions(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"grpc port inside pool range", gcpfunctionsbackend.DefaultPort, 8000, 9000, true}, - {"grpc port outside pool range", gcpfunctionsbackend.DefaultPort, 10000, 10100, false}, - {"ingress port inside pool range", gcpfunctionsbackend.DefaultIngressPort, 8000, 9000, true}, - {"ingress port outside pool range", gcpfunctionsbackend.DefaultIngressPort, 10000, 10100, false}, + {"grpc port inside pool range", gcpfunctionsbackend.DefaultPort, true}, + {"grpc port outside pool range", gcpfunctionsbackend.DefaultPort, false}, + {"ingress port inside pool range", gcpfunctionsbackend.DefaultIngressPort, true}, + {"ingress port outside pool range", gcpfunctionsbackend.DefaultIngressPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, ports := fixedPortPools( + t, tt.wantTaken, gcpfunctionsbackend.DefaultPort, gcpfunctionsbackend.DefaultIngressPort, + ) - cli := CLI{GCPFunctions: gcpfunctionsbackend.Settings{ - Port: gcpfunctionsbackend.DefaultPort, IngressPort: gcpfunctionsbackend.DefaultIngressPort, - }} + cli := CLI{GCPFunctions: gcpfunctionsbackend.Settings{Port: ports[0], IngressPort: ports[1]}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + port := ports[0] + if tt.port == gcpfunctionsbackend.DefaultIngressPort { + port = ports[1] + } + + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpiam_port_reservation_test.go b/internal/app/gcpiam_port_reservation_test.go index c84bb5dbdb..286bd49c0e 100644 --- a/internal/app/gcpiam_port_reservation_test.go +++ b/internal/app/gcpiam_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpiambackend "github.com/blackbirdworks/gopherstack/services/gcpiam" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPIAM(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpiambackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpiambackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpiambackend.DefaultPort, true}, + {"port outside pool range", gcpiambackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPIAM: gcpiambackend.Settings{Port: tt.port}} + cli := CLI{GCPIAM: gcpiambackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpidentitytoolkit_port_reservation_test.go b/internal/app/gcpidentitytoolkit_port_reservation_test.go index 5caf468601..bfdaac615a 100644 --- a/internal/app/gcpidentitytoolkit_port_reservation_test.go +++ b/internal/app/gcpidentitytoolkit_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpidtkbackend "github.com/blackbirdworks/gopherstack/services/gcpidentitytoolkit" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPIdentityToolkit(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpidtkbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpidtkbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpidtkbackend.DefaultPort, true}, + {"port outside pool range", gcpidtkbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPIdentityToolkit: gcpidtkbackend.Settings{Port: tt.port}} + cli := CLI{GCPIdentityToolkit: gcpidtkbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpkms_port_reservation_test.go b/internal/app/gcpkms_port_reservation_test.go index 38f521e76c..a05e65068b 100644 --- a/internal/app/gcpkms_port_reservation_test.go +++ b/internal/app/gcpkms_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpkmsbackend "github.com/blackbirdworks/gopherstack/services/gcpkms" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPKMS(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpkmsbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpkmsbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpkmsbackend.DefaultPort, true}, + {"port outside pool range", gcpkmsbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPKMS: gcpkmsbackend.Settings{Port: tt.port}} + cli := CLI{GCPKMS: gcpkmsbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcplogging_port_reservation_test.go b/internal/app/gcplogging_port_reservation_test.go index d4b72e0eab..d5b6ed39e1 100644 --- a/internal/app/gcplogging_port_reservation_test.go +++ b/internal/app/gcplogging_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcploggingbackend "github.com/blackbirdworks/gopherstack/services/gcplogging" ) @@ -15,26 +13,23 @@ func TestReserveFixedServicePorts_GCPLogging(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, gcploggingbackend.DefaultPort) - cli := CLI{GCPLogging: gcploggingbackend.Settings{Port: gcploggingbackend.DefaultPort}} + cli := CLI{GCPLogging: gcploggingbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(gcploggingbackend.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpmonitoring_port_reservation_test.go b/internal/app/gcpmonitoring_port_reservation_test.go index 84e3c4d7c4..66ee2730e0 100644 --- a/internal/app/gcpmonitoring_port_reservation_test.go +++ b/internal/app/gcpmonitoring_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpmonitoringbackend "github.com/blackbirdworks/gopherstack/services/gcpmonitoring" ) @@ -15,26 +13,23 @@ func TestReserveFixedServicePorts_GCPMonitoring(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, gcpmonitoringbackend.DefaultPort) - cli := CLI{GCPMonitoring: gcpmonitoringbackend.Settings{Port: gcpmonitoringbackend.DefaultPort}} + cli := CLI{GCPMonitoring: gcpmonitoringbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(gcpmonitoringbackend.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpmonitoring_wiring.go b/internal/app/gcpmonitoring_wiring.go index 3aaf8d531a..5831c2b99e 100644 --- a/internal/app/gcpmonitoring_wiring.go +++ b/internal/app/gcpmonitoring_wiring.go @@ -1,7 +1,12 @@ package app import ( + "encoding/json" + "strconv" + "time" + "cloud.google.com/go/logging/apiv2/loggingpb" + "google.golang.org/protobuf/encoding/protojson" "github.com/blackbirdworks/gopherstack/pkgs/service" gcploggingbackend "github.com/blackbirdworks/gopherstack/services/gcplogging" @@ -56,13 +61,68 @@ func (s logMetricSource) metric(parent string, m *loggingpb.LogMetric) gcpmonito for _, p := range pts { lm.Series = append(lm.Series, gcpmonitoringbackend.LogMetricSeries{ Labels: p.Labels, BucketOptions: m.GetBucketOptions(), BucketCounts: p.BucketCounts, - Count: p.Count, Sum: p.Sum, SumSqDev: p.SumSqDev, + Count: p.Count, Sum: p.Sum, SumSqDev: p.SumSqDev, Windows: logWindows(p.Windows), }) } return lm } +func logWindows(in []gcploggingbackend.MetricWindow) []gcpmonitoringbackend.LogMetricWindow { + out := make([]gcpmonitoringbackend.LogMetricWindow, len(in)) + for i, w := range in { + out[i] = gcpmonitoringbackend.LogMetricWindow{ + Start: w.Start, BucketCounts: w.BucketCounts, Count: w.Count, Sum: w.Sum, SumSqDev: w.SumSqDev, + } + } + + return out +} + +// logEntrySource lets Cloud Monitoring log-match conditions read entries from Cloud Logging. +type logEntrySource struct { + b *gcploggingbackend.InMemoryBackend +} + +func (s logEntrySource) LogEntries(project, filter string, notBefore time.Time) []gcpmonitoringbackend.LogEntry { + var out []gcpmonitoringbackend.LogEntry + + token := "" + + for { + entries, next, err := s.b.ListLogEntries(&loggingpb.ListLogEntriesRequest{ + ResourceNames: []string{"projects/" + project}, + Filter: "(" + filter + ") AND timestamp>=" + strconv.Quote(notBefore.UTC().Format(time.RFC3339Nano)), + OrderBy: "timestamp asc", PageSize: logMetricPageSize, PageToken: token, + }) + if err != nil { + return out + } + + for _, e := range entries { + raw, mErr := protojson.Marshal(e) + if mErr != nil { + continue + } + + fields := map[string]any{} + if json.Unmarshal(raw, &fields) != nil { + continue + } + + out = append(out, gcpmonitoringbackend.LogEntry{ + Timestamp: e.GetTimestamp().AsTime(), InsertID: e.GetInsertId(), Fields: fields, + }) + } + + if next == "" { + return out + } + + token = next + } +} + // wireGCPMonitoring connects Cloud Monitoring to Pub/Sub notification channels and Cloud Logging's log-based metrics. func wireGCPMonitoring(byName map[string]service.Registerable) { monH, ok := byName["GCPMonitoring"].(*gcpmonitoringbackend.Handler) @@ -76,5 +136,6 @@ func wireGCPMonitoring(byName map[string]service.Registerable) { if lg, isLG := byName["GCPLogging"].(*gcploggingbackend.Handler); isLG { monH.Backend.SetLogMetricSource(logMetricSource{b: lg.Backend}) + monH.Backend.SetLogEntrySource(logEntrySource{b: lg.Backend}) } } diff --git a/internal/app/gcpmonitoring_wiring_test.go b/internal/app/gcpmonitoring_wiring_test.go index 6d471b500d..b3bae5f812 100644 --- a/internal/app/gcpmonitoring_wiring_test.go +++ b/internal/app/gcpmonitoring_wiring_test.go @@ -34,7 +34,7 @@ func newMonitoringFixture(t *testing.T) *monitoringFixture { monitoring: gcpmonitoringbackend.NewHandler(gcpmonitoringbackend.NewInMemoryBackend()), logging: gcploggingbackend.NewHandler(gcploggingbackend.NewInMemoryBackend()), pubsub: gcppubsubbackend.NewHandler(gcppubsubbackend.NewInMemoryBackend()), - now: time.Date(2030, 1, 1, 12, 0, 0, 0, time.UTC), + now: time.Now().UTC().Truncate(time.Second).Add(2 * time.Minute), } f.monitoring.Backend.SetClock(func() time.Time { return f.now }) diff --git a/internal/app/gcppubsub_port_reservation_test.go b/internal/app/gcppubsub_port_reservation_test.go index 7db6284080..e10969edde 100644 --- a/internal/app/gcppubsub_port_reservation_test.go +++ b/internal/app/gcppubsub_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcppubsubbackend "github.com/blackbirdworks/gopherstack/services/gcppubsub" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPPubSub(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcppubsubbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcppubsubbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcppubsubbackend.DefaultPort, true}, + {"port outside pool range", gcppubsubbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPPubSub: gcppubsubbackend.Settings{Port: tt.port}} + cli := CLI{GCPPubSub: gcppubsubbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpredis_wiring_test.go b/internal/app/gcpredis_wiring_test.go index 90648790db..5b3b1a621c 100644 --- a/internal/app/gcpredis_wiring_test.go +++ b/internal/app/gcpredis_wiring_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" gcpredisbackend "github.com/blackbirdworks/gopherstack/services/gcpredis" gcpstoragebackend "github.com/blackbirdworks/gopherstack/services/gcpstorage" @@ -72,26 +71,23 @@ func TestReserveFixedServicePorts_GCPRedis(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, gcpredisbackend.DefaultPort) - cli := CLI{GCPRedis: gcpredisbackend.Settings{Port: gcpredisbackend.DefaultPort}} + cli := CLI{GCPRedis: gcpredisbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(gcpredisbackend.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpresourcemanager_port_reservation_test.go b/internal/app/gcpresourcemanager_port_reservation_test.go index 754d09d76d..ce0b4925c7 100644 --- a/internal/app/gcpresourcemanager_port_reservation_test.go +++ b/internal/app/gcpresourcemanager_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcprmbackend "github.com/blackbirdworks/gopherstack/services/gcpresourcemanager" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPResourceManager(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcprmbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcprmbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcprmbackend.DefaultPort, true}, + {"port outside pool range", gcprmbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPResourceManager: gcprmbackend.Settings{Port: tt.port}} + cli := CLI{GCPResourceManager: gcprmbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcprun_port_reservation_test.go b/internal/app/gcprun_port_reservation_test.go index 58a2dda4f0..79d0b9b539 100644 --- a/internal/app/gcprun_port_reservation_test.go +++ b/internal/app/gcprun_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcprunbackend "github.com/blackbirdworks/gopherstack/services/gcprun" ) @@ -15,31 +13,31 @@ func TestReserveFixedServicePorts_GCPRun(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"grpc port inside pool range", gcprunbackend.DefaultPort, 8000, 9000, true}, - {"grpc port outside pool range", gcprunbackend.DefaultPort, 10000, 10100, false}, - {"ingress port inside pool range", gcprunbackend.DefaultIngressPort, 8000, 9000, true}, - {"ingress port outside pool range", gcprunbackend.DefaultIngressPort, 10000, 10100, false}, + {"grpc port inside pool range", gcprunbackend.DefaultPort, true}, + {"grpc port outside pool range", gcprunbackend.DefaultPort, false}, + {"ingress port inside pool range", gcprunbackend.DefaultIngressPort, true}, + {"ingress port outside pool range", gcprunbackend.DefaultIngressPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, ports := fixedPortPools(t, tt.wantTaken, gcprunbackend.DefaultPort, gcprunbackend.DefaultIngressPort) - cli := CLI{GCPRun: gcprunbackend.Settings{ - Port: gcprunbackend.DefaultPort, IngressPort: gcprunbackend.DefaultIngressPort, - }} + cli := CLI{GCPRun: gcprunbackend.Settings{Port: ports[0], IngressPort: ports[1]}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + port := ports[0] + if tt.port == gcprunbackend.DefaultIngressPort { + port = ports[1] + } + + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpscheduler_port_reservation_test.go b/internal/app/gcpscheduler_port_reservation_test.go index 8f0ce7cc7d..8ea2bdf8d9 100644 --- a/internal/app/gcpscheduler_port_reservation_test.go +++ b/internal/app/gcpscheduler_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpschedulerbackend "github.com/blackbirdworks/gopherstack/services/gcpscheduler" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPScheduler(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpschedulerbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpschedulerbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpschedulerbackend.DefaultPort, true}, + {"port outside pool range", gcpschedulerbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPScheduler: gcpschedulerbackend.Settings{Port: tt.port}} + cli := CLI{GCPScheduler: gcpschedulerbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpsecretmanager_port_reservation_test.go b/internal/app/gcpsecretmanager_port_reservation_test.go index 32a40f7703..6bb2d45201 100644 --- a/internal/app/gcpsecretmanager_port_reservation_test.go +++ b/internal/app/gcpsecretmanager_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpsecretmanagerbackend "github.com/blackbirdworks/gopherstack/services/gcpsecretmanager" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPSecretManager(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpsecretmanagerbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpsecretmanagerbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpsecretmanagerbackend.DefaultPort, true}, + {"port outside pool range", gcpsecretmanagerbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPSecretManager: gcpsecretmanagerbackend.Settings{Port: tt.port}} + cli := CLI{GCPSecretManager: gcpsecretmanagerbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpspanner_port_reservation_test.go b/internal/app/gcpspanner_port_reservation_test.go index 99f190d8fe..c5e0b93cf0 100644 --- a/internal/app/gcpspanner_port_reservation_test.go +++ b/internal/app/gcpspanner_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpspannerbackend "github.com/blackbirdworks/gopherstack/services/gcpspanner" ) @@ -15,26 +13,23 @@ func TestReserveFixedServicePorts_GCPSpanner(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, gcpspannerbackend.DefaultPort) - cli := CLI{GCPSpanner: gcpspannerbackend.Settings{Port: gcpspannerbackend.DefaultPort}} + cli := CLI{GCPSpanner: gcpspannerbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(gcpspannerbackend.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpsql_port_reservation_test.go b/internal/app/gcpsql_port_reservation_test.go index 18fdec1f05..5fb7757363 100644 --- a/internal/app/gcpsql_port_reservation_test.go +++ b/internal/app/gcpsql_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpsqlbackend "github.com/blackbirdworks/gopherstack/services/gcpsql" ) @@ -15,26 +13,23 @@ func TestReserveFixedServicePorts_GCPSQL(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, gcpsqlbackend.DefaultPort) - cli := CLI{GCPSQL: gcpsqlbackend.Settings{Port: gcpsqlbackend.DefaultPort}} + cli := CLI{GCPSQL: gcpsqlbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(gcpsqlbackend.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpstorage_port_reservation_test.go b/internal/app/gcpstorage_port_reservation_test.go index d05fc8a342..59dad589e4 100644 --- a/internal/app/gcpstorage_port_reservation_test.go +++ b/internal/app/gcpstorage_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpstoragebackend "github.com/blackbirdworks/gopherstack/services/gcpstorage" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPStorage(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpstoragebackend.DefaultPort, 4400, 4500, true}, - {"port outside pool range", gcpstoragebackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpstoragebackend.DefaultPort, true}, + {"port outside pool range", gcpstoragebackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPStorage: gcpstoragebackend.Settings{Port: tt.port}} + cli := CLI{GCPStorage: gcpstoragebackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpstorage_wiring.go b/internal/app/gcpstorage_wiring.go new file mode 100644 index 0000000000..a96591e116 --- /dev/null +++ b/internal/app/gcpstorage_wiring.go @@ -0,0 +1,45 @@ +package app + +import ( + "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + gcpiambackend "github.com/blackbirdworks/gopherstack/services/gcpiam" + gcppubsubbackend "github.com/blackbirdworks/gopherstack/services/gcppubsub" + gcpstoragebackend "github.com/blackbirdworks/gopherstack/services/gcpstorage" +) + +type gcsPubSubNotifier struct { + ps *gcppubsubbackend.InMemoryBackend +} + +func (n gcsPubSubNotifier) TopicExists(topic string) bool { + _, err := n.ps.GetTopic(topic) + + return err == nil +} + +func (n gcsPubSubNotifier) Publish(topic string, data []byte, attrs map[string]string) error { + _, err := n.ps.Publish(&pubsubpb.PublishRequest{ + Topic: topic, Messages: []*pubsubpb.PubsubMessage{{Data: data, Attributes: attrs}}, + }) + + return err +} + +// wireGCPStorageSecurity gives Cloud Storage Pub/Sub delivery for bucket notifications and Cloud IAM service +// account keys for verifying signed URLs. +func wireGCPStorageSecurity(byName map[string]service.Registerable) { + gcs, ok := byName["GCPStorage"].(*gcpstoragebackend.Handler) + if !ok { + return + } + + if ps, isPS := byName["GCPPubSub"].(*gcppubsubbackend.Handler); isPS { + gcs.Backend.SetNotifier(gcsPubSubNotifier{ps: ps.Backend}) + } + + if iam, isIAM := byName["GCPIAM"].(*gcpiambackend.Handler); isIAM { + gcs.Backend.SetKeyResolver(iam.Backend.VerificationKeys) + } +} diff --git a/internal/app/gcpstorage_wiring_test.go b/internal/app/gcpstorage_wiring_test.go new file mode 100644 index 0000000000..ac8e976c93 --- /dev/null +++ b/internal/app/gcpstorage_wiring_test.go @@ -0,0 +1,64 @@ +package app + +import ( + "context" + "testing" + + "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + gcppubsubbackend "github.com/blackbirdworks/gopherstack/services/gcppubsub" + gcpstoragebackend "github.com/blackbirdworks/gopherstack/services/gcpstorage" +) + +func TestGCPStorageNotificationsDeliverToPubSub(t *testing.T) { + t.Parallel() + + gcs := gcpstoragebackend.NewHandler(gcpstoragebackend.NewInMemoryBackend()) + ps := gcppubsubbackend.NewHandler(gcppubsubbackend.NewInMemoryBackend()) + byName := map[string]service.Registerable{"GCPStorage": gcs, "GCPPubSub": ps} + + wireGCPStorageSecurity(byName) + + const ( + topic = "projects/p/topics/events" + sub = "projects/p/subscriptions/events-sub" + ) + + _, err := ps.Backend.CreateTopic(&pubsubpb.Topic{Name: topic}) + require.NoError(t, err) + + _, err = ps.Backend.CreateSubscription(&pubsubpb.Subscription{Name: sub, Topic: topic}) + require.NoError(t, err) + + _, err = gcs.Backend.CreateBucket("p", gcpstoragebackend.Patch{"name": []byte(`"bkt"`)}) + require.NoError(t, err) + + _, err = gcs.Backend.AddNotification("bkt", gcpstoragebackend.Notification{ + Topic: "//pubsub.googleapis.com/" + topic, PayloadFormat: "JSON_API_V1", + }) + require.NoError(t, err) + + _, err = gcs.Backend.PutObject("bkt", "k", []byte("v"), gcpstoragebackend.Patch{}, gcpstoragebackend.Preconds{}) + require.NoError(t, err) + + res, err := ps.SubscriberAPI().Pull(context.Background(), &pubsubpb.PullRequest{ + Subscription: sub, MaxMessages: 10, ReturnImmediately: true, //nolint:staticcheck // immediate poll in tests + }) + require.NoError(t, err) + require.Len(t, res.GetReceivedMessages(), 1) + assert.Equal(t, "OBJECT_FINALIZE", res.GetReceivedMessages()[0].GetMessage().GetAttributes()["eventType"]) +} + +func TestGCPStorageSecurityWiringWithoutPeers(t *testing.T) { + t.Parallel() + + assert.NotPanics(t, func() { wireGCPStorageSecurity(map[string]service.Registerable{}) }) + assert.NotPanics(t, func() { + wireGCPStorageSecurity(map[string]service.Registerable{ + "GCPStorage": gcpstoragebackend.NewHandler(gcpstoragebackend.NewInMemoryBackend()), + }) + }) +} diff --git a/internal/app/gcpworkflows_port_reservation_test.go b/internal/app/gcpworkflows_port_reservation_test.go index 02d5e958c7..72bb75af39 100644 --- a/internal/app/gcpworkflows_port_reservation_test.go +++ b/internal/app/gcpworkflows_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" gcpworkflowsbackend "github.com/blackbirdworks/gopherstack/services/gcpworkflows" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_GCPWorkflows(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", gcpworkflowsbackend.DefaultPort, 8000, 9000, true}, - {"port outside pool range", gcpworkflowsbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", gcpworkflowsbackend.DefaultPort, true}, + {"port outside pool range", gcpworkflowsbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{GCPWorkflows: gcpworkflowsbackend.Settings{Port: tt.port}} + cli := CLI{GCPWorkflows: gcpworkflowsbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/gcpworkflows_wiring_test.go b/internal/app/gcpworkflows_wiring_test.go index 51b3e6fff8..817f1c3b4a 100644 --- a/internal/app/gcpworkflows_wiring_test.go +++ b/internal/app/gcpworkflows_wiring_test.go @@ -142,6 +142,7 @@ func TestWireGCPEventarcStartsWorkflowExecutions(t *testing.T) { t.Parallel() ea := gcpeventarcbackend.NewHandler(gcpeventarcbackend.NewInMemoryBackend()) + ea.Backend.SetOperationDelay(0) ea.Port = 0 ps := gcppubsubbackend.NewHandler(gcppubsubbackend.NewInMemoryBackend()) ps.Port = 0 diff --git a/internal/app/glue_ram_policy_share_wiring_test.go b/internal/app/glue_ram_policy_share_wiring_test.go index e882bfdc1c..358c8528d1 100644 --- a/internal/app/glue_ram_policy_share_wiring_test.go +++ b/internal/app/glue_ram_policy_share_wiring_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" gluebackend "github.com/blackbirdworks/gopherstack/services/glue" rambackend "github.com/blackbirdworks/gopherstack/services/ram" @@ -29,8 +29,7 @@ func TestInitializeServices_GlueRAMPolicyShareWiring(t *testing.T) { t.Parallel() cli := &CLI{AccountID: "000000000000", Region: "us-east-1"} - portAlloc, err := portalloc.New(19200, 19300) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) appCtx := &service.AppContext{ Logger: slog.Default(), diff --git a/internal/app/iam_policy_stream.go b/internal/app/iam_policy_stream.go index fd798e3b8b..b941fb2d7e 100644 --- a/internal/app/iam_policy_stream.go +++ b/internal/app/iam_policy_stream.go @@ -10,10 +10,14 @@ import ( "github.com/labstack/echo/v5" + "github.com/blackbirdworks/gopherstack/pkgs/awsmeta" "github.com/blackbirdworks/gopherstack/pkgs/iampolicystream" "github.com/blackbirdworks/gopherstack/pkgs/service" iambackend "github.com/blackbirdworks/gopherstack/services/iam" + kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" organizationsbackend "github.com/blackbirdworks/gopherstack/services/organizations" + s3backend "github.com/blackbirdworks/gopherstack/services/s3" + sqsbackend "github.com/blackbirdworks/gopherstack/services/sqs" ) const ( @@ -202,3 +206,115 @@ func scpProviderFor(enabled bool, services []service.Registerable) iambackend.SC return nil } + +type bucketOwnerBackend interface { + GetBucketOwnerAccount(ctx context.Context, bucket string) (string, error) +} + +// s3OwnerResolver maps account-less S3 ARNs to the account that created the bucket. +type s3OwnerResolver struct{ backend bucketOwnerBackend } + +func (r s3OwnerResolver) ResourceOwnerAccount(ctx context.Context, resourceARN string) (string, bool) { + rest, ok := strings.CutPrefix(resourceARN, "arn:aws:s3:::") + if !ok { + return "", false + } + + bucket, _, _ := strings.Cut(rest, "/") + if bucket == "" { + return "", false + } + + owner, err := r.backend.GetBucketOwnerAccount(ctx, bucket) + + return owner, err == nil && owner != "" +} + +// resourceOwnersFor returns the owner resolvers RCP evaluation needs, or nil when org policy enforcement is off. +func resourceOwnersFor(enabled bool, services []service.Registerable) []iambackend.ResourceOwnerResolver { + if !enabled { + return nil + } + + var out []iambackend.ResourceOwnerResolver + + for _, svc := range services { + if h, ok := svc.(*s3backend.S3Handler); ok { + if b, isOwner := h.Backend.(bucketOwnerBackend); isOwner { + out = append(out, s3OwnerResolver{backend: b}) + } + } + + if h, ok := svc.(*sqsbackend.Handler); ok { + if b, isOwner := h.Backend.(queueOwnerBackend); isOwner { + out = append(out, sqsOwnerResolver{backend: b}) + } + } + } + + return out +} + +type queueOwnerBackend interface { + QueueOwner(arnAccount, callerAccount, region, name string) string +} + +// sqsOwnerResolver maps a queue ARN to the account whose queue the request addresses. +type sqsOwnerResolver struct{ backend queueOwnerBackend } + +const sqsARNParts = 6 + +func (r sqsOwnerResolver) ResourceOwnerAccount(ctx context.Context, resourceARN string) (string, bool) { + parts := strings.SplitN(resourceARN, ":", sqsARNParts) + if len(parts) < sqsARNParts || parts[2] != "sqs" || parts[4] == "" || parts[5] == "" { + return "", false + } + + return r.backend.QueueOwner(parts[4], awsmeta.Account(ctx), parts[3], parts[5]), true +} + +type managedKeyBackend interface { + IsAWSManagedKey(ctx context.Context, keyID string) bool +} + +// kmsManagedKeyExemption exempts AWS managed keys, which RCPs never restrict. +type kmsManagedKeyExemption struct{ backend managedKeyBackend } + +func (e kmsManagedKeyExemption) RCPExempt(ctx context.Context, resourceARN string) bool { + return strings.HasPrefix(resourceARN, "arn:aws:kms:") && e.backend.IsAWSManagedKey(ctx, resourceARN) +} + +// rcpExemptionsFor returns the resources RCPs must skip, or nil when org policy enforcement is off. +func rcpExemptionsFor(enabled bool, services []service.Registerable) []iambackend.RCPExemptionResolver { + if !enabled { + return nil + } + + var out []iambackend.RCPExemptionResolver + + for _, svc := range services { + if h, ok := svc.(*kmsbackend.Handler); ok { + if b, isKMS := h.Backend.(managedKeyBackend); isKMS { + out = append(out, kmsManagedKeyExemption{backend: b}) + } + } + } + + return out +} + +// wireSimulatorSCPs lets the IAM policy simulator report the SCP verdict (OrganizationsDecisionDetail). +func wireSimulatorSCPs(enabled bool, services []service.Registerable) { + p := scpProviderFor(enabled, services) + if p == nil { + return + } + + for _, svc := range services { + if h, ok := svc.(*iambackend.Handler); ok { + if b, isSim := h.Backend.(interface{ SetSCPProvider(iambackend.SCPProvider) }); isSim { + b.SetSCPProvider(p) + } + } + } +} diff --git a/internal/app/iam_rcp_services_test.go b/internal/app/iam_rcp_services_test.go new file mode 100644 index 0000000000..53dd6ef199 --- /dev/null +++ b/internal/app/iam_rcp_services_test.go @@ -0,0 +1,228 @@ +package app + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const openQueuePolicy = `{"Version":"2012-10-17","Statement":` + + `[{"Effect":"Allow","Principal":"*","Action":"sqs:*","Resource":"*"}]}` + +func (f *scpFixture) sqs(t *testing.T, c scpCaller) *sqs.Client { + t.Helper() + + return sqs.NewFromConfig(scpConfig(t, f.srv.URL, c.key, c.secret, "us-east-1", c.account)) +} + +func (f *scpFixture) queueFor(t *testing.T, owner scpCaller, name string) string { + t.Helper() + + c := f.sqs(t, owner) + out, err := c.CreateQueue(t.Context(), &sqs.CreateQueueInput{QueueName: aws.String(name)}) + require.NoError(t, err) + + _, err = c.SetQueueAttributes(t.Context(), &sqs.SetQueueAttributesInput{ + QueueUrl: out.QueueUrl, + Attributes: map[string]string{"Policy": openQueuePolicy}, + }) + require.NoError(t, err) + + return aws.ToString(out.QueueUrl) +} + +func TestRCPEnforcementSQS(t *testing.T) { + t.Parallel() + + tests := []struct { + run func(t *testing.T, f *scpFixture, member scpCaller) + name string + }{ + { + name: "outsider_denied_peer_and_owner_allowed", + run: func(t *testing.T, f *scpFixture, member scpCaller) { + t.Helper() + + url := f.queueFor(t, member, "rcp-q") + outsider := f.adminUser(t, outsideAccount, "outsider") + send := func(c scpCaller) error { + _, err := f.sqs(t, c).SendMessage(t.Context(), &sqs.SendMessageInput{ + QueueUrl: aws.String(url), MessageBody: aws.String("m"), + }) + + return err + } + + if err := send(outsider); err != nil { + assert.NotContains(t, err.Error(), "resource control policy", "no RCP attached yet") + } + + f.enableRCP(t) + f.attach(t, f.createRCP(t, `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":"*",`+ + `"Action":"sqs:*","Resource":"*","Condition":{"StringNotEqualsIfExists":`+ + `{"aws:PrincipalOrgID":"`+f.orgID(t)+`"}}}]}`), f.rootID) + + err := send(outsider) + require.Error(t, err) + assert.Contains(t, accessDeniedMessage(t, err), "explicit deny in a resource control policy") + + require.NoError(t, send(member)) + + peer := f.adminUser(t, f.accountID, "peer") + require.NoError(t, send(peer)) + }, + }, + { + name: "management_owned_queue_exempt", + run: func(t *testing.T, f *scpFixture, _ scpCaller) { + t.Helper() + + mgmt := f.adminUser(t, "", "mgmt-admin") + url := f.queueFor(t, mgmt, "mgmt-q") + + f.enableRCP(t) + f.attach(t, f.createRCP(t, `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":"*",`+ + `"Action":"sqs:*","Resource":"*"}]}`), f.rootID) + + _, err := f.sqs(t, mgmt).SendMessage(t.Context(), &sqs.SendMessageInput{ + QueueUrl: aws.String(url), MessageBody: aws.String("m"), + }) + require.NoError(t, err) + }, + }, + { + name: "member_create_queue_governed_by_member_rcp", + run: func(t *testing.T, f *scpFixture, member scpCaller) { + t.Helper() + + f.enableRCP(t) + f.attach(t, f.createRCP(t, `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":"*",`+ + `"Action":"sqs:CreateQueue","Resource":"*"}]}`), f.ouID) + + _, err := f.sqs(t, member).CreateQueue(t.Context(), + &sqs.CreateQueueInput{QueueName: aws.String("denied-q")}) + require.Error(t, err) + assert.Contains(t, accessDeniedMessage(t, err), "resource control policy") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := newSCPFixture(t, true) + tt.run(t, f, f.adminUser(t, f.accountID, "member-admin")) + }) + } +} + +func TestRCPEnforcementSTS(t *testing.T) { + t.Parallel() + + f := newSCPFixture(t, true) + member := f.adminUser(t, f.accountID, "member-admin") + outsider := f.adminUser(t, outsideAccount, "outsider") + + const trust = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},` + + `"Action":"sts:AssumeRole"}]}` + + iamc := iam.NewFromConfig(scpConfig(t, f.srv.URL, member.key, member.secret, "us-east-1", member.account)) + role, err := iamc.CreateRole(t.Context(), &iam.CreateRoleInput{ + RoleName: aws.String("target"), AssumeRolePolicyDocument: aws.String(trust), + }) + require.NoError(t, err) + + assume := func(c scpCaller) error { + _, e := sts.NewFromConfig(scpConfig(t, f.srv.URL, c.key, c.secret, "us-east-1", c.account)). + AssumeRole(t.Context(), &sts.AssumeRoleInput{ + RoleArn: role.Role.Arn, RoleSessionName: aws.String("sess"), + }) + + return e + } + + require.NoError(t, assume(outsider), "trust policy allows any principal") + + f.enableRCP(t) + f.attach(t, f.createRCP(t, `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":"*",`+ + `"Action":"sts:AssumeRole","Resource":"*","Condition":{"StringNotEqualsIfExists":`+ + `{"aws:PrincipalOrgID":"`+f.orgID(t)+`"}}}]}`), f.rootID) + + err = assume(outsider) + require.Error(t, err) + assert.Contains(t, accessDeniedMessage(t, err), "explicit deny in a resource control policy") + + require.NoError(t, assume(member)) +} + +func TestSimulatePrincipalPolicyOrganizations(t *testing.T) { + t.Parallel() + + const denyDelete = `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"s3:DeleteBucket","Resource":"*"}]}` + + tests := []struct { + account string + action string + wantDecision string + name string + scpAttached bool + wantDetail bool + wantAllowedByOr bool + }{ + {name: "member_scp_denies", account: "member", action: "s3:DeleteBucket", scpAttached: true, + wantDetail: true, wantAllowedByOr: false, wantDecision: "explicitDeny"}, + {name: "member_scp_allows_other_action", account: "member", action: "s3:GetObject", scpAttached: true, + wantDetail: true, wantAllowedByOr: true, wantDecision: "allowed"}, + {name: "management_account_has_no_detail", account: "mgmt", action: "s3:DeleteBucket", scpAttached: true, + wantDetail: false, wantDecision: "allowed"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := newSCPFixture(t, true) + acct := f.accountID + + if tt.account == "mgmt" { + acct = "000000000000" + } + + callerAccount := acct + if tt.account == "mgmt" { + callerAccount = "" + } + + caller := f.adminUser(t, callerAccount, "sim-user") + if tt.scpAttached { + f.attach(t, f.createSCP(t, "deny-delete", denyDelete), f.ouID) + } + + c := iam.NewFromConfig(scpConfig(t, f.srv.URL, caller.key, caller.secret, "us-east-1", caller.account)) + out, err := c.SimulatePrincipalPolicy(t.Context(), &iam.SimulatePrincipalPolicyInput{ + PolicySourceArn: aws.String("arn:aws:iam::" + acct + ":user/sim-user"), + ActionNames: []string{tt.action}, + }) + require.NoError(t, err) + require.Len(t, out.EvaluationResults, 1) + + res := out.EvaluationResults[0] + assert.Equal(t, tt.wantDecision, string(res.EvalDecision)) + + if !tt.wantDetail { + assert.Nil(t, res.OrganizationsDecisionDetail) + + return + } + + require.NotNil(t, res.OrganizationsDecisionDetail) + assert.Equal(t, tt.wantAllowedByOr, res.OrganizationsDecisionDetail.AllowedByOrganizations) + }) + } +} diff --git a/internal/app/iam_rcp_test.go b/internal/app/iam_rcp_test.go new file mode 100644 index 0000000000..68a804c250 --- /dev/null +++ b/internal/app/iam_rcp_test.go @@ -0,0 +1,216 @@ +package app + +import ( + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/organizations" + orgtypes "github.com/aws/aws-sdk-go-v2/service/organizations/types" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/iampolicystream" +) + +const ( + rcpFullAccessID = "p-RCPFullAWSAccess" + outsideAccount = "222222222222" +) + +func (f *scpFixture) enableRCP(t *testing.T) { + t.Helper() + + _, err := f.orgs.EnablePolicyType(t.Context(), &organizations.EnablePolicyTypeInput{ + RootId: aws.String(f.rootID), PolicyType: orgtypes.PolicyTypeResourceControlPolicy, + }) + require.NoError(t, err) +} + +func (f *scpFixture) createRCP(t *testing.T, doc string) string { + t.Helper() + + out, err := f.orgs.CreatePolicy(t.Context(), &organizations.CreatePolicyInput{ + Name: aws.String("perimeter"), Description: aws.String("perimeter"), Content: aws.String(doc), + Type: orgtypes.PolicyTypeResourceControlPolicy, + }) + require.NoError(t, err) + + return aws.ToString(out.Policy.PolicySummary.Id) +} + +func (f *scpFixture) orgID(t *testing.T) string { + t.Helper() + + out, err := f.orgs.DescribeOrganization(t.Context(), &organizations.DescribeOrganizationInput{}) + require.NoError(t, err) + + return aws.ToString(out.Organization.Id) +} + +func orgPerimeterRCP(orgID string) string { + return `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":"*","Action":"s3:*","Resource":"*",` + + `"Condition":{"StringNotEqualsIfExists":{"aws:PrincipalOrgID":"` + orgID + `"}}}]}` +} + +func (f *scpFixture) bucketWithObject(t *testing.T, owner scpCaller, bucket string) { + t.Helper() + + c := f.s3(t, owner) + _, err := c.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String(bucket)}) + require.NoError(t, err) + _, err = c.PutObject(t.Context(), &s3.PutObjectInput{ + Bucket: aws.String(bucket), Key: aws.String("k"), Body: strings.NewReader("v"), + }) + require.NoError(t, err) +} + +func TestRCPEnforcement(t *testing.T) { + t.Parallel() + + tests := []struct { + run func(t *testing.T, f *scpFixture, member scpCaller) + name string + }{ + { + name: "outside_account_denied_in_org_allowed", + run: func(t *testing.T, f *scpFixture, member scpCaller) { + t.Helper() + + f.bucketWithObject(t, member, "rcp-b") + outsider := f.adminUser(t, outsideAccount, "outsider") + peer := f.adminUser(t, f.accountID, "peer") + + _, err := f.s3(t, outsider).GetObject(t.Context(), &s3.GetObjectInput{ + Bucket: aws.String("rcp-b"), Key: aws.String("k"), + }) + require.NoError(t, err, "no RCP attached yet") + + f.enableRCP(t) + pid := f.createRCP(t, orgPerimeterRCP(f.orgID(t))) + f.attach(t, pid, f.rootID) + + _, err = f.s3(t, outsider).GetObject(t.Context(), &s3.GetObjectInput{ + Bucket: aws.String("rcp-b"), Key: aws.String("k"), + }) + require.Error(t, err) + + msg := accessDeniedMessage(t, err) + assert.Contains(t, msg, "is not authorized to perform: s3:GetObject") + assert.Contains(t, msg, "with an explicit deny in a resource control policy: arn:aws:organizations::") + assert.Contains(t, msg, "/resource_control_policy/"+pid) + + _, err = f.s3(t, peer).GetObject(t.Context(), &s3.GetObjectInput{ + Bucket: aws.String("rcp-b"), Key: aws.String("k"), + }) + require.NoError(t, err) + + _, err = f.s3(t, member).GetObject(t.Context(), &s3.GetObjectInput{ + Bucket: aws.String("rcp-b"), Key: aws.String("k"), + }) + require.NoError(t, err) + + var denied *iampolicystream.Event + + for _, ev := range f.rec.Snapshot(func(iampolicystream.Event) bool { return true }) { + if ev.Decision == iampolicystream.DecisionDeny { + denied = &ev + } + } + + require.NotNil(t, denied) + assert.Equal(t, iampolicystream.ReasonResourceControl, denied.Reason) + assert.Equal(t, iampolicystream.DenyExplicit, denied.DenyType) + require.NotNil(t, denied.Policy) + assert.Equal(t, iampolicystream.PolicyRCP, denied.Policy.Type) + assert.Contains(t, denied.Policy.Arn, pid) + }, + }, + { + name: "management_account_resources_exempt", + run: func(t *testing.T, f *scpFixture, _ scpCaller) { + t.Helper() + + f.enableRCP(t) + f.attach(t, f.createRCP(t, orgPerimeterRCP(f.orgID(t))), f.rootID) + + mgmt := f.adminUser(t, "", "mgmt-admin") + f.bucketWithObject(t, mgmt, "mgmt-b") + + outsider := f.adminUser(t, outsideAccount, "outsider") + _, err := f.s3(t, outsider).GetObject(t.Context(), &s3.GetObjectInput{ + Bucket: aws.String("mgmt-b"), Key: aws.String("k"), + }) + require.NoError(t, err) + }, + }, + { + name: "rcp_type_disabled_has_no_effect", + run: func(t *testing.T, f *scpFixture, member scpCaller) { + t.Helper() + + f.bucketWithObject(t, member, "off-b") + f.enableRCP(t) + pid := f.createRCP(t, orgPerimeterRCP(f.orgID(t))) + f.attach(t, pid, f.rootID) + + _, err := f.orgs.DisablePolicyType(t.Context(), &organizations.DisablePolicyTypeInput{ + RootId: aws.String(f.rootID), PolicyType: orgtypes.PolicyTypeResourceControlPolicy, + }) + require.NoError(t, err) + + outsider := f.adminUser(t, outsideAccount, "outsider") + _, err = f.s3(t, outsider).GetObject(t.Context(), &s3.GetObjectInput{ + Bucket: aws.String("off-b"), Key: aws.String("k"), + }) + require.NoError(t, err) + }, + }, + { + name: "unsupported_service_not_evaluated", + run: func(t *testing.T, f *scpFixture, member scpCaller) { + t.Helper() + + f.enableRCP(t) + f.attach(t, f.createRCP(t, orgPerimeterRCP(f.orgID(t))), f.rootID) + f.bucketWithObject(t, member, "svc-b") + + _, err := f.s3(t, member).ListBuckets(t.Context(), &s3.ListBucketsInput{}) + require.NoError(t, err) + }, + }, + { + name: "default_rcp_attached_and_not_detachable", + run: func(t *testing.T, f *scpFixture, _ scpCaller) { + t.Helper() + + f.enableRCP(t) + + for _, target := range []string{f.rootID, f.ouID, f.accountID} { + out, err := f.orgs.ListPoliciesForTarget(t.Context(), &organizations.ListPoliciesForTargetInput{ + TargetId: aws.String(target), Filter: orgtypes.PolicyTypeResourceControlPolicy, + }) + require.NoError(t, err) + require.Len(t, out.Policies, 1) + assert.Equal(t, rcpFullAccessID, aws.ToString(out.Policies[0].Id)) + } + + _, err := f.orgs.DetachPolicy(t.Context(), &organizations.DetachPolicyInput{ + PolicyId: aws.String(rcpFullAccessID), TargetId: aws.String(f.ouID), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "InvalidInputException") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := newSCPFixture(t, true) + tt.run(t, f, f.adminUser(t, f.accountID, "member-admin")) + }) + } +} diff --git a/internal/app/insight_rule_transformed_test.go b/internal/app/insight_rule_transformed_test.go index cd11fa4842..013da444eb 100644 --- a/internal/app/insight_rule_transformed_test.go +++ b/internal/app/insight_rule_transformed_test.go @@ -48,7 +48,7 @@ func TestInitializeServices_InsightRuleTransformedLogs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - services, err := initializeServicesT(t, newTestAppContext(t, 20000, 20100)) + services, err := initializeServicesT(t, newTestAppContext(t)) require.NoError(t, err) byName := serviceByName(services) diff --git a/internal/app/kinesis_channel_s3_delivery_wiring_test.go b/internal/app/kinesis_channel_s3_delivery_wiring_test.go index c20181d411..553e7b90a4 100644 --- a/internal/app/kinesis_channel_s3_delivery_wiring_test.go +++ b/internal/app/kinesis_channel_s3_delivery_wiring_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" kinesisbackend "github.com/blackbirdworks/gopherstack/services/kinesis" s3backend "github.com/blackbirdworks/gopherstack/services/s3" @@ -35,8 +35,7 @@ func TestInitializeServices_KinesisChannelS3DeliveryWiring(t *testing.T) { t.Parallel() cli := &CLI{AccountID: "000000000000", Region: "us-east-1"} - portAlloc, err := portalloc.New(19200, 19300) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) appCtx := &service.AppContext{ Logger: slog.Default(), diff --git a/internal/app/lambda_ecr_wiring_test.go b/internal/app/lambda_ecr_wiring_test.go index 053a322373..e8dd6048ff 100644 --- a/internal/app/lambda_ecr_wiring_test.go +++ b/internal/app/lambda_ecr_wiring_test.go @@ -15,7 +15,7 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" ecrbackend "github.com/blackbirdworks/gopherstack/services/ecr" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" @@ -36,8 +36,7 @@ func TestInitializeServices_LambdaECRWiring(t *testing.T) { t.Parallel() cli := &CLI{AccountID: "000000000000", Region: "us-east-1"} - portAlloc, err := portalloc.New(19200, 19300) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) appCtx := &service.AppContext{ Logger: slog.Default(), diff --git a/internal/app/lambda_runtime_env.go b/internal/app/lambda_runtime_env.go index 6d6152ad46..3e0411e9f2 100644 --- a/internal/app/lambda_runtime_env.go +++ b/internal/app/lambda_runtime_env.go @@ -57,6 +57,9 @@ func lambdaEndpointURL(dockerHost, port string) string { // wireLambdaRuntimeEnv gives function containers the emulator endpoint and // execution-role credentials. func wireLambdaRuntimeEnv(cli *CLI, byName map[string]service.Registerable) { + wireLambdaWeb(cli, byName) + wireCodeBuildRuntimeEnv(cli, byName) + lambdaH, ok := byName["Lambda"].(*lambdabackend.Handler) if !ok { return diff --git a/internal/app/lambda_telemetry_docker_test.go b/internal/app/lambda_telemetry_docker_test.go new file mode 100644 index 0000000000..471273fe8d --- /dev/null +++ b/internal/app/lambda_telemetry_docker_test.go @@ -0,0 +1,265 @@ +package app + +import ( + "context" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/container" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" + lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" +) + +const telemetryExtensionSource = `package main + +import ( + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" +) + +var ( + cl = new(http.Client) + mu sync.Mutex +) + +func record(s string) { + mu.Lock() + defer mu.Unlock() + f, err := os.OpenFile("/ext-out/telemetry.ndjson", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return + } + defer f.Close() + fmt.Fprintln(f, s) +} + +func main() { + api := os.Getenv("AWS_LAMBDA_RUNTIME_API") + req, _ := http.NewRequest(http.MethodPost, "http://"+api+"/2020-01-01/extension/register", + strings.NewReader(` + "`" + `{"events":["INVOKE","SHUTDOWN"]}` + "`" + `)) + req.Header.Set("Lambda-Extension-Name", filepath.Base(os.Args[0])) + resp, err := cl.Do(req) + if err != nil { + os.Exit(1) + } + id := resp.Header.Get("Lambda-Extension-Identifier") + resp.Body.Close() + + ln, err := net.Listen("tcp", "0.0.0.0:4243") + if err != nil { + os.Exit(1) + } + go http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + record(string(b)) + })) + + sub, _ := http.NewRequest(http.MethodPut, "http://"+api+"/2022-07-01/telemetry", strings.NewReader( + ` + "`" + `{"schemaVersion":"2022-12-13","types":["platform","function","extension"], + "buffering":{"maxItems":1000,"maxBytes":262144,"timeoutMs":25}, + "destination":{"protocol":"HTTP","URI":"http://sandbox.localdomain:4243/"}}` + "`" + `)) + sub.Header.Set("Lambda-Extension-Identifier", id) + sresp, err := cl.Do(sub) + if err != nil || sresp.StatusCode != 200 { + record("SUBSCRIBE-FAILED") + os.Exit(1) + } + sresp.Body.Close() + fmt.Println("extension says hello") + + for { + nreq, _ := http.NewRequest(http.MethodGet, "http://"+api+"/2020-01-01/extension/event/next", nil) + nreq.Header.Set("Lambda-Extension-Identifier", id) + nresp, err := cl.Do(nreq) + if err != nil { + os.Exit(1) + } + raw, _ := io.ReadAll(nresp.Body) + nresp.Body.Close() + if strings.Contains(string(raw), "SHUTDOWN") { + return + } + } +} +` + +const telemetryHandler = `def handler(event, context): + print("hello from function") + return {"ok": True} +` + +const telemetryDockerfile = `FROM ` + efsDockerRuntimeImage + ` +COPY handler.py ${LAMBDA_TASK_ROOT}/ +COPY ext-bin /opt/extensions/ext-bin +CMD ["handler.handler"] +` + +func buildTelemetryExtension(t *testing.T) []byte { + t.Helper() + + dir := t.TempDir() + src := filepath.Join(dir, "main.go") + require.NoError(t, os.WriteFile(src, []byte(telemetryExtensionSource), 0o600)) + + out := filepath.Join(dir, "ext-bin") + cmd := exec.CommandContext(t.Context(), "go", "build", "-o", out, src) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux", "GOFLAGS=", "GOWORK=off", "GO111MODULE=off") + + if combined, err := cmd.CombinedOutput(); err != nil { + t.Skipf("cannot build extension binary: %v: %s", err, combined) + } + + bin, err := os.ReadFile(out) + require.NoError(t, err) + + return bin +} + +func buildExtensionImage(t *testing.T, bin []byte) string { + t.Helper() + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "handler.py"), []byte(telemetryHandler), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "ext-bin"), bin, 0o600)) + require.NoError(t, os.Chmod(filepath.Join(dir, "ext-bin"), 0o755)) // #nosec G302 -- executable extension + require.NoError(t, os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(telemetryDockerfile), 0o600)) + + tag := "gopherstack-telemetry-ext-test:" + strings.ToLower(filepath.Base(dir)) + build := exec.CommandContext(t.Context(), "docker", "build", "-q", "-t", tag, dir) + if out, err := build.CombinedOutput(); err != nil { + t.Skipf("cannot build image: %v: %s", err, out) + } + + t.Cleanup(func() { + _ = exec.CommandContext(context.Background(), "docker", "rmi", "-f", tag).Run() + }) + + return tag +} + +func TestLambdaTelemetryAPIDocker(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("docker test") + } + + if err := exec.CommandContext(t.Context(), "docker", "image", "inspect", efsDockerRuntimeImage).Run(); err != nil { + t.Skipf("lambda runtime image %s not available locally: %v", efsDockerRuntimeImage, err) + } + + bin := buildTelemetryExtension(t) + imageTag := buildExtensionImage(t, bin) + + tests := []struct { + setup func(t *testing.T, bk *lambdabackend.InMemoryBackend, fn *lambdabackend.FunctionConfiguration) + name string + }{ + { + name: "zip with layer", + setup: func(t *testing.T, bk *lambdabackend.InMemoryBackend, fn *lambdabackend.FunctionConfiguration) { + t.Helper() + + layer, err := bk.PublishLayerVersion(&lambdabackend.PublishLayerVersionInput{ + LayerName: "telemetry-layer", + Content: &lambdabackend.LayerVersionContentInput{ZipFile: extensionLayerZip(t, bin)}, + }) + require.NoError(t, err) + + fn.PackageType = lambdabackend.PackageTypeZip + fn.Runtime = "python3.12" + fn.Handler = "tel_handler.handler" + fn.ZipData = zipOf(t, "tel_handler.py", telemetryHandler) + fn.Layers = []*lambdabackend.FunctionLayer{{Arn: layer.LayerVersionArn}} + }, + }, + { + name: "container image", + setup: func(t *testing.T, _ *lambdabackend.InMemoryBackend, fn *lambdabackend.FunctionConfiguration) { + t.Helper() + + fn.PackageType = lambdabackend.PackageTypeImage + fn.ImageURI = imageTag + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rt, err := container.NewRuntime( + container.Config{Logger: slog.Default(), PoolSize: 2, IdleTimeout: time.Minute}, + ) + if err != nil { + t.Skipf("container runtime unavailable: %v", err) + } + + t.Cleanup(func() { _ = rt.Close() }) + + outDir := t.TempDir() + require.NoError(t, os.Chmod(outDir, 0o777)) // #nosec G302 -- container root writes here + + settings := lambdabackend.DefaultSettings() + settings.DockerFlags = "-v " + outDir + ":/ext-out" + + bk := lambdabackend.NewInMemoryBackend(rt, portalloctest.New(t, 20), settings, "000000000000", "us-east-1") + t.Cleanup(func() { bk.Close(context.Background()) }) + + fn := &lambdabackend.FunctionConfiguration{FunctionName: "tel-fn", Timeout: 60, MemorySize: 256} + tt.setup(t, bk, fn) + require.NoError(t, bk.CreateFunction(fn)) + + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute) + defer cancel() + + out, status, invokeErr := bk.InvokeFunction( + ctx, + "tel-fn", + lambdabackend.InvocationTypeRequestResponse, + []byte(`{}`), + ) + require.NoError(t, invokeErr) + require.Equal(t, 200, status) + assert.Contains(t, string(out), "true") + + read := func() string { + b, _ := os.ReadFile(filepath.Join(outDir, "telemetry.ndjson")) + + return string(b) + } + + for _, want := range []string{ + `"type":"platform.initStart"`, `"type":"platform.telemetrySubscription"`, `"type":"platform.extension"`, + `"type":"platform.initRuntimeDone"`, `"type":"platform.initReport"`, `"type":"platform.start"`, + `"type":"platform.runtimeDone"`, `"type":"platform.report"`, + `"type":"function","record":"hello from function"`, + `"type":"extension","record":"extension says hello"`, + `"memorySizeMB":256`, `"functionName":"tel-fn"`, + } { + require.Eventually(t, func() bool { return strings.Contains(read(), want) }, + 30*time.Second, 50*time.Millisecond, "missing %s in %s", want, read()) + } + + assert.NotContains(t, read(), "SUBSCRIBE-FAILED") + assert.Contains(t, read(), `"maxMemoryUsedMB"`) + + require.NoError(t, bk.DeleteFunction("tel-fn")) + }) + } +} diff --git a/internal/app/lambdaweb_wiring.go b/internal/app/lambdaweb_wiring.go new file mode 100644 index 0000000000..2c47ced325 --- /dev/null +++ b/internal/app/lambdaweb_wiring.go @@ -0,0 +1,157 @@ +package app + +import ( + "context" + "errors" + "io" + "maps" + "slices" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + cwlogsbackend "github.com/blackbirdworks/gopherstack/services/cloudwatchlogs" + kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" + lambdawebbackend "github.com/blackbirdworks/gopherstack/services/lambdaweb" + s3backend "github.com/blackbirdworks/gopherstack/services/s3" + stsbackend "github.com/blackbirdworks/gopherstack/services/sts" +) + +// lambdaWebCode serves web function deployment packages from versioned S3 buckets. +type lambdaWebCode struct { + backend s3backend.StorageBackend +} + +func (c lambdaWebCode) ResolveObject(ctx context.Context, bucket, key, versionID string) (string, error) { + ver, err := c.backend.GetBucketVersioning(ctx, &awss3.GetBucketVersioningInput{Bucket: aws.String(bucket)}) + if err != nil { + if errors.Is(err, s3backend.ErrNoSuchBucket) { + return "", lambdawebbackend.ErrBucketNotFound + } + + return "", err + } + + if ver.Status != s3types.BucketVersioningStatusEnabled { + return "", lambdawebbackend.ErrBucketNotVersioned + } + + in := &awss3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)} + if versionID != "" { + in.VersionId = aws.String(versionID) + } + + head, err := c.backend.HeadObject(ctx, in) + if err != nil { + if errors.Is(err, s3backend.ErrNoSuchKey) || errors.Is(err, s3backend.ErrNoSuchBucket) { + return "", lambdawebbackend.ErrObjectNotFound + } + + return "", err + } + + return aws.ToString(head.VersionId), nil +} + +func (c lambdaWebCode) GetObject(ctx context.Context, bucket, key, versionID string) ([]byte, error) { + in := &awss3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)} + if versionID != "" { + in.VersionId = aws.String(versionID) + } + + out, err := c.backend.GetObject(ctx, in) + if err != nil { + return nil, err + } + defer out.Body.Close() + + return io.ReadAll(out.Body) +} + +// lambdaWebCredentials adapts the Lambda STS role-credential issuer to lambdaweb.CredentialIssuer. +type lambdaWebCredentials struct { + inner *lambdaRoleCredentials +} + +func (a lambdaWebCredentials) IssueCredentials(roleARN, sessionName string) (lambdawebbackend.Credentials, error) { + c, err := a.inner.IssueFunctionCredentials(roleARN, sessionName) + if err != nil { + return lambdawebbackend.Credentials{}, err + } + + return lambdawebbackend.Credentials{ + AccessKeyID: c.AccessKeyID, SecretAccessKey: c.SecretAccessKey, SessionToken: c.SessionToken, + }, nil +} + +var errLambdaWebKeyUnusable = errors.New("key is not an enabled symmetric encryption key") + +// lambdaWebKeys checks that a revision's KMS key exists, is enabled and is a symmetric encryption key. +type lambdaWebKeys struct { + backend *kmsbackend.InMemoryBackend +} + +func (k lambdaWebKeys) CheckKey(ctx context.Context, region, keyARN string) error { + out, err := k.backend.DescribeKey(kmsbackend.WithRegion(ctx, region), &kmsbackend.DescribeKeyInput{KeyID: keyARN}) + if err != nil { + return err + } + + meta := out.KeyMetadata + if !meta.Enabled || meta.KeyUsage != kmsbackend.KeyUsageEncryptDecrypt || + (meta.KeySpec != "" && meta.KeySpec != "SYMMETRIC_DEFAULT") { + return errLambdaWebKeyUnusable + } + + return nil +} + +// wireLambdaWeb gives Lambda Web Functions its S3 code store, CloudWatch Logs sink, execution-role +// credentials, emulator endpoint and the SigV4 secret IamAuth endpoints verify against. +func wireLambdaWeb(cli *CLI, byName map[string]service.Registerable) { + h, ok := byName["LambdaWeb"].(*lambdawebbackend.Handler) + if !ok { + return + } + + bk, ok := h.Backend.(*lambdawebbackend.InMemoryBackend) + if !ok { + return + } + + if s3H, s3Ok := byName["S3"].(*s3backend.S3Handler); s3Ok { + bk.SetCodeStore(lambdaWebCode{backend: s3H.Backend}) + } + + if logsH, logsOk := byName["CloudWatchLogs"].(*cwlogsbackend.Handler); logsOk { + if logsBk, bkOk := logsH.Backend.(*cwlogsbackend.InMemoryBackend); bkOk { + bk.SetLogSink(codebuildLogs{backend: logsBk}) + } + } + + bk.SetCredentialIssuer(lambdaWebCredentials{inner: &lambdaRoleCredentials{byName: byName}}) + + if kmsH, kmsOk := byName["KMS"].(*kmsbackend.Handler); kmsOk { + if kmsBk, bkOk := kmsH.Backend.(*kmsbackend.InMemoryBackend); bkOk { + bk.SetKeyChecker(lambdaWebKeys{backend: kmsBk}) + } + } + + if stsH, stsOk := byName["STS"].(*stsbackend.Handler); stsOk { + if stsBk, bkOk := stsH.Backend.(*stsbackend.InMemoryBackend); bkOk { + bk.SetRoleTrust(stsBk) + } + } + + bk.SetPrincipalResolver(buildPrincipalResolvers(slices.Collect(maps.Values(byName)))) + + if cli != nil { + bk.SetSigV4Secret(cli.SigV4Secret) + + if cli.Port != "" { + bk.SetEndpointURL(lambdaEndpointURL(cli.Lambda.DockerHost, cli.Port)) + } + } +} diff --git a/internal/app/lambdaweb_wiring_test.go b/internal/app/lambdaweb_wiring_test.go new file mode 100644 index 0000000000..9eaec8587f --- /dev/null +++ b/internal/app/lambdaweb_wiring_test.go @@ -0,0 +1,58 @@ +package app + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" +) + +func TestLambdaWebKeys_CheckKey(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + spec string + usage string + disable bool + missing bool + wantErr bool + }{ + {name: "symmetric enabled"}, + {name: "disabled", disable: true, wantErr: true}, + {name: "missing", missing: true, wantErr: true}, + {name: "signing key", usage: "SIGN_VERIFY", spec: "RSA_2048", wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + bk := kmsbackend.NewInMemoryBackend() + ctx := t.Context() + + out, err := bk.CreateKey(ctx, &kmsbackend.CreateKeyInput{KeyUsage: tc.usage, KeySpec: tc.spec}) + require.NoError(t, err) + + arn := out.KeyMetadata.Arn + if tc.missing { + arn = "arn:aws:kms:us-east-1:123456789012:key/00000000-0000-0000-0000-000000000000" + } + + if tc.disable { + require.NoError(t, bk.DisableKey(ctx, &kmsbackend.DisableKeyInput{KeyID: out.KeyMetadata.KeyID})) + } + + err = lambdaWebKeys{backend: bk}.CheckKey(ctx, "us-east-1", arn) + if tc.wantErr { + require.Error(t, err) + + return + } + + assert.NoError(t, err) + }) + } +} diff --git a/internal/app/lifecycle.go b/internal/app/lifecycle.go index 7a5a3e436b..5257b1f62f 100644 --- a/internal/app/lifecycle.go +++ b/internal/app/lifecycle.go @@ -50,6 +50,7 @@ import ( kafkaconnectbackend "github.com/blackbirdworks/gopherstack/services/kafkaconnect" kinesisanalyticsv2backend "github.com/blackbirdworks/gopherstack/services/kinesisanalyticsv2" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" + lambdawebbackend "github.com/blackbirdworks/gopherstack/services/lambdaweb" macie2backend "github.com/blackbirdworks/gopherstack/services/macie2" managedblockchainbackend "github.com/blackbirdworks/gopherstack/services/managedblockchain" medialivebackend "github.com/blackbirdworks/gopherstack/services/medialive" @@ -253,6 +254,14 @@ func applyFIS(reg service.Registerable, l cliconfig.LifecycleSettings) { } } +func applyLambdaWeb(reg service.Registerable, l cliconfig.LifecycleSettings) { + if h, ok := reg.(*lambdawebbackend.Handler); ok { + if bk, isBk := h.Backend.(*lambdawebbackend.InMemoryBackend); isBk { + bk.SetLifecycleDelay(l.Effective(l.LambdaWeb)) + } + } +} + func applyDSQL(reg service.Registerable, l cliconfig.LifecycleSettings) { if h, ok := reg.(*dsqlbackend.Handler); ok { if bk, isBk := h.Backend.(*dsqlbackend.InMemoryBackend); isBk { @@ -780,6 +789,7 @@ func wireLifecycleDelays(byName map[string]service.Registerable, l cliconfig.Lif "AppStream": applyAppStream, "FIS": applyFIS, "DSQL": applyDSQL, + "LambdaWeb": applyLambdaWeb, "GlobalAccelerator": applyGlobalAccelerator, "CodeGuruReviewer": applyCodeGuruReviewer, "Connect": applyConnect, diff --git a/internal/app/lifecycle_cmd.go b/internal/app/lifecycle_cmd.go index 090fa06b9c..0b4e5234d6 100644 --- a/internal/app/lifecycle_cmd.go +++ b/internal/app/lifecycle_cmd.go @@ -27,6 +27,7 @@ const ( lifecycleDefaultTail = 100 lifecycleLogTailMaxByte = 1 << 20 lifecycleSocketInside = "/var/run/docker.sock" + lifecycleRootUser = "0:0" // image default 65532 can't write a host bind mount or open the socket lifecycleSchemeHTTPS = "https" lifecycleStopGrace = 2 * lifecycleStopTimeout localstackAvailable = "available" diff --git a/internal/app/lifecycle_cmd_test.go b/internal/app/lifecycle_cmd_test.go index a943b50c5a..25c309ce23 100644 --- a/internal/app/lifecycle_cmd_test.go +++ b/internal/app/lifecycle_cmd_test.go @@ -319,10 +319,12 @@ func TestLifecycleStartCreatesContainer(t *testing.T) { assert.Equal(t, wantPorts, spec.Ports) if tt.mount { + assert.Equal(t, "0:0", spec.User) require.Len(t, spec.Mounts, 1) assert.True(t, strings.HasSuffix(spec.Mounts[0], ":/data")) assert.True(t, strings.HasPrefix(spec.Mounts[0], filepath.Join(dir, "data-n1"))) } else { + assert.Empty(t, spec.User) assert.Empty(t, spec.Mounts) } diff --git a/internal/app/lifecycle_foreground_test.go b/internal/app/lifecycle_foreground_test.go index b9d30e8e1a..8132969dca 100644 --- a/internal/app/lifecycle_foreground_test.go +++ b/internal/app/lifecycle_foreground_test.go @@ -261,6 +261,7 @@ func TestLifecycleDockerSocketMount(t *testing.T) { } assert.Contains(t, spec.Mounts, tt.wantMount) + assert.Equal(t, "0:0", spec.User) assert.Contains(t, spec.Env, "DOCKER_HOST=unix:///var/run/docker.sock") assert.Contains(t, out.String(), "root-equivalent") }) diff --git a/internal/app/lifecycle_run.go b/internal/app/lifecycle_run.go index 9ba2fe5363..18ed776a28 100644 --- a/internal/app/lifecycle_run.go +++ b/internal/app/lifecycle_run.go @@ -246,11 +246,13 @@ func (c *StartCmd) runContainer(ctx context.Context, d *lifecycleDeps, endpoint, if dataDir != "" { env = append(env, "GOPHERSTACK_DATA_DIR="+lifecycleContainerData) spec.Mounts = []string{dataDir + ":" + lifecycleContainerData} + spec.User = lifecycleRootUser } if c.DockerSocket { sock := c.dockerSocketPath(d.Getenv) spec.Mounts = append(spec.Mounts, sock+":"+lifecycleSocketInside) + spec.User = lifecycleRootUser env = append(env, "DOCKER_HOST=unix://"+lifecycleSocketInside) fmt.Fprintf(d.Err, "warning: mounting %s gives the server root-equivalent control of the host Docker daemon\n", sock) diff --git a/internal/app/lightsail_cloudformation_wiring_test.go b/internal/app/lightsail_cloudformation_wiring_test.go index 3c1002aab3..98ef1aa67f 100644 --- a/internal/app/lightsail_cloudformation_wiring_test.go +++ b/internal/app/lightsail_cloudformation_wiring_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" cfnbackend "github.com/blackbirdworks/gopherstack/services/cloudformation" lightsailbackend "github.com/blackbirdworks/gopherstack/services/lightsail" @@ -19,8 +19,7 @@ func TestInitializeServices_LightsailCloudFormationWiring(t *testing.T) { t.Parallel() cli := &CLI{AccountID: "000000000000", Region: "us-east-1"} - portAlloc, err := portalloc.New(19300, 19400) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) appCtx := &service.AppContext{ Logger: slog.Default(), diff --git a/internal/app/localstack_compat_test.go b/internal/app/localstack_compat_test.go index 1d0715c54b..71fd0ea8b0 100644 --- a/internal/app/localstack_compat_test.go +++ b/internal/app/localstack_compat_test.go @@ -223,6 +223,18 @@ func TestLocalstackEnvPlan(t *testing.T) { "GOPHERSTACK_SQS_DISABLE_MESSAGE_RETENTION": "true", }, }, + { + name: "enable_dms_runs_the_engine", environ: []string{"ENABLE_DMS=1"}, + want: map[string]string{"DMS_ENGINE": "docker"}, + }, + { + name: "enable_dms_off", environ: []string{"ENABLE_DMS=0"}, + want: map[string]string{}, + }, + { + name: "enable_dms_keeps_explicit_engine", environ: []string{"ENABLE_DMS=1", "DMS_ENGINE=stub"}, + want: map[string]string{}, + }, { name: "sqs_toggles_on_noop", environ: []string{"SQS_DELAY_PURGE_RETRY=1", "SQS_ENABLE_MESSAGE_RETENTION_PERIOD=1"}, diff --git a/internal/app/localstack_options.go b/internal/app/localstack_options.go index daf8b14c1b..d54aa78d94 100644 --- a/internal/app/localstack_options.go +++ b/internal/app/localstack_options.go @@ -159,6 +159,7 @@ func (p *envPlan) planOptions() { p.planPortRange() p.planSQS() + p.planDMS() p.planLimits() p.planS3Signatures() @@ -183,6 +184,13 @@ func (p *envPlan) planOptions() { } } +// planDMS maps ENABLE_DMS=1, which LocalStack documents as activating DMS, onto the engine that moves data. +func (p *envPlan) planDMS() { + if p.truthy("ENABLE_DMS") { + p.put("DMS_ENGINE", "docker") + } +} + // planSQS inverts LocalStack's positive SQS toggles onto the negative native switches. func (p *envPlan) planSQS() { for _, m := range []struct{ ls, native string }{ diff --git a/internal/app/main_test.go b/internal/app/main_test.go index 45755ec0ce..2f27b24373 100644 --- a/internal/app/main_test.go +++ b/internal/app/main_test.go @@ -2,10 +2,16 @@ package app import ( "context" + "crypto/rand" "fmt" + "io" + "math/big" "net" "net/http" + "net/http/httptest" + "net/url" "strconv" + "strings" "testing" "time" @@ -49,14 +55,10 @@ func TestMultipleServersStartupAndShutdown(t *testing.T) { } } -// startServerRetryingPort starts Gopherstack on a freshly picked ephemeral port and waits for its -// dashboard to answer. freeTCPPort's Listen-then-Close releases the port immediately, but the real -// bind happens much later, at the end of run's init chain -- a wide TOCTOU window in which something -// else can take the same port first. When that happens run exits with a bind error before the -// dashboard ever comes up; this retries on a fresh port rather than burning the rest of the deadline -// on a port already lost (gopherstack-tajh, following the pkgs/dns TOCTOU precedent fixed in -// gopherstack-nn94/7tbt). A plain timeout with no error from run is a genuine hang, not a port race, -// and is not retried -- retrying would not fix it and could make things worse under load. +// startServerRetryingPort starts Gopherstack on a fresh ephemeral port and waits until that attempt's own +// server answers (STS account ID match, so a squatter on the port is not mistaken for it). freeTCPPort +// releases the port before run binds it, so a bind error from run is retried on a new port; a plain +// timeout with no run error is a genuine hang and is not retried. func startServerRetryingPort(t *testing.T, demo bool) (int, chan struct{}, chan error) { t.Helper() @@ -68,14 +70,15 @@ func startServerRetryingPort(t *testing.T, demo bool) (int, chan struct{}, chan for attempt := 1; attempt <= maxAttempts; attempt++ { port := freeTCPPort(t) + accountID := uniqueAccountID(t) stopChan := make(chan struct{}) errChan := make(chan error, 1) go func() { - errChan <- startServerOnPort(t, port, demo, stopChan) + errChan <- startServerOnPort(t, port, accountID, demo, stopChan) }() - up, runErr := waitForDashboard(port, errChan, dashboardWait, dashboardPoll) + up, runErr := waitForInstance(port, accountID, errChan, dashboardWait, dashboardPoll) if up { return port, stopChan, errChan } @@ -102,10 +105,10 @@ func startServerRetryingPort(t *testing.T, demo bool) (int, chan struct{}, chan return 0, nil, nil } -// waitForDashboard polls the dashboard on port until it answers, run exits (delivering to errChan), -// or the deadline elapses. It mirrors require.Eventually's polling shape but, unlike Eventually, -// never marks the test failed itself -- the caller decides whether a given outcome is retryable. -func waitForDashboard(port int, errChan <-chan error, timeout, interval time.Duration) (bool, error) { +// waitForInstance polls until the server on port reports accountID from STS GetCallerIdentity, run +// exits (delivering to errChan), or the deadline elapses. Matching the per-attempt account ID proves +// the responder is this attempt's server, not another test's server that grabbed the port first. +func waitForInstance(port int, accountID string, errChan <-chan error, timeout, interval time.Duration) (bool, error) { client := &http.Client{Timeout: 1 * time.Second} deadline := time.Now().Add(timeout) ticker := time.NewTicker(interval) @@ -118,14 +121,8 @@ func waitForDashboard(port int, errChan <-chan error, timeout, interval time.Dur default: } - resp, err := client.Get(fmt.Sprintf("http://localhost:%d/dashboard", port)) - if err == nil { - ready := resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusInternalServerError - resp.Body.Close() - - if ready { - return true, nil - } + if instanceHasAccount(client, port, accountID) { + return true, nil } if time.Now().After(deadline) { @@ -136,9 +133,35 @@ func waitForDashboard(port int, errChan <-chan error, timeout, interval time.Dur } } +func instanceHasAccount(client *http.Client, port int, accountID string) bool { + resp, err := client.PostForm( + fmt.Sprintf("http://localhost:%d/", port), + url.Values{"Action": {"GetCallerIdentity"}, "Version": {"2011-06-15"}}, + ) + if err != nil { + return false + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + + return err == nil && resp.StatusCode == http.StatusOK && + strings.Contains(string(body), ""+accountID+"") +} + +// uniqueAccountID returns a random 12-digit account ID identifying one server instance. +func uniqueAccountID(t *testing.T) string { + t.Helper() + + n, err := rand.Int(rand.Reader, big.NewInt(1_000_000_000_000)) + require.NoError(t, err) + + return fmt.Sprintf("%012d", n) +} + // startServerOnPort starts Gopherstack on the given port using the CLI run path. // It returns when the stopChan is closed. -func startServerOnPort(t *testing.T, port int, demo bool, stopChan chan struct{}) error { +func startServerOnPort(t *testing.T, port int, accountID string, demo bool, stopChan chan struct{}) error { t.Helper() ctx, cancel := context.WithCancel(t.Context()) defer cancel() @@ -147,6 +170,7 @@ func startServerOnPort(t *testing.T, port int, demo bool, stopChan chan struct{} LogLevel: "info", Port: strconv.Itoa(port), Region: "us-east-1", + AccountID: accountID, Demo: demo, PortRangeStart: 10000, PortRangeEnd: 10100, @@ -177,3 +201,35 @@ func freeTCPPort(t *testing.T) int { return l.Addr().(*net.TCPAddr).Port } + +func TestWaitForInstanceRejectsForeignServer(t *testing.T) { + t.Parallel() + + foreign := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "111111111111") + })) + t.Cleanup(foreign.Close) + + port, err := strconv.Atoi(foreign.URL[strings.LastIndex(foreign.URL, ":")+1:]) + require.NoError(t, err) + + tests := []struct { + name string + accountID string + wantUp bool + }{ + {name: "other instance rejected", accountID: "222222222222", wantUp: false}, + {name: "matching instance accepted", accountID: "111111111111", wantUp: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + up, runErr := waitForInstance( + port, tt.accountID, make(chan error), 300*time.Millisecond, 20*time.Millisecond) + require.NoError(t, runErr) + require.Equal(t, tt.wantUp, up) + }) + } +} diff --git a/internal/app/metrics_wiring.go b/internal/app/metrics_wiring.go index 085c772ce2..1758c3c266 100644 --- a/internal/app/metrics_wiring.go +++ b/internal/app/metrics_wiring.go @@ -16,6 +16,7 @@ import ( firehosebackend "github.com/blackbirdworks/gopherstack/services/firehose" kinesisbackend "github.com/blackbirdworks/gopherstack/services/kinesis" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" + lambdawebbackend "github.com/blackbirdworks/gopherstack/services/lambdaweb" s3backend "github.com/blackbirdworks/gopherstack/services/s3" snsbackend "github.com/blackbirdworks/gopherstack/services/sns" sfnbackend "github.com/blackbirdworks/gopherstack/services/stepfunctions" @@ -90,6 +91,12 @@ func wireServiceMetrics(ctx context.Context, byName map[string]service.Registera h.SetMetricEmitter(e) } + if h, hok := byName["LambdaWeb"].(*lambdawebbackend.Handler); hok { + if bk, bok := h.Backend.(*lambdawebbackend.InMemoryBackend); bok { + bk.SetMetricEmitter(e) + } + } + wireMessagingMetrics(e, byName) wireStorageAndComputeMetrics(e, byName) } diff --git a/internal/app/mgn_s3_import_wiring_test.go b/internal/app/mgn_s3_import_wiring_test.go index 4dce525065..a5639381d6 100644 --- a/internal/app/mgn_s3_import_wiring_test.go +++ b/internal/app/mgn_s3_import_wiring_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" mgnbackend "github.com/blackbirdworks/gopherstack/services/mgn" s3backend "github.com/blackbirdworks/gopherstack/services/s3" @@ -35,8 +35,7 @@ func TestInitializeServices_MGNS3ImportWiring(t *testing.T) { t.Parallel() cli := &CLI{AccountID: "000000000000", Region: "us-east-1"} - portAlloc, err := portalloc.New(19100, 19200) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) appCtx := &service.AppContext{ Logger: slog.Default(), diff --git a/internal/app/neptune_iam_auth.go b/internal/app/neptune_iam_auth.go index 21b569e65c..6d7db103d1 100644 --- a/internal/app/neptune_iam_auth.go +++ b/internal/app/neptune_iam_auth.go @@ -4,15 +4,18 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/service" iambackend "github.com/blackbirdworks/gopherstack/services/iam" "github.com/blackbirdworks/gopherstack/services/neptune" + s3backend "github.com/blackbirdworks/gopherstack/services/s3" stsbackend "github.com/blackbirdworks/gopherstack/services/sts" ) -// wireNeptuneIAMAuth connects Neptune data-plane IAM database authentication to IAM and STS. +// wireNeptuneIAMAuth connects Neptune data-plane IAM database authentication to IAM and STS, and the RDF bulk +// loader / SPARQL LOAD to S3. func wireNeptuneIAMAuth(services []service.Registerable, enforceIAM bool) { var ( nepH *neptune.Handler iamH *iambackend.Handler stsH *stsbackend.Handler + s3H *s3backend.S3Handler ) for _, svc := range services { @@ -23,6 +26,8 @@ func wireNeptuneIAMAuth(services []service.Registerable, enforceIAM bool) { iamH = h case *stsbackend.Handler: stsH = h + case *s3backend.S3Handler: + s3H = h } } @@ -30,6 +35,12 @@ func wireNeptuneIAMAuth(services []service.Registerable, enforceIAM bool) { return } + if nbk, ok := nepH.Backend.(interface{ SetS3Backend(neptune.S3Reader) }); ok && s3H != nil { + if s3bk, isS3 := s3H.Backend.(neptune.S3Reader); isS3 { + nbk.SetS3Backend(s3bk) + } + } + bk, ok := nepH.Backend.(interface{ SetIAMAuth(neptune.IAMAuthResolver) }) if !ok { return diff --git a/internal/app/ociapigateway_wiring.go b/internal/app/ociapigateway_wiring.go index d0ad4cfc96..cf66db6c16 100644 --- a/internal/app/ociapigateway_wiring.go +++ b/internal/app/ociapigateway_wiring.go @@ -2,6 +2,7 @@ package app import ( "context" + "encoding/base64" "errors" "github.com/blackbirdworks/gopherstack/pkgs/service" @@ -9,6 +10,7 @@ import ( ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" ocifunctions "github.com/blackbirdworks/gopherstack/services/ocifunctions" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" + ocivaultbackend "github.com/blackbirdworks/gopherstack/services/ocivault" ) // GetOCIAPIGatewaySettings returns OCI API Gateway settings (ociapigateway.ConfigProvider). @@ -60,4 +62,17 @@ func wireOCIAPIGateway(byName map[string]service.Registerable) { if fnH, found := byName["OCIFunctions"].(*ocifunctions.Handler); found { gwH.Backend.SetFunctionInvoker(ociAPIGatewayFunctions{fn: fnH.Backend}) } + + if vaultH, found := byName["OCIVault"].(*ocivaultbackend.Handler); found { + gwH.Backend.SetSecretResolver(func(id string, version int64) (string, bool) { + w, _, err := vaultH.Backend.GetSecretBundle(id, ocivaultbackend.BundleSelector{VersionNumber: &version}) + if err != nil || w.Content == nil { + return "", false + } + + raw, derr := base64.StdEncoding.DecodeString(w.Content.Content) + + return string(raw), derr == nil + }) + } } diff --git a/internal/app/ociapigateway_wiring_test.go b/internal/app/ociapigateway_wiring_test.go index 4f24bab27c..822e54f59e 100644 --- a/internal/app/ociapigateway_wiring_test.go +++ b/internal/app/ociapigateway_wiring_test.go @@ -1,16 +1,26 @@ package app import ( + "context" + "encoding/base64" + "encoding/json" + "io" "log/slog" + "net/http" + "net/http/httptest" + "strconv" + "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/pkgs/testclient" ociapigateway "github.com/blackbirdworks/gopherstack/services/ociapigateway" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" + ocivaultbackend "github.com/blackbirdworks/gopherstack/services/ocivault" ) func TestReserveFixedServicePorts_OCIAPIGateway(t *testing.T) { @@ -19,26 +29,23 @@ func TestReserveFixedServicePorts_OCIAPIGateway(t *testing.T) { assert.Equal(t, 8163, ociapigateway.DefaultPort) tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, ociapigateway.DefaultPort) - cli := CLI{OCIAPIGateway: ociapigateway.Settings{Port: ociapigateway.DefaultPort}} + cli := CLI{OCIAPIGateway: ociapigateway.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ociapigateway.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } @@ -57,3 +64,106 @@ func TestWireOCIAPIGatewaySettingsAndLinking(t *testing.T) { cli := CLI{OCIAPIGateway: ociapigateway.Settings{Port: 8163}} assert.Equal(t, 8163, cli.GetOCIAPIGatewaySettings().Port) } + +func TestWireOCIAPIGatewayResolvesVaultSecrets(t *testing.T) { + t.Parallel() + + clock := time.Now().UTC() + vaultB := ocivaultbackend.NewInMemoryBackend() + vaultB.SetClock(func() time.Time { return clock }) + vaultB.SetTransitionDelay(0) + + vault, _, err := vaultB.CreateVault(ocivaultbackend.CreateVaultIn{ + CompartmentID: "ocid1.compartment.oc1..c", DisplayName: "v", VaultType: "DEFAULT", + }, "localhost") + require.NoError(t, err) + + key, _, err := vaultB.CreateKey(vault.ID, ocivaultbackend.CreateKeyIn{ + CompartmentID: "ocid1.compartment.oc1..c", DisplayName: "k", ProtectionMode: "SOFTWARE", + KeyShape: ocivaultbackend.KeyShape{Algorithm: "AES", Length: 32}, + }) + require.NoError(t, err) + + clock = clock.Add(time.Hour) + vaultB.Sweep() + + secret, _, err := vaultB.CreateSecret(ocivaultbackend.CreateSecretIn{ + CompartmentID: "ocid1.compartment.oc1..c", VaultID: vault.ID, KeyID: key.ID, SecretName: "client-secret", + SecretContent: &ocivaultbackend.SecretContent{ + ContentType: "BASE64", Content: base64.StdEncoding.EncodeToString([]byte("s3cret")), + }, + }) + require.NoError(t, err) + + idp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/introspect" { + user, pass, _ := r.BasicAuth() + _ = json.NewEncoder(w).Encode(map[string]any{"active": user == "cid" && pass == "s3cret"}) + + return + } + + _ = json.NewEncoder(w).Encode(map[string]string{"introspection_endpoint": "http://" + r.Host + "/introspect"}) + })) + t.Cleanup(idp.Close) + + gwB := ociapigateway.NewInMemoryBackend() + gwH := ociapigateway.NewHandler(gwB) + gwH.Port = 0 + + wireOCIAPIGateway(map[string]service.Registerable{ + "OCIAPIGateway": gwH, "OCIVault": ocivaultbackend.NewHandler(vaultB), + }) + require.NoError(t, gwH.StartWorker(t.Context())) + t.Cleanup(func() { gwH.Shutdown(context.Background()) }) + + base := "http://localhost:" + strconv.Itoa(gwH.Port) + "/20190501" + post := func(path, body string) string { + req, rerr := http.NewRequestWithContext(t.Context(), http.MethodPost, base+path, strings.NewReader(body)) + require.NoError(t, rerr) + + resp, derr := testclient.New(t).Do(req) + require.NoError(t, derr) + + defer resp.Body.Close() + + out, _ := io.ReadAll(resp.Body) + require.Equal(t, http.StatusAccepted, resp.StatusCode, string(out)) + + return string(out) + } + + gwB.SetTransition(0) + + var gw map[string]any + + require.NoError(t, json.Unmarshal([]byte(post("/gateways", + `{"compartmentId":"c","endpointType":"PUBLIC","subnetId":"s"}`)), &gw)) + + gwB.Sweep() + + gwID, _ := gw["id"].(string) + post("/deployments", `{"gatewayId":"`+gwID+`","compartmentId":"c","pathPrefix":"/i","specification":{"routes":[`+ + `{"path":"/x","methods":["GET"],"backend":{"type":"STOCK_RESPONSE_BACKEND","status":200}}],`+ + `"requestPolicies":{"authentication":{"type":"TOKEN_AUTHENTICATION","tokenHeader":"Authorization",`+ + `"tokenAuthScheme":"Bearer","validationPolicy":{"type":"REMOTE_DISCOVERY",`+ + `"clientDetails":{"type":"CUSTOM","clientId":"cid","clientSecretId":"`+secret.ID+`","clientSecretVersionNumber":1},`+ + `"sourceUriDetails":{"type":"DISCOVERY_URI","uri":"`+idp.URL+`/.well-known/openid-configuration"}}}}}}`) + gwB.Sweep() + + req, err := http.NewRequestWithContext( + t.Context(), + http.MethodGet, + base[:len(base)-len("/20190501")]+"/_apigw/"+gwID+"/i/x", + nil, + ) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer opaque") + + resp, err := testclient.New(t).Do(req) + require.NoError(t, err) + + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode, "the vault secret authenticated the introspection call") +} diff --git a/internal/app/ociartifacts_wiring_test.go b/internal/app/ociartifacts_wiring_test.go index 54c4807d82..70e91a6a24 100644 --- a/internal/app/ociartifacts_wiring_test.go +++ b/internal/app/ociartifacts_wiring_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" "github.com/blackbirdworks/gopherstack/services/ociartifacts" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" @@ -20,26 +18,23 @@ func TestReserveFixedServicePorts_OCIArtifacts(t *testing.T) { assert.Equal(t, 8166, ociartifacts.DefaultPort) tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, ociartifacts.DefaultPort) - cli := CLI{OCIArtifacts: ociartifacts.Settings{Port: ociartifacts.DefaultPort}} + cli := CLI{OCIArtifacts: ociartifacts.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ociartifacts.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocicontainerengine_wire_test.go b/internal/app/ocicontainerengine_wire_test.go index ba6ee077b8..0afdc908e9 100644 --- a/internal/app/ocicontainerengine_wire_test.go +++ b/internal/app/ocicontainerengine_wire_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" ocioke "github.com/blackbirdworks/gopherstack/services/ocicontainerengine" ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" @@ -20,26 +19,23 @@ func TestReserveFixedServicePorts_OCIContainerEngine(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, ocioke.DefaultPort) - cli := CLI{OCIContainerEngine: ocioke.Settings{Port: ocioke.DefaultPort}} + cli := CLI{OCIContainerEngine: ocioke.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ocioke.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocicontainerinstances_wiring_test.go b/internal/app/ocicontainerinstances_wiring_test.go index cb1130c9ab..514eb97819 100644 --- a/internal/app/ocicontainerinstances_wiring_test.go +++ b/internal/app/ocicontainerinstances_wiring_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" ocicontainerinstances "github.com/blackbirdworks/gopherstack/services/ocicontainerinstances" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" @@ -19,26 +17,23 @@ func TestReserveFixedServicePorts_OCIContainerInstances(t *testing.T) { assert.Equal(t, 8167, ocicontainerinstances.DefaultPort) tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, ocicontainerinstances.DefaultPort) - cli := CLI{OCIContainerInstances: ocicontainerinstances.Settings{Port: ocicontainerinstances.DefaultPort}} + cli := CLI{OCIContainerInstances: ocicontainerinstances.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ocicontainerinstances.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocicore_port_reservation_test.go b/internal/app/ocicore_port_reservation_test.go index 58055290f9..7a4d65f152 100644 --- a/internal/app/ocicore_port_reservation_test.go +++ b/internal/app/ocicore_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_OCICore(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", ocicorebackend.DefaultPort, 8100, 8200, true}, - {"port outside pool range", ocicorebackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", ocicorebackend.DefaultPort, true}, + {"port outside pool range", ocicorebackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{OCICore: ocicorebackend.Settings{Port: tt.port}} + cli := CLI{OCICore: ocicorebackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocicore_wiring.go b/internal/app/ocicore_wiring.go new file mode 100644 index 0000000000..6e28c270e5 --- /dev/null +++ b/internal/app/ocicore_wiring.go @@ -0,0 +1,17 @@ +package app + +import ( + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + ocivaultbackend "github.com/blackbirdworks/gopherstack/services/ocivault" +) + +// wireOCICoreVault lets OCI Core validate volume and backup kmsKeyId references against OCI Vault keys. +func wireOCICoreVault(byName map[string]service.Registerable) { + coreH, cok := byName["OCICore"].(*ocicorebackend.Handler) + vaultH, vok := byName["OCIVault"].(*ocivaultbackend.Handler) + + if cok && vok { + coreH.Backend.SetKeyChecker(vaultH.Backend.KeyState) + } +} diff --git a/internal/app/ocicore_wiring_test.go b/internal/app/ocicore_wiring_test.go new file mode 100644 index 0000000000..55773d7a8c --- /dev/null +++ b/internal/app/ocicore_wiring_test.go @@ -0,0 +1,53 @@ +package app + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + ocivaultbackend "github.com/blackbirdworks/gopherstack/services/ocivault" +) + +func TestWireOCICoreVault(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + key string + wantErr bool + wire bool + }{ + {"unwired accepts any key", "ocid1.key.oc1.iad.ghost", false, false}, + {"wired rejects unknown key", "ocid1.key.oc1.iad.ghost", true, true}, + {"wired accepts no key", "", false, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + coreH := ocicorebackend.NewHandler(ocicorebackend.NewInMemoryBackend()) + vaultH := ocivaultbackend.NewHandler(ocivaultbackend.NewInMemoryBackend()) + + if tt.wire { + wireOCICoreVault(map[string]service.Registerable{"OCICore": coreH, "OCIVault": vaultH}) + } + + _, _, aerr := coreH.Backend.CreateVolume(ocicorebackend.CreateVolumeIn{ + CompartmentID: "ocid1.compartment.oc1..x", AvailabilityDomain: "Abcd:US-ASHBURN-AD-1", KmsKeyID: tt.key, + }) + + if tt.wantErr { + require.NotNil(t, aerr) + assert.Equal(t, "NotAuthorizedOrNotFound", aerr.Code) + + return + } + + assert.Nil(t, aerr) + }) + } +} diff --git a/internal/app/ocidatabase_wiring.go b/internal/app/ocidatabase_wiring.go new file mode 100644 index 0000000000..36e928c499 --- /dev/null +++ b/internal/app/ocidatabase_wiring.go @@ -0,0 +1,37 @@ +package app + +import ( + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocidatabase" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +// GetOCIDatabaseSettings returns OCI Database settings (ocidatabase.ConfigProvider). +func (c *CLI) GetOCIDatabaseSettings() ocidatabase.Settings { return c.OCIDatabase } + +// wireOCIDatabase shares OCI Identity keys and compartments and checks private endpoint subnets in OCI Core. +func wireOCIDatabase(byName map[string]service.Registerable) { + h, ok := byName["OCIDatabase"].(*ocidatabase.Handler) + if !ok { + return + } + + if identityH, found := byName["OCIIdentity"].(*ociidentitybackend.Handler); found { + h.Backend.Auth.Link(identityH.Backend.Auth) + h.Backend.SetCompartmentChecker(identityH.Backend.CompartmentActive) + } + + if coreH, found := byName["OCICore"].(*ocicorebackend.Handler); found { + h.Backend.SetSubnetResolver(func(id string) (ocidatabase.SubnetInfo, bool) { + ref, live := coreH.Backend.LookupSubnet(id) + + return ocidatabase.SubnetInfo{CIDR: ref.CidrBlock}, live + }) + } +} + +// GetOCIDatabaseHandler returns the OCIDatabase handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetOCIDatabaseHandler() service.Registerable { return c.ociDatabaseHandler } diff --git a/internal/app/ocidatabase_wiring_test.go b/internal/app/ocidatabase_wiring_test.go new file mode 100644 index 0000000000..e5b063749f --- /dev/null +++ b/internal/app/ocidatabase_wiring_test.go @@ -0,0 +1,101 @@ +package app + +import ( + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocidatabase" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +func TestReserveFixedServicePorts_OCIDatabase(t *testing.T) { + t.Parallel() + + assert.Equal(t, 8170, ocidatabase.DefaultPort) + + tests := []struct { + name string + wantTaken bool + }{ + {"port inside pool range", true}, + {"port outside pool range", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + alloc, port := fixedPortPool(t, tt.wantTaken, ocidatabase.DefaultPort) + + settings := ocidatabase.DefaultSettings() + settings.Port = port + cli := CLI{OCIDatabase: settings} + reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) + + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) + }) + } +} + +func TestOCIDatabaseEngineFollowsGlobalSwitch(t *testing.T) { + t.Parallel() + + tests := []struct { + cli func() *CLI + name string + want string + }{ + { + name: "stub-default", want: ocidatabase.EngineStub, + cli: func() *CLI { return &CLI{OCIDatabase: ocidatabase.DefaultSettings()} }, + }, + { + name: "explicit-docker", want: ocidatabase.EngineDocker, + cli: func() *CLI { return &CLI{OCIDatabase: ocidatabase.Settings{Engine: ocidatabase.EngineDocker}} }, + }, + { + name: "engines-switch", want: ocidatabase.EngineDocker, + cli: func() *CLI { + return &CLI{ + OCIDatabase: ocidatabase.DefaultSettings(), + engineModes: map[string]string{"oci-database-engine": ocidatabase.EngineDocker}, + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, tt.cli().GetOCIDatabaseEngine()) + }) + } +} + +func TestWireOCIDatabaseAgainstCoreAndIdentity(t *testing.T) { + t.Parallel() + + idH := ociidentitybackend.NewHandler(ociidentitybackend.NewInMemoryBackend()) + coreH := ocicorebackend.NewHandler(ocicorebackend.NewInMemoryBackend()) + dbH := ocidatabase.NewHandler(ocidatabase.NewInMemoryBackend()) + + assert.NotPanics(t, func() { wireOCIDatabase(map[string]service.Registerable{"OCIDatabase": dbH}) }) + assert.NotPanics(t, func() { wireOCIDatabase(map[string]service.Registerable{}) }) + + wireOCIDatabase(map[string]service.Registerable{"OCIIdentity": idH, "OCICore": coreH, "OCIDatabase": dbH}) + + _, err := dbH.Backend.ListBackups( + ocidatabase.BackupFilter{Compartment: "ocid1.compartment.oc1..ghost"}, + ocidatabase.ListOpts{}, + ) + require.Error(t, err, "compartments are checked against OCI Identity") + + cli := CLI{OCIDatabase: ocidatabase.Settings{Port: 8170}} + assert.Equal(t, 8170, cli.GetOCIDatabaseSettings().Port) +} diff --git a/internal/app/ocidns_wiring.go b/internal/app/ocidns_wiring.go new file mode 100644 index 0000000000..6ddd8ffa34 --- /dev/null +++ b/internal/app/ocidns_wiring.go @@ -0,0 +1,49 @@ +package app + +import ( + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocidns" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +// GetOCIDNSSettings returns OCI DNS settings (ocidns.ConfigProvider). +func (c *CLI) GetOCIDNSSettings() ocidns.Settings { return c.OCIDNS } + +// wireOCIDNS shares OCI Identity keys and compartments and attaches private resolvers to OCI Core VCNs. +func wireOCIDNS(byName map[string]service.Registerable) { + dnsH, ok := byName["OCIDNS"].(*ocidns.Handler) + if !ok { + return + } + + if identityH, found := byName["OCIIdentity"].(*ociidentitybackend.Handler); found { + dnsH.Backend.Auth.Link(identityH.Backend.Auth) + dnsH.Backend.SetCompartmentChecker(identityH.Backend.CompartmentActive) + } + + if coreH, found := byName["OCICore"].(*ocicorebackend.Handler); found { + dnsH.Backend.SetVcnSource(func() []ocidns.VcnInfo { + vcns := coreH.Backend.ListVcns() + out := make([]ocidns.VcnInfo, 0, len(vcns)) + + for _, v := range vcns { + out = append(out, ocidns.VcnInfo{ + ID: v.ID, CompartmentID: v.CompartmentID, DisplayName: v.DisplayName, Created: v.TimeCreated, + }) + } + + return out + }) + dnsH.Backend.SetSubnetResolver(func(id string) (ocidns.SubnetInfo, bool) { + ref, live := coreH.Backend.LookupSubnet(id) + + return ocidns.SubnetInfo{VcnID: ref.VcnID, Cidr: ref.CidrBlock}, live + }) + } +} + +// GetOCIDNSHandler returns the OCIDNS handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetOCIDNSHandler() service.Registerable { return c.ociDNSHandler } diff --git a/internal/app/ocidns_wiring_test.go b/internal/app/ocidns_wiring_test.go new file mode 100644 index 0000000000..dc82100a10 --- /dev/null +++ b/internal/app/ocidns_wiring_test.go @@ -0,0 +1,108 @@ +package app + +import ( + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocidns" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +func TestReserveFixedServicePorts_OCIDNS(t *testing.T) { + t.Parallel() + + assert.Equal(t, 8168, ocidns.DefaultPort) + assert.Equal(t, 8169, ocidns.DefaultDNSPort) + + tests := []struct { + name string + wantTaken bool + }{ + {"ports inside pool range", true}, + {"ports outside pool range", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + alloc, ports := fixedPortPools(t, tt.wantTaken, ocidns.DefaultPort, ocidns.DefaultDNSPort) + + settings := ocidns.DefaultSettings() + settings.Port, settings.DNSPort = ports[0], ports[1] + cli := CLI{OCIDNS: settings} + reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) + + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ports[0])) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ports[1])) + }) + } +} + +func TestWireOCIDNSAgainstCoreAndIdentity(t *testing.T) { + t.Parallel() + + idH := ociidentitybackend.NewHandler(ociidentitybackend.NewInMemoryBackend()) + coreH := ocicorebackend.NewHandler(ocicorebackend.NewInMemoryBackend()) + dnsH := ocidns.NewHandler(ocidns.NewInMemoryBackend()) + + assert.NotPanics(t, func() { wireOCIDNS(map[string]service.Registerable{"OCIDNS": dnsH}) }) + assert.NotPanics(t, func() { wireOCIDNS(map[string]service.Registerable{}) }) + + wireOCIDNS(map[string]service.Registerable{"OCIIdentity": idH, "OCICore": coreH, "OCIDNS": dnsH}) + + _, _, err := dnsH.Backend.CreateView(ocidns.CreateViewIn{CompartmentID: "ocid1.compartment.oc1..ghost"}) + require.Error(t, err, "compartments are checked against OCI Identity") + + const compartment = "ocid1.compartment.oc1..c" + + vcn, _, aerr := coreH.Backend.CreateVcn(ocicorebackend.CreateVcnIn{ + CompartmentID: compartment, CidrBlocks: []string{"10.0.0.0/16"}, DisplayName: "core-vcn", + }) + require.Nil(t, aerr) + + vcnID := vcn.(*ocicorebackend.Vcn).ID //nolint:forcetypeassert // CreateVcn returns *Vcn + + sub, _, aerr := coreH.Backend.CreateSubnet(ocicorebackend.CreateSubnetIn{ + CompartmentID: compartment, VcnID: vcnID, CidrBlock: "10.0.1.0/24", + }) + require.Nil(t, aerr) + + subID := sub.(*ocicorebackend.Subnet).ID //nolint:forcetypeassert // CreateSubnet returns *Subnet + + dnsH.Backend.SyncVcns() + + got, err := dnsH.Backend.ListResolvers(ocidns.ResolverFilter{}, ocidns.ListOpts{}) + require.Error(t, err, "compartmentId is required") + assert.Empty(t, got.Items) + + dnsH.Backend.SetCompartmentChecker(nil) + + got, err = dnsH.Backend.ListResolvers(ocidns.ResolverFilter{Compart: compartment}, ocidns.ListOpts{}) + require.NoError(t, err) + require.Len(t, got.Items, 1, "every OCI Core VCN gets a resolver") + assert.Equal(t, vcnID, got.Items[0].AttachedVcnID) + assert.Equal(t, "core-vcn", got.Items[0].DisplayName) + + yes, no := true, false + + ep, _, err := dnsH.Backend.CreateEndpoint(got.Items[0].ID, ocidns.CreateEndpointIn{ + Name: "ep", SubnetID: subID, IsForwarding: &yes, IsListening: &no, + }) + require.NoError(t, err, "subnets resolve through OCI Core") + assert.Equal(t, "10.0.1.2", *ep.ForwardingAddress) + + _, _, err = dnsH.Backend.CreateEndpoint(got.Items[0].ID, ocidns.CreateEndpointIn{ + Name: "ghost", SubnetID: "ocid1.subnet.oc1..ghost", IsForwarding: &yes, IsListening: &no, + }) + require.Error(t, err) + + cli := CLI{OCIDNS: ocidns.Settings{Port: 8168, DNSPort: 8169}} + assert.Equal(t, 8168, cli.GetOCIDNSSettings().Port) + assert.Equal(t, 8169, cli.GetOCIDNSSettings().DNSPort) +} diff --git a/internal/app/ocievents_wiring_test.go b/internal/app/ocievents_wiring_test.go index 260e433287..0f8ca83ee1 100644 --- a/internal/app/ocievents_wiring_test.go +++ b/internal/app/ocievents_wiring_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" ocievents "github.com/blackbirdworks/gopherstack/services/ocievents" ocifunctions "github.com/blackbirdworks/gopherstack/services/ocifunctions" @@ -24,26 +23,23 @@ func TestReserveFixedServicePorts_OCIEvents(t *testing.T) { assert.Equal(t, 8164, ocievents.DefaultPort) tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, ocievents.DefaultPort) - cli := CLI{OCIEvents: ocievents.Settings{Port: ocievents.DefaultPort}} + cli := CLI{OCIEvents: ocievents.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ocievents.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocifilestorage_wiring.go b/internal/app/ocifilestorage_wiring.go new file mode 100644 index 0000000000..248cf8b3b9 --- /dev/null +++ b/internal/app/ocifilestorage_wiring.go @@ -0,0 +1,85 @@ +package app + +import ( + "encoding/base64" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocifilestorage" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" + ocivaultbackend "github.com/blackbirdworks/gopherstack/services/ocivault" +) + +// GetOCIFileStorageSettings returns OCI File Storage settings (ocifilestorage.ConfigProvider). +func (c *CLI) GetOCIFileStorageSettings() ocifilestorage.Settings { return c.OCIFileStorage } + +// GetOCIFileStorageEngine returns the OCI File Storage engine mode (ocifilestorage.EngineConfigProvider). The NFS +// engine needs no container runtime, so the global --engines switch does not turn it on. +func (c *CLI) GetOCIFileStorageEngine() string { return c.OCIFileStorage.Engine } + +// GetOCIFileStorageHandler returns the OCIFileStorage handler. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetOCIFileStorageHandler() service.Registerable { return c.ociFileStorageHandler } + +// wireOCIFileStorage shares OCI Identity keys and compartments, backs mount targets with OCI Core VNICs and reads +// Kerberos keytabs and LDAP passwords from OCI Vault secrets. +func wireOCIFileStorage(byName map[string]service.Registerable) { + h, ok := byName["OCIFileStorage"].(*ocifilestorage.Handler) + if !ok { + return + } + + if identityH, found := byName["OCIIdentity"].(*ociidentitybackend.Handler); found { + h.Backend.Auth.Link(identityH.Backend.Auth) + h.Backend.SetCompartmentChecker(identityH.Backend.CompartmentActive) + } + + if coreH, found := byName["OCICore"].(*ocicorebackend.Handler); found { + h.Backend.SetNetwork(ocifilestorage.Network{ + Subnet: func(id string) (ocifilestorage.SubnetInfo, bool) { + ref, live := coreH.Backend.LookupSubnet(id) + + return ocifilestorage.SubnetInfo{ + VcnID: ref.VcnID, CIDR: ref.CidrBlock, AD: ref.AvailabilityDomain, CompartmentID: ref.CompartmentID, + }, live + }, + CreateNic: func(r ocifilestorage.NicRequest) (ocifilestorage.Nic, error) { + nic, err := coreH.Backend.CreateServiceNic(ocicorebackend.ServiceNicIn{ + CompartmentID: r.CompartmentID, AD: r.AD, SubnetID: r.SubnetID, DisplayName: r.DisplayName, + HostnameLabel: r.Hostname, PrivateIP: r.IPAddress, NsgIDs: r.NsgIDs, + }) + if err != nil { + return ocifilestorage.Nic{}, err + } + + return ocifilestorage.Nic{ + VnicID: nic.VnicID, + PrivateIPID: nic.PrivateIPID, + IPAddress: nic.IPAddress, + }, nil + }, + DeleteNic: coreH.Backend.DeleteServiceNic, + }) + } + + if vaultH, found := byName["OCIVault"].(*ocivaultbackend.Handler); found { + h.Backend.SetSecretLookup(func(id string, version *int) ([]byte, bool) { + sel := ocivaultbackend.BundleSelector{Stage: "CURRENT"} + + if version != nil { + v := int64(*version) + sel = ocivaultbackend.BundleSelector{VersionNumber: &v} + } + + bundle, _, err := vaultH.Backend.GetSecretBundle(id, sel) + if err != nil || bundle.Content == nil { + return nil, false + } + + raw, derr := base64.StdEncoding.DecodeString(bundle.Content.Content) + + return raw, derr == nil + }) + } +} diff --git a/internal/app/ocifilestorage_wiring_test.go b/internal/app/ocifilestorage_wiring_test.go new file mode 100644 index 0000000000..a2f33b251f --- /dev/null +++ b/internal/app/ocifilestorage_wiring_test.go @@ -0,0 +1,147 @@ +package app + +import ( + "encoding/json" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocifilestorage" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +func TestReserveFixedServicePorts_OCIFileStorage(t *testing.T) { + t.Parallel() + + assert.Equal(t, 8171, ocifilestorage.DefaultPort) + + tests := []struct { + name string + wantTaken bool + }{ + {"port inside pool range", true}, + {"port outside pool range", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + alloc, port := fixedPortPool(t, tt.wantTaken, ocifilestorage.DefaultPort) + + settings := ocifilestorage.DefaultSettings() + settings.Port = port + cli := CLI{OCIFileStorage: settings} + reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) + + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) + }) + } +} + +func TestOCIFileStorageEngineIgnoresGlobalSwitch(t *testing.T) { + t.Parallel() + + tests := []struct { + cli *CLI + name string + want string + }{ + {name: "default", want: ocifilestorage.EngineStub, cli: &CLI{OCIFileStorage: ocifilestorage.DefaultSettings()}}, + { + name: "explicit", + want: ocifilestorage.EngineNFS, + cli: &CLI{OCIFileStorage: ocifilestorage.Settings{Engine: ocifilestorage.EngineNFS}}, + }, + { + name: "global-docker", want: ocifilestorage.EngineStub, + cli: &CLI{ + OCIFileStorage: ocifilestorage.DefaultSettings(), + engineModes: map[string]string{"oci-filestorage-engine": "docker"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, tt.cli.GetOCIFileStorageEngine()) + }) + } +} + +func idOf(t *testing.T, v any) string { + t.Helper() + + raw, err := json.Marshal(v) + require.NoError(t, err) + + var out struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(raw, &out)) + + return out.ID +} + +func TestWireOCIFileStorageAgainstCoreAndIdentity(t *testing.T) { + t.Parallel() + + idH := ociidentitybackend.NewHandler(ociidentitybackend.NewInMemoryBackend()) + coreH := ocicorebackend.NewHandler(ocicorebackend.NewInMemoryBackend()) + fsH := ocifilestorage.NewHandler(ocifilestorage.NewInMemoryBackend()) + + assert.NotPanics(t, func() { wireOCIFileStorage(map[string]service.Registerable{"OCIFileStorage": fsH}) }) + assert.NotPanics(t, func() { wireOCIFileStorage(map[string]service.Registerable{}) }) + + wireOCIFileStorage(map[string]service.Registerable{"OCICore": coreH, "OCIFileStorage": fsH}) + + const comp = "ocid1.compartment.oc1..c" + + vcn, _, aerr := coreH.Backend.CreateVcn( + ocicorebackend.CreateVcnIn{CompartmentID: comp, DisplayName: "v", CidrBlock: "10.0.0.0/16"}, + ) + require.Nil(t, aerr) + + sub, _, aerr := coreH.Backend.CreateSubnet( + ocicorebackend.CreateSubnetIn{CompartmentID: comp, VcnID: idOf(t, vcn), CidrBlock: "10.0.1.0/24"}, + ) + require.Nil(t, aerr) + + create := func() error { + _, _, err := fsH.Backend.CreateMountTarget(ocifilestorage.CreateMountTargetIn{ + AvailabilityDomain: "ad-1", CompartmentID: comp, SubnetID: idOf(t, sub), IPAddress: new("10.0.1.40"), + }, "") + + return err + } + + require.NoError(t, create()) + require.Error(t, create(), "the mount target holds a real private IP in OCI Core") + + mts, err := fsH.Backend.ListMountTargets( + ocifilestorage.MTFilter{Compartment: comp, AD: "ad-1"}, + ocifilestorage.ListOpts{}, + ) + require.NoError(t, err) + require.Len(t, mts.Data, 1) + + require.NoError(t, fsH.Backend.DeleteMountTarget(mts.Data[0]["id"].(string), "", false)) + require.NoError(t, create(), "deleting the mount target releases its VNIC") + + wireOCIFileStorage(map[string]service.Registerable{"OCIIdentity": idH, "OCICore": coreH, "OCIFileStorage": fsH}) + + _, err = fsH.Backend.ListFileSystems( + ocifilestorage.FSFilter{Compartment: "ocid1.compartment.oc1..ghost", AD: "ad-1"}, ocifilestorage.ListOpts{}, + ) + require.Error(t, err, "compartments are checked against OCI Identity") + + cli := CLI{OCIFileStorage: ocifilestorage.Settings{Port: 8171}} + assert.Equal(t, 8171, cli.GetOCIFileStorageSettings().Port) +} diff --git a/internal/app/ocifunctions_port_reservation_test.go b/internal/app/ocifunctions_port_reservation_test.go index 1b3e0333ea..5b77f55867 100644 --- a/internal/app/ocifunctions_port_reservation_test.go +++ b/internal/app/ocifunctions_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" ocifunctionsbackend "github.com/blackbirdworks/gopherstack/services/ocifunctions" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_OCIFunctions(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", ocifunctionsbackend.DefaultPort, 8100, 8200, true}, - {"port outside pool range", ocifunctionsbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", ocifunctionsbackend.DefaultPort, true}, + {"port outside pool range", ocifunctionsbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{OCIFunctions: ocifunctionsbackend.Settings{Port: tt.port}} + cli := CLI{OCIFunctions: ocifunctionsbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ociidentity_port_reservation_test.go b/internal/app/ociidentity_port_reservation_test.go index d8bf42da14..be55c94663 100644 --- a/internal/app/ociidentity_port_reservation_test.go +++ b/internal/app/ociidentity_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_OCIIdentity(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", ociidentitybackend.DefaultPort, 8100, 8200, true}, - {"port outside pool range", ociidentitybackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", ociidentitybackend.DefaultPort, true}, + {"port outside pool range", ociidentitybackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{OCIIdentity: ociidentitybackend.Settings{Port: tt.port}} + cli := CLI{OCIIdentity: ociidentitybackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ociloadbalancer_wiring_test.go b/internal/app/ociloadbalancer_wiring_test.go index 389abcb2b9..2b9ca11176 100644 --- a/internal/app/ociloadbalancer_wiring_test.go +++ b/internal/app/ociloadbalancer_wiring_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" ociloadbalancer "github.com/blackbirdworks/gopherstack/services/ociloadbalancer" @@ -19,26 +17,23 @@ func TestReserveFixedServicePorts_OCILoadBalancer(t *testing.T) { assert.Equal(t, 8165, ociloadbalancer.DefaultPort) tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, ociloadbalancer.DefaultPort) - cli := CLI{OCILoadBalancer: ociloadbalancer.Settings{Port: ociloadbalancer.DefaultPort}} + cli := CLI{OCILoadBalancer: ociloadbalancer.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ociloadbalancer.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocilogging_wiring_test.go b/internal/app/ocilogging_wiring_test.go index 9cf2fb222d..710cdd6e9e 100644 --- a/internal/app/ocilogging_wiring_test.go +++ b/internal/app/ocilogging_wiring_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" ocifunctionsbackend "github.com/blackbirdworks/gopherstack/services/ocifunctions" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" @@ -24,47 +23,28 @@ func TestReserveFixedServicePorts_OCILoggingMonitoring(t *testing.T) { tests := []struct { name string - cli CLI - port int - rangeStart int - rangeEnd int + monitoring bool wantTaken bool }{ - { - "logging inside pool", - CLI{OCILogging: ocilogging.Settings{Port: ocilogging.DefaultPort}}, - ocilogging.DefaultPort, - 8100, - 8200, - true, - }, - { - "logging outside pool", - CLI{OCILogging: ocilogging.Settings{Port: ocilogging.DefaultPort}}, - ocilogging.DefaultPort, - 10000, - 10100, - false, - }, - { - "monitoring inside pool", CLI{OCIMonitoring: ocimonitoring.Settings{Port: ocimonitoring.DefaultPort}}, - ocimonitoring.DefaultPort, 8100, 8200, true, - }, - { - "monitoring outside pool", CLI{OCIMonitoring: ocimonitoring.Settings{Port: ocimonitoring.DefaultPort}}, - ocimonitoring.DefaultPort, 10000, 10100, false, - }, + {"logging inside pool", false, true}, + {"logging outside pool", false, false}, + {"monitoring inside pool", true, true}, + {"monitoring outside pool", true, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + def, cli := ocilogging.DefaultPort, func(p int) CLI { return CLI{OCILogging: ocilogging.Settings{Port: p}} } + if tt.monitoring { + def, cli = ocimonitoring.DefaultPort, func(p int) CLI { return CLI{OCIMonitoring: ocimonitoring.Settings{Port: p}} } + } + + alloc, port := fixedPortPool(t, tt.wantTaken, def) - reserveFixedServicePorts(t.Context(), slog.Default(), alloc, tt.cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli(port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocimysql_wire_test.go b/internal/app/ocimysql_wire_test.go index 08187589be..cfee016baa 100644 --- a/internal/app/ocimysql_wire_test.go +++ b/internal/app/ocimysql_wire_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" ocimysqlbackend "github.com/blackbirdworks/gopherstack/services/ocimysql" @@ -19,26 +18,23 @@ func TestReserveFixedServicePorts_OCIMySQL(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, ocimysqlbackend.DefaultPort) - cli := CLI{OCIMySQL: ocimysqlbackend.Settings{Port: ocimysqlbackend.DefaultPort}} + cli := CLI{OCIMySQL: ocimysqlbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ocimysqlbackend.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocinosql_wiring_test.go b/internal/app/ocinosql_wiring_test.go index e3db02324a..2135bf3d81 100644 --- a/internal/app/ocinosql_wiring_test.go +++ b/internal/app/ocinosql_wiring_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" ocinosql "github.com/blackbirdworks/gopherstack/services/ocinosql" @@ -19,26 +17,23 @@ func TestReserveFixedServicePorts_OCINoSQL(t *testing.T) { assert.Equal(t, 8162, ocinosql.DefaultPort) tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, ocinosql.DefaultPort) - cli := CLI{OCINoSQL: ocinosql.Settings{Port: ocinosql.DefaultPort}} + cli := CLI{OCINoSQL: ocinosql.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ocinosql.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocinotifications_port_reservation_test.go b/internal/app/ocinotifications_port_reservation_test.go index 86fbb286bd..92156b61ce 100644 --- a/internal/app/ocinotifications_port_reservation_test.go +++ b/internal/app/ocinotifications_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" ocinotificationsbackend "github.com/blackbirdworks/gopherstack/services/ocinotifications" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_OCINotifications(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", ocinotificationsbackend.DefaultPort, 8100, 8200, true}, - {"port outside pool range", ocinotificationsbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", ocinotificationsbackend.DefaultPort, true}, + {"port outside pool range", ocinotificationsbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{OCINotifications: ocinotificationsbackend.Settings{Port: tt.port}} + cli := CLI{OCINotifications: ocinotificationsbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ociobjectstorage_port_reservation_test.go b/internal/app/ociobjectstorage_port_reservation_test.go index c75f8aa2b5..0a80837862 100644 --- a/internal/app/ociobjectstorage_port_reservation_test.go +++ b/internal/app/ociobjectstorage_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" ociobjectstoragebackend "github.com/blackbirdworks/gopherstack/services/ociobjectstorage" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_OCIObjectStorage(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", ociobjectstoragebackend.DefaultPort, 8100, 8200, true}, - {"port outside pool range", ociobjectstoragebackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", ociobjectstoragebackend.DefaultPort, true}, + {"port outside pool range", ociobjectstoragebackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{OCIObjectStorage: ociobjectstoragebackend.Settings{Port: tt.port}} + cli := CLI{OCIObjectStorage: ociobjectstoragebackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ociqueue_port_reservation_test.go b/internal/app/ociqueue_port_reservation_test.go index 7a830fb20a..1087af4568 100644 --- a/internal/app/ociqueue_port_reservation_test.go +++ b/internal/app/ociqueue_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" ociqueuebackend "github.com/blackbirdworks/gopherstack/services/ociqueue" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_OCIQueue(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", ociqueuebackend.DefaultPort, 8100, 8200, true}, - {"port outside pool range", ociqueuebackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", ociqueuebackend.DefaultPort, true}, + {"port outside pool range", ociqueuebackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{OCIQueue: ociqueuebackend.Settings{Port: tt.port}} + cli := CLI{OCIQueue: ociqueuebackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocistreaming_port_reservation_test.go b/internal/app/ocistreaming_port_reservation_test.go index a78dcc7bee..fa82aa27d7 100644 --- a/internal/app/ocistreaming_port_reservation_test.go +++ b/internal/app/ocistreaming_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" ocistreamingbackend "github.com/blackbirdworks/gopherstack/services/ocistreaming" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_OCIStreaming(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", ocistreamingbackend.DefaultPort, 8100, 8200, true}, - {"port outside pool range", ocistreamingbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", ocistreamingbackend.DefaultPort, true}, + {"port outside pool range", ocistreamingbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{OCIStreaming: ocistreamingbackend.Settings{Port: tt.port}} + cli := CLI{OCIStreaming: ocistreamingbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/ocivault_port_reservation_test.go b/internal/app/ocivault_port_reservation_test.go index af9102d921..e25cc4491d 100644 --- a/internal/app/ocivault_port_reservation_test.go +++ b/internal/app/ocivault_port_reservation_test.go @@ -5,9 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" ocivaultbackend "github.com/blackbirdworks/gopherstack/services/ocivault" ) @@ -15,27 +13,24 @@ func TestReserveFixedServicePorts_OCIVault(t *testing.T) { t.Parallel() tests := []struct { - name string - port int - rangeStart int - rangeEnd int - wantTaken bool + name string + port int + wantTaken bool }{ - {"port inside pool range", ocivaultbackend.DefaultPort, 8100, 8200, true}, - {"port outside pool range", ocivaultbackend.DefaultPort, 10000, 10100, false}, + {"port inside pool range", ocivaultbackend.DefaultPort, true}, + {"port outside pool range", ocivaultbackend.DefaultPort, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, tt.port) - cli := CLI{OCIVault: ocivaultbackend.Settings{Port: tt.port}} + cli := CLI{OCIVault: ocivaultbackend.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(tt.port)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/omics_s3_wiring_test.go b/internal/app/omics_s3_wiring_test.go index fc781246d6..f35e0f3324 100644 --- a/internal/app/omics_s3_wiring_test.go +++ b/internal/app/omics_s3_wiring_test.go @@ -156,7 +156,7 @@ func TestInitializeServices_OmicsS3Wiring(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - services, err := initializeServicesT(t, newTestAppContext(t, 19600, 19700)) + services, err := initializeServicesT(t, newTestAppContext(t)) require.NoError(t, err) byName := serviceByName(services) diff --git a/internal/app/portpool_helpers_test.go b/internal/app/portpool_helpers_test.go new file mode 100644 index 0000000000..52c0b74fae --- /dev/null +++ b/internal/app/portpool_helpers_test.go @@ -0,0 +1,56 @@ +package app + +import ( + "testing" + + "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" +) + +const fixedPortPoolSpan = 10 + +// fixedPortPools returns a probed-free pool and the ports to configure services with: distinct +// ports inside the pool when inPool, else the given ports (which must not fall in the test range). +func fixedPortPools(tb testing.TB, inPool bool, ports ...int) (*portalloc.Allocator, []int) { + tb.Helper() + + alloc := portalloctest.New(tb, fixedPortPoolSpan) + if !inPool { + return alloc, ports + } + + return alloc, poolPorts(tb, alloc, len(ports)) +} + +// fixedPortPool is fixedPortPools for a single port. +func fixedPortPool(tb testing.TB, inPool bool, port int) (*portalloc.Allocator, int) { + tb.Helper() + + alloc, ports := fixedPortPools(tb, inPool, port) + + return alloc, ports[0] +} + +// poolPorts returns n distinct ports inside alloc's range, left unallocated. +func poolPorts(tb testing.TB, alloc *portalloc.Allocator, n int) []int { + tb.Helper() + + ports := make([]int, 0, n) + + for range n { + p, err := alloc.Acquire("probe") + if err != nil { + tb.Fatalf("acquire: %v", err) + } + + ports = append(ports, p) + } + + for _, p := range ports { + if err := alloc.Release(p); err != nil { + tb.Fatalf("release: %v", err) + } + } + + return ports +} diff --git a/internal/app/redshift_serverless_data_test.go b/internal/app/redshift_serverless_data_test.go index 4868557915..ab685ef223 100644 --- a/internal/app/redshift_serverless_data_test.go +++ b/internal/app/redshift_serverless_data_test.go @@ -2,6 +2,7 @@ package app import ( "net/http/httptest" + "sync/atomic" "testing" "time" @@ -16,9 +17,10 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/chaos" "github.com/blackbirdworks/gopherstack/pkgs/service" + redshiftbackend "github.com/blackbirdworks/gopherstack/services/redshift" ) -const serverlessDwell = 150 * time.Millisecond +const serverlessDwell = time.Hour func TestRedshiftServerlessDataAPI(t *testing.T) { t.Parallel() @@ -37,6 +39,15 @@ func TestRedshiftServerlessDataAPI(t *testing.T) { wireRedshiftServerlessData(byName) applyRedshiftServerless(byName["RedshiftServerless"], serverlessDwell) + var offset atomic.Int64 + + slHandler, ok := byName["RedshiftServerless"].(*redshiftbackend.ServerlessHandler) + require.True(t, ok) + + slHandler.Backend.SetServerlessClock(func() time.Time { + return time.Now().Add(time.Duration(offset.Load())) + }) + e := buildEchoServer(t.Context(), log, nil, services, cli) require.NoError(t, setupChaosAndRegistry(e, log, &cli, services)) @@ -79,6 +90,8 @@ func TestRedshiftServerlessDataAPI(t *testing.T) { require.ErrorAs(t, execute("wg-data"), &apiErr) assert.Equal(t, "ValidationException", apiErr.ErrorCode()) + offset.Store(int64(serverlessDwell)) + require.Eventually(t, func() bool { got, gerr := rs.GetWorkgroup( t.Context(), diff --git a/internal/app/region_wiring_test.go b/internal/app/region_wiring_test.go index f6cd071af2..3cce5a2ce1 100644 --- a/internal/app/region_wiring_test.go +++ b/internal/app/region_wiring_test.go @@ -14,7 +14,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/awsmeta" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" athenabackend "github.com/blackbirdworks/gopherstack/services/athena" ecrbackend "github.com/blackbirdworks/gopherstack/services/ecr" @@ -68,8 +68,7 @@ func TestInitializeServices_RegionAwareCrossServiceWiring(t *testing.T) { t.Parallel() cli := &CLI{AccountID: "000000000000", Region: "us-east-1"} - portAlloc, err := portalloc.New(19300, 19400) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) cli.faultStore = chaos.NewFaultStore() diff --git a/internal/app/registry_wiring_test.go b/internal/app/registry_wiring_test.go index 6df81ec725..262b1bfe54 100644 --- a/internal/app/registry_wiring_test.go +++ b/internal/app/registry_wiring_test.go @@ -19,7 +19,6 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/pkgs/ociregistry/ocitest" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" "github.com/blackbirdworks/gopherstack/pkgs/service" "github.com/blackbirdworks/gopherstack/services/azurearm" acr "github.com/blackbirdworks/gopherstack/services/azurecontainerregistry" @@ -289,26 +288,23 @@ func TestReserveFixedServicePorts_AzureContainerRegistry(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 10000, 10100, true}, - {"port outside pool range", 20000, 20100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, acr.DefaultPort) - cli := CLI{AzureACR: acr.Settings{Port: acr.DefaultPort}} + cli := CLI{AzureACR: acr.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(acr.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } @@ -338,26 +334,23 @@ func TestReserveFixedServicePorts_GCPArtifactRegistry(t *testing.T) { t.Parallel() tests := []struct { - name string - rangeStart int - rangeEnd int - wantTaken bool + name string + wantTaken bool }{ - {"port inside pool range", 8100, 8200, true}, - {"port outside pool range", 10000, 10100, false}, + {"port inside pool range", true}, + {"port outside pool range", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) - require.NoError(t, err) + alloc, port := fixedPortPool(t, tt.wantTaken, gcpartifactregistry.DefaultPort) - cli := CLI{GCPArtifactRegistry: gcpartifactregistry.Settings{Port: gcpartifactregistry.DefaultPort}} + cli := CLI{GCPArtifactRegistry: gcpartifactregistry.Settings{Port: port}} reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) - assert.Equal(t, tt.wantTaken, alloc.IsAllocated(gcpartifactregistry.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(port)) }) } } diff --git a/internal/app/routing_equivalence_test.go b/internal/app/routing_equivalence_test.go index d1637dc61b..68a90c1ca1 100644 --- a/internal/app/routing_equivalence_test.go +++ b/internal/app/routing_equivalence_test.go @@ -243,7 +243,8 @@ func TestRoutingCorpusCoversServices(t *testing.T) { var missing []string neverMatch := []string{ - "AzureAppConfig", "AzureARM", "AzureBlob", "AzureEventGrid", "AzureEventHubs", "AzureKeyVault", "AzureQueue", + "AzureAppConfig", "AzureSearch", + "AzureARM", "AzureBlob", "AzureEventGrid", "AzureEventHubs", "AzureKeyVault", "AzureQueue", "AzureFile", "AzureGraph", "AzureMonitor", "AzureServiceBus", "AzureStorageVHost", "AzureTable", "CosmosDB", "GCPStorage", "GCPPubSub", "GCPSecretManager", "GCPKMS", "GCPCloudTasks", "GCPScheduler", "GCPFirestore", "GCPIAM", "GCPRun", "GCPFunctions", "GCPBigQuery", "GCPSQL", "GCPLogging", @@ -254,7 +255,8 @@ func TestRoutingCorpusCoversServices(t *testing.T) { "AzureContainerRegistry", "AzureFunctions", "GCPArtifactRegistry", "OCIObjectStorage", "OCIVault", "OCIStreaming", "OCIIdentity", "OCIFunctions", "OCICore", "OCIQueue", "OCINotifications", "OCIContainerEngine", "OCIMySQL", "OCILogging", "OCIMonitoring", "OCINoSQL", "OCIAPIGateway", "OCIEvents", - "OCILoadBalancer", "OCIArtifacts", "OCIContainerInstances", + "OCILoadBalancer", "OCIArtifacts", "OCIContainerInstances", "OCIDNS", "OCIDatabase", + "OCIFileStorage", } for _, e := range entries { diff --git a/internal/app/servicecatalog_wiring.go b/internal/app/servicecatalog_wiring.go index 0de2513b58..79f31306a2 100644 --- a/internal/app/servicecatalog_wiring.go +++ b/internal/app/servicecatalog_wiring.go @@ -167,7 +167,7 @@ func (s *serviceCatalogStacks) CreateStack( return servicecatalogbackend.StackInfo{}, err } - return stackInfoOf(st), nil + return settledStackInfo(ctx, bk, st) } func (s *serviceCatalogStacks) UpdateStack( @@ -184,7 +184,7 @@ func (s *serviceCatalogStacks) UpdateStack( return servicecatalogbackend.StackInfo{}, err } - return stackInfoOf(st), nil + return settledStackInfo(ctx, bk, st) } func (s *serviceCatalogStacks) DeleteStack(ctx context.Context, nameOrID string) error { @@ -193,7 +193,28 @@ func (s *serviceCatalogStacks) DeleteStack(ctx context.Context, nameOrID string) return err } - return bk.DeleteStack(ctx, nameOrID) + if err = bk.DeleteStack(ctx, nameOrID); err != nil { + return err + } + + _, err = bk.WaitForStack(ctx, nameOrID) + if errors.Is(err, cfnbackend.ErrStackNotFound) { + return nil + } + + return err +} + +// settledStackInfo waits out the stack's async provisioning, as callers here need its final status. +func settledStackInfo( + ctx context.Context, bk cfnbackend.StorageBackend, st *cfnbackend.Stack, +) (servicecatalogbackend.StackInfo, error) { + settled, err := bk.WaitForStack(ctx, st.StackID) + if err != nil { + return servicecatalogbackend.StackInfo{}, err + } + + return stackInfoOf(settled), nil } func (s *serviceCatalogStacks) DescribeStack( @@ -279,7 +300,13 @@ func (s *serviceCatalogStacks) ExecuteChangeSet(ctx context.Context, stackName, return err } - return bk.ExecuteChangeSet(ctx, stackName, changeSetName, false, false) + if err = bk.ExecuteChangeSet(ctx, stackName, changeSetName, false, false); err != nil { + return err + } + + _, err = bk.WaitForStack(ctx, stackName) + + return err } func (s *serviceCatalogStacks) DeleteChangeSet(ctx context.Context, stackName, changeSetName string) error { diff --git a/internal/app/subnet_wiring.go b/internal/app/subnet_wiring.go index 5baecc4a99..a4a03bd5f6 100644 --- a/internal/app/subnet_wiring.go +++ b/internal/app/subnet_wiring.go @@ -181,6 +181,7 @@ func (a macSecSecretAdapter) CreateMacSecSecret(region, name, secretString strin // wireDirectConnectMacSecSecrets gives Direct Connect a Secrets Manager store for MACsec keys. func wireDirectConnectMacSecSecrets(byName map[string]service.Registerable) { wireTimestreamInfluxDBIntegrations(byName) + wireTimestreamQueryIntegrations(byName) dcH, ok := byName["DirectConnect"].(*directconnectbackend.Handler) if !ok { diff --git a/internal/app/textract_rekognition_s3_wiring_test.go b/internal/app/textract_rekognition_s3_wiring_test.go index c58743b500..b0f060997b 100644 --- a/internal/app/textract_rekognition_s3_wiring_test.go +++ b/internal/app/textract_rekognition_s3_wiring_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/pkgs/chaos" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" "github.com/blackbirdworks/gopherstack/pkgs/service" rekognitionbackend "github.com/blackbirdworks/gopherstack/services/rekognition" s3backend "github.com/blackbirdworks/gopherstack/services/s3" @@ -52,12 +52,11 @@ func doJSONTargetRequest( // newTestAppContext builds a minimal service.AppContext for driving the real // initializeServices composition root, matching the pattern used by the // other *_wiring_test.go files (e.g. TestInitializeServices_LambdaS3CodeWiring). -func newTestAppContext(t *testing.T, portFrom, portTo int) *service.AppContext { +func newTestAppContext(t *testing.T) *service.AppContext { t.Helper() cli := &CLI{AccountID: "000000000000", Region: "us-east-1", faultStore: chaos.NewFaultStore()} - portAlloc, err := portalloc.New(portFrom, portTo) - require.NoError(t, err) + portAlloc := portalloctest.New(t, 100) return &service.AppContext{ Logger: slog.Default(), @@ -75,7 +74,7 @@ func newTestAppContext(t *testing.T, portFrom, portTo int) *service.AppContext { func TestInitializeServices_TextractS3Wiring(t *testing.T) { t.Parallel() - appCtx := newTestAppContext(t, 19400, 19500) + appCtx := newTestAppContext(t) services, err := initializeServicesT(t, appCtx) require.NoError(t, err) @@ -129,7 +128,7 @@ func TestInitializeServices_TextractS3Wiring(t *testing.T) { func TestInitializeServices_RekognitionS3Wiring(t *testing.T) { t.Parallel() - appCtx := newTestAppContext(t, 19500, 19600) + appCtx := newTestAppContext(t) services, err := initializeServicesT(t, appCtx) require.NoError(t, err) diff --git a/internal/app/timestream_source.go b/internal/app/timestream_source.go index e7a735b2de..34cc986bbb 100644 --- a/internal/app/timestream_source.go +++ b/internal/app/timestream_source.go @@ -1,6 +1,7 @@ package app import ( + "errors" "sort" "strconv" "strings" @@ -22,7 +23,10 @@ type timestreamRecordSource struct { timestreamCatalog } -var _ timestreamquerybackend.RecordSource = (*timestreamRecordSource)(nil) +var ( + _ timestreamquerybackend.RecordSource = (*timestreamRecordSource)(nil) + _ timestreamquerybackend.SchemaSource = (*timestreamRecordSource)(nil) +) func newTimestreamRecordSource(h *timestreamwritebackend.Handler) *timestreamRecordSource { return ×treamRecordSource{timestreamCatalog{handler: h}} @@ -44,6 +48,27 @@ func (s *timestreamRecordSource) TableRecords( return out, nil } +func (s *timestreamRecordSource) TableShapes(region, database, table string) []timestreamquerybackend.SourceRecord { + shapes := s.handler.BackendFor(region).TableShapes(database, table) + out := make([]timestreamquerybackend.SourceRecord, 0, len(shapes)) + + for _, sh := range shapes { + r := timestreamquerybackend.SourceRecord{MeasureName: sh.MeasureName, MeasureValueType: sh.MeasureValueType} + + for _, d := range sh.Dimensions { + r.Dimensions = append(r.Dimensions, timestreamquerybackend.SourceDimension{Name: d}) + } + + for _, m := range sh.Attributes { + r.Measures = append(r.Measures, timestreamquerybackend.SourceMeasure{Name: m.Name, Type: m.Type}) + } + + out = append(out, r) + } + + return out +} + func (s *timestreamRecordSource) ListDatabases(region string) []string { dbs := s.handler.BackendFor(region).ListDatabases() names := make([]string, 0, len(dbs)) @@ -83,6 +108,18 @@ func (s *timestreamRecordSource) AppendRecords( _, err := s.handler.BackendFor(region).WriteRecords(database, table, recs) + if rejected, ok := errors.AsType[*timestreamwritebackend.RejectedRecordsError](err); ok { + out := ×treamquerybackend.RejectedError{} + for _, r := range rejected.RejectedRecords { + out.Rejected = append( + out.Rejected, + timestreamquerybackend.RejectedRecord{Index: r.RecordIndex, Reason: r.Reason}, + ) + } + + return out + } + return err } diff --git a/internal/app/timestreamquery_scheduler_wiring_test.go b/internal/app/timestreamquery_scheduler_wiring_test.go new file mode 100644 index 0000000000..acbe5e0ca1 --- /dev/null +++ b/internal/app/timestreamquery_scheduler_wiring_test.go @@ -0,0 +1,394 @@ +package app + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "io" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/sns" + "github.com/aws/aws-sdk-go-v2/service/sqs" + tqsdk "github.com/aws/aws-sdk-go-v2/service/timestreamquery" + tqtypes "github.com/aws/aws-sdk-go-v2/service/timestreamquery/types" + twsdk "github.com/aws/aws-sdk-go-v2/service/timestreamwrite" + twtypes "github.com/aws/aws-sdk-go-v2/service/timestreamwrite/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/parquetio" + timestreamquerybackend "github.com/blackbirdworks/gopherstack/services/timestreamquery" + timestreamwritebackend "github.com/blackbirdworks/gopherstack/services/timestreamwrite" +) + +const sensorsDB = "sensors" + +type tsqStack struct { + fx *sfnFixture + query *tqsdk.Client + write *twsdk.Client + s3 *s3.Client + qb *timestreamquerybackend.InMemoryBackend + now time.Time +} + +func newTSQStack(t *testing.T, start time.Time) *tsqStack { + t.Helper() + + fx := newSFNFixture(t) + byName := serviceByName(fx.services) + + st := &tsqStack{ + fx: fx, + query: tqsdk.NewFromConfig(fx.cfg), + write: twsdk.NewFromConfig(fx.cfg), + s3: s3.NewFromConfig(fx.cfg, func(o *s3.Options) { o.UsePathStyle = true }), + now: start, + } + + qh, ok := byName["TimestreamQuery"].(*timestreamquerybackend.Handler) + require.True(t, ok) + + st.qb, ok = qh.Backend.(*timestreamquerybackend.InMemoryBackend) + require.True(t, ok) + + wh, ok := byName["TimestreamWrite"].(*timestreamwritebackend.Handler) + require.True(t, ok) + + clock := func() time.Time { return st.now } + st.qb.SetClock(clock) + wh.Backend.SetClock(clock) + + return st +} + +func (s *tsqStack) rows(t *testing.T, q string) [][]string { + t.Helper() + + out, err := s.query.Query(t.Context(), &tqsdk.QueryInput{QueryString: aws.String(q)}) + require.NoError(t, err, q) + + return rowsOf(out) +} + +func (s *tsqStack) seed( + t *testing.T, + table string, + props *twtypes.RetentionProperties, + magnetic bool, + recs ...twtypes.Record, +) { + t.Helper() + + _, err := s.write.CreateDatabase(t.Context(), &twsdk.CreateDatabaseInput{DatabaseName: aws.String(sensorsDB)}) + if err != nil { + require.ErrorContains(t, err, "ConflictException") + } + + _, err = s.write.CreateTable(t.Context(), &twsdk.CreateTableInput{ + DatabaseName: aws.String(sensorsDB), + TableName: aws.String(table), + RetentionProperties: props, + MagneticStoreWriteProperties: &twtypes.MagneticStoreWriteProperties{ + EnableMagneticStoreWrites: aws.Bool(magnetic), + }, + }) + require.NoError(t, err) + + if len(recs) > 0 { + _, err = s.write.WriteRecords(t.Context(), &twsdk.WriteRecordsInput{ + DatabaseName: aws.String(sensorsDB), TableName: aws.String(table), Records: recs, + }) + require.NoError(t, err) + } +} + +func (s *tsqStack) object(t *testing.T, uri string) []byte { + t.Helper() + + bucket, key, _ := strings.Cut(strings.TrimPrefix(uri, "s3://"), "/") + + out, err := s.s3.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)}) + require.NoError(t, err, uri) + + defer func() { _ = out.Body.Close() }() + + b, err := io.ReadAll(out.Body) + require.NoError(t, err) + + return b +} + +func (s *tsqStack) bucket(t *testing.T, name string) { + t.Helper() + + _, err := s.s3.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String(name)}) + require.NoError(t, err) +} + +func tsqRetention(memHours, magDays int64) *twtypes.RetentionProperties { + return &twtypes.RetentionProperties{ + MemoryStoreRetentionPeriodInHours: aws.Int64(memHours), MagneticStoreRetentionPeriodInDays: aws.Int64(magDays), + } +} + +func rollupTarget(db, table string) *tqtypes.TargetConfiguration { + return &tqtypes.TargetConfiguration{TimestreamConfiguration: &tqtypes.TimestreamConfiguration{ + DatabaseName: aws.String(db), TableName: aws.String(table), TimeColumn: aws.String("t"), + DimensionMappings: []tqtypes.DimensionMapping{ + {Name: aws.String("host"), DimensionValueType: tqtypes.DimensionValueTypeVarchar}, + }, + MultiMeasureMappings: &tqtypes.MultiMeasureMappings{ + TargetMultiMeasureName: aws.String("cpu_stats"), + MultiMeasureAttributeMappings: []tqtypes.MultiMeasureAttributeMapping{{ + SourceColumn: aws.String("avg_cpu"), MeasureValueType: tqtypes.ScalarMeasureValueTypeDouble, + }}, + }, + }} +} + +func (s *tsqStack) createSchedule( + t *testing.T, name, topic, bucket string, target *tqtypes.TargetConfiguration, +) string { + t.Helper() + + out, err := s.query.CreateScheduledQuery(t.Context(), &tqsdk.CreateScheduledQueryInput{ + Name: aws.String(name), + QueryString: aws.String( + `SELECT host, bin(time, 1h) AS t, avg(measure_value::double) AS avg_cpu FROM sensors.readings + WHERE measure_name = 'cpu' AND time BETWEEN @scheduled_runtime - 2h AND @scheduled_runtime + GROUP BY host, bin(time, 1h)`, + ), + ScheduledQueryExecutionRoleArn: aws.String("arn:aws:iam::000000000000:role/tsq"), + ScheduleConfiguration: &tqtypes.ScheduleConfiguration{ScheduleExpression: aws.String("rate(1 hour)")}, + NotificationConfiguration: &tqtypes.NotificationConfiguration{ + SnsConfiguration: &tqtypes.SnsConfiguration{TopicArn: aws.String(topic)}, + }, + ErrorReportConfiguration: &tqtypes.ErrorReportConfiguration{S3Configuration: &tqtypes.S3Configuration{ + BucketName: aws.String(bucket), ObjectKeyPrefix: aws.String("reports"), + }}, + TargetConfiguration: target, + }) + require.NoError(t, err) + + return aws.ToString(out.Arn) +} + +func TestTimestreamQuery_FullStack_ScheduleNotifyAndErrorReport(t *testing.T) { + t.Parallel() + + t0 := time.Date(2026, 10, 31, 12, 0, 0, 0, time.UTC) + st := newTSQStack(t, t0) + ctx := t.Context() + dbl := twtypes.MeasureValueTypeDouble + + q, err := sqs.NewFromConfig(st.fx.cfg).CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("tsq-events")}) + require.NoError(t, err) + + topic, err := sns.NewFromConfig(st.fx.cfg).CreateTopic(ctx, &sns.CreateTopicInput{Name: aws.String("tsq")}) + require.NoError(t, err) + + _, err = sns.NewFromConfig(st.fx.cfg).Subscribe(ctx, &sns.SubscribeInput{ + TopicArn: topic.TopicArn, Protocol: aws.String("sqs"), + Endpoint: aws.String("arn:aws:sqs:us-east-1:000000000000:tsq-events"), + }) + require.NoError(t, err) + + st.bucket(t, "tsq-errors") + st.seed(t, "readings", tsqRetention(24, 365), true, + single("cpu", "10", dbl, t0.Add(-30*time.Minute), dim("host", "a")), + single("cpu", "30", dbl, t0.Add(-10*time.Minute), dim("host", "a")), + ) + st.seed(t, "rollup", tsqRetention(24, 365), true) + st.seed(t, "strict", tsqRetention(1, 1), false) + + good := st.createSchedule(t, "good", aws.ToString(topic.TopicArn), "tsq-errors", rollupTarget("sensors", "rollup")) + bad := st.createSchedule(t, "bad", aws.ToString(topic.TopicArn), "tsq-errors", rollupTarget("sensors", "strict")) + + st.now = t0.Add(time.Hour) + st.qb.RunDue(ctx) + + assert.Equal(t, [][]string{{"a", "cpu_stats", "20.0", "2026-10-31 11:00:00.000000000"}}, + st.rows(t, `SELECT host, measure_name, avg_cpu, time FROM sensors.rollup`)) + + desc, err := st.query.DescribeScheduledQuery( + ctx, + &tqsdk.DescribeScheduledQueryInput{ScheduledQueryArn: aws.String(good)}, + ) + require.NoError(t, err) + assert.Equal(t, tqtypes.ScheduledQueryRunStatusAutoTriggerSuccess, desc.ScheduledQuery.LastRunSummary.RunStatus) + assert.Equal(t, t0.Add(2*time.Hour), desc.ScheduledQuery.NextInvocationTime.UTC()) + + failed, err := st.query.DescribeScheduledQuery( + ctx, + &tqsdk.DescribeScheduledQueryInput{ScheduledQueryArn: aws.String(bad)}, + ) + require.NoError(t, err) + require.NotNil(t, failed.ScheduledQuery.LastRunSummary) + assert.Equal(t, tqtypes.ScheduledQueryRunStatusAutoTriggerFailure, failed.ScheduledQuery.LastRunSummary.RunStatus) + require.Len(t, failed.ScheduledQuery.RecentlyFailedRuns, 1) + + loc := failed.ScheduledQuery.LastRunSummary.ErrorReportLocation.S3ReportLocation + report := st.object(t, "s3://"+aws.ToString(loc.BucketName)+"/"+aws.ToString(loc.ObjectKey)) + + var parsed struct { + Errors []struct { + Reason string `json:"reason"` + } `json:"errors"` + } + + require.NoError(t, json.Unmarshal(report, &parsed)) + require.Len(t, parsed.Errors, 1) + assert.Contains(t, parsed.Errors[0].Reason, "retention") + + msgs, err := sqs.NewFromConfig(st.fx.cfg).ReceiveMessage(ctx, &sqs.ReceiveMessageInput{ + QueueUrl: q.QueueUrl, MaxNumberOfMessages: 10, + }) + require.NoError(t, err) + + got := make([]string, 0, len(msgs.Messages)) + + for _, m := range msgs.Messages { + var env struct { + Message string `json:"Message"` + } + + require.NoError(t, json.Unmarshal([]byte(aws.ToString(m.Body)), &env)) + + var inner struct { + Type string `json:"type"` + } + + require.NoError(t, json.Unmarshal([]byte(env.Message), &inner)) + got = append(got, inner.Type) + } + + assert.ElementsMatch( + t, + []string{ + "SCHEDULED_QUERY_CREATING", + "SCHEDULED_QUERY_CREATING", + "AUTO_TRIGGER_SUCCESS", + "AUTO_TRIGGER_FAILURE", + }, + got, + ) +} + +func TestTimestreamQuery_FullStack_UnloadToS3(t *testing.T) { + t.Parallel() + + t0 := time.Date(2026, 10, 31, 12, 0, 0, 0, time.UTC) + st := newTSQStack(t, t0) + dbl := twtypes.MeasureValueTypeDouble + + st.bucket(t, "unloads") + st.seed(t, "readings", tsqRetention(24, 365), true, + single("cpu", "10", dbl, t0.Add(-30*time.Minute), dim("host", "a")), + single("cpu", "30", dbl, t0.Add(-20*time.Minute), dim("host", "a")), + single("cpu", "50", dbl, t0.Add(-10*time.Minute), dim("host", "b")), + ) + + run := func(with string) []string { + out, err := st.query.Query(t.Context(), &tqsdk.QueryInput{QueryString: aws.String( + `UNLOAD (SELECT host, measure_value::double AS cpu FROM sensors.readings ORDER BY time) ` + + `TO 's3://unloads/out' WITH (` + with + `)`)}) + require.NoError(t, err) + require.Len(t, out.Rows, 1) + assert.Equal(t, "rows", aws.ToString(out.ColumnInfo[0].Name)) + assert.Equal(t, tqtypes.ScalarTypeBigint, out.ColumnInfo[0].Type.ScalarType) + + return rowsOf(out)[0] + } + + cells := run(`format = 'CSV', include_header = 'true'`) + assert.Equal(t, "3", cells[0]) + + manifest := st.object(t, cells[2]) + + var man struct { + ResultFiles []struct { + URL string `json:"url"` + } `json:"result_files"` + } + + require.NoError(t, json.Unmarshal(manifest, &man)) + require.Len(t, man.ResultFiles, 1) + + zr, err := gzip.NewReader(bytes.NewReader(st.object(t, man.ResultFiles[0].URL))) + require.NoError(t, err) + + csvBody, err := io.ReadAll(zr) + require.NoError(t, err) + assert.Equal(t, "host,cpu\na,10.0\na,30.0\nb,50.0\n", string(csvBody)) + + cells = run(`format = 'PARQUET', partitioned_by = ARRAY['host']`) + assert.Equal(t, "3", cells[0]) + + require.NoError(t, json.Unmarshal(st.object(t, cells[2]), &man)) + require.Len(t, man.ResultFiles, 2) + + var total float64 + + for _, f := range man.ResultFiles { + assert.Contains(t, f.URL, "/results/host=") + + pf, perr := parquetio.Open(st.object(t, f.URL)) + require.NoError(t, perr) + require.NoError(t, pf.ForEach([]string{"cpu"}, func(v []any) error { + total += v[0].(float64) + + return nil + })) + } + + assert.InDelta(t, 90.0, total, 0) + + meta := st.object(t, cells[1]) + assert.Contains(t, string(meta), `"MetadataFileVersion":"1.0"`) + + _, err = st.query.Query(t.Context(), &tqsdk.QueryInput{QueryString: aws.String( + `UNLOAD (SELECT 1) TO 's3://does-not-exist/x'`)}) + require.ErrorContains(t, err, "ValidationException") +} + +func TestTimestreamQuery_FullStack_SchemaOutlivesRetention(t *testing.T) { + t.Parallel() + + t0 := time.Date(2026, 10, 31, 12, 0, 0, 0, time.UTC) + st := newTSQStack(t, t0) + dbl := twtypes.MeasureValueTypeDouble + + st.seed(t, "short", tsqRetention(1, 1), true, + single("cpu", "10", dbl, t0.Add(-5*time.Minute), dim("host", "a"), dim("region", "us")), + ) + + require.Equal(t, [][]string{{"1"}}, st.rows(t, `SELECT count(*) FROM sensors.short`)) + + st.now = t0.Add(72 * time.Hour) + + assert.Equal(t, [][]string{{"0"}}, st.rows(t, `SELECT count(*) FROM sensors.short`)) + assert.Equal(t, [][]string{ + {"host", "varchar", "DIMENSION"}, {"region", "varchar", "DIMENSION"}, + {"measure_name", "varchar", "MEASURE_NAME"}, {"time", "timestamp", "TIMESTAMP"}, + {"measure_value::double", "double", "MEASURE_VALUE"}, + }, st.rows(t, `DESCRIBE sensors.short`)) + assert.Equal(t, [][]string{ + { + "cpu", + "double", + `[{"data_type":"varchar","dimension_name":"host"},{"data_type":"varchar","dimension_name":"region"}]`, + }, + }, st.rows(t, `SHOW MEASURES FROM sensors.short`)) + assert.Empty(t, st.rows(t, `SELECT host, region FROM sensors.short`)) + + tbl, err := st.write.DescribeTable(t.Context(), &twsdk.DescribeTableInput{ + DatabaseName: aws.String("sensors"), TableName: aws.String("short"), + }) + require.NoError(t, err) + assert.Equal(t, "short", aws.ToString(tbl.Table.TableName)) +} diff --git a/internal/app/timestreamquery_wiring.go b/internal/app/timestreamquery_wiring.go new file mode 100644 index 0000000000..20d8e2f812 --- /dev/null +++ b/internal/app/timestreamquery_wiring.go @@ -0,0 +1,45 @@ +package app + +import ( + "context" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + s3backend "github.com/blackbirdworks/gopherstack/services/s3" + snsbackend "github.com/blackbirdworks/gopherstack/services/sns" + timestreamquerybackend "github.com/blackbirdworks/gopherstack/services/timestreamquery" +) + +// tsqNotifier publishes scheduled query notifications to SNS. +type tsqNotifier struct{ sns *snsbackend.InMemoryBackend } + +func (n tsqNotifier) Publish(ctx context.Context, topicARN, message string) error { + _, err := n.sns.PublishContext(ctx, topicARN, message, "", "", nil) + + return err +} + +// wireTimestreamQueryIntegrations gives Timestream Query its SNS notifier and the S3 store used for +// scheduled query error reports and UNLOAD output. +func wireTimestreamQueryIntegrations(byName map[string]service.Registerable) { + h, ok := byName["TimestreamQuery"].(*timestreamquerybackend.Handler) + if !ok { + return + } + + bk, ok := h.Backend.(*timestreamquerybackend.InMemoryBackend) + if !ok { + return + } + + if snsH, sok := byName["SNS"].(*snsbackend.Handler); sok { + if snsBk, bok := snsH.Backend.(*snsbackend.InMemoryBackend); bok { + bk.SetNotifier(tsqNotifier{sns: snsBk}) + } + } + + if s3H, sok := byName["S3"].(*s3backend.S3Handler); sok { + if s3Bk, bok := s3H.Backend.(*s3backend.InMemoryBackend); bok { + bk.SetObjectStore(s3Bk) + } + } +} diff --git a/internal/app/transcribe_output.go b/internal/app/transcribe_output.go new file mode 100644 index 0000000000..802e596de8 --- /dev/null +++ b/internal/app/transcribe_output.go @@ -0,0 +1,19 @@ +package app + +import ( + "net/http" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + transcribebackend "github.com/blackbirdworks/gopherstack/services/transcribe" +) + +// registerTranscribeEndpoints serves the fetch URLs of service-managed Transcribe output. +func registerTranscribeEndpoints(e *echo.Echo, services []service.Registerable) { + for _, svc := range services { + if h, ok := svc.(*transcribebackend.Handler); ok { + e.Add(http.MethodGet, "/_gopherstack/transcribe/output/*", h.ServeManagedOutput) + } + } +} diff --git a/internal/app/transcribe_output_test.go b/internal/app/transcribe_output_test.go new file mode 100644 index 0000000000..d1d343e486 --- /dev/null +++ b/internal/app/transcribe_output_test.go @@ -0,0 +1,48 @@ +package app + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + transcribebackend "github.com/blackbirdworks/gopherstack/services/transcribe" +) + +func TestRegisterTranscribeEndpoints(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + wantCode int + }{ + { + name: "unknown job", + path: "/_gopherstack/transcribe/output/us-east-1/transcription/nope/tok/asrOutput.json", + wantCode: http.StatusForbidden, + }, + {name: "short path", path: "/_gopherstack/transcribe/output/us-east-1/x", wantCode: http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := transcribebackend.NewHandler(transcribebackend.NewInMemoryBackend()) + h.EnableRegions() + + e := echo.New() + registerTranscribeEndpoints(e, []service.Registerable{h}) + + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, tt.path, nil)) + require.Equal(t, tt.wantCode, rec.Code) + assert.Empty(t, rec.Body.String()) + }) + } +} diff --git a/internal/cliconfig/lifecycle.go b/internal/cliconfig/lifecycle.go index 830bfde521..c8fb5df5b2 100644 --- a/internal/cliconfig/lifecycle.go +++ b/internal/cliconfig/lifecycle.go @@ -63,6 +63,7 @@ type LifecycleSettings struct { AppStream time.Duration `name:"appstream" env:"APPSTREAM_LIFECYCLE_DELAY" default:"0s" help:"AppStream fleet and image builder STARTING/STOPPING/PENDING window."` //nolint:lll // config struct tags are intentionally verbose FIS time.Duration `name:"fis" env:"FIS_LIFECYCLE_DELAY" default:"0s" help:"FIS experiment state step delay (built-in 10ms)."` //nolint:lll // config struct tags are intentionally verbose DSQL time.Duration `name:"dsql" env:"DSQL_LIFECYCLE_DELAY" default:"0s" help:"Aurora DSQL cluster and stream transitional window (built-in 750ms/500ms)."` //nolint:lll // config struct tags are intentionally verbose + LambdaWeb time.Duration `name:"lambdaweb" env:"LAMBDAWEB_LIFECYCLE_DELAY" default:"0s" help:"Lambda Web Functions function, revision and endpoint transitional window (built-in 500ms-1s)."` //nolint:lll // config struct tags are intentionally verbose GlobalAccelerator time.Duration `name:"globalaccelerator" env:"GLOBALACCELERATOR_LIFECYCLE_DELAY" default:"0s" help:"Global Accelerator transitional window (built-in 500ms)."` //nolint:lll // config struct tags are intentionally verbose CodeGuruReviewer time.Duration `name:"codegurureviewer" env:"CODEGURUREVIEWER_LIFECYCLE_DELAY" default:"0s" help:"CodeGuru Reviewer repository association and code review transitional window (built-in 500ms)."` //nolint:lll // config struct tags are intentionally verbose Connect time.Duration `name:"connect" env:"CONNECT_LIFECYCLE_DELAY" default:"0s" help:"Amazon Connect instance CREATION_IN_PROGRESS window (built-in 1s)."` //nolint:lll // config struct tags are intentionally verbose diff --git a/internal/dockercompat/client/client.go b/internal/dockercompat/client/client.go index 7c6119f4ff..5e22554ebc 100644 --- a/internal/dockercompat/client/client.go +++ b/internal/dockercompat/client/client.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "regexp" + "strings" "time" cerrdefs "github.com/containerd/errdefs" @@ -73,6 +74,22 @@ func (c *Client) ImageTag(ctx context.Context, source, target string) error { return err } +// ImageDigest returns the repo digest ("sha256:...") of a local image, or its image ID when it has none. +func (c *Client) ImageDigest(ctx context.Context, ref string) (string, error) { + info, err := c.inner.ImageInspect(ctx, ref) + if err != nil { + return "", err + } + + for _, rd := range info.RepoDigests { + if _, d, ok := strings.Cut(rd, "@"); ok { + return d, nil + } + } + + return info.ID, nil +} + // ImagePush pushes ref to its registry and returns the pushed manifest digest when the daemon reports one. func (c *Client) ImagePush(ctx context.Context, ref string) (string, error) { resp, err := c.inner.ImagePush(ctx, ref, mobyclient.ImagePushOptions{RegistryAuth: pushAnonymousAuth}) diff --git a/internal/linklocal/linklocal.go b/internal/linklocal/linklocal.go index f75b67e6ad..49ebfe86d7 100644 --- a/internal/linklocal/linklocal.go +++ b/internal/linklocal/linklocal.go @@ -45,7 +45,11 @@ const ( startWait = 10 * time.Second pollEvery = 100 * time.Millisecond gwPriority = -1 - relayScript = "printf '%s\\n' \"$LL_SCRIPT\" > /relay.sh && chmod +x /relay.sh && " + + + // DefaultUpstreamNetwork is the network the relay also joins to reach the emulator on the Docker host. + DefaultUpstreamNetwork = "bridge" + upstreamGwPriority = 1 + relayScript = "printf '%s\\n' \"$LL_SCRIPT\" > /relay.sh && chmod +x /relay.sh && " + "exec nc -lk -p " + relayPort + " -e /relay.sh" ) @@ -74,6 +78,10 @@ type Spec struct { // UpstreamHost and UpstreamPort are where the relay forwards, as seen from inside the network. UpstreamHost string UpstreamPort string + // UpstreamNetwork is a second network the relay joins so its upstream connections leave from an + // address the host can route back to. Over the link-local network alone the source is the proxy IP, + // and a cloud VM's own host route for 169.254.169.254 (EC2, Azure, GCP) sends the reply astray. + UpstreamNetwork string // PathPrefix is prepended to every forwarded request path. PathPrefix string // OwnerLabel is the container label naming the owning entity, used to recover owners after a restart. @@ -83,26 +91,28 @@ type Spec struct { // ECSSpec is the 169.254.170.2 task-credentials relay. func ECSSpec(image, host, port string) Spec { return Spec{ - Name: "ecs", - Image: image, - Subnet: netip.MustParsePrefix("169.254.170.0/24"), - ProxyIP: netip.MustParseAddr("169.254.170.2"), - UpstreamHost: host, - UpstreamPort: port, + Name: "ecs", + Image: image, + Subnet: netip.MustParsePrefix("169.254.170.0/24"), + ProxyIP: netip.MustParseAddr("169.254.170.2"), + UpstreamHost: host, + UpstreamPort: port, + UpstreamNetwork: DefaultUpstreamNetwork, } } // IMDSSpec is the 169.254.169.254 instance-metadata relay. func IMDSSpec(image, host, port, ownerLabel string) Spec { return Spec{ - Name: "imds", - Image: image, - Subnet: netip.MustParsePrefix("169.254.169.0/24"), - ProxyIP: netip.MustParseAddr("169.254.169.254"), - UpstreamHost: host, - UpstreamPort: port, - PathPrefix: IMDSPathPrefix, - OwnerLabel: ownerLabel, + Name: "imds", + Image: image, + Subnet: netip.MustParsePrefix("169.254.169.0/24"), + ProxyIP: netip.MustParseAddr("169.254.169.254"), + UpstreamHost: host, + UpstreamPort: port, + UpstreamNetwork: DefaultUpstreamNetwork, + PathPrefix: IMDSPathPrefix, + OwnerLabel: ownerLabel, } } @@ -314,9 +324,33 @@ func (m *Manager) createRelay(ctx context.Context) error { return fmt.Errorf("linklocal: relay create: %w", err) } + if err = m.joinUpstream(ctx, created.ID); err != nil { + _, _ = m.api.ContainerRemove( + ctx, created.ID, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true}, + ) + + return err + } + return m.startRelay(ctx, created.ID) } +func (m *Manager) joinUpstream(ctx context.Context, id string) error { + if m.spec.UpstreamNetwork == "" { + return nil + } + + _, err := m.api.NetworkConnect(ctx, m.spec.UpstreamNetwork, mobyclient.NetworkConnectOptions{ + Container: id, + EndpointConfig: &mobynetwork.EndpointSettings{GwPriority: upstreamGwPriority}, + }) + if err != nil { + return fmt.Errorf("linklocal: relay join %s: %w", m.spec.UpstreamNetwork, err) + } + + return nil +} + func (m *Manager) startRelay(ctx context.Context, id string) error { if _, err := m.api.ContainerStart(ctx, id, mobyclient.ContainerStartOptions{}); err != nil { return fmt.Errorf("linklocal: relay start: %w", err) @@ -488,7 +522,7 @@ func (m *Manager) Remove(ctx context.Context) error { func specHash(s Spec) string { return hex.EncodeToString(hashStrings(s.Image, s.Subnet.String(), s.ProxyIP.String(), - s.UpstreamHost, s.UpstreamPort, s.PathPrefix, relaySource, relayScript)) + s.UpstreamHost, s.UpstreamPort, s.UpstreamNetwork, s.PathPrefix, relaySource, relayScript)) } func hashStrings(parts ...string) []byte { diff --git a/internal/linklocal/linklocal_test.go b/internal/linklocal/linklocal_test.go index 13584aa28f..146ce718ac 100644 --- a/internal/linklocal/linklocal_test.go +++ b/internal/linklocal/linklocal_test.go @@ -17,10 +17,14 @@ import ( "github.com/blackbirdworks/gopherstack/internal/linklocal" ) -var errNoSuchImage = errors.New("no such image") +var ( + errNoSuchImage = errors.New("no such image") + errNoSuchNetwork = errors.New("no such network") +) type fakeDocker struct { imageErr error + connectErr error endpoints map[string]mobynetwork.EndpointResource labels map[string]map[string]string netCreated []mobyclient.NetworkCreateOptions @@ -71,7 +75,7 @@ func (f *fakeDocker) NetworkConnect( defer f.mu.Unlock() f.connected = append(f.connected, o) - return mobyclient.NetworkConnectResult{}, nil + return mobyclient.NetworkConnectResult{}, f.connectErr } func (f *fakeDocker) NetworkRemove( @@ -210,6 +214,52 @@ func TestManagerRelaySpec(t *testing.T) { c.NetworkingConfig.EndpointsConfig["gopherstack-linklocal-imds"].IPAMConfig.IPv4Address) } +func TestManagerRelayJoinsUpstreamNetwork(t *testing.T) { + t.Parallel() + + noUpstream := linklocal.IMDSSpec("", "172.17.0.1", "8123", "") + noUpstream.UpstreamNetwork = "" + + tests := []struct { + spec linklocal.Spec + connectErr error + name string + wantNetwork string + wantRemoved int + wantActive bool + }{ + {name: "imds", spec: linklocal.IMDSSpec("", "172.17.0.1", "8123", ""), wantNetwork: "bridge", wantActive: true}, + {name: "ecs", spec: linklocal.ECSSpec("", "172.17.0.1", "8123"), wantNetwork: "bridge", wantActive: true}, + {name: "disabled", spec: noUpstream, wantActive: true}, + { + name: "join_failure_removes_relay", spec: linklocal.IMDSSpec("", "172.17.0.1", "8123", ""), + connectErr: errNoSuchNetwork, wantRemoved: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := &fakeDocker{connectErr: tt.connectErr} + m := linklocal.New(f, tt.spec) + + assert.Equal(t, tt.wantActive, m.Active(t.Context())) + assert.Len(t, f.removedCtrs, tt.wantRemoved) + + if tt.wantNetwork == "" && tt.connectErr == nil { + assert.Empty(t, f.connected) + + return + } + + require.Len(t, f.connected, 1) + assert.Equal(t, "relay1", f.connected[0].Container) + assert.Positive(t, f.connected[0].EndpointConfig.GwPriority, "upstream network carries the default route") + }) + } +} + func TestManagerAttachAndOwner(t *testing.T) { t.Parallel() @@ -223,8 +273,8 @@ func TestManagerAttachAndOwner(t *testing.T) { m := linklocal.New(f, linklocal.IMDSSpec("", "h", "1", "gopherstack.ec2.instance")) require.NoError(t, m.Attach(t.Context(), "c-attached", "i-attached")) - require.Len(t, f.connected, 1) - assert.Equal(t, -1, f.connected[0].EndpointConfig.GwPriority) + require.NotEmpty(t, f.connected) + assert.Equal(t, -1, f.connected[len(f.connected)-1].EndpointConfig.GwPriority) tests := []struct { name string diff --git a/internal/routegates/gates.go b/internal/routegates/gates.go index 36eee80d3e..5b9a60efb1 100644 --- a/internal/routegates/gates.go +++ b/internal/routegates/gates.go @@ -143,7 +143,8 @@ func AltGates() map[string]service.AltGate { "/.well-known/jwks.json", "/.well-known/openid-configuration", "/saml2/metadata", }, }, - "Lambda": {Target: []string{"AWSLambda"}, Path: []string{"/20"}, Host: []string{".lambda-url."}}, + "Lambda": {Target: []string{"AWSLambda"}, Path: []string{"/20"}, Host: []string{".lambda-url."}}, + "LambdaWeb": {Path: []string{"/2025-03-07/"}, Host: []string{".lambda-web."}}, "EventBridge": { Target: []string{"AmazonEventBridge.", "AWSEvents.", "AWSSchemas."}, Path: []string{"/v1"}, }, diff --git a/pkgs/aadauth/discovery.go b/pkgs/aadauth/discovery.go index fff6b2b940..734dc3aa4a 100644 --- a/pkgs/aadauth/discovery.go +++ b/pkgs/aadauth/discovery.go @@ -26,7 +26,7 @@ func BuildOpenIDConfiguration(baseURL, tenant string) OpenIDConfiguration { ResponseTypesSupp: []string{"code", "id_token", "code id_token", "token"}, SubjectTypesSupp: []string{"pairwise"}, IDTokenSigningAlgs: []string{"RS256"}, - TokenEndpointAuthMeth: []string{"client_secret_post", "client_secret_basic"}, + TokenEndpointAuthMeth: []string{"client_secret_post", "client_secret_basic", "private_key_jwt"}, } } diff --git a/pkgs/amqp1/conn.go b/pkgs/amqp1/conn.go index 869dd17caf..37b74b14e2 100644 --- a/pkgs/amqp1/conn.go +++ b/pkgs/amqp1/conn.go @@ -74,6 +74,12 @@ func (c *conn) closeWithError(e *Error) { _ = c.writePerformative(FrameAMQP, 0, &Close{Error: e}) } +// internalError closes just this connection after a recovered panic. +func (c *conn) internalError(any) { + c.closeWithError(&Error{Condition: CondInternal, Description: "internal server error"}) + c.shutdown() +} + func (c *conn) closeSessions() { c.smu.Lock() all := make([]*session, 0, len(c.sessions)) diff --git a/pkgs/amqp1/link.go b/pkgs/amqp1/link.go index d28ae4afff..a120f45529 100644 --- a/pkgs/amqp1/link.go +++ b/pkgs/amqp1/link.go @@ -1,6 +1,10 @@ package amqp1 -import "context" +import ( + "context" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" +) func (s *session) attach(a *Attach) bool { s.mu.Lock() @@ -81,6 +85,8 @@ func (l *sendLink) open(a *Attach) bool { } func (l *sendLink) openAsync(d DeferredAttach, a *Attach) { + defer logger.Recover(l.ctx, "amqp1.openasync", l.s.c.internalError) + e := d.Ready(l.ctx) if l.ctx.Err() != nil { l.ep.Close() diff --git a/pkgs/amqp1/sendlink.go b/pkgs/amqp1/sendlink.go index fe40d1f61b..aae6fdd04f 100644 --- a/pkgs/amqp1/sendlink.go +++ b/pkgs/amqp1/sendlink.go @@ -6,6 +6,8 @@ import ( "errors" "math" "sync" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" ) type sendLink struct { @@ -39,7 +41,7 @@ func (l *sendLink) onFlow(f *Flow) { credit := peerCount + *f.LinkCredit - l.count if credit > math.MaxInt32 { - credit = 0 + credit = clampWindow(*f.LinkCredit) } l.credit = credit @@ -107,6 +109,7 @@ func (l *sendLink) finishDrain() { func (l *sendLink) run() { defer l.finish() + defer logger.Recover(l.ctx, "amqp1.sendlink", l.s.c.internalError) for l.waitCredit() { nctx, stop := context.WithCancel(l.ctx) diff --git a/pkgs/amqp1/server.go b/pkgs/amqp1/server.go index d240c62a5b..b3df054428 100644 --- a/pkgs/amqp1/server.go +++ b/pkgs/amqp1/server.go @@ -11,6 +11,8 @@ import ( "net" "sync" "time" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" ) // Defaults for Config. @@ -148,6 +150,7 @@ func (s *Server) handle(nc net.Conn) { defer s.untrack(c) defer c.shutdown() + defer logger.Recover(context.Background(), "amqp1.conn", c.internalError) if err := c.handshake(); err != nil { s.cfg.Logger.Debug("amqp1: handshake failed", "error", err) @@ -164,7 +167,7 @@ func (c *conn) authenticateSASL(r *bufio.Reader) error { return err } - mechs := &SASLMechanisms{Mechanisms: []Symbol{"ANONYMOUS", "PLAIN"}} + mechs := &SASLMechanisms{Mechanisms: []Symbol{"ANONYMOUS", "PLAIN", "MSSBCBS"}} if err := c.writePerformative(FrameSASL, 0, mechs); err != nil { return err } diff --git a/pkgs/amqp1/server_test.go b/pkgs/amqp1/server_test.go index 6ff71669f8..a69581f041 100644 --- a/pkgs/amqp1/server_test.go +++ b/pkgs/amqp1/server_test.go @@ -16,7 +16,7 @@ import ( type fakeHandler struct{ got chan []byte } func (h *fakeHandler) Authenticate(mech amqp1.Symbol, _, pass string) bool { - return mech == "ANONYMOUS" || pass == "ok" + return mech == "ANONYMOUS" || mech == "MSSBCBS" || pass == "ok" } func (h *fakeHandler) NewConn() amqp1.ConnHandler { return &fakeConn{got: h.got} } @@ -180,11 +180,13 @@ func TestServer_SASLPlain(t *testing.T) { tests := []struct { name string + mech amqp1.Symbol pass string wantCode uint8 }{ - {name: "good credentials", pass: "ok", wantCode: amqp1.SASLOK}, - {name: "bad credentials", pass: "nope", wantCode: amqp1.SASLAuth}, + {name: "good credentials", mech: "PLAIN", pass: "ok", wantCode: amqp1.SASLOK}, + {name: "bad credentials", mech: "PLAIN", pass: "nope", wantCode: amqp1.SASLAuth}, + {name: "claims based security", mech: "MSSBCBS", wantCode: amqp1.SASLOK}, } for _, tt := range tests { @@ -204,8 +206,14 @@ func TestServer_SASLPlain(t *testing.T) { mechs, ok := c.recvSASL().(*amqp1.SASLMechanisms) require.True(t, ok) assert.Contains(t, mechs.Mechanisms, amqp1.Symbol("PLAIN")) + assert.Contains(t, mechs.Mechanisms, amqp1.Symbol("MSSBCBS"), "the .NET SDKs only speak MSSBCBS") - c.sendSASL(&amqp1.SASLInit{Mechanism: "PLAIN", InitialResponse: []byte("\x00user\x00" + tt.pass)}) + init := &amqp1.SASLInit{Mechanism: tt.mech} + if tt.mech == "PLAIN" { + init.InitialResponse = []byte("\x00user\x00" + tt.pass) + } + + c.sendSASL(init) out, ok := c.recvSASL().(*amqp1.SASLOutcome) require.True(t, ok) diff --git a/pkgs/amqp1/session.go b/pkgs/amqp1/session.go index bd3a27d328..cdb72b7baa 100644 --- a/pkgs/amqp1/session.go +++ b/pkgs/amqp1/session.go @@ -88,13 +88,23 @@ func (s *session) handle(p Performative) bool { return true } +// clampWindow resolves a wrapped window: a peer advertising a huge incoming window (the .NET SDKs send 2^32-1) +// means "effectively unlimited", while a wrapped small window means the peer is behind and gets none. +func clampWindow(advertised uint32) uint32 { + if advertised > math.MaxInt32 { + return math.MaxInt32 + } + + return 0 +} + func (s *session) onFlow(f *Flow) { s.mu.Lock() if f.NextIncomingID != nil { w := *f.NextIncomingID + f.IncomingWindow - s.nextOutgoingID if w > math.MaxInt32 { - w = 0 + w = clampWindow(f.IncomingWindow) } s.remoteWindow = int64(w) diff --git a/pkgs/amqp1/window_test.go b/pkgs/amqp1/window_test.go new file mode 100644 index 0000000000..642c6afe83 --- /dev/null +++ b/pkgs/amqp1/window_test.go @@ -0,0 +1,119 @@ +package amqp1_test + +import ( + "bufio" + "context" + "math" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/amqp1" +) + +type oneShotHandler struct{} + +func (oneShotHandler) Authenticate(amqp1.Symbol, string, string) bool { return true } + +func (oneShotHandler) NewConn() amqp1.ConnHandler { return oneShotConn{} } + +type oneShotConn struct{} + +func (oneShotConn) Close() {} + +func (oneShotConn) AttachSender(*amqp1.Attach) (amqp1.SenderEndpoint, *amqp1.Error) { + return &oneShotSender{sent: make(chan struct{}, 1)}, nil +} + +func (oneShotConn) AttachReceiver(*amqp1.Attach) (amqp1.ReceiverEndpoint, *amqp1.Error) { + return nil, &amqp1.Error{Condition: amqp1.CondNotImplemented} +} + +type oneShotSender struct{ sent chan struct{} } + +func (s *oneShotSender) Next(ctx context.Context) (*amqp1.Outgoing, error) { + select { + case s.sent <- struct{}{}: + return &amqp1.Outgoing{Tag: []byte{1}, Payload: []byte("payload")}, nil + default: + } + + <-ctx.Done() + + return nil, ctx.Err() +} + +func (*oneShotSender) Settle(*amqp1.Outgoing, amqp1.DeliveryState) {} +func (*oneShotSender) Release(*amqp1.Outgoing) {} +func (*oneShotSender) Close() {} + +func TestServer_SenderHonoursHugePeerWindows(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + window uint32 + credit uint32 + }{ + {"small", 100, 10}, + {"max incoming window", math.MaxUint32, 10}, + {"max window and credit", math.MaxUint32, math.MaxUint32}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var lc net.ListenConfig + + ln, err := lc.Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + + srv := amqp1.NewServer(amqp1.Config{Handler: oneShotHandler{}, MaxMessageSize: 4096, MaxFrameSize: 1024}) + + go func() { _ = srv.Serve(ln, false) }() + + t.Cleanup(srv.Close) + + var d net.Dialer + + nc, err := d.DialContext(t.Context(), "tcp", ln.Addr().String()) + require.NoError(t, err) + t.Cleanup(func() { _ = nc.Close() }) + require.NoError(t, nc.SetDeadline(time.Now().Add(10*time.Second))) + + c := &rawClient{t: t, nc: nc, r: bufio.NewReader(nc)} + + _, err = nc.Write([]byte(amqp1.HeaderAMQP)) + require.NoError(t, err) + + hdr := make([]byte, 8) + _, err = c.r.Read(hdr) + require.NoError(t, err) + + c.send(&amqp1.Open{ContainerID: "c", MaxFrameSize: 1024}, nil) + require.IsType(t, &amqp1.Open{}, c.recv()) + + c.send(&amqp1.Begin{IncomingWindow: tt.window, OutgoingWindow: 100, HandleMax: 10}, nil) + require.IsType(t, &amqp1.Begin{}, c.recv()) + + c.send(&amqp1.Attach{Name: "l", Handle: 0, Role: true, Source: &amqp1.Source{Address: "q"}, + Target: &amqp1.Target{}}, nil) + require.IsType(t, &amqp1.Attach{}, c.recv()) + + zero, handle := uint32(0), uint32(0) + c.send(&amqp1.Flow{ + NextIncomingID: &zero, IncomingWindow: tt.window, NextOutgoingID: 1, OutgoingWindow: 100, + Handle: &handle, DeliveryCount: &zero, LinkCredit: &tt.credit, + }, nil) + + tr, ok := c.recv().(*amqp1.Transfer) + require.True(t, ok, "the message is delivered") + assert.Equal(t, uint32(0), handle) + assert.NotNil(t, tr.DeliveryID) + }) + } +} diff --git a/pkgs/bolt/bolt_test.go b/pkgs/bolt/bolt_test.go index 2b9723545c..5fd833c349 100644 --- a/pkgs/bolt/bolt_test.go +++ b/pkgs/bolt/bolt_test.go @@ -57,6 +57,8 @@ func (s *fakeSession) Run(_ context.Context, q string, _ map[string]any) (*bolt. return nil, &bolt.Error{Code: bolt.CodeSyntax, Message: "bad query"} case "PLAIN": return nil, errBoom + case "PANIC": + panic("injected backend panic") } return &bolt.Result{Fields: []string{"n"}, Rows: s.f.rows, Write: strings.HasPrefix(q, "CREATE")}, nil diff --git a/pkgs/bolt/recover_test.go b/pkgs/bolt/recover_test.go new file mode 100644 index 0000000000..a41528e28a --- /dev/null +++ b/pkgs/bolt/recover_test.go @@ -0,0 +1,52 @@ +package bolt_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/bolt" +) + +func TestBackendPanicAnswersFailureAndKeepsServing(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + ver bolt.Version + }{ + {name: "v5.4", query: "PANIC", ver: bolt.Version{Major: 5, Minor: 4}}, + {name: "v4.4", query: "PANIC", ver: bolt.Version{Major: 4, Minor: 4}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := &fakeBackend{rows: rowsOf(2)} + + bad := startServer(t, b, tt.ver) + login(t, bad) + + other := startServer(t, b, tt.ver) + login(t, other) + + r, err := bad.Call(bolt.TagRun, tt.query, map[string]any{}, map[string]any{}) + require.NoError(t, err) + require.Equal(t, bolt.TagFailure, r.Tag) + assert.Equal(t, bolt.CodeDatabaseError, r.Meta()["code"]) + + r, err = bad.Call(bolt.TagReset) + require.NoError(t, err) + require.Equal(t, bolt.TagSuccess, r.Tag) + + for _, c := range []*bolt.Client{bad, other} { + rows, _, qerr := c.Query("MATCH (n) RETURN n", nil) + require.NoError(t, qerr) + assert.Len(t, rows, 2) + } + }) + } +} diff --git a/pkgs/bolt/server.go b/pkgs/bolt/server.go index 27a18122ec..d05da1d0bb 100644 --- a/pkgs/bolt/server.go +++ b/pkgs/bolt/server.go @@ -9,6 +9,8 @@ import ( "strconv" "github.com/google/uuid" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" ) // Message tags. @@ -118,6 +120,7 @@ type conn struct { // Serve speaks Bolt on nc until the client leaves or ctx ends, then closes nc. func Serve(ctx context.Context, nc net.Conn, cfg Config) { defer func() { _ = nc.Close() }() + defer logger.Recover(ctx, "bolt.conn", nil) ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -156,7 +159,7 @@ func (c *conn) step(ctx context.Context) error { return err } - if derr := c.dispatch(ctx, tag, fields); derr != nil { + if derr := c.guardedDispatch(ctx, tag, fields); derr != nil { return derr } @@ -167,6 +170,21 @@ func (c *conn) step(ctx context.Context) error { return nil } +func (c *conn) guardedDispatch(ctx context.Context, tag byte, fields []any) error { + var err error + + func() { + defer logger.Recover(ctx, "bolt.message", func(any) { + c.w.Reset(c.nc) + err = c.fail(&Error{Code: CodeDatabaseError, Message: "internal server error"}) + }) + + err = c.dispatch(ctx, tag, fields) + }() + + return err +} + func parseMessage(raw []byte) (byte, []any, error) { v, err := NewDecoder(raw).Decode() if err != nil { diff --git a/pkgs/bolt/split.go b/pkgs/bolt/split.go index aaf5fd5cb8..829bfb3b75 100644 --- a/pkgs/bolt/split.go +++ b/pkgs/bolt/split.go @@ -2,10 +2,13 @@ package bolt import ( "bufio" + "context" "errors" "net" "sync" "time" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" ) const sniffWindow = 30 * time.Second @@ -48,6 +51,8 @@ func (s *splitter) run() { } func (s *splitter) route(c net.Conn) { + defer logger.Recover(context.Background(), "bolt.split", func(any) { _ = c.Close() }) + br := bufio.NewReader(c) _ = c.SetReadDeadline(time.Now().Add(sniffWindow)) diff --git a/pkgs/container/container.go b/pkgs/container/container.go index eef1410951..d01f9f0f09 100644 --- a/pkgs/container/container.go +++ b/pkgs/container/container.go @@ -41,21 +41,38 @@ const ( RuntimePodman RuntimeName = "podman" // RuntimeAuto auto-detects the available runtime by probing sockets. RuntimeAuto RuntimeName = "auto" + // RuntimeKubernetes runs containers as Pods through the Kubernetes API. + RuntimeKubernetes RuntimeName = "kubernetes" ) // Config holds configuration for the container runtime layer. type Config struct { Logger *slog.Logger // Injection overrides the env-derived DNS/CA injection for spawned containers. - Injection *Injection + Injection *Injection + // Kubernetes configures RuntimeKubernetes; unset fields come from the environment. + Kubernetes *KubernetesConfig Runtime RuntimeName PoolSize int IdleTimeout time.Duration } +// Archive is a zip delivered to Dest inside the container; runtimes without bind mounts unpack it for the caller. +type Archive struct { + Dest string + Zip []byte +} + +// ArchiveDeliverer is implemented by runtimes that deliver Spec.Archives and have no host bind mounts. +type ArchiveDeliverer interface { + DeliversArchives() bool +} + // Spec holds the specification for creating a container. type Spec struct { - Labels map[string]string + Labels map[string]string + // Archives are unpacked read-only at their Dest before the container starts. + Archives []Archive Image string Name string Network string diff --git a/pkgs/container/digest_test.go b/pkgs/container/digest_test.go new file mode 100644 index 0000000000..ef300de83e --- /dev/null +++ b/pkgs/container/digest_test.go @@ -0,0 +1,60 @@ +package container_test + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/container" +) + +var errInspectFailed = errors.New("inspect failed") + +type digestAPI struct { + *mockAPI + + err error + digest string +} + +func (d *digestAPI) ImageDigest(_ context.Context, _ string) (string, error) { return d.digest, d.err } + +func TestDockerRuntime_ImageDigest(t *testing.T) { + t.Parallel() + + tests := []struct { + api container.APIClient + wantErr error + name string + wantDigest string + }{ + {name: "digest", api: &digestAPI{mockAPI: &mockAPI{}, digest: "sha256:abc"}, wantDigest: "sha256:abc"}, + { + name: "inspect failure", + api: &digestAPI{mockAPI: &mockAPI{}, err: errInspectFailed}, + wantErr: errInspectFailed, + }, + {name: "client cannot inspect", api: &mockAPI{}, wantErr: container.ErrPushUnsupported}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rt := container.NewDockerRuntimeWithAPI(tt.api, container.Config{PoolSize: 1}) + + digest, err := rt.ImageDigest(t.Context(), "img:1") + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + + return + } + + require.NoError(t, err) + assert.Equal(t, tt.wantDigest, digest) + }) + } +} diff --git a/pkgs/container/docker_runtime.go b/pkgs/container/docker_runtime.go index 9a9e661144..63f958e121 100644 --- a/pkgs/container/docker_runtime.go +++ b/pkgs/container/docker_runtime.go @@ -104,6 +104,10 @@ type realDockerClient struct { c *client.Client } +func (r *realDockerClient) ImageDigest(ctx context.Context, ref string) (string, error) { + return r.c.ImageDigest(ctx, ref) +} + func (r *realDockerClient) ImagePush(ctx context.Context, ref string) (string, error) { return r.c.ImagePush(ctx, ref) } @@ -400,6 +404,10 @@ func (r *DockerRuntime) HasImage(ctx context.Context, imageRef string) (bool, er // CreateAndStart creates a new container from spec and starts it. // Returns the container ID. func (r *DockerRuntime) CreateAndStart(ctx context.Context, spec Spec) (string, error) { + if len(spec.Archives) > 0 { + return "", ErrArchivesUnsupported + } + cfg := &dockercontainer.Config{ Image: spec.Image, Env: spec.Env, diff --git a/pkgs/container/factory.go b/pkgs/container/factory.go index e097a9f06e..790cefab8a 100644 --- a/pkgs/container/factory.go +++ b/pkgs/container/factory.go @@ -42,8 +42,10 @@ func NewRuntime(cfg Config) (Runtime, error) { return newPodmanRuntime(cfg) case RuntimeAuto: return autoDetectRuntime(cfg) + case RuntimeKubernetes: + return newKubernetesRuntime(cfg) default: - return nil, fmt.Errorf("%w %q; valid values: docker, podman, auto", ErrUnknownRuntime, name) + return nil, fmt.Errorf("%w %q; valid values: docker, podman, auto, kubernetes", ErrUnknownRuntime, name) } } diff --git a/pkgs/container/k8s_code.go b/pkgs/container/k8s_code.go new file mode 100644 index 0000000000..3a59b92812 --- /dev/null +++ b/pkgs/container/k8s_code.go @@ -0,0 +1,163 @@ +package container + +import ( + "archive/zip" + "bytes" + "context" + "crypto/rand" + "crypto/subtle" + "encoding/hex" + "errors" + "fmt" + "net" + "net/http" + "path" + "strings" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" +) + +const ( + archivePathPrefix = "/archives/" + bearerPrefix = "Bearer " + codeReadHeaderWait = 10 * time.Second + codeIDBytes = 16 + codeTokenBytes = 32 +) + +// ErrUnsafeArchive is returned for a zip with absolute or parent-escaping entry names. +var ErrUnsafeArchive = errors.New("archive has an unsafe entry name") + +type codeBlob struct { + token string + data []byte +} + +// codeServer hands archives to the fetch initContainers of pods this process creates. Each pod gets its own +// token, valid only for its own archives and revoked once the pod is up or gone. +type codeServer struct { + mu *lockmetrics.RWMutex + blobs map[string]codeBlob + ln net.Listener + srv *http.Server + advertise string +} + +func newCodeServer(ln net.Listener, advertise string) *codeServer { + return &codeServer{ + mu: lockmetrics.New("container.k8s.code"), + blobs: map[string]codeBlob{}, + ln: ln, + advertise: advertise, + } +} + +func (c *codeServer) start() { + c.srv = &http.Server{Handler: http.HandlerFunc(c.serve), ReadHeaderTimeout: codeReadHeaderWait} + + go func() { _ = c.srv.Serve(c.ln) }() +} + +func (c *codeServer) close(ctx context.Context) { + if c.srv != nil { + _ = c.srv.Shutdown(ctx) + } +} + +func (c *codeServer) serve(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || !strings.HasPrefix(r.URL.Path, archivePathPrefix) { + http.NotFound(w, r) + + return + } + + id := strings.TrimPrefix(r.URL.Path, archivePathPrefix) + + c.mu.RLock("serve") + blob, ok := c.blobs[id] + c.mu.RUnlock() + + got := r.Header.Get("Authorization") + if !ok || !strings.HasPrefix(got, bearerPrefix) || + subtle.ConstantTimeCompare([]byte(strings.TrimPrefix(got, bearerPrefix)), []byte(blob.token)) != 1 { + http.Error(w, "unauthorized", http.StatusUnauthorized) + + return + } + + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(blob.data) +} + +// grant is a pod's fetch credentials: the token and one URL per archive, in order. +type grant struct { + token string + urls []string + ids []string +} + +// register stores the archives under a fresh token and returns the credentials to give the pod. +func (c *codeServer) register(archives []Archive) (grant, error) { + token, err := randomToken(codeTokenBytes) + if err != nil { + return grant{}, err + } + + g := grant{token: token} + + c.mu.Lock("register") + defer c.mu.Unlock() + + for _, a := range archives { + id, idErr := randomToken(codeIDBytes) + if idErr != nil { + for _, done := range g.ids { + delete(c.blobs, done) + } + + return grant{}, idErr + } + + c.blobs[id] = codeBlob{token: token, data: a.Zip} + g.ids = append(g.ids, id) + g.urls = append(g.urls, "http://"+c.advertise+archivePathPrefix+id) + } + + return g, nil +} + +func (c *codeServer) revoke(g grant) { + c.mu.Lock("revoke") + defer c.mu.Unlock() + + for _, id := range g.ids { + delete(c.blobs, id) + } +} + +func randomToken(n int) (string, error) { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("random token: %w", err) + } + + return hex.EncodeToString(b), nil +} + +// validateArchive rejects zips whose entries could escape the extraction directory. +func validateArchive(data []byte) error { + r, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return fmt.Errorf("open archive: %w", err) + } + + for _, f := range r.File { + name := strings.ReplaceAll(f.Name, "\\", "/") + if strings.HasPrefix(name, "/") || path.Clean(name) == ".." || strings.HasPrefix(path.Clean(name), "../") { + return fmt.Errorf("%w: %q", ErrUnsafeArchive, f.Name) + } + } + + return nil +} diff --git a/pkgs/container/k8s_exec.go b/pkgs/container/k8s_exec.go new file mode 100644 index 0000000000..36cdc23e2e --- /dev/null +++ b/pkgs/container/k8s_exec.go @@ -0,0 +1,174 @@ +package container + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "sync" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" +) + +// ErrExecNoConfig is returned when pods/exec is requested on a runtime built from an injected client without +// an Executor. +var ErrExecNoConfig = errors.New("pods/exec needs a cluster connection or a PodExecutor") + +// TermSize is a terminal size in character cells. +type TermSize struct{ Cols, Rows uint } + +// ExecRequest is one pods/exec stream with a TTY: stdout and stderr share Stdout. +type ExecRequest struct { + Stdin io.Reader + Stdout io.Writer + Resize <-chan TermSize + Pod string + Container string + Command []string +} + +// PodExecutor streams a command in a Pod container until it ends or ctx is cancelled. +type PodExecutor interface { + StreamExec(ctx context.Context, req ExecRequest) error +} + +// PodShell is an interactive command attached to a Pod container. +type PodShell interface { + io.ReadWriteCloser + Resize(ctx context.Context, cols, rows uint) error +} + +type spdyExecutor struct{ r *KubernetesRuntime } + +func (e spdyExecutor) StreamExec(ctx context.Context, req ExecRequest) error { + r := e.r + + u := r.cs.CoreV1().RESTClient().Post().Resource("pods").Namespace(r.ns).Name(req.Pod).SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: req.Container, Command: req.Command, Stdin: true, Stdout: true, TTY: true, + }, scheme.ParameterCodec).URL() + + ex, err := remotecommand.NewSPDYExecutor(r.restCfg, "POST", u) + if err != nil { + return fmt.Errorf("pods/exec executor: %w", err) + } + + err = ex.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: req.Stdin, Stdout: req.Stdout, Tty: true, TerminalSizeQueue: sizeQueue{done: ctx.Done(), ch: req.Resize}, + }) + if err != nil { + return fmt.Errorf("pods/exec %s/%s: %w", req.Pod, req.Container, err) + } + + return nil +} + +type sizeQueue struct { + done <-chan struct{} + ch <-chan TermSize +} + +func (q sizeQueue) Next() *remotecommand.TerminalSize { + select { + case s := <-q.ch: + return &remotecommand.TerminalSize{Width: clampU16(s.Cols), Height: clampU16(s.Rows)} + case <-q.done: + return nil + } +} + +func clampU16(v uint) uint16 { return uint16(min(v, math.MaxUint16)) } + +func (r *KubernetesRuntime) executor() (PodExecutor, error) { + if r.exec != nil { + return r.exec, nil + } + + if r.restCfg == nil { + return nil, ErrExecNoConfig + } + + return spdyExecutor{r: r}, nil +} + +type podShell struct { + in *io.PipeWriter + out *io.PipeReader + resize chan TermSize + cancel context.CancelFunc + done chan struct{} + once sync.Once +} + +// ExecInteractive runs command in a Pod container with a TTY. Closing the shell ends the command. +func (r *KubernetesRuntime) ExecInteractive( + ctx context.Context, pod, containerName string, command []string, cols, rows uint, +) (PodShell, error) { + ex, err := r.executor() + if err != nil { + return nil, err + } + + inR, inW := io.Pipe() + outR, outW := io.Pipe() + sh := &podShell{in: inW, out: outR, resize: make(chan TermSize, 1), done: make(chan struct{})} + + ctx, sh.cancel = context.WithCancel(context.WithoutCancel(ctx)) + sh.resize <- TermSize{Cols: cols, Rows: rows} + + go func() { + defer close(sh.done) + + streamErr := ex.StreamExec(ctx, ExecRequest{ + Pod: pod, Container: containerName, Command: command, Stdin: inR, Stdout: outW, Resize: sh.resize, + }) + + _ = inR.Close() + _ = outW.CloseWithError(streamErr) + }() + + return sh, nil +} + +func (s *podShell) Read(p []byte) (int, error) { + n, err := s.out.Read(p) + if err != nil && !errors.Is(err, io.EOF) { + return n, io.EOF + } + + return n, err //nolint:wrapcheck // io.Reader contract +} + +func (s *podShell) Write(p []byte) (int, error) { + n, err := s.in.Write(p) + if err != nil { + return n, fmt.Errorf("pod shell write: %w", err) + } + + return n, nil +} + +func (s *podShell) Resize(ctx context.Context, cols, rows uint) error { + select { + case s.resize <- TermSize{Cols: cols, Rows: rows}: + case <-s.done: + case <-ctx.Done(): + return fmt.Errorf("pod shell resize: %w", ctx.Err()) + } + + return nil +} + +func (s *podShell) Close() error { + s.once.Do(func() { + s.cancel() + _ = s.in.Close() + _ = s.out.Close() + <-s.done + }) + + return nil +} diff --git a/pkgs/container/k8s_logs.go b/pkgs/container/k8s_logs.go new file mode 100644 index 0000000000..d470abb2b2 --- /dev/null +++ b/pkgs/container/k8s_logs.go @@ -0,0 +1,72 @@ +package container + +import ( + "context" + "fmt" + "io" + "maps" + "strconv" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const managedSelector = LabelManaged + "=true" + +// StreamLogs follows the Pod's output; Pods merge stdout and stderr, so everything goes to stdout. +func (r *KubernetesRuntime) StreamLogs(ctx context.Context, containerID string, stdout, stderr io.Writer) error { + return r.ReadLogs(ctx, containerID, "", true, stdout, stderr) +} + +// ReadLogs reads the Pod's logs; Pods merge stdout and stderr, so everything goes to stdout. +func (r *KubernetesRuntime) ReadLogs( + ctx context.Context, + containerID, tail string, + follow bool, + stdout, _ io.Writer, +) error { + opts := &corev1.PodLogOptions{Container: podMainContainer, Follow: follow} + + if n, err := strconv.ParseInt(tail, 10, 64); err == nil { + opts.TailLines = &n + } + + rc, err := r.cs.CoreV1().Pods(r.ns).GetLogs(containerID, opts).Stream(ctx) + if err != nil { + return fmt.Errorf("pod logs %q: %w", containerID, err) + } + defer rc.Close() + + if _, err = io.Copy(stdout, rc); err != nil && ctx.Err() == nil { + return fmt.Errorf("pod logs %q: %w", containerID, err) + } + + return nil +} + +// ListManaged lists the Pods carrying LabelManaged; labels are read back from annotations, which keep values +// (such as pid:start) that Kubernetes label syntax rejects. +func (r *KubernetesRuntime) ListManaged(ctx context.Context) ([]ManagedContainer, error) { + list, err := r.cs.CoreV1().Pods(r.ns).List(ctx, metav1.ListOptions{LabelSelector: managedSelector}) + if err != nil { + return nil, fmt.Errorf("list pods: %w", err) + } + + out := make([]ManagedContainer, 0, len(list.Items)) + + for i := range list.Items { + p := &list.Items[i] + if p.DeletionTimestamp != nil { + continue + } + + out = append(out, ManagedContainer{ + ID: p.Name, + Name: p.Name, + Labels: maps.Clone(p.Annotations), + Running: p.Status.Phase == corev1.PodRunning || p.Status.Phase == corev1.PodPending, + }) + } + + return out, nil +} diff --git a/pkgs/container/k8s_pod.go b/pkgs/container/k8s_pod.go new file mode 100644 index 0000000000..36173fc992 --- /dev/null +++ b/pkgs/container/k8s_pod.go @@ -0,0 +1,271 @@ +package container + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation" +) + +const ( + podMainContainer = "main" + podFetchContainer = "fetch" + podFetchTokenEnv = "FETCH_TOKEN" + maxPodNameLen = 63 + podNameHashLen = 8 + nanoCPUsPerMilli = 1_000_000 + envKVParts = 2 + podTerminationSec = 2 +) + +// ErrSpecUnsupported is returned for Spec fields a Pod cannot honour. +var ErrSpecUnsupported = errors.New("container spec field is not supported by the kubernetes runtime") + +// ErrArchivesUnsupported is returned by runtimes that cannot deliver Spec.Archives. +var ErrArchivesUnsupported = errors.New("container archives are not supported by this runtime") + +var nonDNSLabel = regexp.MustCompile(`[^a-z0-9-]+`) + +// fetchScript downloads and unpacks each "URL DEST" argument pair; the token comes from the environment. +const fetchScript = `set -eu +while [ "$#" -gt 1 ]; do + wget -q -O /tmp/archive.zip --header "Authorization: Bearer ${FETCH_TOKEN}" "$1" + unzip -q -o /tmp/archive.zip -d "$2" + rm -f /tmp/archive.zip + shift 2 +done +` + +func podName(name string) string { + clean := strings.Trim(nonDNSLabel.ReplaceAllString(strings.ToLower(name), "-"), "-") + + if clean == name && len(clean) <= maxPodNameLen && clean != "" { + return clean + } + + sum := sha256.Sum256([]byte(name)) + suffix := hex.EncodeToString(sum[:])[:podNameHashLen] + + if clean == "" { + clean = "gopherstack" + } + + if keep := maxPodNameLen - podNameHashLen - 1; len(clean) > keep { + clean = strings.TrimRight(clean[:keep], "-") + } + + return clean + "-" + suffix +} + +func rejectUnsupported(spec Spec) error { + var bad []string + + for field, set := range map[string]bool{ + "Mounts": len(spec.Mounts) > 0, + "Ports": len(spec.Ports) > 0, + "Privileged": spec.Privileged, + "Network": spec.Network != "", + "Platform": spec.Platform != "", + "GroupAdd": len(spec.GroupAdd) > 0, + } { + if set { + bad = append(bad, field) + } + } + + if len(bad) > 0 { + return fmt.Errorf("%w: %s", ErrSpecUnsupported, strings.Join(bad, ", ")) + } + + return nil +} + +func splitEnv(env []string) []corev1.EnvVar { + out := make([]corev1.EnvVar, 0, len(env)) + + for _, e := range env { + k, v, _ := strings.Cut(e, "=") + out = append(out, corev1.EnvVar{Name: k, Value: v}) + } + + return out +} + +func podMetadata(spec Spec) (map[string]string, map[string]string) { + all := spec.launchLabels() + labels := map[string]string{} + annotations := make(map[string]string, len(all)) + + for k, v := range all { + annotations[k] = v + + if len(validation.IsQualifiedName(k)) == 0 && len(validation.IsValidLabelValue(v)) == 0 { + labels[k] = v + } + } + + return labels, annotations +} + +func resourcesFor(spec Spec) corev1.ResourceRequirements { + req := corev1.ResourceRequirements{Limits: make(corev1.ResourceList)} + + if spec.MemoryBytes > 0 { + q := resource.NewQuantity(spec.MemoryBytes, resource.BinarySI) + req.Limits[corev1.ResourceMemory] = *q + req.Requests = make(corev1.ResourceList) + req.Requests[corev1.ResourceMemory] = *q + } + + if spec.NanoCPUs > 0 { + req.Limits[corev1.ResourceCPU] = *resource.NewMilliQuantity(spec.NanoCPUs/nanoCPUsPerMilli, resource.DecimalSI) + } + + if len(req.Limits) == 0 { + req.Limits = nil + } + + return req +} + +// buildPod maps spec onto a Pod. g carries the fetch credentials when spec has archives. +func (r *KubernetesRuntime) buildPod(spec Spec, name string, g grant) (*corev1.Pod, error) { + if err := rejectUnsupported(spec); err != nil { + return nil, err + } + + labels, annotations := podMetadata(spec) + + main := corev1.Container{ + Name: podMainContainer, + Image: spec.Image, + Command: spec.Entrypoint, + Args: spec.Cmd, + Env: splitEnv(spec.Env), + WorkingDir: spec.WorkingDir, + Resources: resourcesFor(spec), + SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: new(false)}, + } + + if uid, ok := numericUser(spec.User); ok { + main.SecurityContext.RunAsUser = &uid + } else if spec.User != "" { + return nil, fmt.Errorf("%w: User %q must be numeric", ErrSpecUnsupported, spec.User) + } + + pod := &corev1.Pod{ + Name: name, Namespace: r.ns, Labels: labels, Annotations: annotations, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + AutomountServiceAccountToken: new(false), + EnableServiceLinks: new(false), + SecurityContext: &corev1.PodSecurityContext{ + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{main}, + }, + } + + if spec.OCIRuntime != "" { + pod.Spec.RuntimeClassName = new(spec.OCIRuntime) + } + + if len(spec.DNS) > 0 { + pod.Spec.DNSConfig = &corev1.PodDNSConfig{Nameservers: spec.DNS} + } + + for _, h := range spec.ExtraHosts { + host, ip, ok := strings.Cut(h, ":") + if !ok { + return nil, fmt.Errorf("%w: ExtraHosts %q", ErrSpecUnsupported, h) + } + + pod.Spec.HostAliases = append(pod.Spec.HostAliases, corev1.HostAlias{IP: ip, Hostnames: []string{host}}) + } + + for i, path := range spec.Tmpfs { + addVolume( + pod, + fmt.Sprintf("tmpfs-%d", i), + path, + false, + corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}, + ) + } + + r.addArchives(pod, spec.Archives, g) + + return pod, nil +} + +func numericUser(user string) (int64, bool) { + uid, _, _ := strings.Cut(user, ":") + n, err := strconv.ParseInt(uid, 10, 64) + + return n, err == nil +} + +func addVolume(pod *corev1.Pod, name, mountPath string, readOnly bool, src corev1.EmptyDirVolumeSource) { + pod.Spec.Volumes = append( + pod.Spec.Volumes, + corev1.Volume{Name: name, EmptyDir: &src}, + ) + pod.Spec.Containers[0].VolumeMounts = append( + pod.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: name, MountPath: mountPath, ReadOnly: readOnly}, + ) +} + +// addArchives adds one emptyDir per distinct Dest, filled by a fetch initContainer and mounted read-only. +func (r *KubernetesRuntime) addArchives(pod *corev1.Pod, archives []Archive, g grant) { + if len(archives) == 0 { + return + } + + vols := map[string]string{} + args := []string{"-c", fetchScript, "fetch"} + fetch := corev1.Container{ + Name: podFetchContainer, + Image: r.fetchImage, + Command: []string{"/bin/sh"}, + Env: []corev1.EnvVar{{Name: podFetchTokenEnv, Value: g.token}}, + SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: new(false)}, + } + + for i, a := range archives { + vol, ok := vols[a.Dest] + if !ok { + vol = fmt.Sprintf("archive-%d", len(vols)) + vols[a.Dest] = vol + + addVolume(pod, vol, a.Dest, true, corev1.EmptyDirVolumeSource{}) + } + + mount := "/mnt/" + vol + if !hasMount(fetch.VolumeMounts, vol) { + fetch.VolumeMounts = append(fetch.VolumeMounts, corev1.VolumeMount{Name: vol, MountPath: mount}) + } + + args = append(args, g.urls[i], mount) + } + + fetch.Args = args + pod.Spec.InitContainers = []corev1.Container{fetch} +} + +func hasMount(mounts []corev1.VolumeMount, name string) bool { + for _, m := range mounts { + if m.Name == name { + return true + } + } + + return false +} diff --git a/pkgs/container/k8s_runtime.go b/pkgs/container/k8s_runtime.go new file mode 100644 index 0000000000..e5c4976065 --- /dev/null +++ b/pkgs/container/k8s_runtime.go @@ -0,0 +1,526 @@ +package container + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "os" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" +) + +const ( + // EnvK8sKubeconfig selects a kubeconfig file; unset means in-cluster config when running in a Pod. + EnvK8sKubeconfig = "CONTAINER_K8S_KUBECONFIG" + // EnvK8sNamespace is the namespace Pods are created in; defaults to the Pod's own namespace. + EnvK8sNamespace = "CONTAINER_K8S_NAMESPACE" + // EnvK8sAdvertiseHost is the address function Pods use to reach this process (usually its Pod IP). + EnvK8sAdvertiseHost = "CONTAINER_K8S_ADVERTISE_HOST" + // EnvK8sCodePort is the TCP port the archive server listens on. + EnvK8sCodePort = "CONTAINER_K8S_CODE_PORT" + // EnvK8sFetchImage is the image of the initContainer that downloads and unpacks archives. + EnvK8sFetchImage = "CONTAINER_K8S_FETCH_IMAGE" + + defaultK8sCodePort = 4599 + defaultK8sFetchImage = "busybox:1.37.0" + defaultK8sStartWait = 5 * time.Minute + k8sNamespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + k8sShutdownWait = 5 * time.Second +) + +// ErrK8sUnsupported is returned for operations a Pod cannot offer. +var ErrK8sUnsupported = errors.New("operation is not supported by the kubernetes runtime") + +// ErrPodFailed is returned when a Pod cannot reach Running. +var ErrPodFailed = errors.New("pod failed to start") + +// ErrWatchClosed is returned when a Pod watch ends before the awaited transition. +var ErrWatchClosed = errors.New("pod watch closed") + +// ErrNoAdvertiseHost is returned when archives are requested but pods have no address to fetch them from. +var ErrNoAdvertiseHost = errors.New("archive delivery needs " + EnvK8sAdvertiseHost) + +// KubernetesConfig configures RuntimeKubernetes. Zero values fall back to the environment. +type KubernetesConfig struct { + Client kubernetes.Interface + CodeListener net.Listener + Executor PodExecutor + Kubeconfig string + Namespace string + AdvertiseHost string + FetchImage string + CodePort int + StartTimeout time.Duration +} + +// KubernetesRuntime runs containers as Pods; the container ID is the Pod name. +type KubernetesRuntime struct { + cs kubernetes.Interface + exec PodExecutor + log *slog.Logger + code *codeServer + codeMu *lockmetrics.RWMutex + restCfg *rest.Config + ns string + fetchImage string + codeCfg KubernetesConfig + start time.Duration +} + +func (c KubernetesConfig) withEnv() KubernetesConfig { + c.Kubeconfig = firstNonEmpty(c.Kubeconfig, os.Getenv(EnvK8sKubeconfig)) + c.Namespace = firstNonEmpty(c.Namespace, os.Getenv(EnvK8sNamespace)) + c.AdvertiseHost = firstNonEmpty(c.AdvertiseHost, os.Getenv(EnvK8sAdvertiseHost)) + c.FetchImage = firstNonEmpty(c.FetchImage, os.Getenv(EnvK8sFetchImage), defaultK8sFetchImage) + + if c.CodePort == 0 { + if p, err := strconv.Atoi(os.Getenv(EnvK8sCodePort)); err == nil { + c.CodePort = p + } else { + c.CodePort = defaultK8sCodePort + } + } + + if c.StartTimeout == 0 { + c.StartTimeout = defaultK8sStartWait + } + + return c +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + + return "" +} + +func newKubernetesRuntime(cfg Config) (*KubernetesRuntime, error) { + kc := KubernetesConfig{} + if cfg.Kubernetes != nil { + kc = *cfg.Kubernetes + } + + kc = kc.withEnv() + + cs := kc.Client + ns := kc.Namespace + + var restCfg *rest.Config + + if cs == nil { + var err error + + cs, restCfg, ns, err = connectKubernetes(kc) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrUnavailable, err) + } + } + + if ns == "" { + ns = "default" + } + + return &KubernetesRuntime{ + cs: cs, + log: cfg.Logger, + codeMu: lockmetrics.New("container.k8s.runtime"), + codeCfg: kc, + restCfg: restCfg, + exec: kc.Executor, + ns: ns, + fetchImage: kc.FetchImage, + start: kc.StartTimeout, + }, nil +} + +// connectKubernetes builds a clientset from the kubeconfig, else the in-cluster config, else the default +// kubeconfig search path, and resolves the namespace Pods go to. +func connectKubernetes(kc KubernetesConfig) (kubernetes.Interface, *rest.Config, string, error) { + ns := kc.Namespace + + var restCfg *rest.Config + + switch { + case kc.Kubeconfig != "": + rules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kc.Kubeconfig} + cc := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, &clientcmd.ConfigOverrides{}) + + c, err := cc.ClientConfig() + if err != nil { + return nil, nil, "", fmt.Errorf("kubeconfig %q: %w", kc.Kubeconfig, err) + } + + restCfg = c + + if ns == "" { + ns, _, _ = cc.Namespace() + } + case os.Getenv("KUBERNETES_SERVICE_HOST") != "": + c, err := rest.InClusterConfig() + if err != nil { + return nil, nil, "", fmt.Errorf("in-cluster config: %w", err) + } + + restCfg = c + + if ns == "" { + if b, readErr := os.ReadFile(k8sNamespaceFile); readErr == nil { + ns = strings.TrimSpace(string(b)) + } + } + default: + cc := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}) + + c, err := cc.ClientConfig() + if err != nil { + return nil, nil, "", fmt.Errorf("no kubeconfig and not running in a cluster: %w", err) + } + + restCfg = c + + if ns == "" { + ns, _, _ = cc.Namespace() + } + } + + cs, err := kubernetes.NewForConfig(restCfg) + if err != nil { + return nil, nil, "", fmt.Errorf("kubernetes client: %w", err) + } + + return cs, restCfg, ns, nil +} + +// DeliversArchives reports that Spec.Archives are unpacked by the runtime. +func (r *KubernetesRuntime) DeliversArchives() bool { return true } + +// Namespace is the namespace Pods are created in. +func (r *KubernetesRuntime) Namespace() string { return r.ns } + +// Ping lists one managed Pod, which also proves the RBAC the runtime needs. +func (r *KubernetesRuntime) Ping(ctx context.Context) error { + _, err := r.cs.CoreV1().Pods(r.ns).List(ctx, metav1.ListOptions{Limit: 1, LabelSelector: managedSelector}) + if err != nil { + return fmt.Errorf("%w: %w", ErrUnavailable, err) + } + + return nil +} + +// PullImage is a no-op: the kubelet pulls the image when the Pod is scheduled. +func (r *KubernetesRuntime) PullImage(context.Context, string) error { return nil } + +// HasImage reports true: whether the image is present is the kubelet's concern, so callers never pre-pull. +func (r *KubernetesRuntime) HasImage(context.Context, string) (bool, error) { return true, nil } + +// CreateAndStart creates the Pod and waits, by watch, until it is running. +func (r *KubernetesRuntime) CreateAndStart(ctx context.Context, spec Spec) (string, error) { + name := podName(spec.Name) + + var g grant + + if len(spec.Archives) > 0 { + var err error + + if g, err = r.grantArchives(ctx, spec.Archives); err != nil { + return "", err + } + + defer r.code.revoke(g) + } + + pod, err := r.buildPod(spec, name, g) + if err != nil { + return "", err + } + + ctx, cancel := context.WithTimeout(ctx, r.start) + defer cancel() + + pods := r.cs.CoreV1().Pods(r.ns) + + w, err := pods.Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name}) + if err != nil { + return "", fmt.Errorf("watch pod %q: %w", name, err) + } + defer func() { w.Stop() }() + + if _, err = pods.Create(ctx, pod, metav1.CreateOptions{}); err != nil { + return "", fmt.Errorf("create pod %q: %w", name, err) + } + + if err = r.awaitPod(ctx, name, &w, podStarted); err != nil { + _ = r.StopAndRemove(context.WithoutCancel(ctx), name) + + return "", err + } + + return name, nil +} + +func (r *KubernetesRuntime) grantArchives(ctx context.Context, archives []Archive) (grant, error) { + for _, a := range archives { + if err := validateArchive(a.Zip); err != nil { + return grant{}, err + } + } + + if err := r.ensureCodeServer(ctx); err != nil { + return grant{}, err + } + + return r.code.register(archives) +} + +func (r *KubernetesRuntime) ensureCodeServer(ctx context.Context) error { + r.codeMu.Lock("ensureCodeServer") + defer r.codeMu.Unlock() + + if r.code != nil { + return nil + } + + if r.codeCfg.AdvertiseHost == "" { + return ErrNoAdvertiseHost + } + + ln := r.codeCfg.CodeListener + if ln == nil { + var err error + + ln, err = (&net.ListenConfig{}).Listen(ctx, "tcp", ":"+strconv.Itoa(r.codeCfg.CodePort)) + if err != nil { + return fmt.Errorf("archive server listen: %w", err) + } + } + + adv := r.codeCfg.AdvertiseHost + if _, _, err := net.SplitHostPort(adv); err != nil { + adv = net.JoinHostPort(adv, strconv.Itoa(r.codeCfg.CodePort)) + } + + r.code = newCodeServer(ln, adv) + r.code.start() + + return nil +} + +// awaitPod consumes watch events until check reports the Pod launched or failed. A closed watch is +// re-established and the Pod re-read, so a server-side watch timeout does not lose the transition. +func (r *KubernetesRuntime) awaitPod( + ctx context.Context, name string, w *watch.Interface, check podCheck, +) error { + for { + select { + case <-ctx.Done(): + return fmt.Errorf("%w: pod %q: %w", ErrPodFailed, name, ctx.Err()) + case ev, ok := <-(*w).ResultChan(): + var ( + done bool + err error + ) + + if ok { + done, err = handlePodEvent(name, ev, check) + } else { + done, err = r.resync(ctx, name, w, check) + } + + if done || err != nil { + return err + } + } + } +} + +// resync reopens the watch after it closed and re-reads the Pod so no transition is missed. +func (r *KubernetesRuntime) resync( + ctx context.Context, name string, w *watch.Interface, check podCheck, +) (bool, error) { + pods := r.cs.CoreV1().Pods(r.ns) + + nw, err := pods.Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name}) + if err != nil { + return false, fmt.Errorf("watch pod %q: %w", name, err) + } + + *w = nw + + cur, err := pods.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("get pod %q: %w", name, err) + } + + return check(cur) +} + +// podCheck reports whether a Pod reached the state a launch waits for, or why it never will. +type podCheck func(*corev1.Pod) (bool, error) + +func handlePodEvent(name string, ev watch.Event, check podCheck) (bool, error) { + pod, ok := ev.Object.(*corev1.Pod) + if !ok || pod.Name != name { + return false, nil + } + + if ev.Type == watch.Deleted { + return false, fmt.Errorf("%w: pod %q was deleted", ErrPodFailed, name) + } + + return check(pod) +} + +var waitingFailures = map[string]bool{ //nolint:gochecknoglobals // fixed lookup set + "ErrImagePull": true, "ImagePullBackOff": true, "InvalidImageName": true, + "CreateContainerConfigError": true, "CreateContainerError": true, "RunContainerError": true, +} + +// podStarted reports whether pod is running, or why it never will. +func podStarted(pod *corev1.Pod) (bool, error) { + switch pod.Status.Phase { + case corev1.PodRunning, corev1.PodSucceeded: + return true, nil + case corev1.PodFailed: + return false, fmt.Errorf( + "%w: pod %q failed: %s", + ErrPodFailed, + pod.Name, + firstNonEmpty(pod.Status.Reason, pod.Status.Message), + ) + case corev1.PodPending, corev1.PodUnknown: + } + + all := append( + append([]corev1.ContainerStatus{}, pod.Status.InitContainerStatuses...), + pod.Status.ContainerStatuses...) + for _, cs := range all { + if w := cs.State.Waiting; w != nil && waitingFailures[w.Reason] { + return false, fmt.Errorf( + "%w: pod %q container %q: %s: %s", + ErrPodFailed, + pod.Name, + cs.Name, + w.Reason, + w.Message, + ) + } + + if t := cs.State.Terminated; t != nil && t.ExitCode != 0 { + return false, fmt.Errorf( + "%w: pod %q container %q exited %d: %s", + ErrPodFailed, + pod.Name, + cs.Name, + t.ExitCode, + t.Reason, + ) + } + } + + return false, nil +} + +// StopAndRemove deletes the Pod and waits, by watch, until it is gone; a missing Pod is not an error. +func (r *KubernetesRuntime) StopAndRemove(ctx context.Context, containerID string) error { + grace := int64(podTerminationSec) + + return r.deletePod(ctx, containerID, &grace) +} + +// deletePod deletes the Pod and waits for it to be gone; a nil grace uses the Pod's own terminationGracePeriodSeconds. +func (r *KubernetesRuntime) deletePod(ctx context.Context, containerID string, grace *int64) error { + pods := r.cs.CoreV1().Pods(r.ns) + + w, err := pods.Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + containerID}) + if err != nil { + return fmt.Errorf("watch pod %q: %w", containerID, err) + } + defer w.Stop() + + err = pods.Delete(ctx, containerID, metav1.DeleteOptions{GracePeriodSeconds: grace}) + if apierrors.IsNotFound(err) { + return nil + } + + if err != nil { + return fmt.Errorf("delete pod %q: %w", containerID, err) + } + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("wait for pod %q deletion: %w", containerID, ctx.Err()) + case ev, ok := <-w.ResultChan(): + if !ok { + if _, getErr := pods.Get(ctx, containerID, metav1.GetOptions{}); apierrors.IsNotFound(getErr) { + return nil + } + + return fmt.Errorf("%w before pod %q was deleted", ErrWatchClosed, containerID) + } + + if p, isPod := ev.Object.(*corev1.Pod); ev.Type == watch.Deleted && isPod && p.Name == containerID { + return nil + } + } + } +} + +// StopContainer is unsupported: a Pod cannot be stopped and restarted in place. +func (r *KubernetesRuntime) StopContainer(context.Context, string) error { return ErrK8sUnsupported } + +// StartContainer is unsupported: a Pod cannot be stopped and restarted in place. +func (r *KubernetesRuntime) StartContainer(context.Context, string) error { return ErrK8sUnsupported } + +// Exec is unsupported; the chart grants no pods/exec. +func (r *KubernetesRuntime) Exec(context.Context, string, []string) (ExecResult, error) { + return ExecResult{}, ErrExecUnsupported +} + +// AcquireWarm is unsupported: Pods are not pooled. +func (r *KubernetesRuntime) AcquireWarm(context.Context, Spec) (*PooledContainer, error) { + return nil, ErrK8sUnsupported +} + +// ReleaseContainer is unsupported: Pods are not pooled. +func (r *KubernetesRuntime) ReleaseContainer(string) error { return ErrK8sUnsupported } + +// ReapIdleContainers does nothing: there is no pool to reap. +func (r *KubernetesRuntime) ReapIdleContainers(context.Context) {} + +// StartReaper does nothing: there is no pool to reap. +func (r *KubernetesRuntime) StartReaper(context.Context, time.Duration) {} + +// Close stops the archive server. +func (r *KubernetesRuntime) Close() error { + r.codeMu.RLock("Close") + code := r.code + r.codeMu.RUnlock() + + if code != nil { + ctx, cancel := context.WithTimeout(context.Background(), k8sShutdownWait) + defer cancel() + + code.close(ctx) + } + + return nil +} diff --git a/pkgs/container/k8s_runtime_test.go b/pkgs/container/k8s_runtime_test.go new file mode 100644 index 0000000000..134d2cb379 --- /dev/null +++ b/pkgs/container/k8s_runtime_test.go @@ -0,0 +1,627 @@ +package container_test + +import ( + "archive/zip" + "bytes" + "context" + "errors" + "io" + "net" + "net/http" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/blackbirdworks/gopherstack/pkgs/container" + "github.com/blackbirdworks/gopherstack/pkgs/testclient" +) + +const testNS = "gs-test" + +var errList = errors.New("forbidden") + +// podDriver reacts to every pod added to cs by passing it through onAdded and storing the result. +type podDriver struct { + cs *fake.Clientset + onAdded func(*corev1.Pod) +} + +func (d *podDriver) run(t *testing.T) { + t.Helper() + + ctx, cancel := context.WithCancel(t.Context()) + w, err := d.cs.CoreV1().Pods(testNS).Watch(ctx, metav1.ListOptions{}) + require.NoError(t, err) + + var wg sync.WaitGroup + + wg.Go(func() { + for ev := range w.ResultChan() { + pod, ok := ev.Object.(*corev1.Pod) + if !ok || ev.Type != watch.Added { + continue + } + + upd := pod.DeepCopy() + d.onAdded(upd) + + _, _ = d.cs.CoreV1().Pods(testNS).UpdateStatus(ctx, upd, metav1.UpdateOptions{}) + } + }) + + t.Cleanup(func() { + cancel() + w.Stop() + wg.Wait() + }) +} + +func running(p *corev1.Pod) { p.Status.Phase = corev1.PodRunning } + +func newK8s( + t *testing.T, + onAdded func(*corev1.Pod), + kc container.KubernetesConfig, +) (*container.KubernetesRuntime, *fake.Clientset) { + t.Helper() + + cs := fake.NewClientset() + if onAdded != nil { + (&podDriver{cs: cs, onAdded: onAdded}).run(t) + } + + kc.Client = cs + kc.Namespace = testNS + + rt, err := container.NewRuntime(container.Config{Runtime: container.RuntimeKubernetes, Kubernetes: &kc}) + require.NoError(t, err) + t.Cleanup(func() { _ = rt.Close() }) + + k, ok := rt.(*container.KubernetesRuntime) + require.True(t, ok) + + return k, cs +} + +func onlyPod(t *testing.T, cs *fake.Clientset) corev1.Pod { + t.Helper() + + list, err := cs.CoreV1().Pods(testNS).List(t.Context(), metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, list.Items, 1) + + return list.Items[0] +} + +func TestKubernetesRuntime_PodSpec(t *testing.T) { + t.Parallel() + + tests := []struct { + check func(*testing.T, corev1.Pod) + name string + spec container.Spec + }{ + { + name: "env cmd entrypoint resources", + spec: container.Spec{ + Image: "img:1", Name: "fn", Env: []string{"A=1", "B=x=y"}, Cmd: []string{"app.handler"}, + Entrypoint: []string{"/bin/run"}, MemoryBytes: 256 << 20, NanoCPUs: 500_000_000, WorkingDir: "/w", + }, + check: func(t *testing.T, p corev1.Pod) { + t.Helper() + c := p.Spec.Containers[0] + assert.Equal(t, "img:1", c.Image) + assert.Equal(t, []string{"/bin/run"}, c.Command) + assert.Equal(t, []string{"app.handler"}, c.Args) + assert.Equal(t, "/w", c.WorkingDir) + assert.Equal(t, []corev1.EnvVar{{Name: "A", Value: "1"}, {Name: "B", Value: "x=y"}}, c.Env) + assert.Equal(t, int64(256<<20), c.Resources.Limits.Memory().Value()) + assert.Equal(t, int64(256<<20), c.Resources.Requests.Memory().Value()) + assert.Equal(t, int64(500), c.Resources.Limits.Cpu().MilliValue()) + assert.Equal(t, corev1.RestartPolicyNever, p.Spec.RestartPolicy) + assert.False(t, *p.Spec.AutomountServiceAccountToken) + assert.False(t, *c.SecurityContext.AllowPrivilegeEscalation) + assert.Empty(t, p.Spec.InitContainers) + }, + }, + { + name: "runtime class user dns hosts tmpfs", + spec: container.Spec{ + Image: "img", Name: "fn", OCIRuntime: "gvisor", User: "1000:1000", DNS: []string{"10.0.0.2"}, + ExtraHosts: []string{"host.test:10.1.1.1"}, Tmpfs: []string{"/scratch"}, + }, + check: func(t *testing.T, p corev1.Pod) { + t.Helper() + assert.Equal(t, "gvisor", *p.Spec.RuntimeClassName) + assert.Equal(t, int64(1000), *p.Spec.Containers[0].SecurityContext.RunAsUser) + assert.Equal(t, []string{"10.0.0.2"}, p.Spec.DNSConfig.Nameservers) + assert.Equal(t, "10.1.1.1", p.Spec.HostAliases[0].IP) + require.Len(t, p.Spec.Volumes, 1) + assert.Equal(t, corev1.StorageMediumMemory, p.Spec.Volumes[0].EmptyDir.Medium) + assert.Equal(t, "/scratch", p.Spec.Containers[0].VolumeMounts[0].MountPath) + }, + }, + { + name: "labels", + spec: container.Spec{Image: "img", Name: "fn", Labels: map[string]string{ + container.LabelResource: "my resource", container.LabelPID: "12:34", + }}, + check: func(t *testing.T, p corev1.Pod) { + t.Helper() + assert.Equal(t, "true", p.Labels[container.LabelManaged]) + assert.Equal(t, "12:34", p.Annotations[container.LabelPID]) + assert.Equal(t, "my resource", p.Annotations[container.LabelResource]) + + for k, v := range p.Labels { + assert.Empty(t, validation.IsQualifiedName(k), k) + assert.Empty(t, validation.IsValidLabelValue(v), v) + } + + assert.NotContains(t, p.Labels, container.LabelPID) + }, + }, + { + name: "name sanitised", + spec: container.Spec{Image: "img", Name: "gopherstack-lambda-My_Func-ab12cd34"}, + check: func(t *testing.T, p corev1.Pod) { + t.Helper() + assert.Empty(t, validation.IsDNS1123Label(p.Name), p.Name) + assert.True(t, strings.HasPrefix(p.Name, "gopherstack-lambda-my-func-ab12cd34-")) + }, + }, + { + name: "long name", + spec: container.Spec{Image: "img", Name: strings.Repeat("a", 100)}, + check: func(t *testing.T, p corev1.Pod) { + t.Helper() + assert.Empty(t, validation.IsDNS1123Label(p.Name), p.Name) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rt, cs := newK8s(t, running, container.KubernetesConfig{}) + + id, err := rt.CreateAndStart(t.Context(), tt.spec) + require.NoError(t, err) + + pod := onlyPod(t, cs) + assert.Equal(t, pod.Name, id) + tt.check(t, pod) + }) + } +} + +func TestKubernetesRuntime_UnsupportedSpec(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + spec container.Spec + }{ + {"mounts", container.Spec{Image: "i", Mounts: []string{"/a:/b"}}}, + {"ports", container.Spec{Image: "i", Ports: []string{"80:80"}}}, + {"privileged", container.Spec{Image: "i", Privileged: true}}, + {"network", container.Spec{Image: "i", Network: "n"}}, + {"group add", container.Spec{Image: "i", GroupAdd: []string{"g"}}}, + {"named user", container.Spec{Image: "i", User: "nobody"}}, + {"bad extra host", container.Spec{Image: "i", ExtraHosts: []string{"nocolon"}}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rt, cs := newK8s(t, running, container.KubernetesConfig{}) + + _, err := rt.CreateAndStart(t.Context(), tt.spec) + require.ErrorIs(t, err, container.ErrSpecUnsupported) + + list, listErr := cs.CoreV1().Pods(testNS).List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + assert.Empty(t, list.Items) + }) + } +} + +func TestKubernetesRuntime_StartFailures(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(*corev1.Pod) + name string + want string + }{ + { + name: "image pull backoff", + mutate: func(p *corev1.Pod) { + p.Status.Phase = corev1.PodPending + p.Status.ContainerStatuses = []corev1.ContainerStatus{ + { + Name: "main", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}, + }, + }, + } + }, + want: "ImagePullBackOff", + }, + { + name: "init container fails", + mutate: func(p *corev1.Pod) { + p.Status.Phase = corev1.PodPending + p.Status.InitContainerStatuses = []corev1.ContainerStatus{ + { + Name: "fetch", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ExitCode: 8, Reason: "Error"}, + }, + }, + } + }, + want: "exited 8", + }, + { + name: "pod failed", + mutate: func(p *corev1.Pod) { p.Status.Phase = corev1.PodFailed; p.Status.Reason = "Evicted" }, + want: "Evicted", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rt, cs := newK8s(t, tt.mutate, container.KubernetesConfig{}) + + _, err := rt.CreateAndStart(t.Context(), container.Spec{Image: "i", Name: "fn"}) + require.ErrorIs(t, err, container.ErrPodFailed) + assert.Contains(t, err.Error(), tt.want) + + list, listErr := cs.CoreV1().Pods(testNS).List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + assert.Empty(t, list.Items, "failed pod must be removed") + }) + } +} + +func TestKubernetesRuntime_CreateErrors(t *testing.T) { + t.Parallel() + + rt, cs := newK8s(t, nil, container.KubernetesConfig{}) + cs.PrependReactor( + "create", + "pods", + func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, errList }, + ) + + _, err := rt.CreateAndStart(t.Context(), container.Spec{Image: "i", Name: "fn"}) + require.ErrorIs(t, err, errList) +} + +func TestKubernetesRuntime_ContextCancelRemovesPod(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + + rt, cs := newK8s(t, func(*corev1.Pod) { cancel() }, container.KubernetesConfig{}) + + _, err := rt.CreateAndStart(ctx, container.Spec{Image: "i", Name: "fn"}) + require.ErrorIs(t, err, container.ErrPodFailed) + + list, listErr := cs.CoreV1().Pods(testNS).List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + assert.Empty(t, list.Items) +} + +func TestKubernetesRuntime_StopAndRemove(t *testing.T) { + t.Parallel() + + rt, cs := newK8s(t, running, container.KubernetesConfig{}) + + id, err := rt.CreateAndStart(t.Context(), container.Spec{Image: "i", Name: "fn"}) + require.NoError(t, err) + + require.NoError(t, rt.StopAndRemove(t.Context(), id)) + + list, listErr := cs.CoreV1().Pods(testNS).List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + assert.Empty(t, list.Items) + + require.NoError(t, rt.StopAndRemove(t.Context(), id), "removing a missing pod is not an error") +} + +func TestKubernetesRuntime_Unsupported(t *testing.T) { + t.Parallel() + + rt, _ := newK8s(t, nil, container.KubernetesConfig{}) + + require.ErrorIs(t, rt.StopContainer(t.Context(), "x"), container.ErrK8sUnsupported) + require.ErrorIs(t, rt.StartContainer(t.Context(), "x"), container.ErrK8sUnsupported) + require.ErrorIs(t, rt.ReleaseContainer("x"), container.ErrK8sUnsupported) + + _, err := rt.Exec(t.Context(), "x", []string{"ls"}) + require.ErrorIs(t, err, container.ErrExecUnsupported) + + _, err = rt.AcquireWarm(t.Context(), container.Spec{}) + require.ErrorIs(t, err, container.ErrK8sUnsupported) + + assert.True(t, rt.DeliversArchives()) +} + +func TestKubernetesRuntime_Ping(t *testing.T) { + t.Parallel() + + tests := []struct { + err error + name string + }{ + {name: "ok"}, + {name: "forbidden", err: errList}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rt, cs := newK8s(t, nil, container.KubernetesConfig{}) + if tt.err != nil { + cs.PrependReactor( + "list", + "pods", + func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, tt.err }, + ) + } + + err := rt.Ping(t.Context()) + if tt.err == nil { + require.NoError(t, err) + + return + } + + require.ErrorIs(t, err, container.ErrUnavailable) + require.ErrorIs(t, err, tt.err) + }) + } +} + +func TestKubernetesRuntime_Logs(t *testing.T) { + t.Parallel() + + rt, _ := newK8s(t, running, container.KubernetesConfig{}) + + id, err := rt.CreateAndStart(t.Context(), container.Spec{Image: "i", Name: "fn"}) + require.NoError(t, err) + + var out bytes.Buffer + + require.NoError(t, rt.ReadLogs(t.Context(), id, "10", false, &out, io.Discard)) + assert.Equal(t, "fake logs", out.String()) + + out.Reset() + require.NoError(t, rt.StreamLogs(t.Context(), id, &out, io.Discard)) + assert.Equal(t, "fake logs", out.String()) +} + +func TestKubernetesRuntime_ListManagedAndReconcile(t *testing.T) { + t.Parallel() + + rt, cs := newK8s(t, running, container.KubernetesConfig{}) + + _, err := cs.CoreV1().Pods(testNS).Create(t.Context(), &corev1.Pod{ + Name: "unrelated", Namespace: testNS, + }, metav1.CreateOptions{}) + require.NoError(t, err) + + dead := map[string]string{ + container.LabelOwner: "run-dead", container.LabelEphemeral: "true", container.LabelPID: "2147483646:0", + container.LabelService: "lambda", + } + _, err = rt.CreateAndStart(t.Context(), container.Spec{Image: "i", Name: "orphan", Labels: dead}) + require.NoError(t, err) + + got, err := rt.ListManaged(t.Context()) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "orphan", got[0].Name) + assert.True(t, got[0].Running) + assert.Equal(t, "2147483646:0", got[0].Labels[container.LabelPID]) + + _, err = container.Reconcile(t.Context(), rt, container.Identity{Owner: "run-new", Ephemeral: true}, nil) + require.NoError(t, err) + + got, err = rt.ListManaged(t.Context()) + require.NoError(t, err) + assert.Empty(t, got, "dead ephemeral owner's pod is swept") + + _, err = cs.CoreV1().Pods(testNS).Get(t.Context(), "unrelated", metav1.GetOptions{}) + require.NoError(t, err, "unmanaged pods are never touched") +} + +func zipOf(t *testing.T, files map[string]string) []byte { + t.Helper() + + var buf bytes.Buffer + + zw := zip.NewWriter(&buf) + + for name, body := range files { + f, err := zw.Create(name) + require.NoError(t, err) + + _, err = f.Write([]byte(body)) + require.NoError(t, err) + } + + require.NoError(t, zw.Close()) + + return buf.Bytes() +} + +func httpGet(t *testing.T, url, auth string) (int, []byte) { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, nil) + require.NoError(t, err) + + if auth != "" { + req.Header.Set("Authorization", auth) + } + + resp, err := testclient.New(t).Do(req) + require.NoError(t, err) + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + return resp.StatusCode, body +} + +func TestKubernetesRuntime_ArchiveDelivery(t *testing.T) { + t.Parallel() + + ln, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + + code := zipOf(t, map[string]string{"app.py": "print(1)"}) + layer := zipOf(t, map[string]string{"lib/x.py": "x=1"}) + + type fetched struct { + body []byte + wrong, none int + status int + } + + res := make(chan fetched, 1) + + var initEnvToken string + + onAdded := func(p *corev1.Pod) { + args := p.Spec.InitContainers[0].Args + initEnvToken = p.Spec.InitContainers[0].Env[0].Value + + f := fetched{} + f.wrong, _ = httpGet(t, args[3], "Bearer wrong") + f.none, _ = httpGet(t, args[3], "") + f.status, f.body = httpGet(t, args[3], "Bearer "+initEnvToken) + res <- f + + p.Status.Phase = corev1.PodRunning + } + + rt, cs := newK8s(t, onAdded, container.KubernetesConfig{ + CodeListener: ln, AdvertiseHost: ln.Addr().String(), FetchImage: "fetch:1", + }) + + _, err = rt.CreateAndStart(t.Context(), container.Spec{ + Image: "py", + Name: "fn", + Archives: []container.Archive{ + {Dest: "/var/task", Zip: code}, + {Dest: "/opt", Zip: layer}, + {Dest: "/var/task", Zip: layer}, + }, + }) + require.NoError(t, err) + + f := <-res + assert.Equal(t, http.StatusUnauthorized, f.wrong) + assert.Equal(t, http.StatusUnauthorized, f.none) + assert.Equal(t, http.StatusOK, f.status) + assert.Equal(t, code, f.body) + + pod := onlyPod(t, cs) + require.Len(t, pod.Spec.InitContainers, 1) + init := pod.Spec.InitContainers[0] + assert.Equal(t, "fetch:1", init.Image) + assert.Len(t, init.VolumeMounts, 2, "one volume per distinct destination") + assert.Len(t, init.Args, 3+2*3) + + main := pod.Spec.Containers[0] + require.Len(t, main.VolumeMounts, 2) + + for _, m := range main.VolumeMounts { + assert.True(t, m.ReadOnly) + } + + assert.Equal(t, "/var/task", main.VolumeMounts[0].MountPath) + assert.Equal(t, "/opt", main.VolumeMounts[1].MountPath) + + for _, a := range init.Args[3:] { + if strings.HasPrefix(a, "http://") { + status, _ := httpGet(t, a, "Bearer "+initEnvToken) + assert.Equal(t, http.StatusUnauthorized, status, "grant is revoked once the pod runs") + } + } +} + +func TestKubernetesRuntime_ArchiveErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + want error + name string + host string + archives []container.Archive + }{ + { + name: "no advertise host", + want: container.ErrNoAdvertiseHost, + archives: []container.Archive{{Dest: "/a", Zip: nil}}, + }, + { + name: "zip slip", host: "127.0.0.1:1", want: container.ErrUnsafeArchive, + archives: []container.Archive{{Dest: "/a", Zip: zipOf(t, map[string]string{"../evil": "x"})}}, + }, + { + name: "absolute path", host: "127.0.0.1:1", want: container.ErrUnsafeArchive, + archives: []container.Archive{{Dest: "/a", Zip: zipOf(t, map[string]string{"/etc/passwd": "x"})}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rt, cs := newK8s(t, running, container.KubernetesConfig{AdvertiseHost: tt.host}) + + if tt.name == "no advertise host" { + tt.archives[0].Zip = zipOf(t, map[string]string{"ok": "x"}) + } + + _, err := rt.CreateAndStart(t.Context(), container.Spec{Image: "i", Name: "fn", Archives: tt.archives}) + require.ErrorIs(t, err, tt.want) + + list, listErr := cs.CoreV1().Pods(testNS).List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + assert.Empty(t, list.Items) + }) + } +} + +func TestDockerRuntime_RejectsArchives(t *testing.T) { + t.Parallel() + + rt := container.NewDockerRuntimeWithAPI(&mockAPI{}, container.Config{}) + + _, err := rt.CreateAndStart(t.Context(), container.Spec{Image: "i", Archives: []container.Archive{{Dest: "/a"}}}) + require.ErrorIs(t, err, container.ErrArchivesUnsupported) +} diff --git a/pkgs/container/k8s_task.go b/pkgs/container/k8s_task.go new file mode 100644 index 0000000000..aeb133a683 --- /dev/null +++ b/pkgs/container/k8s_task.go @@ -0,0 +1,317 @@ +package container + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" +) + +// PodLaunch describes a multi-container Pod started as one unit, for callers (ECS tasks) that map their own +// container model onto a PodSpec. +type PodLaunch struct { + // Labels are added to the managed identity labels; values Kubernetes rejects as labels land in annotations. + Labels map[string]string + // Secret, when non-empty, becomes a Secret named after the Pod that the PodSpec references and the Pod owns. + Secret map[string][]byte + Name string + Spec corev1.PodSpec +} + +// PodFailedError is returned when a Pod cannot reach a running state. Reason is the Kubernetes waiting or +// termination reason (ErrImagePull, ImagePullBackOff, Error, ...). +type PodFailedError struct { + Container string + Reason string + Message string + Pod string +} + +func (f *PodFailedError) Error() string { + return fmt.Sprintf("pod %q container %q: %s: %s", f.Pod, f.Container, f.Reason, f.Message) +} + +// Is reports ErrPodFailed so callers can match every launch failure with errors.Is. +func (f *PodFailedError) Is(target error) bool { return target == ErrPodFailed } + +// podLaunched reports a Pod that is running or finished; failed init containers and image errors are failures. +func podLaunched(pod *corev1.Pod) (bool, error) { + for _, cs := range pod.Status.InitContainerStatuses { + if t := cs.State.Terminated; t != nil && t.ExitCode != 0 { + return false, &PodFailedError{ + Pod: pod.Name, Container: cs.Name, + Reason: firstNonEmpty(t.Reason, "Error"), Message: fmt.Sprintf("exited with code %d", t.ExitCode), + } + } + } + + all := append( + append([]corev1.ContainerStatus{}, pod.Status.InitContainerStatuses...), + pod.Status.ContainerStatuses...) + for _, cs := range all { + if w := cs.State.Waiting; w != nil && waitingFailures[w.Reason] { + return false, &PodFailedError{Pod: pod.Name, Container: cs.Name, Reason: w.Reason, Message: w.Message} + } + } + + switch pod.Status.Phase { + case corev1.PodRunning, corev1.PodSucceeded: + return true, nil + case corev1.PodFailed: + if hasTerminated(pod.Status.ContainerStatuses) { + return true, nil + } + + return false, &PodFailedError{ + Pod: pod.Name, Reason: firstNonEmpty(pod.Status.Reason, "Failed"), Message: pod.Status.Message, + } + case corev1.PodPending, corev1.PodUnknown: + } + + return false, nil +} + +func hasTerminated(statuses []corev1.ContainerStatus) bool { + for _, cs := range statuses { + if cs.State.Terminated != nil { + return true + } + } + + return false +} + +func hardenPod(spec *corev1.PodSpec) { + spec.RestartPolicy = corev1.RestartPolicyNever + spec.AutomountServiceAccountToken = new(false) + spec.EnableServiceLinks = new(false) + + if spec.SecurityContext == nil { + spec.SecurityContext = &corev1.PodSecurityContext{} + } + + if spec.SecurityContext.SeccompProfile == nil { + spec.SecurityContext.SeccompProfile = &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault} + } +} + +// LaunchPod creates the Pod (and its Secret) and waits, by watch, until it is running or has run. A failed launch +// removes both. The returned Pod is the last observed state. +func (r *KubernetesRuntime) LaunchPod(ctx context.Context, l PodLaunch) (*corev1.Pod, error) { + hardenPod(&l.Spec) + + labels, annotations := podMetadata(Spec{Labels: l.Labels}) + pod := &corev1.Pod{ + Name: l.Name, Namespace: r.ns, Labels: labels, Annotations: annotations, + Spec: l.Spec, + } + + ctx, cancel := context.WithTimeout(ctx, r.start) + defer cancel() + + hasSecret := len(l.Secret) > 0 + secrets := r.cs.CoreV1().Secrets(r.ns) + + if hasSecret { + sec := &corev1.Secret{ + Name: l.Name, Namespace: r.ns, Labels: map[string]string{LabelManaged: labelTrue}, + Type: corev1.SecretTypeOpaque, + Data: l.Secret, + } + + if _, err := secrets.Create(ctx, sec, metav1.CreateOptions{}); err != nil { + return nil, fmt.Errorf("create secret %q: %w", l.Name, err) + } + } + + cleanup := func(cause error) error { + _ = r.DeletePod(context.WithoutCancel(ctx), l.Name, hasSecret) + + return cause + } + + pods := r.cs.CoreV1().Pods(r.ns) + + w, err := pods.Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + l.Name}) + if err != nil { + return nil, cleanup(fmt.Errorf("watch pod %q: %w", l.Name, err)) + } + defer func() { w.Stop() }() + + created, err := pods.Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return nil, cleanup(fmt.Errorf("create pod %q: %w", l.Name, err)) + } + + if hasSecret { + if err = r.ownSecret(ctx, created); err != nil { + return nil, cleanup(err) + } + } + + if err = r.awaitPod(ctx, l.Name, &w, podLaunched); err != nil { + return nil, cleanup(err) + } + + cur, err := pods.Get(ctx, l.Name, metav1.GetOptions{}) + if err != nil { + return nil, cleanup(fmt.Errorf("get pod %q: %w", l.Name, err)) + } + + return cur, nil +} + +// ownSecret makes the Pod the Secret's owner so the garbage collector removes it with the Pod. +func (r *KubernetesRuntime) ownSecret(ctx context.Context, pod *corev1.Pod) error { + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{ + "ownerReferences": []metav1.OwnerReference{{ + APIVersion: "v1", Kind: "Pod", Name: pod.Name, UID: pod.UID, + }}, + }, + }) + if err != nil { + return fmt.Errorf("marshal owner reference: %w", err) + } + + _, err = r.cs.CoreV1().Secrets(r.ns).Patch(ctx, pod.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("own secret %q: %w", pod.Name, err) + } + + return nil +} + +// DeletePod deletes the Pod, giving its containers their terminationGracePeriodSeconds, and waits until it is gone. +// withSecret also deletes the Secret LaunchPod created. A missing Pod or Secret is not an error. +func (r *KubernetesRuntime) DeletePod(ctx context.Context, name string, withSecret bool) error { + err := r.deletePod(ctx, name, nil) + + if withSecret { + serr := r.cs.CoreV1().Secrets(r.ns).Delete(ctx, name, metav1.DeleteOptions{}) + if serr != nil && !apierrors.IsNotFound(serr) { + err = errors.Join(err, fmt.Errorf("delete secret %q: %w", name, serr)) + } + } + + return err +} + +// GetPod returns the Pod's current state. +func (r *KubernetesRuntime) GetPod(ctx context.Context, name string) (*corev1.Pod, error) { + pod, err := r.cs.CoreV1().Pods(r.ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get pod %q: %w", name, err) + } + + return pod, nil +} + +// FollowPod calls fn with the Pod's current state, then with every change, until fn returns true, the Pod is +// deleted (fn then sees gone) or ctx ends. A watch that closes is re-established and the Pod re-read. +func (r *KubernetesRuntime) FollowPod( + ctx context.Context, name string, fn func(pod *corev1.Pod, gone bool) bool, +) error { + pods := r.cs.CoreV1().Pods(r.ns) + + for { + w, err := pods.Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name}) + if err != nil { + return fmt.Errorf("watch pod %q: %w", name, err) + } + + cur, err := pods.Get(ctx, name, metav1.GetOptions{}) + + switch { + case apierrors.IsNotFound(err): + w.Stop() + fn(&corev1.Pod{Name: name}, true) + + return nil + case err != nil: + w.Stop() + + return fmt.Errorf("get pod %q: %w", name, err) + } + + if fn(cur, false) { + w.Stop() + + return nil + } + + done := drainPodEvents(ctx, w, name, fn) + w.Stop() + + if done { + return nil + } + + select { + case <-ctx.Done(): + return nil + default: + } + } +} + +// drainPodEvents feeds events for name to fn until fn stops, the Pod is deleted, ctx ends or the watch closes. +func drainPodEvents( + ctx context.Context, w watch.Interface, name string, fn func(*corev1.Pod, bool) bool, +) bool { + for { + select { + case <-ctx.Done(): + return true + case ev, ok := <-w.ResultChan(): + if !ok { + return false + } + + pod, isPod := ev.Object.(*corev1.Pod) + if !isPod || pod.Name != name { + continue + } + + gone := ev.Type == watch.Deleted + if fn(pod, gone) || gone { + return true + } + } + } +} + +// StreamPodLogs follows one container's output into w until the container ends or ctx is cancelled. +func (r *KubernetesRuntime) StreamPodLogs(ctx context.Context, pod, containerName string, w io.Writer) error { + opts := &corev1.PodLogOptions{Container: containerName, Follow: true} + + rc, err := r.cs.CoreV1().Pods(r.ns).GetLogs(pod, opts).Stream(ctx) + if err != nil { + return fmt.Errorf("pod logs %q/%q: %w", pod, containerName, err) + } + defer rc.Close() + + if _, err = io.Copy(w, rc); err != nil && ctx.Err() == nil { + return fmt.Errorf("pod logs %q/%q: %w", pod, containerName, err) + } + + return nil +} + +// SplitContainerRef splits a "pod/container" reference. +func SplitContainerRef(ref string) (string, string, bool) { + pod, c, ok := strings.Cut(ref, "/") + if !ok || pod == "" || c == "" { + return "", "", false + } + + return pod, c, true +} diff --git a/pkgs/container/k8s_task_test.go b/pkgs/container/k8s_task_test.go new file mode 100644 index 0000000000..781b90bb8d --- /dev/null +++ b/pkgs/container/k8s_task_test.go @@ -0,0 +1,376 @@ +package container_test + +import ( + "context" + "errors" + "io" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/blackbirdworks/gopherstack/pkgs/container" +) + +func twoContainerSpec() corev1.PodSpec { + return corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "web", Image: "nginx"}, + {Name: "side", Image: "busybox"}, + }, + } +} + +func waiting(name, reason string) corev1.ContainerStatus { + return corev1.ContainerStatus{ + Name: name, + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: reason, Message: "boom"}}, + } +} + +func terminated(name string, code int32) corev1.ContainerStatus { + return corev1.ContainerStatus{ + Name: name, + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: code, Reason: "Error"}}, + } +} + +func TestKubernetesRuntime_LaunchPod(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(*corev1.Pod) + secret map[string][]byte + name string + wantReason string + wantErr bool + wantSecret bool + }{ + { + name: "running with secret", + mutate: func(p *corev1.Pod) { + p.UID = types.UID("uid-1") + p.Status.Phase = corev1.PodRunning + p.Status.PodIP = "10.1.2.3" + }, + secret: map[string][]byte{"web.TOKEN": []byte("s3cret")}, + wantSecret: true, + }, + { + name: "main container already failed is a launch", + mutate: func(p *corev1.Pod) { + p.Status.Phase = corev1.PodFailed + p.Status.ContainerStatuses = []corev1.ContainerStatus{terminated("web", 3)} + }, + }, + { + name: "image pull failure removes pod and secret", + mutate: func(p *corev1.Pod) { + p.Status.Phase = corev1.PodPending + p.Status.ContainerStatuses = []corev1.ContainerStatus{waiting("web", "ImagePullBackOff")} + }, + secret: map[string][]byte{"k": []byte("v")}, + wantErr: true, + wantReason: "ImagePullBackOff", + }, + { + name: "init container failure", + mutate: func(p *corev1.Pod) { + p.Status.Phase = corev1.PodPending + p.Status.InitContainerStatuses = []corev1.ContainerStatus{terminated("setup", 2)} + }, + wantErr: true, + wantReason: "Error", + }, + { + name: "evicted", + mutate: func(p *corev1.Pod) { p.Status.Phase = corev1.PodFailed; p.Status.Reason = "Evicted" }, + wantErr: true, + wantReason: "Evicted", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rt, cs := newK8s(t, tt.mutate, container.KubernetesConfig{}) + + pod, err := rt.LaunchPod(t.Context(), container.PodLaunch{ + Name: "ecs-abc", Spec: twoContainerSpec(), Secret: tt.secret, + Labels: map[string]string{"gopherstack.ecs.task": "arn:aws:ecs:us-east-1:1:task/c/abc"}, + }) + + pods, listErr := cs.CoreV1().Pods(testNS).List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + + sec, secErr := cs.CoreV1().Secrets(testNS).Get(t.Context(), "ecs-abc", metav1.GetOptions{}) + + if tt.wantErr { + require.ErrorIs(t, err, container.ErrPodFailed) + + pf, ok := errors.AsType[*container.PodFailedError](err) + require.True(t, ok) + assert.Equal(t, tt.wantReason, pf.Reason) + assert.Empty(t, pods.Items, "failed pod must be removed") + assert.True(t, apierrors.IsNotFound(secErr), "secret must be removed with the pod") + + return + } + + require.NoError(t, err) + require.Len(t, pods.Items, 1) + assert.Equal(t, "ecs-abc", pod.Name) + assert.Equal(t, corev1.RestartPolicyNever, pods.Items[0].Spec.RestartPolicy) + assert.False(t, *pods.Items[0].Spec.AutomountServiceAccountToken) + assert.Equal(t, "true", pods.Items[0].Labels[container.LabelManaged]) + assert.Equal(t, "arn:aws:ecs:us-east-1:1:task/c/abc", pods.Items[0].Annotations["gopherstack.ecs.task"]) + + if !tt.wantSecret { + assert.True(t, apierrors.IsNotFound(secErr)) + + return + } + + require.NoError(t, secErr) + assert.Equal(t, tt.secret, sec.Data) + require.Len(t, sec.OwnerReferences, 1) + assert.Equal(t, "Pod", sec.OwnerReferences[0].Kind) + assert.Equal(t, "ecs-abc", sec.OwnerReferences[0].Name) + }) + } +} + +func TestKubernetesRuntime_DeletePod(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + withSecret bool + }{ + {name: "pod and secret", withSecret: true}, + {name: "pod only"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + secret := map[string][]byte{"k": []byte("v")} + rt, cs := newK8s(t, running, container.KubernetesConfig{}) + + _, err := rt.LaunchPod( + t.Context(), + container.PodLaunch{Name: "ecs-del", Spec: twoContainerSpec(), Secret: secret}, + ) + require.NoError(t, err) + + require.NoError(t, rt.DeletePod(t.Context(), "ecs-del", tt.withSecret)) + require.NoError(t, rt.DeletePod(t.Context(), "ecs-del", tt.withSecret), "deleting twice is fine") + + _, getErr := cs.CoreV1().Pods(testNS).Get(t.Context(), "ecs-del", metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(getErr)) + + _, secErr := cs.CoreV1().Secrets(testNS).Get(t.Context(), "ecs-del", metav1.GetOptions{}) + assert.Equal(t, tt.withSecret, apierrors.IsNotFound(secErr)) + }) + } +} + +type podEvent struct { + pod corev1.Pod + gone bool +} + +func TestKubernetesRuntime_FollowPod(t *testing.T) { + t.Parallel() + + rt, cs := newK8s(t, running, container.KubernetesConfig{}) + + _, err := rt.LaunchPod(t.Context(), container.PodLaunch{Name: "ecs-follow", Spec: twoContainerSpec()}) + require.NoError(t, err) + + var ( + mu sync.Mutex + events []podEvent + ) + + seen := make(chan struct{}, 8) + done := make(chan error, 1) + + go func() { + done <- rt.FollowPod(t.Context(), "ecs-follow", func(p *corev1.Pod, gone bool) bool { + mu.Lock() + events = append(events, podEvent{pod: *p, gone: gone}) + mu.Unlock() + + seen <- struct{}{} + + return gone + }) + }() + + <-seen + + cur, err := cs.CoreV1().Pods(testNS).Get(t.Context(), "ecs-follow", metav1.GetOptions{}) + require.NoError(t, err) + + cur.Status.ContainerStatuses = []corev1.ContainerStatus{terminated("side", 0)} + _, err = cs.CoreV1().Pods(testNS).UpdateStatus(t.Context(), cur, metav1.UpdateOptions{}) + require.NoError(t, err) + + <-seen + + require.NoError(t, cs.CoreV1().Pods(testNS).Delete(t.Context(), "ecs-follow", metav1.DeleteOptions{})) + require.NoError(t, <-done) + + mu.Lock() + defer mu.Unlock() + + require.GreaterOrEqual(t, len(events), 3) + assert.False(t, events[0].gone) + assert.Empty(t, events[0].pod.Status.ContainerStatuses) + assert.True(t, events[len(events)-1].gone) + + exited := false + + for _, ev := range events { + for _, cs := range ev.pod.Status.ContainerStatuses { + exited = exited || (cs.Name == "side" && cs.State.Terminated != nil) + } + } + + assert.True(t, exited, "the update must be delivered") +} + +func TestKubernetesRuntime_FollowPodMissing(t *testing.T) { + t.Parallel() + + rt, _ := newK8s(t, nil, container.KubernetesConfig{}) + + var gone bool + + require.NoError(t, rt.FollowPod(t.Context(), "nope", func(_ *corev1.Pod, g bool) bool { + gone = g + + return true + })) + assert.True(t, gone) +} + +func TestKubernetesRuntime_StreamPodLogs(t *testing.T) { + t.Parallel() + + rt, _ := newK8s(t, running, container.KubernetesConfig{}) + + _, err := rt.LaunchPod(t.Context(), container.PodLaunch{Name: "ecs-logs", Spec: twoContainerSpec()}) + require.NoError(t, err) + + var sb stringsBuilder + + require.NoError(t, rt.StreamPodLogs(t.Context(), "ecs-logs", "web", &sb)) + assert.Equal(t, "fake logs", sb.String()) +} + +type stringsBuilder struct{ b []byte } + +func (s *stringsBuilder) Write(p []byte) (int, error) { + s.b = append(s.b, p...) + + return len(p), nil +} +func (s *stringsBuilder) String() string { return string(s.b) } + +type fakeExecutor struct { + got chan container.ExecRequest +} + +func (f *fakeExecutor) StreamExec(ctx context.Context, req container.ExecRequest) error { + f.got <- req + + buf := make([]byte, 64) + + n, err := req.Stdin.Read(buf) + if err != nil { + return nil //nolint:nilerr // stdin closed ends the session + } + + _, _ = req.Stdout.Write(append([]byte("echo:"), buf[:n]...)) + + size := <-req.Resize + _, _ = req.Stdout.Write([]byte{byte(size.Cols), byte(size.Rows)}) + + <-ctx.Done() + + return nil +} + +func TestKubernetesRuntime_ExecInteractive(t *testing.T) { + t.Parallel() + + ex := &fakeExecutor{got: make(chan container.ExecRequest, 1)} + rt, _ := newK8s(t, nil, container.KubernetesConfig{Executor: ex}) + + sh, err := rt.ExecInteractive(t.Context(), "ecs-x", "web", []string{"sh"}, 80, 24) + require.NoError(t, err) + + req := <-ex.got + assert.Equal(t, "ecs-x", req.Pod) + assert.Equal(t, "web", req.Container) + assert.Equal(t, []string{"sh"}, req.Command) + + _, err = sh.Write([]byte("hi")) + require.NoError(t, err) + + buf := make([]byte, 7) + _, err = io.ReadFull(sh, buf) + require.NoError(t, err) + assert.Equal(t, "echo:hi", string(buf)) + + size := make([]byte, 2) + _, err = io.ReadFull(sh, size) + require.NoError(t, err) + assert.Equal(t, []byte{80, 24}, size) + + require.NoError(t, sh.Close()) + require.NoError(t, sh.Close(), "close is idempotent") +} + +func TestKubernetesRuntime_ExecWithoutCluster(t *testing.T) { + t.Parallel() + + rt, _ := newK8s(t, nil, container.KubernetesConfig{}) + + _, err := rt.ExecInteractive(t.Context(), "p", "c", []string{"sh"}, 1, 1) + require.ErrorIs(t, err, container.ErrExecNoConfig) +} + +func TestSplitContainerRef(t *testing.T) { + t.Parallel() + + tests := []struct { + ref, pod, name string + ok bool + }{ + {ref: "ecs-a/web", pod: "ecs-a", name: "web", ok: true}, + {ref: "web"}, + {ref: "/web"}, + {ref: "pod/"}, + } + + for _, tt := range tests { + t.Run(tt.ref, func(t *testing.T) { + t.Parallel() + + pod, name, ok := container.SplitContainerRef(tt.ref) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.pod, pod) + assert.Equal(t, tt.name, name) + }) + } +} diff --git a/pkgs/container/push.go b/pkgs/container/push.go index efc25c2646..e06b9488a4 100644 --- a/pkgs/container/push.go +++ b/pkgs/container/push.go @@ -39,3 +39,23 @@ func (r *DockerRuntime) TagAndPush(ctx context.Context, source, target string) ( return digest, nil } + +// ImageDigester is implemented by runtimes that can report the content digest ("sha256:...") of a local image. +type ImageDigester interface { + ImageDigest(ctx context.Context, image string) (string, error) +} + +// ImageDigest returns the digest of the local image ref. +func (r *DockerRuntime) ImageDigest(ctx context.Context, ref string) (string, error) { + api, ok := r.docker.(ImageDigester) + if !ok { + return "", ErrPushUnsupported + } + + d, err := api.ImageDigest(ctx, ref) + if err != nil { + return "", fmt.Errorf("inspect %q: %w", ref, err) + } + + return d, nil +} diff --git a/pkgs/devtls/ca.go b/pkgs/devtls/ca.go index 8488a7a304..744dffd0e5 100644 --- a/pkgs/devtls/ca.go +++ b/pkgs/devtls/ca.go @@ -65,6 +65,9 @@ func NewCA() (*CA, error) { // CertPEM returns the CA certificate in PEM form, for clients to trust. func (c *CA) CertPEM() []byte { return append([]byte(nil), c.certPEM...) } +// Certificate returns the parsed CA certificate. +func (c *CA) Certificate() *x509.Certificate { return c.cert } + // Pool returns a cert pool containing only this CA. func (c *CA) Pool() *x509.CertPool { p := x509.NewCertPool() @@ -76,6 +79,21 @@ func (c *CA) Pool() *x509.CertPool { // Issue signs a server leaf certificate covering the given DNS names and IPs // (each entry is tried as an IP first). func (c *CA) Issue(hosts ...string) (*tls.Certificate, error) { + return c.issue([]x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, orgName, hosts) +} + +// IssueServerClient signs a leaf valid for both server and client authentication, for peers that dial each other +// over mutually verified TLS (a Redis cluster bus). +func (c *CA) IssueServerClient(hosts ...string) (*tls.Certificate, error) { + return c.issue([]x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, orgName, hosts) +} + +// IssueClient signs a client-authentication leaf certificate with the given common name. +func (c *CA) IssueClient(commonName string) (*tls.Certificate, error) { + return c.issue([]x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, commonName, nil) +} + +func (c *CA) issue(usage []x509.ExtKeyUsage, commonName string, hosts []string) (*tls.Certificate, error) { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { return nil, fmt.Errorf("devtls: generate leaf key: %w", err) @@ -90,11 +108,11 @@ func (c *CA) Issue(hosts ...string) (*tls.Certificate, error) { tmpl := &x509.Certificate{ SerialNumber: serial, - Subject: pkix.Name{CommonName: orgName, Organization: []string{orgName}}, + Subject: pkix.Name{CommonName: commonName, Organization: []string{orgName}}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(DefaultValidity), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + ExtKeyUsage: usage, DNSNames: dnsNames, IPAddresses: ips, } diff --git a/pkgs/devtls/cert_test.go b/pkgs/devtls/cert_test.go index 024c6593aa..b77f32232e 100644 --- a/pkgs/devtls/cert_test.go +++ b/pkgs/devtls/cert_test.go @@ -127,3 +127,41 @@ func TestLoadOrCreateCA_Persists(t *testing.T) { _, err = parsed.Verify(x509.VerifyOptions{Roots: first.Pool(), DNSName: "x.amazonaws.com"}) require.NoError(t, err) } + +func TestCA_IssueClient(t *testing.T) { + t.Parallel() + + ca, err := devtls.NewCA() + require.NoError(t, err) + + tests := []struct { + name string + usage x509.ExtKeyUsage + want bool + }{ + {name: "client-auth", usage: x509.ExtKeyUsageClientAuth, want: true}, + {name: "server-auth", usage: x509.ExtKeyUsageServerAuth, want: false}, + } + + leaf, err := ca.IssueClient("wallet-user") + require.NoError(t, err) + + parsed, err := x509.ParseCertificate(leaf.Certificate[0]) + require.NoError(t, err) + assert.Equal(t, "wallet-user", parsed.Subject.CommonName) + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, verr := parsed.Verify(x509.VerifyOptions{Roots: ca.Pool(), KeyUsages: []x509.ExtKeyUsage{tc.usage}}) + if tc.want { + require.NoError(t, verr) + } else { + require.Error(t, verr) + } + }) + } + + assert.Equal(t, "gopherstack local CA", ca.Certificate().Subject.CommonName) +} diff --git a/pkgs/dmsengine/dmstypes.go b/pkgs/dmsengine/dmstypes.go new file mode 100644 index 0000000000..9ad0b92b61 --- /dev/null +++ b/pkgs/dmsengine/dmstypes.go @@ -0,0 +1,161 @@ +package dmsengine + +import ( + "fmt" + "strings" +) + +// DMSType is a documented AWS DMS data type with its length, precision and scale. +type DMSType struct { + Name string + Length int + Precision int + Scale int +} + +// dmsTypeKinds maps the DMS data types the transformation rules and S3 sources accept to a value class. +func dmsTypeKinds() map[string]Kind { + return map[string]Kind{ + tBytes: KindBytes, tBlob: KindBytes, + tDate: KindTime, tTime: KindTime, "datetime": KindTime, + tInt1: KindInt, tInt2: KindInt, tInt4: KindInt, tInt8: KindInt, + "uint1": KindInt, "uint2": KindInt, "uint4": KindInt, "uint8": KindInt, + "numeric": KindDecimal, tReal4: KindFloat, tReal8: KindFloat, + "string": KindString, "wstring": KindString, tClob: KindString, tNclob: KindString, + "boolean": KindBool, + } +} + +// ParseDMSType validates a DMS data type name. +func ParseDMSType(name string) (DMSType, error) { + n := strings.ToLower(strings.TrimSpace(name)) + if _, ok := dmsTypeKinds()[n]; !ok { + return DMSType{}, fmt.Errorf("%w: unsupported data-type %q", ErrMapping, name) + } + + return DMSType{Name: n}, nil +} + +// Kind is the value class of the type. +func (d DMSType) Kind() Kind { return dmsTypeKinds()[d.Name] } + +// Column derives a nullable column of the type. +func (d DMSType) Column(name string) Column { + return Column{Name: name, Native: d.Name, Kind: d.Kind(), Nullable: true, DMS: d} +} + +// dmsTypeOf maps an engine-native type to a DMS type name; empty when unknown. +func dmsTypeOf(native string) string { + p := parseNative(native) + m := map[string]string{ + tTinyint: tInt1, tSmallint: tInt2, tInt2: tInt2, tMediumint: tInt4, tInt: tInt4, tInteger: tInt4, + tInt4: tInt4, tSerial: tInt4, tBigint: tInt8, tInt8: tInt8, tBigserial: tInt8, + tDecimal: tNumeric, tNumeric: tNumeric, tFloat: tReal4, tReal: tReal4, tFloat4: tReal4, + tDouble: tReal8, tDoublePrec: tReal8, "float8": tReal8, tBoolean: tBoolean, tBool: tBoolean, + tDate: tDate, tTime: tTime, tDatetime: tDatetime, tTimestamp: tDatetime, tTimestampN: tDatetime, + tTimestampZ: tDatetime, tVarchar: tString, tChar: tString, tText: tString, "character varying": tString, + "character": tString, "bpchar": tString, tLongtext: tClob, "mediumtext": tClob, "tinytext": tString, + tBlob: tBlob, tLongblob: tBlob, tBytea: tBlob, tBinary: tBytes, tVarbinary: tBytes, + } + + return m[p.base] +} + +const ( + maxDatetimeScale = 6 + maxMySQLVarchar = 16383 + maxMySQLVarbin = 65535 +) + +// fixedSQLTypes are the DMS types whose SQL form takes no length, precision or scale: {PostgreSQL, MySQL}. +func fixedSQLTypes() map[string][2]string { + return map[string][2]string{ + tInt1: { + tSmallint, + "tinyint", + }, tInt2: {tSmallint, tSmallint}, tInt4: {tInteger, tInt}, tInt8: {tBigint, tBigint}, + "uint1": {tSmallint, "tinyint unsigned"}, "uint2": {tInteger, "smallint unsigned"}, + "uint4": {tBigint, "int unsigned"}, "uint8": {"numeric(20)", "bigint unsigned"}, + tReal4: {tReal, tFloat}, tReal8: {tDoublePrec, tDouble}, tBoolean: {tBoolean, tTinyint1}, + tDate: {tDate, tDate}, tTime: {tTime, tTime}, + } +} + +// SQLType renders the DMS type for a SQL target of family fam. +func (d DMSType) SQLType(fam Family) string { + pg := fam == FamilyPostgres + + if v, ok := fixedSQLTypes()[d.Name]; ok { + return pick(pg, v[0], v[1]) + } + + switch d.Name { + case tNumeric: + return d.numeric() + case tDatetime: + return d.datetime(pg) + case tBytes, tBlob: + return d.binary(pg) + default: + return d.text(pg) + } +} + +func pick(first bool, a, b string) string { + if first { + return a + } + + return b +} + +func (d DMSType) numeric() string { + switch { + case d.Precision > 0 && d.Scale > 0: + return fmt.Sprintf("numeric(%d,%d)", d.Precision, d.Scale) + case d.Precision > 0: + return fmt.Sprintf("numeric(%d)", d.Precision) + default: + return tNumeric + } +} + +func (d DMSType) datetime(pg bool) string { + scale := d.Scale + if scale == 0 { + scale = d.Precision + } + + base := pick(pg, tTimestamp, tDatetime) + if scale > 0 { + return fmt.Sprintf("%s(%d)", base, min(scale, maxDatetimeScale)) + } + + return base +} + +func (d DMSType) binary(pg bool) string { + switch { + case pg: + return tBytea + case d.Name == tBytes && d.Length > 0 && d.Length <= maxMySQLVarbin: + return fmt.Sprintf("varbinary(%d)", d.Length) + default: + return tLongblob + } +} + +func (d DMSType) text(pg bool) string { + long := d.Name == tClob || d.Name == tNclob || d.Length <= 0 + + switch { + case pg && long: + return tText + case pg: + return fmt.Sprintf("varchar(%d)", d.Length) + case long || d.Length > maxMySQLVarchar: + return tLongtext + default: + return fmt.Sprintf("varchar(%d)", d.Length) + } +} diff --git a/pkgs/dmsengine/endpoint.go b/pkgs/dmsengine/endpoint.go index d4591ed036..596f53eaa6 100644 --- a/pkgs/dmsengine/endpoint.go +++ b/pkgs/dmsengine/endpoint.go @@ -32,7 +32,13 @@ type EndpointConfig struct { // SlotName names the PostgreSQL logical replication slot for source endpoints. SlotName string // S3Settings, KinesisSettings and DynamoDBSettings carry the raw settings JSON objects. - S3Settings string + S3Settings string + // ExternalTableDefinition is the S3 source's table-definition JSON (the endpoint member or S3Settings' + // ExternalTableDefinition). + ExternalTableDefinition string + // SecretID and SecretRoleARN are SecretsManagerSecretId and SecretsManagerAccessRoleArn from the engine settings. + SecretID string + SecretRoleARN string KinesisSettings string DynamoDBSettings string Port int @@ -51,6 +57,9 @@ func (c EndpointConfig) addr() string { // ObjectStore writes objects into the emulator's S3. type ObjectStore interface { PutObject(ctx context.Context, bucket, key string, body []byte) error + // ListObjects returns the keys under prefix in lexicographic order. + ListObjects(ctx context.Context, bucket, prefix string) ([]string, error) + GetObject(ctx context.Context, bucket, key string) ([]byte, error) CheckBucket(ctx context.Context, bucket string) error } @@ -100,7 +109,7 @@ type Opener interface { func SupportedSource(engine string) bool { f := FamilyOf(engine) - return f == FamilyMySQL || f == FamilyPostgres + return f == FamilyMySQL || f == FamilyPostgres || f == FamilyS3 } // SupportedTarget reports whether the engine can write to the endpoint engine. @@ -119,6 +128,8 @@ func (o *defaultOpener) OpenSource(ctx context.Context, ep EndpointConfig) (Sour return openMySQLSource(ctx, ep) case FamilyPostgres: return openPGSource(ctx, ep) + case FamilyS3: + return newS3Source(ep, o.stores.Objects) default: return nil, fmt.Errorf("%w: %s as a source", ErrUnsupportedEngine, ep.Engine) } diff --git a/pkgs/dmsengine/expr.go b/pkgs/dmsengine/expr.go new file mode 100644 index 0000000000..9536e66db6 --- /dev/null +++ b/pkgs/dmsengine/expr.go @@ -0,0 +1,1351 @@ +package dmsengine + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "math" + "math/rand/v2" + "strconv" + "strings" + "time" + "unicode" +) + +const ( + opWordInsert = "INSERT" + arityTwo = 2 + arityThree = 3 + blobRank = 2 + decimalBase = 10 + nanosPerMilli = 1_000_000 + daysInWeek = 7 +) + +// ErrExpression reports an add-column expression the engine cannot evaluate. +var ErrExpression = fmt.Errorf("%w: unsupported expression", ErrMapping) + +// EvalCtx is what an expression sees: the source row and the change header values. +type EvalCtx struct { + Time time.Time + Now time.Time + Op Op + Position string + Row Row + Full bool +} + +// Expr is a compiled SQLite-style expression from a transformation rule. +type Expr struct{ eval node } + +type node func(*EvalCtx) any + +// Eval evaluates the expression: nil, int64, float64, string or []byte. +func (e *Expr) Eval(c *EvalCtx) any { return e.eval(c) } + +type tokKind int + +const ( + tokEOF tokKind = iota + tokNum + tokStr + tokIdent + tokRef + tokOp +) + +type token struct { + text string + kind tokKind +} + +func lex(src string) ([]token, error) { + var out []token + + rs := []rune(src) + + for i := 0; i < len(rs); { + r := rs[i] + + switch { + case unicode.IsSpace(r): + i++ + case r == '\'': + s, n, err := lexString(rs[i:]) + if err != nil { + return nil, err + } + + out, i = append(out, token{s, tokStr}), i+n + case unicode.IsDigit(r) || (r == '.' && i+1 < len(rs) && unicode.IsDigit(rs[i+1])): + n := scanWhile(rs[i:], func(c rune) bool { return unicode.IsDigit(c) || c == '.' }) + out, i = append(out, token{string(rs[i : i+n]), tokNum}), i+n + case r == '$': + n := scanWhile(rs[i+1:], isIdentRune) + if n == 0 { + return nil, fmt.Errorf("%w: empty column reference", ErrExpression) + } + + out, i = append(out, token{string(rs[i+1 : i+1+n]), tokRef}), i+1+n + case isIdentRune(r): + n := scanWhile(rs[i:], isIdentRune) + out, i = append(out, token{string(rs[i : i+n]), tokIdent}), i+n + default: + op, n := lexOp(rs[i:]) + if n == 0 { + return nil, fmt.Errorf("%w: unexpected character %q", ErrExpression, r) + } + + out, i = append(out, token{op, tokOp}), i+n + } + } + + return append(out, token{kind: tokEOF}), nil +} + +func isIdentRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' } + +func scanWhile(rs []rune, f func(rune) bool) int { + n := 0 + for n < len(rs) && f(rs[n]) { + n++ + } + + return n +} + +func lexString(rs []rune) (string, int, error) { + var sb strings.Builder + + for i := 1; i < len(rs); i++ { + if rs[i] != '\'' { + sb.WriteRune(rs[i]) + + continue + } + + if i+1 < len(rs) && rs[i+1] == '\'' { + sb.WriteRune('\'') + + i++ + + continue + } + + return sb.String(), i + 1, nil + } + + return "", 0, fmt.Errorf("%w: unterminated string literal", ErrExpression) +} + +func lexOp(rs []rune) (string, int) { + for _, op := range []string{"||", "<>", "!=", "<=", ">=", "=="} { + if strings.HasPrefix(string(rs), op) { + return op, len(op) + } + } + + if strings.ContainsRune("+-*/%=<>(),", rs[0]) { + return string(rs[0]), 1 + } + + return "", 0 +} + +type parser struct { + table Table + toks []token + pos int +} + +// CompileExpr compiles an expression against the source table whose columns $name refers to. +func CompileExpr(src string, t Table) (*Expr, error) { + toks, err := lex(src) + if err != nil { + return nil, err + } + + p := &parser{toks: toks, table: t} + + n, err := p.parseOr() + if err != nil { + return nil, err + } + + if p.peek().kind != tokEOF { + return nil, fmt.Errorf("%w: unexpected %q", ErrExpression, p.peek().text) + } + + return &Expr{eval: n}, nil +} + +func (p *parser) peek() token { return p.toks[p.pos] } + +func (p *parser) next() token { + t := p.toks[p.pos] + if t.kind != tokEOF { + p.pos++ + } + + return t +} + +func (p *parser) isKeyword(kw string) bool { + t := p.peek() + + return t.kind == tokIdent && strings.EqualFold(t.text, kw) +} + +func (p *parser) acceptKeyword(kw string) bool { + if p.isKeyword(kw) { + p.pos++ + + return true + } + + return false +} + +func (p *parser) acceptOp(ops ...string) (string, bool) { + t := p.peek() + if t.kind != tokOp { + return "", false + } + + for _, o := range ops { + if t.text == o { + p.pos++ + + return o, true + } + } + + return "", false +} + +func (p *parser) expectOp(op string) error { + if _, ok := p.acceptOp(op); !ok { + return fmt.Errorf("%w: expected %q, found %q", ErrExpression, op, p.peek().text) + } + + return nil +} + +func (p *parser) parseOr() (node, error) { + l, err := p.parseAnd() + if err != nil { + return nil, err + } + + for p.acceptKeyword("or") { + r, rerr := p.parseAnd() + if rerr != nil { + return nil, rerr + } + + l = logic(l, r, true) + } + + return l, nil +} + +func (p *parser) parseAnd() (node, error) { + l, err := p.parseNot() + if err != nil { + return nil, err + } + + for p.acceptKeyword("and") { + r, rerr := p.parseNot() + if rerr != nil { + return nil, rerr + } + + l = logic(l, r, false) + } + + return l, nil +} + +func (p *parser) parseNot() (node, error) { + if !p.acceptKeyword("not") { + return p.parseCompare() + } + + n, err := p.parseNot() + if err != nil { + return nil, err + } + + return func(c *EvalCtx) any { + v := n(c) + if v == nil { + return nil + } + + return boolInt(!truthy(v)) + }, nil +} + +func (p *parser) parseCompare() (node, error) { + l, err := p.parseAdd() + if err != nil { + return nil, err + } + + for { + if p.acceptKeyword("is") { + l, err = p.parseIs(l) + if err != nil { + return nil, err + } + + continue + } + + op, ok := p.acceptOp("=", "==", "<>", "!=", "<", "<=", ">", ">=") + if !ok { + return l, nil + } + + r, rerr := p.parseAdd() + if rerr != nil { + return nil, rerr + } + + l = compareNode(op, l, r) + } +} + +func (p *parser) parseIs(l node) (node, error) { + neg := p.acceptKeyword("not") + + if p.acceptKeyword("null") { + return func(c *EvalCtx) any { return boolInt((l(c) == nil) != neg) }, nil + } + + r, err := p.parseAdd() + if err != nil { + return nil, err + } + + return func(c *EvalCtx) any { + a, b := l(c), r(c) + same := (a == nil && b == nil) || (a != nil && b != nil && compareValues(a, b) == 0) + + return boolInt(same != neg) + }, nil +} + +func (p *parser) parseAdd() (node, error) { + l, err := p.parseMul() + if err != nil { + return nil, err + } + + for { + op, ok := p.acceptOp("+", "-") + if !ok { + return l, nil + } + + r, rerr := p.parseMul() + if rerr != nil { + return nil, rerr + } + + l = arith(op, l, r) + } +} + +func (p *parser) parseMul() (node, error) { + l, err := p.parseConcat() + if err != nil { + return nil, err + } + + for { + op, ok := p.acceptOp("*", "/", "%") + if !ok { + return l, nil + } + + r, rerr := p.parseConcat() + if rerr != nil { + return nil, rerr + } + + l = arith(op, l, r) + } +} + +func (p *parser) parseConcat() (node, error) { + l, err := p.parseUnary() + if err != nil { + return nil, err + } + + for { + if _, ok := p.acceptOp("||"); !ok { + return l, nil + } + + r, rerr := p.parseUnary() + if rerr != nil { + return nil, rerr + } + + a, b := l, r + l = func(c *EvalCtx) any { + x, y := a(c), b(c) + if x == nil || y == nil { + return nil + } + + return text(x) + text(y) + } + } +} + +func (p *parser) parseUnary() (node, error) { + if op, ok := p.acceptOp("-", "+"); ok { + n, err := p.parseUnary() + if err != nil { + return nil, err + } + + if op == "+" { + return n, nil + } + + return func(c *EvalCtx) any { return arithValues("*", int64(-1), n(c)) }, nil + } + + return p.parsePrimary() +} + +func (p *parser) parsePrimary() (node, error) { + t := p.next() + + switch t.kind { + case tokNum: + return numLiteral(t.text) + case tokStr: + return func(*EvalCtx) any { return t.text }, nil + case tokRef: + return p.reference(t.text) + case tokIdent: + return p.identifier(t.text) + case tokOp: + if t.text == "(" { + n, err := p.parseOr() + if err != nil { + return nil, err + } + + return n, p.expectOp(")") + } + case tokEOF: + return nil, fmt.Errorf("%w: unexpected end of expression", ErrExpression) + } + + return nil, fmt.Errorf("%w: unexpected %q", ErrExpression, t.text) +} + +func numLiteral(s string) (node, error) { + if i, err := strconv.ParseInt(s, 10, 64); err == nil { + return func(*EvalCtx) any { return i }, nil + } + + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, fmt.Errorf("%w: bad number %q", ErrExpression, s) + } + + return func(*EvalCtx) any { return f }, nil +} + +func (p *parser) identifier(name string) (node, error) { + switch strings.ToLower(name) { + case "null": + return func(*EvalCtx) any { return nil }, nil + case "case": + return p.parseCase() + } + + if _, ok := p.acceptOp("("); !ok { + return nil, fmt.Errorf("%w: unexpected identifier %q", ErrExpression, name) + } + + return p.call(strings.ToLower(name)) +} + +func (p *parser) call(name string) (node, error) { + var args []node + + if _, ok := p.acceptOp(")"); !ok { + for { + a, err := p.parseOr() + if err != nil { + return nil, err + } + + args = append(args, a) + + if _, more := p.acceptOp(","); !more { + break + } + } + + if err := p.expectOp(")"); err != nil { + return nil, err + } + } + + spec, ok := exprFuncs()[name] + if !ok { + return nil, fmt.Errorf("%w: function %s() is not supported", ErrExpression, name) + } + + if len(args) < spec.min || (spec.max >= 0 && len(args) > spec.max) { + return nil, fmt.Errorf("%w: wrong number of arguments to %s()", ErrExpression, name) + } + + return func(c *EvalCtx) any { + vals := make([]any, len(args)) + for i, a := range args { + vals[i] = a(c) + } + + return spec.fn(c, vals) + }, nil +} + +type caseArm struct{ when, then node } + +func (p *parser) parseCase() (node, error) { + var subject node + + if !p.isKeyword("when") { + s, err := p.parseOr() + if err != nil { + return nil, err + } + + subject = s + } + + var arms []caseArm + + for p.acceptKeyword("when") { + arm, err := p.parseArm() + if err != nil { + return nil, err + } + + arms = append(arms, arm) + } + + var els node + + if p.acceptKeyword("else") { + e, err := p.parseOr() + if err != nil { + return nil, err + } + + els = e + } + + if len(arms) == 0 || !p.acceptKeyword("end") { + return nil, fmt.Errorf("%w: malformed CASE expression", ErrExpression) + } + + return func(c *EvalCtx) any { return evalCase(c, subject, arms, els) }, nil +} + +func (p *parser) parseArm() (caseArm, error) { + w, err := p.parseOr() + if err != nil { + return caseArm{}, err + } + + if !p.acceptKeyword("then") { + return caseArm{}, fmt.Errorf("%w: CASE WHEN needs THEN", ErrExpression) + } + + th, err := p.parseOr() + + return caseArm{when: w, then: th}, err +} + +func evalCase(c *EvalCtx, subject node, arms []caseArm, els node) any { + var base any + if subject != nil { + base = subject(c) + } + + for _, a := range arms { + w := a.when(c) + + matched := w != nil && truthy(w) + if subject != nil { + matched = base != nil && w != nil && compareValues(base, w) == 0 + } + + if matched { + return a.then(c) + } + } + + if els != nil { + return els(c) + } + + return nil +} + +func (p *parser) reference(name string) (node, error) { + if strings.HasPrefix(name, "AR_") { + return p.header(name) + } + + for i, c := range p.table.Columns { + if c.Name == name { + kind := c.Kind + + return func(ctx *EvalCtx) any { + if i >= len(ctx.Row) { + return nil + } + + return fromText(ctx.Row[i], kind) + }, nil + } + } + + return nil, fmt.Errorf("%w: column $%s does not exist in %s", ErrExpression, name, p.table.Key()) +} + +const headerTimeLayout = "2006-01-02 15:04:05.000000" + +func (p *parser) header(name string) (node, error) { + switch name { + case "AR_H_STREAM_POSITION": + return func(c *EvalCtx) any { + if c.Full { + return "" + } + + return c.Position + }, nil + case "AR_H_TIMESTAMP", "AR_H_COMMIT_TIMESTAMP": + return func(c *EvalCtx) any { + if c.Full || c.Time.IsZero() { + return c.Now.UTC().Format(headerTimeLayout) + } + + return c.Time.UTC().Format(headerTimeLayout) + }, nil + case "AR_H_OPERATION": + return func(c *EvalCtx) any { + if c.Full { + return opWordInsert + } + + return strings.ToUpper(string(c.Op)) + }, nil + case "AR_H_USER": + return func(*EvalCtx) any { return "" }, nil + case "AR_M_SOURCE_SCHEMA": + s := p.table.Schema + + return func(*EvalCtx) any { return s }, nil + case "AR_M_SOURCE_TABLE_NAME": + s := p.table.Name + + return func(*EvalCtx) any { return s }, nil + } + + return nil, fmt.Errorf("%w: $%s is not supported", ErrExpression, name) +} + +func fromText(v any, k Kind) any { + switch x := v.(type) { + case nil: + return nil + case []byte: + return x + case string: + switch k { + case KindInt: + if i, err := strconv.ParseInt(x, 10, 64); err == nil { + return i + } + case KindDecimal, KindFloat: + if f, err := strconv.ParseFloat(x, 64); err == nil { + return f + } + case KindString, KindBool, KindTime, KindBytes, KindJSON: + } + + return x + } + + return fmt.Sprint(v) +} + +func boolInt(b bool) any { + if b { + return int64(1) + } + + return int64(0) +} + +func truthy(v any) bool { + switch x := toNumber(v).(type) { + case int64: + return x != 0 + case float64: + return x != 0 + } + + return false +} + +// toNumber follows SQLite's numeric prefix rule: text that is not a number reads as 0. +func toNumber(v any) any { + switch x := v.(type) { + case int64, float64: + return x + case []byte: + return toNumber(string(x)) + case string: + s := strings.TrimSpace(x) + if i, err := strconv.ParseInt(s, 10, 64); err == nil { + return i + } + + if f, err := strconv.ParseFloat(s, 64); err == nil { + return f + } + + return numericPrefix(s) + } + + return int64(0) +} + +func numericPrefix(s string) any { + for end := len(s); end > 0; end-- { + if f, err := strconv.ParseFloat(s[:end], 64); err == nil { + if i, ierr := strconv.ParseInt(s[:end], 10, 64); ierr == nil { + return i + } + + return f + } + } + + return int64(0) +} + +func asFloat(v any) float64 { + switch x := toNumber(v).(type) { + case int64: + return float64(x) + case float64: + return x + } + + return 0 +} + +func logic(l, r node, or bool) node { + return func(c *EvalCtx) any { + a, b := l(c), r(c) + at, bt := a != nil && truthy(a), b != nil && truthy(b) + + switch { + case or && (at || bt): + return int64(1) + case !or && ((a != nil && !at) || (b != nil && !bt)): + return int64(0) + case a == nil || b == nil: + return nil + case or: + return int64(0) + } + + return int64(1) + } +} + +// compareValues orders values SQLite-style: numbers before text before blobs. +func compareValues(a, b any) int { + switch x := a.(type) { + case string: + if y, ok := b.(string); ok { + return strings.Compare(x, y) + } + case []byte: + if y, ok := b.([]byte); ok { + return bytes.Compare(x, y) + } + default: + if valueRank(b) == 0 { + return compareNumbers(a, b) + } + } + + return valueRank(a) - valueRank(b) +} + +func compareNumbers(a, b any) int { + ia, aInt := a.(int64) + ib, bInt := b.(int64) + + if aInt && bInt { + return cmpOrdered(ia, ib) + } + + return cmpOrdered(asFloat(a), asFloat(b)) +} + +func cmpOrdered[T int64 | float64](a, b T) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + } + + return 0 +} + +func valueRank(v any) int { + switch v.(type) { + case string: + return 1 + case []byte: + return blobRank + } + + return 0 +} + +func compareNode(op string, l, r node) node { + return func(c *EvalCtx) any { + a, b := l(c), r(c) + if a == nil || b == nil { + return nil + } + + d := compareValues(a, b) + + switch op { + case "=", "==": + return boolInt(d == 0) + case "<>", "!=": + return boolInt(d != 0) + case "<": + return boolInt(d < 0) + case "<=": + return boolInt(d <= 0) + case ">": + return boolInt(d > 0) + } + + return boolInt(d >= 0) + } +} + +func arith(op string, l, r node) node { + return func(c *EvalCtx) any { return arithValues(op, l(c), r(c)) } +} + +func arithValues(op string, a, b any) any { + if a == nil || b == nil { + return nil + } + + na, nb := toNumber(a), toNumber(b) + ia, aInt := na.(int64) + ib, bInt := nb.(int64) + + if aInt && bInt { + return intArith(op, ia, ib) + } + + fa, fb := asFloat(na), asFloat(nb) + + switch op { + case "+": + return fa + fb + case "-": + return fa - fb + case "*": + return fa * fb + case "/": + if fb == 0 { + return nil + } + + return fa / fb + } + + if fb == 0 { + return nil + } + + return math.Mod(fa, fb) +} + +func intArith(op string, a, b int64) any { + switch op { + case "+": + return a + b + case "-": + return a - b + case "*": + return a * b + case "/": + if b == 0 { + return nil + } + + return a / b + } + + if b == 0 { + return nil + } + + return a % b +} + +type funcSpec struct { + fn func(*EvalCtx, []any) any + min, max int +} + +func exprFuncs() map[string]funcSpec { + return map[string]funcSpec{ + "lower": {min: 1, max: 1, fn: func(_ *EvalCtx, a []any) any { return mapNil(a[0], asciiCase(false)) }}, + "upper": {min: 1, max: 1, fn: func(_ *EvalCtx, a []any) any { return mapNil(a[0], asciiCase(true)) }}, + "ltrim": {min: 1, max: arityTwo, fn: trimFn(true, false)}, + "rtrim": {min: 1, max: arityTwo, fn: trimFn(false, true)}, + "trim": {min: 1, max: arityTwo, fn: trimFn(true, true)}, + "replace": {min: arityThree, max: arityThree, fn: func(_ *EvalCtx, a []any) any { + if a[0] == nil || a[1] == nil || a[2] == nil { + return nil + } + + if text(a[1]) == "" { + return text(a[0]) + } + + return strings.ReplaceAll(text(a[0]), text(a[1]), text(a[2])) + }}, + "substr": {min: arityTwo, max: arityThree, fn: substrFn}, + "hex": {min: 1, max: 1, fn: hexFn}, + "abs": {min: 1, max: 1, fn: absFn}, + "round": {min: 1, max: arityTwo, fn: roundFn}, + "random": {min: 0, max: 0, fn: randomInt}, + "max": {min: arityTwo, max: -1, fn: extremum(1)}, + "min": {min: arityTwo, max: -1, fn: extremum(-1)}, + "coalesce": {min: arityTwo, max: -1, fn: coalesceFn}, + "ifnull": {min: arityTwo, max: arityTwo, fn: coalesceFn}, + "nullif": {min: arityTwo, max: arityTwo, fn: func(_ *EvalCtx, a []any) any { + if a[0] != nil && a[1] != nil && compareValues(a[0], a[1]) == 0 { + return nil + } + + return a[0] + }}, + "zeroblob": {min: 1, max: 1, fn: func(_ *EvalCtx, a []any) any { + return make([]byte, max(0, int(asFloat(a[0])))) + }}, + "randomblob": {min: 1, max: 1, fn: randomBlob}, + "hash_sha256": {min: 1, max: 1, fn: func(_ *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + sum := sha256.Sum256([]byte(text(a[0]))) + + return hex.EncodeToString(sum[:]) + }}, + "operation_indicator": {min: arityThree, max: arityThree, fn: operationIndicator}, + tDate: {min: 0, max: 1, fn: dateFn("2006-01-02")}, + tTime: {min: 0, max: 1, fn: dateFn("15:04:05")}, + "datetime": {min: 0, max: 1, fn: dateFn("2006-01-02 15:04:05")}, + "julianday": {min: 0, max: 1, fn: julianFn}, + "strftime": {min: 1, max: arityTwo, fn: strftimeFn}, + } +} + +func text(v any) string { + switch x := v.(type) { + case nil: + return "" + case string: + return x + case []byte: + return string(x) + case int64: + return strconv.FormatInt(x, 10) + case float64: + return formatFloat(x) + } + + return fmt.Sprint(v) +} + +func formatFloat(f float64) string { + if math.IsInf(f, 0) || math.IsNaN(f) { + return "" + } + + s := strconv.FormatFloat(f, 'g', 15, 64) + if !strings.ContainsAny(s, ".eE") { + s += ".0" + } + + return s +} + +func mapNil(v any, f func(string) string) any { + if v == nil { + return nil + } + + return f(text(v)) +} + +func asciiCase(upper bool) func(string) string { + return func(s string) string { + b := []byte(s) + for i, c := range b { + switch { + case upper && c >= 'a' && c <= 'z': + b[i] = c - 'a' + 'A' + case !upper && c >= 'A' && c <= 'Z': + b[i] = c - 'A' + 'a' + } + } + + return string(b) + } +} + +func trimFn(left, right bool) func(*EvalCtx, []any) any { + return func(_ *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + cut := " " + if len(a) > 1 { + if a[1] == nil { + return nil + } + + cut = text(a[1]) + } + + s := text(a[0]) + if left { + s = strings.TrimLeft(s, cut) + } + + if right { + s = strings.TrimRight(s, cut) + } + + return s + } +} + +func substrFn(_ *EvalCtx, a []any) any { + if a[0] == nil || a[1] == nil || (len(a) == arityThree && a[2] == nil) { + return nil + } + + rs := []rune(text(a[0])) + start := int(asFloat(a[1])) + length := len(rs) + + if len(a) == arityThree { + length = int(asFloat(a[2])) + } + + if start < 0 { + start = len(rs) + start + 1 + } + + from, to := start, start+length + + if length < 0 { + from, to = start+length, start + } + + from, to = max(from, 1), min(to, len(rs)+1) + if from >= to { + return "" + } + + return string(rs[from-1 : to-1]) +} + +func hexFn(_ *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + return strings.ToUpper(hex.EncodeToString([]byte(text(a[0])))) +} + +func absFn(_ *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + switch x := toNumber(a[0]).(type) { + case int64: + return max(x, -x) + case float64: + return math.Abs(x) + } + + return float64(0) +} + +func roundFn(_ *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + digits := 0 + if len(a) == arityTwo { + digits = int(asFloat(a[1])) + } + + scale := math.Pow(decimalBase, float64(digits)) + + return math.Round(asFloat(a[0])*scale) / scale +} + +func extremum(dir int) func(*EvalCtx, []any) any { + return func(_ *EvalCtx, a []any) any { + best := a[0] + + for _, v := range a { + if v == nil { + return nil + } + + if compareValues(v, best)*dir > 0 { + best = v + } + } + + return best + } +} + +func coalesceFn(_ *EvalCtx, a []any) any { + for _, v := range a { + if v != nil { + return v + } + } + + return nil +} + +func randomInt(*EvalCtx, []any) any { + return int64(rand.Uint64()) //nolint:gosec // SQL random() is not a security primitive +} + +func randomBlob(_ *EvalCtx, a []any) any { + b := make([]byte, max(1, int(asFloat(a[0])))) + for i := range b { + b[i] = byte(rand.UintN(256)) //nolint:gosec,mnd // SQL randomblob() + } + + return b +} + +func operationIndicator(c *EvalCtx, a []any) any { + if c.Full { + return a[2] + } + + switch c.Op { + case OpDelete: + return a[0] + case OpUpdate: + return a[1] + case OpInsert: + } + + return a[2] +} + +func parseTimeString(c *EvalCtx, v any) (time.Time, bool) { + s := strings.TrimSpace(text(v)) + if strings.EqualFold(s, "now") { + return c.Now.UTC(), true + } + + for _, layout := range []string{ + "2006-01-02 15:04:05.999999999", "2006-01-02T15:04:05.999999999", "2006-01-02 15:04", "2006-01-02", "15:04:05", + } { + if t, err := time.ParseInLocation(layout, s, time.UTC); err == nil { + return t, true + } + } + + return time.Time{}, false +} + +func timeArg(c *EvalCtx, a []any) (time.Time, bool) { + if len(a) == 0 { + return c.Now.UTC(), true + } + + if a[0] == nil { + return time.Time{}, false + } + + return parseTimeString(c, a[0]) +} + +func dateFn(layout string) func(*EvalCtx, []any) any { + return func(c *EvalCtx, a []any) any { + t, ok := timeArg(c, a) + if !ok { + return nil + } + + return t.Format(layout) + } +} + +const ( + secondsPerDay = 86400.0 + unixEpochJulis = 2440587.5 +) + +func julianFn(c *EvalCtx, a []any) any { + t, ok := timeArg(c, a) + if !ok { + return nil + } + + return float64(t.UnixNano())/1e9/secondsPerDay + unixEpochJulis +} + +func strftimeFn(c *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + t, ok := timeArg(c, a[1:]) + if !ok { + return nil + } + + var sb strings.Builder + + f := []rune(text(a[0])) + for i := 0; i < len(f); i++ { + if f[i] != '%' || i+1 == len(f) { + sb.WriteRune(f[i]) + + continue + } + + i++ + + s, known := strftimeVar(f[i], t) + if !known { + return nil + } + + sb.WriteString(s) + } + + return sb.String() +} + +func strftimeVar(v rune, t time.Time) (string, bool) { + switch v { + case 'd': + return t.Format("02"), true + case 'H': + return t.Format("15"), true + case 'f': + return fmt.Sprintf("%02d.%03d", t.Second(), t.Nanosecond()/nanosPerMilli), true + case 'j': + return fmt.Sprintf("%03d", t.YearDay()), true + case 'J': + return formatFloat(float64(t.UnixNano())/1e9/secondsPerDay + unixEpochJulis), true + case 'm': + return t.Format("01"), true + case 'M': + return t.Format("04"), true + case 's': + return strconv.FormatInt(t.Unix(), 10), true + case 'S': + return t.Format("05"), true + case 'w': + return strconv.Itoa(int(t.Weekday())), true + case 'W': + return fmt.Sprintf("%02d", (t.YearDay()+daysInWeek-1-int(t.Weekday()))/daysInWeek), true + case 'Y': + return t.Format("2006"), true + case '%': + return "%", true + } + + return "", false +} + +// Cast renders an expression result as the text form of a column of DMS type d. +func (d DMSType) Cast(v any) any { + if v == nil { + return nil + } + + switch d.Kind() { + case KindInt: + switch x := toNumber(v).(type) { + case float64: + return strconv.FormatInt(int64(x), 10) + case int64: + return strconv.FormatInt(x, 10) + } + case KindDecimal, KindFloat: + f := asFloat(v) + if i, ok := toNumber(v).(int64); ok { + return strconv.FormatInt(i, 10) + } + + return strconv.FormatFloat(f, 'f', -1, 64) + case KindBool: + return strconv.FormatBool(truthy(v)) + case KindBytes: + if b, ok := v.([]byte); ok { + return b + } + + return []byte(text(v)) + case KindString, KindTime, KindJSON: + } + + return text(v) +} diff --git a/pkgs/dmsengine/filter.go b/pkgs/dmsengine/filter.go new file mode 100644 index 0000000000..9086bcebc7 --- /dev/null +++ b/pkgs/dmsengine/filter.go @@ -0,0 +1,278 @@ +package dmsengine + +import ( + "fmt" + "strconv" + "strings" +) + +// Selection-rule filter operators (DMS User Guide, "Using source filters"). +const ( + opLTE = "lte" + opSTE = "ste" + opGTE = "gte" + opEQ = "eq" + opNotEQ = "noteq" + opBetween = "between" + opNotBetween = "notbetween" + opNull = "null" + opNotNull = "notnull" +) + +type filterCondition struct { + Op string `json:"filter-operator"` + Value flexString `json:"value"` + Start flexString `json:"start-value"` + End flexString `json:"end-value"` +} + +// RuleFilter is one entry of a selection rule's filters array. +type RuleFilter struct { + Type string `json:"filter-type"` + Column string `json:"column-name"` + Conditions []filterCondition `json:"filter-conditions"` +} + +func validateFilter(f RuleFilter) error { + if f.Type != "source" { + return fmt.Errorf("%w: unsupported filter-type %q", ErrMapping, f.Type) + } + + if f.Column == "" || len(f.Conditions) == 0 { + return fmt.Errorf("%w: a filter needs column-name and filter-conditions", ErrMapping) + } + + for _, c := range f.Conditions { + switch c.Op { + case opLTE, opSTE, opGTE, opEQ, opNotEQ, opBetween, opNotBetween, opNull, opNotNull: + default: + return fmt.Errorf("%w: unsupported filter-operator %q", ErrMapping, c.Op) + } + } + + return nil +} + +func (c filterCondition) negative() bool { return c.Op == opNotEQ || c.Op == opNotBetween } + +func (c filterCondition) inclusive() bool { + switch c.Op { + case opLTE, opSTE, opGTE, opEQ, opBetween: + return true + } + + return false +} + +// effective applies the documented NULL notes: with a negative and a null condition in the same filter, NULL rows are +// never replicated, and without an inclusive condition nothing is. +func (f RuleFilter) effective() ([]filterCondition, bool) { + var neg, null, incl bool + + for _, c := range f.Conditions { + neg, null, incl = neg || c.negative(), null || c.Op == opNull, incl || c.inclusive() + } + + if !neg || !null { + return f.Conditions, true + } + + if !incl { + return nil, false + } + + out := make([]filterCondition, 0, len(f.Conditions)) + + for _, c := range f.Conditions { + if c.Op != opNull { + out = append(out, c) + } + } + + return out, true +} + +// TableFilter is the compiled filters of one selection rule bound to a source table. +type TableFilter struct { + table Table + filters []boundFilter +} + +type boundFilter struct { + conds []filterCondition + col Column + idx int + none bool +} + +func newTableFilter(t Table, fs []RuleFilter) (*TableFilter, error) { + if len(fs) == 0 { + return nil, nil //nolint:nilnil // no filters + } + + tf := &TableFilter{table: t} + + for _, f := range fs { + idx := -1 + + for i, c := range t.Columns { + if c.Name == f.Column { + idx = i + } + } + + if idx < 0 { + return nil, fmt.Errorf("%w: filter column %q does not exist in %s", ErrMapping, f.Column, t.Key()) + } + + conds, ok := f.effective() + tf.filters = append(tf.filters, boundFilter{conds: conds, col: t.Columns[idx], idx: idx, none: !ok}) + } + + return tf, nil +} + +// Match reports whether a source row passes every filter. +func (f *TableFilter) Match(r Row) bool { + if f == nil { + return true + } + + for _, bf := range f.filters { + if bf.none || bf.idx >= len(r) || !bf.match(r[bf.idx]) { + return false + } + } + + return true +} + +func (b boundFilter) match(v any) bool { + for _, c := range b.conds { + if b.matchCond(c, v) { + return true + } + } + + return false +} + +func (b boundFilter) matchCond(c filterCondition, v any) bool { + switch c.Op { + case opNull: + return v == nil + case opNotNull: + return v != nil + } + + if v == nil { + return false + } + + cmp := func(s flexString) int { return compareFilterValue(b.col.Kind, v, string(s)) } + + switch c.Op { + case opLTE, opSTE: + return cmp(c.Value) <= 0 + case opGTE: + return cmp(c.Value) >= 0 + case opEQ: + return cmp(c.Value) == 0 + case opNotEQ: + return cmp(c.Value) != 0 + case opBetween: + return cmp(c.Start) >= 0 && cmp(c.End) <= 0 + } + + return cmp(c.Start) < 0 || cmp(c.End) > 0 +} + +// compareFilterValue compares a column value with a rule literal: numerically for numeric columns, otherwise bytewise +// (SQLite BINARY), which orders the ISO date and time forms DMS filters use. +func compareFilterValue(k Kind, v any, lit string) int { + var s string + + switch x := v.(type) { + case string: + s = x + case []byte: + s = string(x) + } + + if k == KindInt || k == KindDecimal || k == KindFloat { + a, aerr := strconv.ParseFloat(strings.TrimSpace(s), 64) + b, berr := strconv.ParseFloat(strings.TrimSpace(lit), 64) + + if aerr == nil && berr == nil { + return cmpOrdered(a, b) + } + } + + return strings.Compare(s, lit) +} + +// SQL renders the filters as a WHERE predicate with positional arguments for the dialect. +func (f *TableFilter) SQL(fam Family, firstArg int) (string, []any) { + if f == nil { + return "", nil + } + + var ( + parts []string + args []any + ) + + ph := func() string { + args = append(args, nil) + + if fam == FamilyPostgres { + return "$" + strconv.Itoa(firstArg+len(args)-1) + } + + return "?" + } + bind := func(s flexString) string { + p := ph() + args[len(args)-1] = string(s) + + return p + } + + for _, bf := range f.filters { + if bf.none { + return "1 = 0", nil + } + + col := QuoteIdent(fam, bf.col.Name) + alts := make([]string, 0, len(bf.conds)) + + for _, c := range bf.conds { + alts = append(alts, condSQL(c, col, bind)) + } + + parts = append(parts, "("+strings.Join(alts, " OR ")+")") + } + + return strings.Join(parts, " AND "), args +} + +func condSQL(c filterCondition, col string, bind func(flexString) string) string { + switch c.Op { + case opNull: + return col + " IS NULL" + case opNotNull: + return col + " IS NOT NULL" + case opLTE, opSTE: + return col + " <= " + bind(c.Value) + case opGTE: + return col + " >= " + bind(c.Value) + case opEQ: + return col + " = " + bind(c.Value) + case opNotEQ: + return col + " <> " + bind(c.Value) + case opBetween: + return col + " BETWEEN " + bind(c.Start) + " AND " + bind(c.End) + } + + return col + " NOT BETWEEN " + bind(c.Start) + " AND " + bind(c.End) +} diff --git a/pkgs/dmsengine/mapping.go b/pkgs/dmsengine/mapping.go index ee83aa71d7..a8fc9cf924 100644 --- a/pkgs/dmsengine/mapping.go +++ b/pkgs/dmsengine/mapping.go @@ -22,9 +22,20 @@ const ( actExclude = "exclude" actRename = "rename" actRemoveColumn = "remove-column" + actIncludeColumn = "include-column" + actAddColumn = "add-column" + actChangeType = "change-data-type" + actBeforeImage = "add-before-image-columns" + actDefinePK = "define-primary-key" targetSchema = "schema" targetTable = "table" targetColumn = "column" + pkOriginUnique = "unique-index" + pkOriginPrimary = "primary-key" + imageFilterPK = "pk-only" + imageFilterNonLOB = "non-lob" + imageFilterAll = "all" + defaultImagePrefix = "BI_" ) type flexString string @@ -47,20 +58,81 @@ func (f *flexString) UnmarshalJSON(b []byte) error { return nil } +type flexInt int + +func (f *flexInt) UnmarshalJSON(b []byte) error { + var s flexString + if err := s.UnmarshalJSON(b); err != nil { + return err + } + + if strings.TrimSpace(string(s)) == "" { + *f = 0 + + return nil + } + + n, err := strconv.Atoi(strings.TrimSpace(string(s))) + if err != nil { + return fmt.Errorf("%w: %q is not an integer", ErrMapping, string(s)) + } + + *f = flexInt(n) + + return nil +} + +type ruleDataType struct { + Type string `json:"type"` + Length flexInt `json:"length"` + Precision flexInt `json:"precision"` + Scale flexInt `json:"scale"` +} + +func (d ruleDataType) parse() (DMSType, error) { + t, err := ParseDMSType(d.Type) + if err != nil { + return t, err + } + + t.Length, t.Precision, t.Scale = int(d.Length), int(d.Precision), int(d.Scale) + + return t, nil +} + +type primaryKeyDef struct { + Name string `json:"name"` + Origin string `json:"origin"` + Columns []string `json:"columns"` +} + +type beforeImageDef struct { + Prefix *string `json:"column-prefix"` + Suffix string `json:"column-suffix"` + Filter string `json:"column-filter"` +} + // Rule is one entry of a table-mappings document. type Rule struct { Locator struct { - Schema string `json:"schema-name"` - Table string `json:"table-name"` - Column string `json:"column-name"` + Schema string `json:"schema-name"` + Table string `json:"table-name"` + Column string `json:"column-name"` + DataType string `json:"data-type"` } `json:"object-locator"` - ID flexString `json:"rule-id"` - Name flexString `json:"rule-name"` - Type string `json:"rule-type"` - Action string `json:"rule-action"` - Target string `json:"rule-target"` - Value string `json:"value"` - Old string `json:"old-value"` + DataType *ruleDataType `json:"data-type"` + PrimaryKey *primaryKeyDef `json:"primary-key-def"` + BeforeImage *beforeImageDef `json:"before-image-def"` + ID flexString `json:"rule-id"` + Name flexString `json:"rule-name"` + LoadOrder flexString `json:"load-order"` + Type string `json:"rule-type"` + Action string `json:"rule-action"` + Target string `json:"rule-target"` + Value string `json:"value"` + Old string `json:"old-value"` + Expression string `json:"expression"` + Filters []RuleFilter `json:"filters"` } // Mappings is a parsed table-mappings document. @@ -101,20 +173,41 @@ func ruleOrder(r Rule) int { return n } +func hasWildcard(s string) bool { return strings.ContainsAny(s, "%_") } + func validateRule(r Rule) error { switch r.Type { case ruleSelection: - switch r.Action { - case actInclude, actExclude, actExplicit: - return nil - } + return validateSelection(r) case ruleTransformation: return validateTransformation(r) default: return fmt.Errorf("%w: unsupported rule-type %q", ErrMapping, r.Type) } +} - return fmt.Errorf("%w: unsupported selection rule-action %q", ErrMapping, r.Action) +func validateSelection(r Rule) error { + switch r.Action { + case actInclude, actExclude, actExplicit: + default: + return fmt.Errorf("%w: unsupported selection rule-action %q", ErrMapping, r.Action) + } + + if r.Action != actInclude { + return nil + } + + if len(r.Filters) > 0 && (hasWildcard(r.Locator.Schema) || hasWildcard(r.Locator.Table)) { + return fmt.Errorf("%w: filters need an exact schema-name and table-name", ErrMapping) + } + + for _, f := range r.Filters { + if err := validateFilter(f); err != nil { + return err + } + } + + return nil } func validateTransformation(r Rule) error { @@ -128,31 +221,130 @@ func validateTransformation(r Rule) error { case actRename, "add-prefix", "remove-prefix", "replace-prefix", "add-suffix", "remove-suffix", "replace-suffix", "convert-lowercase", "convert-uppercase": return nil - case actRemoveColumn: + case actRemoveColumn, actIncludeColumn: if r.Target == targetColumn { return nil } + case actAddColumn, actChangeType, actBeforeImage: + return validateColumnRule(r) + case actDefinePK: + return validatePrimaryKeyRule(r) } return fmt.Errorf("%w: unsupported transformation rule-action %q for %s", ErrMapping, r.Action, r.Target) } -// MatchPattern matches s against a DMS locator pattern where % matches any run of characters. +func validateColumnRule(r Rule) error { + if r.Target != targetColumn { + return fmt.Errorf("%w: %s needs rule-target column", ErrMapping, r.Action) + } + + switch r.Action { + case actAddColumn, actChangeType: + if r.DataType == nil { + return fmt.Errorf("%w: %s needs a data-type", ErrMapping, r.Action) + } + + if _, err := r.DataType.parse(); err != nil { + return err + } + + if r.Action == actAddColumn && r.Value == "" { + return fmt.Errorf("%w: add-column needs a value naming the new column", ErrMapping) + } + default: + return validateBeforeImage(r) + } + + return nil +} + +func validateBeforeImage(r Rule) error { + def := r.imageDef() + + switch def.Filter { + case imageFilterPK, imageFilterNonLOB, imageFilterAll: + default: + return fmt.Errorf("%w: unsupported column-filter %q", ErrMapping, def.Filter) + } + + if *def.Prefix == "" && def.Suffix == "" { + return fmt.Errorf("%w: before-image-def needs a column-prefix or column-suffix", ErrMapping) + } + + return nil +} + +func (r Rule) imageDef() beforeImageDef { + def := beforeImageDef{Filter: imageFilterPK} + prefix := defaultImagePrefix + def.Prefix = &prefix + + if r.BeforeImage != nil { + def.Suffix = r.BeforeImage.Suffix + + if r.BeforeImage.Prefix != nil { + def.Prefix = r.BeforeImage.Prefix + } + + if r.BeforeImage.Filter != "" { + def.Filter = r.BeforeImage.Filter + } + } + + return def +} + +func validatePrimaryKeyRule(r Rule) error { + if r.Target != targetTable { + return fmt.Errorf("%w: define-primary-key needs rule-target table", ErrMapping) + } + + if hasWildcard(r.Locator.Schema) || hasWildcard(r.Locator.Table) { + return fmt.Errorf("%w: define-primary-key takes no wildcards in schema-name or table-name", ErrMapping) + } + + if r.PrimaryKey == nil || len(r.PrimaryKey.Columns) == 0 { + return fmt.Errorf("%w: define-primary-key needs primary-key-def.columns", ErrMapping) + } + + switch r.PrimaryKey.Origin { + case "", pkOriginPrimary, pkOriginUnique: + return nil + } + + return fmt.Errorf("%w: unsupported primary-key-def origin %q", ErrMapping, r.PrimaryKey.Origin) +} + +// MatchPattern matches s against a DMS locator pattern where % matches any run of characters and _ any one +// character (DMS User Guide, "Wildcards in table mapping"). func MatchPattern(pattern, s string) bool { if pattern == "" { pattern = "%" } - if !strings.Contains(pattern, "%") { + if !hasWildcard(pattern) { return pattern == s } - parts := strings.Split(pattern, "%") - for i, p := range parts { - parts[i] = regexp.QuoteMeta(p) + var sb strings.Builder + + sb.WriteString("^") + + for _, r := range pattern { + switch r { + case '%': + sb.WriteString(".*") + case '_': + sb.WriteString(".") + default: + sb.WriteString(regexp.QuoteMeta(string(r))) + } } - return regexp.MustCompile("^" + strings.Join(parts, ".*") + "$").MatchString(s) + sb.WriteString("$") + + return regexp.MustCompile("(?s)" + sb.String()).MatchString(s) } func applyNameAction(action, name, value, old string) string { @@ -184,28 +376,48 @@ func applyNameAction(action, name, value, old string) string { return name } -// Select returns the source tables chosen by the selection rules, in source order. -func (m Mappings) Select(tables []Table) []Table { - chosen := make(map[string]bool, len(tables)) +// SelectedTable is a source table chosen by the selection rules with its source filter. +type SelectedTable struct { + Filter *TableFilter + Table Table +} - for _, r := range m.Rules { +func (m Mappings) chosen(tables []Table) map[string]*Rule { + chosen := make(map[string]*Rule, len(tables)) + + for i := range m.Rules { + r := &m.Rules[i] if r.Type != ruleSelection { continue } for _, t := range tables { - if !MatchPattern(r.Locator.Schema, t.Schema) || !MatchPattern(r.Locator.Table, t.Name) { + if !r.selects(t) { continue } - chosen[t.Key()] = r.Action != actExclude + if r.Action == actExclude { + delete(chosen, t.Key()) + } else { + chosen[t.Key()] = r + } } } + return chosen +} + +func (r Rule) selects(t Table) bool { + return MatchPattern(r.Locator.Schema, t.Schema) && MatchPattern(r.Locator.Table, t.Name) +} + +// Select returns the source tables chosen by the selection rules, in source order. +func (m Mappings) Select(tables []Table) []Table { + chosen := m.chosen(tables) out := make([]Table, 0, len(chosen)) for _, t := range tables { - if chosen[t.Key()] { + if _, ok := chosen[t.Key()]; ok { out = append(out, t) } } @@ -213,9 +425,43 @@ func (m Mappings) Select(tables []Table) []Table { return out } +// SelectFiltered is Select plus each table's compiled source filter; tables with a higher load-order come first. +func (m Mappings) SelectFiltered(tables []Table) ([]SelectedTable, error) { + chosen := m.chosen(tables) + out := make([]SelectedTable, 0, len(chosen)) + + for _, t := range tables { + r, ok := chosen[t.Key()] + if !ok { + continue + } + + var fs []RuleFilter + if r.Action == actInclude { + fs = r.Filters + } + + f, err := newTableFilter(t, fs) + if err != nil { + return nil, err + } + + out = append(out, SelectedTable{Table: t, Filter: f}) + } + + order := func(st SelectedTable) int { + n, _ := strconv.Atoi(string(chosen[st.Table.Key()].LoadOrder)) + + return n + } + sort.SliceStable(out, func(i, j int) bool { return order(out[i]) > order(out[j]) }) + + return out, nil +} + // Plan maps a selected source table to its target shape. overrideSchema (TargetMetadata.TargetSchema) wins // over schema renames when set. -func (m Mappings) Plan(t Table, src Family, overrideSchema string) TargetTable { +func (m Mappings) Plan(t Table, src Family, overrideSchema string) (TargetTable, error) { out := TargetTable{ SourceFamily: src, Schema: t.Schema, Name: t.Name, SourceSchema: t.Schema, SourceName: t.Name, } @@ -241,53 +487,9 @@ func (m Mappings) Plan(t Table, src Family, overrideSchema string) TargetTable { out.Schema = overrideSchema } - out.Columns, out.PK = m.planColumns(t) - - return out -} - -func (m Mappings) planColumns(t Table) ([]TargetColumn, []string) { - cols := make([]TargetColumn, 0, len(t.Columns)) - rename := make(map[string]string, len(t.Columns)) - - for i, c := range t.Columns { - name, removed := c.Name, false - - for _, r := range m.Rules { - if r.Type != ruleTransformation || r.Target != targetColumn || !m.columnRuleApplies(r, t, c.Name) { - continue - } - - if r.Action == actRemoveColumn { - removed = true - - break - } - - name = applyNameAction(r.Action, name, r.Value, r.Old) - } - - if removed { - continue - } - - rename[c.Name] = name - c.Name = name - cols = append(cols, TargetColumn{Column: c, Source: i}) - } - - pk := make([]string, 0, len(t.PK)) - - for _, p := range t.PK { - if n, ok := rename[p]; ok { - pk = append(pk, n) - } + if err := m.planColumns(t, &out); err != nil { + return out, err } - return cols, pk -} - -func (m Mappings) columnRuleApplies(r Rule, t Table, col string) bool { - return MatchPattern(r.Locator.Schema, t.Schema) && MatchPattern(r.Locator.Table, t.Name) && - MatchPattern(r.Locator.Column, col) + return out, m.definePrimaryKey(t, &out) } diff --git a/pkgs/dmsengine/mapping_test.go b/pkgs/dmsengine/mapping_test.go index 83925277c3..52333c50cb 100644 --- a/pkgs/dmsengine/mapping_test.go +++ b/pkgs/dmsengine/mapping_test.go @@ -162,7 +162,8 @@ func TestMappingsPlan(t *testing.T) { m, err := dmsengine.ParseMappings(doc) require.NoError(t, err) - got := m.Plan(sampleTables()[0], dmsengine.FamilyMySQL, tt.override) + got, perr := m.Plan(sampleTables()[0], dmsengine.FamilyMySQL, tt.override) + require.NoError(t, perr) assert.Equal(t, tt.wantSchema, got.Schema) assert.Equal(t, tt.wantTable, got.Name) diff --git a/pkgs/dmsengine/model.go b/pkgs/dmsengine/model.go index 701c8a5ca8..86a5305d93 100644 --- a/pkgs/dmsengine/model.go +++ b/pkgs/dmsengine/model.go @@ -58,6 +58,7 @@ func FamilyOf(engine string) Family { type Column struct { Name string Native string + DMS DMSType Kind Kind Nullable bool } @@ -136,33 +137,66 @@ const ( // TargetColumn is a column on the target; Source indexes the originating source column. type TargetColumn struct { + Expr *Expr Column Source int + // Computed marks an add-column column evaluated from Expr; Before marks a before-image column. + Computed bool + Before bool } // TargetTable is a source table after mapping rules. type TargetTable struct { - Schema string - Name string - SourceSchema string - SourceName string - Columns []TargetColumn - PK []string - SourceFamily Family + Schema string + Name string + SourceSchema string + SourceName string + KeyName string + Columns []TargetColumn + PK []string + SourceFamily Family + KeyUnique bool + HasBeforeImage bool } // Key is "schema.table" on the target. func (t TargetTable) Key() string { return t.Schema + "." + t.Name } -// Project selects and orders a source row's values for the target. +// ProjectCtx carries what computed and before-image columns need to fill in one row. +type ProjectCtx struct { + Time time.Time + Now time.Time + Op Op + Position string + Before Row + Full bool +} + +// Project selects and orders a full-load source row's values for the target. func (t TargetTable) Project(r Row) Row { + return t.ProjectWith(r, ProjectCtx{Full: true, Now: time.Now()}) +} + +// ProjectWith is Project for a change row: r is the row image, pc.Before feeds the before-image columns. +func (t TargetTable) ProjectWith(r Row, pc ProjectCtx) Row { if r == nil { return nil } + ec := &EvalCtx{Row: r, Op: pc.Op, Time: pc.Time, Now: pc.Now, Position: pc.Position, Full: pc.Full} out := make(Row, len(t.Columns)) + for i, c := range t.Columns { - if c.Source < len(r) { + switch { + case c.Computed: + if c.Expr != nil { + out[i] = c.DMS.Cast(c.Expr.Eval(ec)) + } + case c.Before: + if pc.Before != nil && c.Source < len(pc.Before) && !pc.Full { + out[i] = pc.Before[c.Source] + } + case c.Source < len(r): out[i] = r[c.Source] } } diff --git a/pkgs/dmsengine/mysql_source.go b/pkgs/dmsengine/mysql_source.go index 44150fd4c2..79a2f4340b 100644 --- a/pkgs/dmsengine/mysql_source.go +++ b/pkgs/dmsengine/mysql_source.go @@ -147,6 +147,11 @@ func (s *mysqlSource) attachKeys(ctx context.Context, tables []Table, index map[ } func (s *mysqlSource) Scan(ctx context.Context, t Table, fn func(Row) error) error { + return s.ScanWhere(ctx, t, nil, fn) +} + +// ScanWhere is Scan restricted by a selection-rule filter, evaluated by the database. +func (s *mysqlSource) ScanWhere(ctx context.Context, t Table, f *TableFilter, fn func(Row) error) error { names := make([]string, len(t.Columns)) for i, c := range t.Columns { names[i] = QuoteIdent(FamilyMySQL, c.Name) @@ -155,7 +160,13 @@ func (s *mysqlSource) Scan(ctx context.Context, t Table, fn func(Row) error) err q := stmt("SELECT ", strings.Join(names, ", "), " FROM ", QuoteIdent(FamilyMySQL, t.Schema), ".", QuoteIdent(FamilyMySQL, t.Name)) - rows, err := s.db.QueryContext(ctx, q) + var args []any + + if where, wargs := f.SQL(FamilyMySQL, 1); where != "" { + q, args = q+" WHERE "+where, wargs + } + + rows, err := s.db.QueryContext(ctx, q, args...) if err != nil { return err } diff --git a/pkgs/dmsengine/pg_source.go b/pkgs/dmsengine/pg_source.go index e7eeea4e5d..c46e090052 100644 --- a/pkgs/dmsengine/pg_source.go +++ b/pkgs/dmsengine/pg_source.go @@ -135,6 +135,11 @@ func (s *pgSource) Tables(ctx context.Context) ([]Table, error) { } func (s *pgSource) Scan(ctx context.Context, t Table, fn func(Row) error) error { + return s.ScanWhere(ctx, t, nil, fn) +} + +// ScanWhere is Scan restricted by a selection-rule filter, evaluated by the database. +func (s *pgSource) ScanWhere(ctx context.Context, t Table, f *TableFilter, fn func(Row) error) error { names := make([]string, len(t.Columns)) for i, c := range t.Columns { @@ -149,7 +154,13 @@ func (s *pgSource) Scan(ctx context.Context, t Table, fn func(Row) error) error q := stmt("SELECT ", strings.Join(names, ", "), " FROM ", QuoteIdent(FamilyPostgres, t.Schema), ".", QuoteIdent(FamilyPostgres, t.Name)) - rows, err := s.db.QueryContext(ctx, q) + var args []any + + if where, wargs := f.SQL(FamilyPostgres, 1); where != "" { + q, args = q+" WHERE "+where, wargs + } + + rows, err := s.db.QueryContext(ctx, q, args...) if err != nil { return err } @@ -334,7 +345,7 @@ func parseDecoded(lsn, data string, tables func(string, string) (Table, bool)) ( c := Change{Schema: t.Schema, Table: t.Name} switch opName { - case "INSERT": + case opWordInsert: c.Op, c.After = OpInsert, tupleRow(parseTuple(body), t) case "DELETE": c.Op, c.Before = OpDelete, tupleRow(parseTuple(body), t) diff --git a/pkgs/dmsengine/plan.go b/pkgs/dmsengine/plan.go new file mode 100644 index 0000000000..861c015343 --- /dev/null +++ b/pkgs/dmsengine/plan.go @@ -0,0 +1,232 @@ +package dmsengine + +import ( + "fmt" + "slices" + "strings" +) + +func (m Mappings) tableRules(t Table, target string) []Rule { + var out []Rule + + for _, r := range m.Rules { + if r.Type == ruleTransformation && r.Target == target && MatchPattern(r.Locator.Schema, t.Schema) && + MatchPattern(r.Locator.Table, t.Name) { + out = append(out, r) + } + } + + return out +} + +func columnMatches(r Rule, c Column) bool { + if !MatchPattern(r.Locator.Column, c.Name) { + return false + } + + if r.Locator.DataType == "" { + return true + } + + return strings.EqualFold(r.Locator.DataType, c.DMS.Name) || + (c.DMS.Name == "" && strings.EqualFold(r.Locator.DataType, dmsTypeOf(c.Native))) +} + +type colPlan struct { + out []TargetColumn + rename map[string]string + rules []Rule + include bool +} + +func (m Mappings) planColumns(t Table, out *TargetTable) error { + cp := colPlan{rules: m.tableRules(t, targetColumn), rename: make(map[string]string, len(t.Columns))} + + for _, r := range cp.rules { + cp.include = cp.include || r.Action == actIncludeColumn + } + + for i, c := range t.Columns { + tc, keep, err := cp.sourceColumn(i, c) + if err != nil { + return err + } + + if keep { + cp.rename[c.Name] = tc.Name + cp.out = append(cp.out, tc) + } + } + + if err := cp.addColumns(t); err != nil { + return err + } + + cp.beforeImages(t) + + if err := checkDuplicates(cp.out, out.Key()); err != nil { + return err + } + + out.Columns = cp.out + out.PK = make([]string, 0, len(t.PK)) + + for _, p := range t.PK { + if n, ok := cp.rename[p]; ok { + out.PK = append(out.PK, n) + } + } + + for _, c := range cp.out { + out.HasBeforeImage = out.HasBeforeImage || c.Before + } + + return nil +} + +func checkDuplicates(cols []TargetColumn, table string) error { + seen := make(map[string]bool, len(cols)) + + for _, c := range cols { + if seen[c.Name] { + return fmt.Errorf("%w: column %q appears twice on %s", ErrMapping, c.Name, table) + } + + seen[c.Name] = true + } + + return nil +} + +func (cp *colPlan) sourceColumn(i int, c Column) (TargetColumn, bool, error) { + name, included, removed := c.Name, !cp.include, false + tc := TargetColumn{Column: c, Source: i} + + for _, r := range cp.rules { + if !columnMatches(r, c) { + continue + } + + switch r.Action { + case actRemoveColumn: + removed = true + case actIncludeColumn: + included = true + case actChangeType: + dt, err := r.DataType.parse() + if err != nil { + return tc, false, err + } + + tc.Native, tc.Kind, tc.DMS = dt.Name, dt.Kind(), dt + case actAddColumn, actBeforeImage: + default: + name = applyNameAction(r.Action, name, r.Value, r.Old) + } + } + + tc.Name = name + + return tc, included && !removed, nil +} + +func (cp *colPlan) addColumns(t Table) error { + for _, r := range cp.rules { + if r.Action != actAddColumn { + continue + } + + dt, err := r.DataType.parse() + if err != nil { + return err + } + + col := TargetColumn{Column: dt.Column(r.Value), Source: -1, Computed: true} + + if strings.TrimSpace(r.Expression) != "" { + e, eerr := CompileExpr(r.Expression, t) + if eerr != nil { + return fmt.Errorf("add-column %q on %s: %w", r.Value, t.Key(), eerr) + } + + col.Expr = e + } + + cp.out = append(cp.out, col) + } + + return nil +} + +func isLOB(c Column) bool { + switch c.DMS.Name { + case tBlob, "clob", tNclob: + return true + } + + return slices.Contains([]string{ + tBlob, tTinyblob, tMediumblob, "longblob", "text", "tinytext", "mediumtext", "longtext", "json", "jsonb", + "bytea", + }, parseNative(c.Native).base) +} + +func (cp *colPlan) beforeImages(t Table) { + for _, r := range cp.rules { + if r.Action != actBeforeImage { + continue + } + + def := r.imageDef() + + for i, c := range t.Columns { + if _, kept := cp.rename[c.Name]; !kept || !imageWanted(def.Filter, t, c) { + continue + } + + col := c + col.Name, col.Nullable = *def.Prefix+c.Name+def.Suffix, true + cp.out = append(cp.out, TargetColumn{Column: col, Source: i, Before: true}) + } + } +} + +func imageWanted(filter string, t Table, c Column) bool { + if isLOB(c) { + return false + } + + return filter != imageFilterPK || slices.Contains(t.PK, c.Name) +} + +func (m Mappings) definePrimaryKey(t Table, out *TargetTable) error { + for _, r := range m.tableRules(t, targetTable) { + if r.Action != actDefinePK || r.Locator.Schema != t.Schema || r.Locator.Table != t.Name { + continue + } + + cols := make([]string, 0, len(r.PrimaryKey.Columns)) + + for _, name := range r.PrimaryKey.Columns { + target, ok := out.columnNamed(t, name) + if !ok { + return fmt.Errorf("%w: primary-key-def column %q does not exist on %s", ErrMapping, name, t.Key()) + } + + cols = append(cols, target) + } + + out.PK, out.KeyName, out.KeyUnique = cols, r.PrimaryKey.Name, r.PrimaryKey.Origin == pkOriginUnique + } + + return nil +} + +func (tt TargetTable) columnNamed(t Table, name string) (string, bool) { + for _, c := range tt.Columns { + if c.Name == name || (!c.Computed && !c.Before && t.Columns[c.Source].Name == name) { + return c.Name, true + } + } + + return "", false +} diff --git a/pkgs/dmsengine/rules_test.go b/pkgs/dmsengine/rules_test.go new file mode 100644 index 0000000000..9b2b8214ae --- /dev/null +++ b/pkgs/dmsengine/rules_test.go @@ -0,0 +1,283 @@ +package dmsengine_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/dmsengine" +) + +func employeeTable() dmsengine.Table { + return dmsengine.Table{Schema: "test", Name: "employee", PK: []string{"empid"}, Columns: []dmsengine.Column{ + {Name: "empid", Native: "int", Kind: dmsengine.KindInt}, + {Name: "dept", Native: "varchar(20)", Kind: dmsengine.KindString, Nullable: true}, + {Name: "salary", Native: "decimal(10,2)", Kind: dmsengine.KindDecimal, Nullable: true}, + {Name: "started", Native: "date", Kind: dmsengine.KindTime, Nullable: true}, + {Name: "notes", Native: "text", Kind: dmsengine.KindString, Nullable: true}, + }} +} + +func filterRule(filters string) string { + return `{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1", + "object-locator":{"schema-name":"test","table-name":"employee"},"rule-action":"include","filters":[` + + filters + `]}]}` +} + +func srcFilter(col, conds string) string { + return `{"filter-type":"source","column-name":"` + col + `","filter-conditions":[` + conds + `]}` +} + +func TestSelectionFilterMatch(t *testing.T) { + t.Parallel() + + row := func(id, dept, salary, started string) dmsengine.Row { + r := dmsengine.Row{id, dept, salary, started, nil} + if dept == "" { + r[1] = nil + } + + return r + } + + tests := []struct { + name string + filters string + rows []dmsengine.Row + want []bool + }{ + { + name: "gte", + filters: srcFilter("empid", `{"filter-operator":"gte","value":"100"}`), + rows: []dmsengine.Row{ + row("99", "a", "1", "2020-01-01"), + row("100", "a", "1", "2020-01-01"), + row("1000", "a", "1", "2020-01-01"), + }, + want: []bool{false, true, true}, + }, + { + name: "or of conditions in one filter", + filters: srcFilter( + "empid", + `{"filter-operator":"lte","value":"10"},{"filter-operator":"between","start-value":"50",`+ + `"end-value":"75"},{"filter-operator":"gte","value":"100"}`, + ), + rows: []dmsengine.Row{ + row("5", "a", "1", "2020-01-01"), + row("20", "a", "1", "2020-01-01"), + row("75", "a", "1", "2020-01-01"), + row("120", "a", "1", "2020-01-01"), + }, + want: []bool{true, false, true, true}, + }, + { + name: "and of filters", + filters: srcFilter("empid", `{"filter-operator":"ste","value":"100"}`) + "," + + srcFilter("dept", `{"filter-operator":"eq","value":"tech"}`), + rows: []dmsengine.Row{ + row("50", "tech", "1", "2020-01-01"), + row("50", "ops", "1", "2020-01-01"), + row("500", "tech", "1", "2020-01-01"), + }, + want: []bool{true, false, false}, + }, + { + name: "null and notnull", + filters: srcFilter("dept", `{"filter-operator":"null"}`), + rows: []dmsengine.Row{row("1", "", "1", "2020-01-01"), row("2", "ops", "1", "2020-01-01")}, + want: []bool{true, false}, + }, + { + name: "notbetween skips null", + filters: srcFilter("salary", `{"filter-operator":"notbetween","start-value":"50","end-value":"75"}`), + rows: []dmsengine.Row{ + row("1", "a", "49.5", "2020-01-01"), + row("2", "a", "60", "2020-01-01"), + {"3", "a", nil, nil, nil}, + }, + want: []bool{true, false, false}, + }, + { + name: "noteq numeric compare not text", + filters: srcFilter("empid", `{"filter-operator":"noteq","value":"9"}`), + rows: []dmsengine.Row{row("9", "a", "1", "2020-01-01"), row("10", "a", "1", "2020-01-01")}, + want: []bool{false, true}, + }, + { + name: "date", + filters: srcFilter("started", `{"filter-operator":"gte","value":"2002-01-01"}`), + rows: []dmsengine.Row{row("1", "a", "1", "2001-12-31"), row("2", "a", "1", "2002-01-01")}, + want: []bool{false, true}, + }, + { + name: "negative with null matches nothing", + filters: srcFilter("empid", `{"filter-operator":"noteq","value":"1"},{"filter-operator":"null"}`), + rows: []dmsengine.Row{row("2", "a", "1", "2020-01-01")}, + want: []bool{false}, + }, + { + name: "inclusive negative and null drop null rows", + filters: srcFilter( + "dept", + `{"filter-operator":"eq","value":"a"},{"filter-operator":"noteq","value":"b"},`+ + `{"filter-operator":"null"}`, + ), + rows: []dmsengine.Row{row("1", "c", "1", "2020-01-01"), row("2", "", "1", "2020-01-01")}, + want: []bool{true, false}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(filterRule(tt.filters)) + require.NoError(t, err) + + sel, err := m.SelectFiltered([]dmsengine.Table{employeeTable()}) + require.NoError(t, err) + require.Len(t, sel, 1) + + for i, r := range tt.rows { + assert.Equal(t, tt.want[i], sel[0].Filter.Match(r), "row %d", i) + } + }) + } +} + +func TestSelectionFilterSQL(t *testing.T) { + t.Parallel() + + filters := srcFilter( + "empid", + `{"filter-operator":"lte","value":"10"},{"filter-operator":"between","start-value":"50","end-value":"75"}`, + ) + + "," + srcFilter( + "dept", + `{"filter-operator":"notnull"}`, + ) + + m, err := dmsengine.ParseMappings(filterRule(filters)) + require.NoError(t, err) + + sel, err := m.SelectFiltered([]dmsengine.Table{employeeTable()}) + require.NoError(t, err) + + where, args := sel[0].Filter.SQL(dmsengine.FamilyPostgres, 1) + assert.Equal(t, `("empid" <= $1 OR "empid" BETWEEN $2 AND $3) AND ("dept" IS NOT NULL)`, where) + assert.Equal(t, []any{"10", "50", "75"}, args) + + where, _ = sel[0].Filter.SQL(dmsengine.FamilyMySQL, 1) + assert.Equal(t, "(`empid` <= ? OR `empid` BETWEEN ? AND ?) AND (`dept` IS NOT NULL)", where) +} + +func TestParseMappingsRuleErrors(t *testing.T) { + t.Parallel() + + sel := `{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"%",` + + `"table-name":"%"},"rule-action":"include"}` + tests := []struct { + name string + rule string + }{ + { + "filter on wildcard table", + `{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"test",` + + `"table-name":"emp%"},"rule-action":"include","filters":[` + srcFilter( + "a", + `{"filter-operator":"eq","value":"1"}`, + ) + `]}`, + }, + { + "bad operator", + `{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"test",` + + `"table-name":"employee"},"rule-action":"include","filters":[` + srcFilter( + "a", + `{"filter-operator":"like","value":"1"}`, + ) + `]}`, + }, + { + "add-column without data-type", + `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"add-column",` + + `"rule-target":"column","object-locator":{"schema-name":"%","table-name":"%"},"value":"x"}`, + }, + { + "unsupported data type", + `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"change-data-type",` + + `"rule-target":"column","object-locator":{"schema-name":"%","table-name":"%","column-name":"%"},` + + `"data-type":{"type":"map"}}`, + }, + { + "define-primary-key wildcard", + `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"define-primary-key",` + + `"rule-target":"table","object-locator":{"schema-name":"%","table-name":"t"},` + + `"primary-key-def":{"name":"pk","columns":["a"]}}`, + }, + { + "before image both empty", + `{"rule-type":"transformation","rule-id":"2","rule-name":"2",` + + `"rule-action":"add-before-image-columns","rule-target":"column",` + + `"object-locator":{"schema-name":"%","table-name":"%"},` + + `"before-image-def":{"column-prefix":"","column-suffix":""}}`, + }, + { + "data masking", + `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"data-masking-hash-mask",` + + `"rule-target":"column","object-locator":{"schema-name":"%","table-name":"%","column-name":"%"}}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := dmsengine.ParseMappings(`{"rules":[` + sel + `,` + tt.rule + `]}`) + require.ErrorIs(t, err, dmsengine.ErrMapping) + }) + } +} + +func TestMatchPatternWildcards(t *testing.T) { + t.Parallel() + + tests := []struct { + pattern, s string + want bool + }{ + {"emp_", "empx", true}, + {"emp_", "emp", false}, + {"emp_", "empxx", false}, + {"e_p%", "excess", false}, + {"e_p%", "exp_anything", true}, + {"%", "", true}, + {"a.c", "abc", false}, + {"a%c", "a\nc", true}, + } + + for _, tt := range tests { + t.Run(tt.pattern+"/"+tt.s, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, dmsengine.MatchPattern(tt.pattern, tt.s)) + }) + } +} + +func TestSelectLoadOrderAndExplicit(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(`{"rules":[ + {"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"app", + "table-name":"items"},"rule-action":"include","load-order":"1"}, + {"rule-type":"selection","rule-id":"2","rule-name":"2","object-locator":{"schema-name":"app", + "table-name":"orders"},"rule-action":"explicit","load-order":"2"}]}`) + require.NoError(t, err) + + sel, err := m.SelectFiltered(twoTables().tables) + require.NoError(t, err) + require.Len(t, sel, 2) + assert.Equal(t, "orders", sel[0].Table.Name) + assert.Equal(t, "items", sel[1].Table.Name) +} diff --git a/pkgs/dmsengine/s3_source.go b/pkgs/dmsengine/s3_source.go new file mode 100644 index 0000000000..cb13998777 --- /dev/null +++ b/pkgs/dmsengine/s3_source.go @@ -0,0 +1,518 @@ +package dmsengine + +import ( + "bytes" + "context" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strconv" + "strings" + "time" +) + +// Errors reported by the S3 source. +var ( + ErrS3Source = errors.New("invalid S3 source") + errS3SourceBucket = fmt.Errorf("%w: S3Settings.BucketName is required", ErrS3Source) + errS3SourceDef = fmt.Errorf("%w: an ExternalTableDefinition is required", ErrS3Source) + errS3SourceCDCPath = fmt.Errorf("%w: S3Settings.CdcPath is required to capture changes", ErrS3Source) + errS3SourceParquet = fmt.Errorf("%w: Parquet source files are not supported", ErrS3Source) + errS3SourceDelim = fmt.Errorf("%w: CsvDelimiter must be a single character", ErrS3Source) + errS3SourceNoStore = fmt.Errorf("%w: %w: S3", ErrS3Source, ErrNotConfigured) +) + +const ( + s3SourceLoadPrefix = "load" + s3SourceCSVSuffix = ".csv" + s3SourceMinCDCCells = 3 +) + +// jsonScalar reads a JSON string, number or boolean as its text; the external table definition quotes everything. +type jsonScalar string + +func (s *jsonScalar) UnmarshalJSON(b []byte) error { + var v any + if err := json.Unmarshal(b, &v); err != nil { + return err + } + + switch x := v.(type) { + case nil: + *s = "" + case string: + *s = jsonScalar(x) + default: + *s = jsonScalar(strings.Trim(string(b), `"`)) + } + + return nil +} + +func (s jsonScalar) bool() bool { return strings.EqualFold(strings.TrimSpace(string(s)), "true") } + +func (s jsonScalar) int() int { + n, _ := strconv.Atoi(strings.TrimSpace(string(s))) + + return n +} + +type s3ColumnDef struct { + Name string `json:"ColumnName"` + Type string `json:"ColumnType"` + Length jsonScalar `json:"ColumnLength"` + Precision jsonScalar `json:"ColumnPrecision"` + Scale jsonScalar `json:"ColumnScale"` + Nullable jsonScalar `json:"ColumnNullable"` + IsPK jsonScalar `json:"ColumnIsPk"` + DateFormat string `json:"ColumnDateFormat"` +} + +type s3TableDef struct { + Name string `json:"TableName"` + Path string `json:"TablePath"` + Owner string `json:"TableOwner"` + ColumnsTotal jsonScalar `json:"TableColumnsTotal"` + Columns []s3ColumnDef `json:"TableColumns"` +} + +type s3SourceOptions struct { + Rfc4180 *bool `json:"Rfc4180"` + BucketName string `json:"BucketName"` + BucketFolder string `json:"BucketFolder"` + CdcPath string `json:"CdcPath"` + CsvDelimiter string `json:"CsvDelimiter"` + CsvRowDelimiter string `json:"CsvRowDelimiter"` + CsvNullValue string `json:"CsvNullValue"` + DataFormat string `json:"DataFormat"` + ExternalTableDef string `json:"ExternalTableDefinition"` + IgnoreHeaderRows int `json:"IgnoreHeaderRows"` +} + +type s3Source struct { + store ObjectStore + paths map[string]string + dates map[string][]string + opts s3SourceOptions + tables []Table + poll time.Duration + delim rune +} + +func newS3Source(ep EndpointConfig, store ObjectStore) (Source, error) { + if store == nil { + return nil, errS3SourceNoStore + } + + var o s3SourceOptions + if ep.S3Settings != "" { + if err := json.Unmarshal([]byte(ep.S3Settings), &o); err != nil { + return nil, fmt.Errorf("%w: S3Settings: %w", ErrS3Source, err) + } + } + + if o.BucketName == "" { + return nil, errS3SourceBucket + } + + if strings.EqualFold(o.DataFormat, "parquet") { + return nil, errS3SourceParquet + } + + s := &s3Source{store: store, opts: o, poll: ep.PollInterval, delim: ',', paths: map[string]string{}, + dates: map[string][]string{}} + + if o.CsvDelimiter != "" { + rs := []rune(o.CsvDelimiter) + if len(rs) != 1 { + return nil, errS3SourceDelim + } + + s.delim = rs[0] + } + + def := ep.ExternalTableDefinition + if def == "" { + def = o.ExternalTableDef + } + + if err := s.loadDefinition(def); err != nil { + return nil, err + } + + if s.poll <= 0 { + s.poll = defaultPGPoll + } + + return s, nil +} + +func (s *s3Source) loadDefinition(def string) error { + if strings.TrimSpace(def) == "" { + return errS3SourceDef + } + + var doc struct { + Tables []s3TableDef `json:"Tables"` + } + + if err := json.Unmarshal([]byte(def), &doc); err != nil { + return fmt.Errorf("%w: ExternalTableDefinition: %w", ErrS3Source, err) + } + + for _, td := range doc.Tables { + t, err := td.table() + if err != nil { + return err + } + + s.tables = append(s.tables, t) + s.paths[t.Key()] = td.Path + + fm := make([]string, len(td.Columns)) + for i, c := range td.Columns { + fm[i] = c.DateFormat + } + + s.dates[t.Key()] = fm + } + + return nil +} + +func (td s3TableDef) table() (Table, error) { + if td.Name == "" || len(td.Columns) == 0 { + return Table{}, fmt.Errorf("%w: a table needs TableName and TableColumns", ErrS3Source) + } + + if td.ColumnsTotal != "" && td.ColumnsTotal.int() != len(td.Columns) { + return Table{}, fmt.Errorf("%w: %s: TableColumnsTotal %s does not match its %d columns", + ErrS3Source, td.Name, td.ColumnsTotal, len(td.Columns)) + } + + t := Table{Schema: td.Owner, Name: td.Name} + + for _, c := range td.Columns { + kind := strings.ToLower(c.Type) + if kind == "byte" { + kind = tBytes + } + + dt, err := ParseDMSType(kind) + if err != nil { + return t, fmt.Errorf("%w: %s.%s: %w", ErrS3Source, td.Name, c.Name, err) + } + + dt.Length, dt.Precision, dt.Scale = c.Length.int(), c.Precision.int(), c.Scale.int() + col := dt.Column(c.Name) + col.Nullable = c.Nullable.bool() + t.Columns = append(t.Columns, col) + + if c.IsPK.bool() { + t.PK = append(t.PK, c.Name) + } + } + + return t, nil +} + +func (s *s3Source) Ping(ctx context.Context) error { + return s.store.CheckBucket(ctx, s.opts.BucketName) +} + +func (s *s3Source) Close() error { return nil } + +func (s *s3Source) Release(context.Context) error { return nil } + +func (s *s3Source) Tables(context.Context) ([]Table, error) { return s.tables, nil } + +func (s *s3Source) Position(context.Context) (string, error) { return "", nil } + +func (s *s3Source) key(parts ...string) string { + all := []string{} + if f := strings.Trim(s.opts.BucketFolder, "/"); f != "" { + all = append(all, f) + } + + for _, p := range parts { + if p = strings.Trim(p, "/"); p != "" { + all = append(all, p) + } + } + + return strings.Join(all, "/") +} + +func (s *s3Source) tablePath(t Table) string { + if p := s.paths[t.Key()]; p != "" { + return p + } + + return t.Schema + "/" + t.Name +} + +func (s *s3Source) Scan(ctx context.Context, t Table, fn func(Row) error) error { + dir := s.key(s.tablePath(t)) + "/" + + keys, err := s.store.ListObjects(ctx, s.opts.BucketName, dir) + if err != nil { + return err + } + + for _, k := range keys { + base := strings.ToLower(strings.TrimPrefix(k, dir)) + if strings.Contains(base, "/") || !strings.HasPrefix(base, s3SourceLoadPrefix) || + !strings.HasSuffix(base, s3SourceCSVSuffix) { + continue + } + + recs, rerr := s.records(ctx, k) + if rerr != nil { + return rerr + } + + for i, rec := range recs { + row, rowErr := s.fullLoadRow(t, rec) + if rowErr != nil { + return fmt.Errorf("%s record %d: %w", k, i+1, rowErr) + } + + if err = fn(row); err != nil { + return err + } + } + } + + return nil +} + +func (s *s3Source) records(ctx context.Context, key string) ([][]string, error) { + body, err := s.store.GetObject(ctx, s.opts.BucketName, key) + if err != nil { + return nil, err + } + + if rd := s.opts.CsvRowDelimiter; rd != "" && rd != "\n" && rd != "\r\n" { + body = bytes.ReplaceAll(body, []byte(rd), []byte("\n")) + } + + r := csv.NewReader(bytes.NewReader(body)) + r.Comma, r.FieldsPerRecord, r.LazyQuotes = s.delim, -1, s.opts.Rfc4180 != nil && !*s.opts.Rfc4180 + + var out [][]string + + for { + rec, rerr := r.Read() + if errors.Is(rerr, io.EOF) { + break + } + + if rerr != nil { + return nil, fmt.Errorf("%s: %w", key, rerr) + } + + out = append(out, rec) + } + + return out[min(s.opts.IgnoreHeaderRows, len(out)):], nil +} + +func isInsertOp(s string) bool { + return strings.EqualFold(s, "I") || strings.EqualFold(s, opWordInsert) +} + +func (s *s3Source) fullLoadRow(t Table, rec []string) (Row, error) { + if len(rec) == len(t.Columns)+1 && isInsertOp(rec[0]) { + rec = rec[1:] + } + + return s.row(t, rec) +} + +func (s *s3Source) row(t Table, rec []string) (Row, error) { + if len(rec) != len(t.Columns) { + return nil, fmt.Errorf( + "%w: %s has %d columns, the record has %d", + ErrS3Source, + t.Key(), + len(t.Columns), + len(rec), + ) + } + + fm := s.dates[t.Key()] + row := make(Row, len(rec)) + + for i, f := range rec { + switch { + case f == s.opts.CsvNullValue: + row[i] = nil + case t.Columns[i].Kind == KindBytes: + row[i] = []byte(f) + default: + row[i] = normalizeDate(f, fm[i], t.Columns[i].DMS.Name) + } + } + + return row, nil +} + +func dateTokens() *strings.Replacer { + return strings.NewReplacer( + "YYYY", + "2006", + "MM", + "01", + "dd", + "02", + "HH", + "15", + "mm", + "04", + "ss", + "05", + ".F", + ".999999999", + ) +} + +// normalizeDate rewrites a ColumnDateFormat value to the ISO form databases accept. +func normalizeDate(v, format, dmsType string) string { + if format == "" { + return v + } + + t, err := time.Parse(dateTokens().Replace(format), v) + if err != nil { + return v + } + + switch dmsType { + case tDate: + return t.Format("2006-01-02") + case tTime: + return t.Format("15:04:05") + } + + return t.Format("2006-01-02 15:04:05.999999999") +} + +func (s *s3Source) Stream( + ctx context.Context, start StreamStart, tables func(string, string) (Table, bool), fn func(Change) error, +) error { + if s.opts.CdcPath == "" { + return errS3SourceCDCPath + } + + last := start.Position + tick := time.NewTicker(s.poll) + + defer tick.Stop() + + for { + var err error + if last, err = s.drain(ctx, last, tables, fn); err != nil { + if ctx.Err() != nil { + return nil + } + + return err + } + + select { + case <-ctx.Done(): + return nil + case <-tick.C: + } + } +} + +// drain applies the CDC files that sort after last, in order, and returns the last key applied. +func (s *s3Source) drain( + ctx context.Context, last string, tables func(string, string) (Table, bool), fn func(Change) error, +) (string, error) { + dir := s.key(s.opts.CdcPath) + "/" + + keys, err := s.store.ListObjects(ctx, s.opts.BucketName, dir) + if err != nil { + return last, err + } + + for _, k := range keys { + base := strings.ToLower(path.Base(k)) + if k <= last || strings.Contains(strings.TrimPrefix(k, dir), "/") || + !strings.HasSuffix(base, s3SourceCSVSuffix) { + continue + } + + if err = s.applyFile(ctx, k, tables, fn); err != nil { + return last, err + } + + last = k + } + + return last, nil +} + +func (s *s3Source) applyFile( + ctx context.Context, key string, tables func(string, string) (Table, bool), fn func(Change) error, +) error { + recs, err := s.records(ctx, key) + if err != nil { + return err + } + + for i, rec := range recs { + c, ok, cerr := s.change(rec, tables) + if cerr != nil { + return fmt.Errorf("%s record %d: %w", key, i+1, cerr) + } + + if !ok { + continue + } + + c.Position, c.Time = key, time.Now().UTC() + if err = fn(c); err != nil { + return err + } + } + + return fn(Change{Commit: true, Position: key, Time: time.Now().UTC()}) +} + +// change reads one CDC record: operation, table name, schema name, then the row. +func (s *s3Source) change(rec []string, tables func(string, string) (Table, bool)) (Change, bool, error) { + if len(rec) < s3SourceMinCDCCells { + return Change{}, false, fmt.Errorf("%w: a CDC record needs operation, table, schema and data", ErrS3Source) + } + + t, ok := tables(rec[2], rec[1]) + if !ok { + return Change{}, false, nil + } + + row, err := s.row(t, rec[s3SourceMinCDCCells:]) + if err != nil { + return Change{}, false, err + } + + c := Change{Schema: t.Schema, Table: t.Name} + + switch strings.ToUpper(rec[0]) { + case "I", opWordInsert: + c.Op, c.After = OpInsert, row + case "U", "UPDATE": + c.Op, c.After = OpUpdate, row + case "D", "DELETE": + c.Op, c.Before = OpDelete, row + default: + return Change{}, false, fmt.Errorf("%w: unknown operation %q", ErrS3Source, rec[0]) + } + + return c, true, nil +} diff --git a/pkgs/dmsengine/s3_source_test.go b/pkgs/dmsengine/s3_source_test.go new file mode 100644 index 0000000000..3f49cd3ca1 --- /dev/null +++ b/pkgs/dmsengine/s3_source_test.go @@ -0,0 +1,307 @@ +package dmsengine_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/dmsengine" +) + +const employeeDefinition = `{ + "TableCount": "1", + "Tables": [{ + "TableName": "employee", "TablePath": "hr/employee/", "TableOwner": "hr", + "TableColumns": [ + {"ColumnName": "Id", "ColumnType": "INT8", "ColumnNullable": "false", "ColumnIsPk": "true"}, + {"ColumnName": "LastName", "ColumnType": "STRING", "ColumnLength": "20", "ColumnNullable": "true"}, + {"ColumnName": "HireDate", "ColumnType": "DATETIME", "ColumnScale": "3", "ColumnNullable": "true"}, + {"ColumnName": "Rate", "ColumnType": "NUMERIC", "ColumnPrecision": "5", "ColumnScale": "2", + "ColumnNullable": "true"} + ], + "TableColumnsTotal": "4" + }] +}` + +func s3SourceEndpoint(settings string) dmsengine.EndpointConfig { + return dmsengine.EndpointConfig{ + Engine: "s3", S3Settings: settings, ExternalTableDefinition: employeeDefinition, + PollInterval: time.Millisecond, + } +} + +func openS3Source(t *testing.T, objs *memObjects, settings string) dmsengine.Source { + t.Helper() + + src, err := dmsengine.NewOpener(dmsengine.Stores{Objects: objs}).OpenSource(t.Context(), s3SourceEndpoint(settings)) + require.NoError(t, err) + + return src +} + +func TestS3SourceTablesAndScan(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + settings string + objects map[string]string + want []dmsengine.Row + }{ + { + name: "plain files under the table path", + settings: `{"BucketName":"b"}`, + objects: map[string]string{ + "b/hr/employee/LOAD001.csv": "101,Smith,2014-06-04 10:00:00,12.50\n102,,2015-10-08 00:00:00,7\n", + "b/hr/employee/LOAD002.csv": "103,Jones,2017-03-13 00:00:00,\n", + "b/hr/employee/other.txt": "ignored", + "b/hr/department/LOAD1.csv": "ignored", + }, + want: []dmsengine.Row{ + {"101", "Smith", "2014-06-04 10:00:00", "12.50"}, + {"102", nil, "2015-10-08 00:00:00", "7"}, + {"103", "Jones", "2017-03-13 00:00:00", nil}, + }, + }, + { + name: "bucket folder, custom delimiters, header and null marker", + settings: `{"BucketName":"b","BucketFolder":"src","CsvDelimiter":"|","CsvRowDelimiter":";", + "IgnoreHeaderRows":1,"CsvNullValue":"\\N"}`, + objects: map[string]string{ + "b/src/hr/employee/LOAD001.csv": "Id|LastName|HireDate|Rate;101|\\N|2014-06-04 10:00:00|1.5;", + }, + want: []dmsengine.Row{{"101", nil, "2014-06-04 10:00:00", "1.5"}}, + }, + { + name: "leading operation column from a DMS S3 target", + settings: `{"BucketName":"b"}`, + objects: map[string]string{"b/hr/employee/LOAD001.csv": "I,101,\"Smith, Jr\",2014-06-04 10:00:00,1\n"}, + want: []dmsengine.Row{{"101", "Smith, Jr", "2014-06-04 10:00:00", "1"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + src := openS3Source(t, &memObjects{objs: tt.objects}, tt.settings) + tables, err := src.Tables(t.Context()) + require.NoError(t, err) + require.Len(t, tables, 1) + + tbl := tables[0] + assert.Equal(t, "hr.employee", tbl.Key()) + assert.Equal(t, []string{"Id"}, tbl.PK) + assert.Equal(t, "int8", tbl.Columns[0].DMS.Name) + assert.Equal(t, 20, tbl.Columns[1].DMS.Length) + assert.Equal(t, 2, tbl.Columns[3].DMS.Scale) + assert.False(t, tbl.Columns[0].Nullable) + + var got []dmsengine.Row + + require.NoError(t, src.Scan(t.Context(), tbl, func(r dmsengine.Row) error { + got = append(got, r) + + return nil + })) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestS3SourceCDCFiles(t *testing.T) { + t.Parallel() + + objs := &memObjects{objs: map[string]string{ + "b/chg/CDC00001.csv": "INSERT,employee,hr,101,Smith,2014-06-04 10:00:00,1\n" + + "U,employee,hr,101,Smith,2015-10-08 00:00:00,2\n", + "b/chg/CDC00003.csv": "DELETE,employee,hr,101,Smith,2015-10-08 00:00:00,2\nI,other,hr,1,x,,\n", + }} + src := openS3Source(t, objs, `{"BucketName":"b","CdcPath":"chg"}`) + + tables, err := src.Tables(t.Context()) + require.NoError(t, err) + + lookup := func(schema, name string) (dmsengine.Table, bool) { + if schema == "hr" && name == "employee" { + return tables[0], true + } + + return dmsengine.Table{}, false + } + + type seen struct { + op dmsengine.Op + pos string + row dmsengine.Row + } + + ctx, cancel := context.WithCancel(t.Context()) + got := make(chan seen, 16) + done := make(chan error, 1) + + go func() { + done <- src.Stream(ctx, dmsengine.StreamStart{}, lookup, func(c dmsengine.Change) error { + row := c.After + if c.Op == dmsengine.OpDelete { + row = c.Before + } + + got <- seen{op: c.Op, row: row, pos: c.Position} + + return nil + }) + }() + + want := []seen{ + { + op: dmsengine.OpInsert, + row: dmsengine.Row{"101", "Smith", "2014-06-04 10:00:00", "1"}, + pos: "chg/CDC00001.csv", + }, + { + op: dmsengine.OpUpdate, + row: dmsengine.Row{"101", "Smith", "2015-10-08 00:00:00", "2"}, + pos: "chg/CDC00001.csv", + }, + {pos: "chg/CDC00001.csv"}, + { + op: dmsengine.OpDelete, + row: dmsengine.Row{"101", "Smith", "2015-10-08 00:00:00", "2"}, + pos: "chg/CDC00003.csv", + }, + {pos: "chg/CDC00003.csv"}, + } + + for _, w := range want { + assert.Equal(t, w, <-got) + } + + // A file that sorts before the last one applied is skipped, one that sorts after is picked up. + require.NoError(t, objs.PutObject(t.Context(), "b", "chg/CDC00002.csv", []byte("I,employee,hr,9,late,,\n"))) + require.NoError(t, objs.PutObject(t.Context(), "b", "chg/CDC00004.csv", []byte("I,employee,hr,4,fresh,,\n"))) + + assert.Equal( + t, + seen{op: dmsengine.OpInsert, row: dmsengine.Row{"4", "fresh", nil, nil}, pos: "chg/CDC00004.csv"}, + <-got, + ) + assert.Equal(t, seen{pos: "chg/CDC00004.csv"}, <-got) + + cancel() + require.NoError(t, <-done) +} + +func TestS3SourceOpenErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + store dmsengine.ObjectStore + name string + ep dmsengine.EndpointConfig + }{ + { + name: "no bucket", + ep: dmsengine.EndpointConfig{ + Engine: "s3", + S3Settings: `{}`, + ExternalTableDefinition: employeeDefinition, + }, + store: &memObjects{}, + }, + { + name: "no definition", + ep: dmsengine.EndpointConfig{Engine: "s3", S3Settings: `{"BucketName":"b"}`}, + store: &memObjects{}, + }, + {name: "parquet", ep: dmsengine.EndpointConfig{ + Engine: "s3", + S3Settings: `{"BucketName":"b","DataFormat":"parquet"}`, + ExternalTableDefinition: employeeDefinition, + }, store: &memObjects{}}, + {name: "bad type", ep: dmsengine.EndpointConfig{ + Engine: "s3", + S3Settings: `{"BucketName":"b"}`, + ExternalTableDefinition: `{"Tables":[{"TableName":"t","TableOwner":"o", + "TableColumns":[{"ColumnName":"c","ColumnType":"MAP"}]}]}`, + }, store: &memObjects{}}, + {name: "column total mismatch", ep: dmsengine.EndpointConfig{ + Engine: "s3", + S3Settings: `{"BucketName":"b"}`, + ExternalTableDefinition: `{"Tables":[{"TableName":"t","TableOwner":"o","TableColumnsTotal":"2", + "TableColumns":[{"ColumnName":"c","ColumnType":"INT4"}]}]}`, + }, store: &memObjects{}}, + {name: "no store", ep: dmsengine.EndpointConfig{ + Engine: "s3", + S3Settings: `{"BucketName":"b"}`, + ExternalTableDefinition: employeeDefinition, + }, store: nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := dmsengine.NewOpener(dmsengine.Stores{Objects: tt.store}).OpenSource(t.Context(), tt.ep) + require.Error(t, err) + }) + } +} + +func TestS3SourceStreamNeedsCdcPath(t *testing.T) { + t.Parallel() + + src := openS3Source(t, &memObjects{}, `{"BucketName":"b"}`) + + err := src.Stream(t.Context(), dmsengine.StreamStart{}, nil, func(dmsengine.Change) error { return nil }) + require.ErrorIs(t, err, dmsengine.ErrS3Source) +} + +func TestApplySecret(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + secret string + want dmsengine.EndpointConfig + wantErr bool + }{ + { + name: "numeric port", + secret: `{"username":"admin","password":"pw","host":"db.example","port":3306,"engine":"mysql","dbname":"ignored"}`, + want: dmsengine.EndpointConfig{User: "admin", Password: "pw", Server: "db.example", Port: 3306}, + }, + { + name: "string port as the guide shows", + secret: `{"username":"admin","password":"pw","port":"8190","host":"db.example"}`, + want: dmsengine.EndpointConfig{User: "admin", Password: "pw", Server: "db.example", Port: 8190}, + }, + {name: "not json", secret: `plain`, wantErr: true}, + {name: "no host", secret: `{"username":"a","password":"b","port":1}`, wantErr: true}, + {name: "no port", secret: `{"username":"a","password":"b","host":"h"}`, wantErr: true}, + {name: "bad port", secret: `{"username":"a","password":"b","host":"h","port":"x"}`, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ep := dmsengine.EndpointConfig{Database: "appdb"} + err := dmsengine.ApplySecret(&ep, tt.secret) + + if tt.wantErr { + require.ErrorIs(t, err, dmsengine.ErrSecret) + + return + } + + require.NoError(t, err) + + tt.want.Database = "appdb" + assert.Equal(t, tt.want, ep) + }) + } +} diff --git a/pkgs/dmsengine/secret.go b/pkgs/dmsengine/secret.go new file mode 100644 index 0000000000..32624a9345 --- /dev/null +++ b/pkgs/dmsengine/secret.go @@ -0,0 +1,57 @@ +package dmsengine + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" +) + +// ErrSecret reports a Secrets Manager secret that cannot authenticate an endpoint. +var ErrSecret = errors.New("invalid endpoint secret") + +// ApplySecret fills the endpoint's credentials from a Secrets Manager secret string. DMS reads username, password, +// host and port; other members are ignored ("Using secrets to access AWS Database Migration Service endpoints"). +func ApplySecret(ep *EndpointConfig, secret string) error { + var raw struct { + Username string `json:"username"` + Password string `json:"password"` + Host string `json:"host"` + Port json.RawMessage `json:"port"` + } + + if err := json.Unmarshal([]byte(secret), &raw); err != nil { + return fmt.Errorf("%w: the secret is not a JSON object: %w", ErrSecret, err) + } + + if raw.Username == "" || raw.Host == "" { + return fmt.Errorf("%w: the secret needs username, password, host and port members", ErrSecret) + } + + port, err := secretPort(raw.Port) + if err != nil { + return err + } + + ep.User, ep.Password, ep.Server, ep.Port = raw.Username, raw.Password, raw.Host, port + + return nil +} + +func secretPort(raw json.RawMessage) (int, error) { + if len(raw) == 0 { + return 0, fmt.Errorf("%w: the secret needs a port member", ErrSecret) + } + + var s json.Number + if err := json.Unmarshal(raw, &s); err != nil { + return 0, fmt.Errorf("%w: port: %w", ErrSecret, err) + } + + n, err := strconv.Atoi(s.String()) + if err != nil { + return 0, fmt.Errorf("%w: port %q is not a number", ErrSecret, s.String()) + } + + return n, nil +} diff --git a/pkgs/dmsengine/sql_target.go b/pkgs/dmsengine/sql_target.go index 5d426d1db2..51a22f63ce 100644 --- a/pkgs/dmsengine/sql_target.go +++ b/pkgs/dmsengine/sql_target.go @@ -65,7 +65,13 @@ func (t *sqlTarget) ph(n int) string { } func (t *sqlTarget) columnType(tt TargetTable, c TargetColumn) string { - typ := TargetType(tt.SourceFamily, t.fam, c.Native) + var typ string + + if c.DMS.Name != "" { + typ = c.DMS.SQLType(t.fam) + } else { + typ = TargetType(tt.SourceFamily, t.fam, c.Native) + } if t.fam == FamilyMySQL && (typ == tLongtext || typ == tText) && slicesContain(tt.PK, c.Name) { return varcharKey @@ -81,7 +87,7 @@ func (t *sqlTarget) createSQL(tt TargetTable) string { for _, c := range tt.Columns { def := t.q(c.Name) + " " + t.columnType(tt, c) - if !c.Nullable || slicesContain(tt.PK, c.Name) { + if !c.Nullable || (slicesContain(tt.PK, c.Name) && !tt.KeyUnique) { def += " NOT NULL" } @@ -89,12 +95,26 @@ func (t *sqlTarget) createSQL(tt TargetTable) string { } if len(tt.PK) > 0 { - defs = append(defs, "PRIMARY KEY ("+t.quoteAll(tt.PK)+")") + defs = append(defs, t.keyDef(tt)) } return "CREATE TABLE IF NOT EXISTS " + t.qualified(tt) + " (" + strings.Join(defs, ", ") + ")" } +func (t *sqlTarget) keyDef(tt TargetTable) string { + kind := "PRIMARY KEY" + if tt.KeyUnique { + kind = "UNIQUE" + } + + def := kind + " (" + t.quoteAll(tt.PK) + ")" + if tt.KeyName != "" { + def = "CONSTRAINT " + t.q(tt.KeyName) + " " + def + } + + return def +} + func (t *sqlTarget) quoteAll(names []string) string { q := make([]string, len(names)) for i, n := range names { diff --git a/pkgs/dmsengine/targets_test.go b/pkgs/dmsengine/targets_test.go index 0e17e83597..0966f9664e 100644 --- a/pkgs/dmsengine/targets_test.go +++ b/pkgs/dmsengine/targets_test.go @@ -34,6 +34,35 @@ func (m *memObjects) PutObject(_ context.Context, bucket, key string, body []byt return nil } +func (m *memObjects) ListObjects(_ context.Context, bucket, prefix string) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + + var keys []string + + for k := range m.objs { + if rest, ok := strings.CutPrefix(k, bucket+"/"); ok && strings.HasPrefix(rest, prefix) { + keys = append(keys, rest) + } + } + + sort.Strings(keys) + + return keys, nil +} + +func (m *memObjects) GetObject(_ context.Context, bucket, key string) ([]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + + body, ok := m.objs[bucket+"/"+key] + if !ok { + return nil, errBoom + } + + return []byte(body), nil +} + func (m *memObjects) CheckBucket(_ context.Context, bucket string) error { if bucket == "missing" { return errBoom diff --git a/pkgs/dmsengine/task.go b/pkgs/dmsengine/task.go index 0686e10cc7..9bc7fd41a0 100644 --- a/pkgs/dmsengine/task.go +++ b/pkgs/dmsengine/task.go @@ -85,9 +85,12 @@ type TaskConfig struct { Mappings Mappings PingBackoff time.Duration SourceFamily Family + // TargetFamily gates target-specific rules such as add-before-image-columns; FamilyOther skips the check. + TargetFamily Family } type planned struct { + filter *TableFilter src Table target TargetTable stat TableStat @@ -212,12 +215,26 @@ func (t *Task) setup(ctx context.Context) error { state = tableBeforeLoad } - for _, st := range t.cfg.Mappings.Select(all) { + selected, err := t.cfg.Mappings.SelectFiltered(all) + if err != nil { + return err + } + + for _, sel := range selected { + st := sel.Table + + target, perr := t.cfg.Mappings.Plan(st, t.cfg.SourceFamily, t.cfg.Settings.TargetSchema) + if perr != nil { + return perr + } + + if target.HasBeforeImage && !beforeImageTarget(t.cfg.TargetFamily) { + return fmt.Errorf("%w: add-before-image-columns is supported for Kinesis and S3 targets only", ErrMapping) + } + pl := &planned{ - src: st, - target: t.cfg.Mappings.Plan(st, t.cfg.SourceFamily, t.cfg.Settings.TargetSchema), - active: true, - stat: TableStat{Schema: st.Schema, Name: st.Name, State: state}, + src: st, filter: sel.Filter, target: target, active: true, + stat: TableStat{Schema: st.Schema, Name: st.Name, State: state}, } t.plans[st.Key()] = pl t.order = append(t.order, pl) @@ -226,6 +243,10 @@ func (t *Task) setup(ctx context.Context) error { return nil } +func beforeImageTarget(f Family) bool { + return f == FamilyOther || f == FamilyS3 || f == FamilyKinesis +} + func (t *Task) fullLoad(ctx context.Context) error { t.pmu.Lock() t.prog.FullLoadStart = t.cfg.Now() @@ -294,14 +315,16 @@ func (t *Task) copyTable(ctx context.Context, pl *planned, mode PrepMode, reload return nil } - err := t.cfg.Source.Scan(ctx, pl.src, func(r Row) error { - batch = append(batch, pl.target.Project(r)) + add := func(r Row) error { + batch = append(batch, pl.target.ProjectWith(r, ProjectCtx{Full: true, Now: t.cfg.Now()})) if len(batch) >= cap(batch) { return flush() } return nil - }) + } + + err := t.scan(ctx, pl, add) if err == nil { err = flush() } @@ -320,6 +343,29 @@ func (t *Task) copyTable(ctx context.Context, pl *planned, mode PrepMode, reload return nil } +// FilteredScanner is implemented by sources that push a selection filter into the full-load query. +type FilteredScanner interface { + ScanWhere(ctx context.Context, t Table, f *TableFilter, fn func(Row) error) error +} + +func (t *Task) scan(ctx context.Context, pl *planned, fn func(Row) error) error { + if pl.filter == nil { + return t.cfg.Source.Scan(ctx, pl.src, fn) + } + + if fs, ok := t.cfg.Source.(FilteredScanner); ok { + return fs.ScanWhere(ctx, pl.src, pl.filter, fn) + } + + return t.cfg.Source.Scan(ctx, pl.src, func(r Row) error { + if !pl.filter.Match(r) { + return nil + } + + return fn(r) + }) +} + // Reload re-copies tables while change capture keeps running. func (t *Task) Reload(ctx context.Context, refs []TableRef) error { t.applyMu.Lock() @@ -455,8 +501,14 @@ func (t *Task) apply(ctx context.Context, c Change) error { return nil } + c, ok := filterChange(pl.filter, c) + if !ok { + return nil + } + + pc := ProjectCtx{Op: c.Op, Time: c.Time, Now: t.cfg.Now(), Position: c.Position, Before: c.Before} tc := c - tc.Before, tc.After = pl.target.Project(c.Before), pl.target.Project(c.After) + tc.Before, tc.After = pl.target.ProjectWith(c.Before, pc), pl.target.ProjectWith(c.After, pc) if err := t.cfg.Target.Apply(ctx, pl.target, tc); err != nil { if ctx.Err() != nil { @@ -482,6 +534,47 @@ func (t *Task) apply(ctx context.Context, c Change) error { return nil } +// filterChange applies a selection filter to a change: rows entering the filter are inserted, rows leaving it are +// deleted, and rows that never matched are dropped. +func filterChange(f *TableFilter, c Change) (Change, bool) { + if f == nil { + return c, true + } + + switch c.Op { + case OpInsert: + return c, f.Match(c.After) + case OpDelete: + return c, c.Before == nil || f.Match(c.Before) + case OpUpdate: + return filterUpdate(f, c) + } + + return c, true +} + +func filterUpdate(f *TableFilter, c Change) (Change, bool) { + switch { + case f.Match(c.After): + if c.Before != nil && !f.Match(c.Before) { + c.Op = OpInsert + } + + return c, true + case c.Before == nil || f.Match(c.Before): + c.Op = OpDelete + if c.Before == nil { + c.Before = c.After + } + + c.After = nil + + return c, true + } + + return c, false +} + func rowWidth(src Table, c Change) string { for _, r := range []Row{c.Before, c.After} { if r != nil && len(r) != len(src.Columns) { diff --git a/pkgs/dmsengine/task_rules_test.go b/pkgs/dmsengine/task_rules_test.go new file mode 100644 index 0000000000..ab704bad9d --- /dev/null +++ b/pkgs/dmsengine/task_rules_test.go @@ -0,0 +1,188 @@ +package dmsengine_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/dmsengine" +) + +func intTable() *fakeSource { + items := dmsengine.Table{Schema: "app", Name: "items", PK: []string{"id"}, Columns: []dmsengine.Column{ + {Name: "id", Native: "int", Kind: dmsengine.KindInt}, + {Name: "name", Native: "varchar(20)", Kind: dmsengine.KindString, Nullable: true}, + }} + + return &fakeSource{ + tables: []dmsengine.Table{items}, + rows: map[string][]dmsengine.Row{ + "app.items": {{"1", "alpha"}, {"2", "beta"}, {"30", "gamma"}}, + }, + changes: make(chan dmsengine.Change, 16), + } +} + +const filteredWithColumn = `{"rules":[ + {"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"app","table-name":"items"}, + "rule-action":"include","filters":[{"filter-type":"source","column-name":"id", + "filter-conditions":[{"filter-operator":"ste","value":"10"}]}]}, + {"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"add-column","rule-target":"column", + "object-locator":{"schema-name":"app","table-name":"items"},"value":"label", + "expression":"upper($name)||'-'||$id","data-type":{"type":"string","length":30}}]}` + +func TestTaskFilterAndAddColumnOnLoadAndCDC(t *testing.T) { + t.Parallel() + + src, tgt, log := intTable(), newFakeTarget(), newProgressLog() + mappings, err := dmsengine.ParseMappings(filteredWithColumn) + require.NoError(t, err) + + task := dmsengine.NewTask(dmsengine.TaskConfig{ + Source: src, Target: tgt, MigrationType: dmsengine.MigrationFullLoadAndCDC, Mappings: mappings, + Settings: defaultSettings(t), SourceFamily: dmsengine.FamilyMySQL, OnProgress: log.push, + }) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan dmsengine.Result, 1) + + go func() { done <- task.Run(ctx) }() + + ins := func(id, name string) dmsengine.Change { + return dmsengine.Change{Schema: "app", Table: "items", Op: dmsengine.OpInsert, After: dmsengine.Row{id, name}} + } + upd := func(bid, bname, aid, aname string) dmsengine.Change { + return dmsengine.Change{ + Schema: "app", Table: "items", Op: dmsengine.OpUpdate, + Before: dmsengine.Row{bid, bname}, After: dmsengine.Row{aid, aname}, + } + } + + src.changes <- ins("5", "five") + src.changes <- ins("50", "fifty") + src.changes <- upd("5", "five", "5", "FIVE") + src.changes <- upd("5", "five", "99", "moved-out") + src.changes <- upd("60", "x", "3", "moved-in") + src.changes <- upd("70", "x", "71", "never") + src.changes <- dmsengine.Change{ + Schema: "app", Table: "items", Op: dmsengine.OpDelete, Before: dmsengine.Row{"80", "x"}, + } + src.changes <- dmsengine.Change{Commit: true, Position: "p"} + log.waitFor(t, func(p dmsengine.Progress) bool { return p.Checkpoint == "p" }) + cancel() + <-done + + assert.Equal(t, []dmsengine.Row{{"1", "alpha", "ALPHA-1"}, {"2", "beta", "BETA-2"}}, tgt.loaded["app.items"]) + + type applied struct { + op dmsengine.Op + row dmsengine.Row + } + + got := make([]applied, 0, len(tgt.applied)) + + for _, c := range tgt.applied { + img := c.After + if c.Op == dmsengine.OpDelete { + img = c.Before + } + + got = append(got, applied{c.Op, img}) + } + + assert.Equal(t, []applied{ + {dmsengine.OpInsert, dmsengine.Row{"5", "five", "FIVE-5"}}, + {dmsengine.OpUpdate, dmsengine.Row{"5", "FIVE", "FIVE-5"}}, + {dmsengine.OpDelete, dmsengine.Row{"5", "five", "FIVE-5"}}, + {dmsengine.OpInsert, dmsengine.Row{"3", "moved-in", "MOVED-IN-3"}}, + }, got) +} + +func TestTaskRuleErrorsFailTheTask(t *testing.T) { + t.Parallel() + + addCol := func(expr string) string { + return `{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1",` + + `"object-locator":{"schema-name":"app","table-name":"items"},"rule-action":"include"}, + {"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"add-column","rule-target":"column", + "object-locator":{"schema-name":"app","table-name":"items"},"value":"x","expression":"` + expr + `", + "data-type":{"type":"string"}}]}` + } + + tests := []struct { + name string + doc string + want string + target dmsengine.Family + }{ + {name: "unsupported function", doc: addCol("soundex($name)"), target: dmsengine.FamilyOther, want: "soundex"}, + {name: "filter column missing", doc: `{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1",` + + `"object-locator":{"schema-name":"app","table-name":"items"},"rule-action":"include",` + + `"filters":[{"filter-type":"source","column-name":"nope",` + + `"filter-conditions":[{"filter-operator":"null"}]}]}]}`, target: dmsengine.FamilyOther, want: "nope"}, + {name: "before image on a SQL target", doc: `{"rules":[{"rule-type":"selection","rule-id":"1",` + + `"rule-name":"1","object-locator":{"schema-name":"app","table-name":"items"},"rule-action":"include"}, + {"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"add-before-image-columns", + "rule-target":"column","object-locator":{"schema-name":"app","table-name":"items"}}]}`, + target: dmsengine.FamilyPostgres, want: "Kinesis and S3"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + mappings, err := dmsengine.ParseMappings(tt.doc) + require.NoError(t, err) + + task := dmsengine.NewTask(dmsengine.TaskConfig{ + Source: intTable(), Target: newFakeTarget(), MigrationType: dmsengine.MigrationFullLoad, + Mappings: mappings, Settings: defaultSettings(t), SourceFamily: dmsengine.FamilyMySQL, + TargetFamily: tt.target, + }) + + res := task.Run(t.Context()) + require.ErrorIs(t, res.Err, dmsengine.ErrMapping) + assert.Equal(t, dmsengine.StopFatalError, res.Reason) + assert.Contains(t, res.Err.Error(), tt.want) + }) + } +} + +func TestTaskBeforeImageOnKinesisLikeTarget(t *testing.T) { + t.Parallel() + + mappings, err := dmsengine.ParseMappings( + `{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"app",` + + `"table-name":"items"},"rule-action":"include"}, + {"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"add-before-image-columns", + "rule-target":"column","object-locator":{"schema-name":"app","table-name":"items"}}]}`, + ) + require.NoError(t, err) + + src, tgt, log := intTable(), newFakeTarget(), newProgressLog() + task := dmsengine.NewTask(dmsengine.TaskConfig{ + Source: src, Target: tgt, MigrationType: dmsengine.MigrationFullLoadAndCDC, Mappings: mappings, + Settings: defaultSettings(t), SourceFamily: dmsengine.FamilyMySQL, TargetFamily: dmsengine.FamilyKinesis, + OnProgress: log.push, + }) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan dmsengine.Result, 1) + + go func() { done <- task.Run(ctx) }() + + src.changes <- dmsengine.Change{ + Schema: "app", Table: "items", Op: dmsengine.OpUpdate, + Before: dmsengine.Row{"1", "alpha"}, After: dmsengine.Row{"1", "ALPHA"}, + } + src.changes <- dmsengine.Change{Commit: true, Position: "p"} + log.waitFor(t, func(p dmsengine.Progress) bool { return p.Checkpoint == "p" }) + cancel() + require.NoError(t, (<-done).Err) + + assert.Equal(t, dmsengine.Row{"1", "alpha", nil}, tgt.loaded["app.items"][0]) + require.Len(t, tgt.applied, 1) + assert.Equal(t, dmsengine.Row{"1", "ALPHA", "1"}, tgt.applied[0].After) +} diff --git a/pkgs/dmsengine/transform_test.go b/pkgs/dmsengine/transform_test.go new file mode 100644 index 0000000000..c2dfe2f9c3 --- /dev/null +++ b/pkgs/dmsengine/transform_test.go @@ -0,0 +1,293 @@ +package dmsengine_test + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/dmsengine" +) + +func transformDoc(rules ...string) string { + sel := `{"rule-type":"selection","rule-id":"1","rule-name":"1", + "object-locator":{"schema-name":"test","table-name":"employee"},"rule-action":"include"}` + + return `{"rules":[` + strings.Join(append([]string{sel}, rules...), ",") + `]}` +} + +func addColumn(id, name, expr, dataType string) string { + return `{"rule-type":"transformation","rule-id":"` + id + `","rule-name":"` + id + `","rule-action":"add-column", + "rule-target":"column","object-locator":{"schema-name":"test","table-name":"employee"},"value":"` + name + `", + "expression":"` + expr + `","data-type":` + dataType + `}` +} + +func TestAddColumnExpressions(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + row := dmsengine.Row{"7", "Tech", "20500.50", "2002-01-15", nil} + + tests := []struct { + want any + name string + expr string + typ string + }{ + {name: "concat", expr: `$dept||'_'||$empid`, typ: `{"type":"string","length":50}`, want: "Tech_7"}, + { + name: "case round", + expr: `CASE WHEN round($salary)>=20000 THEN 'SENIOR' ELSE 'JUNIOR' END`, + typ: `{"type":"string","length":50}`, + want: "SENIOR", + }, + {name: "round int8", expr: `round($salary)`, typ: `{"type":"int8"}`, want: "20501"}, + {name: "arithmetic", expr: `$empid*2+1`, typ: `{"type":"int4"}`, want: "15"}, + {name: "lower upper", expr: `lower($dept)||upper($dept)`, typ: `{"type":"string"}`, want: "techTECH"}, + {name: "substr hex", expr: `substr(hex($dept),1,4)`, typ: `{"type":"string"}`, want: "5465"}, + {name: "sha256", expr: `hash_sha256($dept)`, typ: `{"type":"string","length":64}`, want: ""}, + {name: "coalesce null", expr: `coalesce($notes,'none')`, typ: `{"type":"string"}`, want: "none"}, + {name: "datetime now", expr: `datetime()`, typ: `{"type":"datetime",` + + `"precision":6}`, want: "2026-03-04 05:06:07"}, + {name: "strftime", expr: `strftime('%Y/%m/%d', $started)`, typ: `{"type":"string"}`, want: "2002/01/15"}, + {name: "header operation", expr: `$AR_H_OPERATION`, typ: `{"type":"string"}`, want: "INSERT"}, + {name: "operation indicator", expr: `operation_indicator('D','U','I')`, typ: `{"type":"string"}`, want: "I"}, + { + name: "source metadata", + expr: `$AR_M_SOURCE_SCHEMA||'.'||$AR_M_SOURCE_TABLE_NAME`, + typ: `{"type":"string"}`, + want: "test.employee", + }, + {name: "boolean", expr: `$empid > 5`, typ: `{"type":"boolean"}`, want: "true"}, + {name: "replace trim", expr: `trim(replace($dept,'e',' '))`, typ: `{"type":"string"}`, want: "T ch"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(transformDoc(addColumn("2", "added", tt.expr, tt.typ))) + require.NoError(t, err) + + plan, err := m.Plan(employeeTable(), dmsengine.FamilyMySQL, "") + require.NoError(t, err) + + out := plan.ProjectWith(row, dmsengine.ProjectCtx{Full: true, Now: now}) + require.Len(t, out, 6) + + if tt.name == "sha256" { + assert.Len(t, out[5], 64) + + return + } + + assert.Equal(t, tt.want, out[5]) + }) + } +} + +func TestAddColumnRejectsUnsupported(t *testing.T) { + t.Parallel() + + tests := []struct{ name, expr string }{ + {"unknown function", `soundex($dept)`}, + {"aggregate", `count($dept)`}, + {"missing column", `$nope`}, + {"change seq header", `$AR_H_CHANGE_SEQ`}, + {"syntax", `$dept ||`}, + {"modifiers", `date('now','+1 day')`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(transformDoc(addColumn("2", "added", tt.expr, `{"type":"string"}`))) + require.NoError(t, err) + + _, err = m.Plan(employeeTable(), dmsengine.FamilyMySQL, "") + require.ErrorIs(t, err, dmsengine.ErrMapping) + assert.Contains(t, err.Error(), "added") + }) + } +} + +func TestChangeDataTypeAndKeys(t *testing.T) { + t.Parallel() + + tests := []struct { + check func(*testing.T, dmsengine.TargetTable) + name string + rule string + wantCols []string + wantPK []string + }{ + { + name: "change type", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"change-data-type",` + + `"rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee","column-name":"empid"}, + "data-type":{"type":"string","length":"12"}}`, + wantCols: []string{"empid", "dept", "salary", "started", "notes"}, wantPK: []string{"empid"}, + check: func(t *testing.T, tt dmsengine.TargetTable) { + t.Helper() + assert.Equal(t, dmsengine.KindString, tt.Columns[0].Kind) + assert.Equal(t, "varchar(12)", tt.Columns[0].DMS.SQLType(dmsengine.FamilyPostgres)) + assert.Equal(t, "string", tt.Columns[0].Native) + }, + }, + { + name: "change type by existing data type", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"change-data-type",` + + `"rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee","column-name":"%", + "data-type":"numeric"},"data-type":{"type":"real8"}}`, + wantCols: []string{"empid", "dept", "salary", "started", "notes"}, wantPK: []string{"empid"}, + check: func(t *testing.T, tt dmsengine.TargetTable) { + t.Helper() + assert.Equal(t, dmsengine.KindFloat, tt.Columns[2].Kind) + assert.Equal(t, dmsengine.KindInt, tt.Columns[0].Kind) + }, + }, + { + name: "include-column", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"include-column",` + + `"rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee","column-name":"dept"}}`, + wantCols: []string{"dept"}, wantPK: []string{}, + }, + { + name: "define-primary-key", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2",` + + `"rule-action":"define-primary-key","rule-target":"table", + "object-locator":{"schema-name":"test","table-name":"employee"},"primary-key-def":{"name":"emp_dept", + "origin":"unique-index","columns":["dept","started"]}}`, + wantCols: []string{"empid", "dept", "salary", "started", "notes"}, wantPK: []string{"dept", "started"}, + check: func(t *testing.T, tt dmsengine.TargetTable) { + t.Helper() + assert.Equal(t, "emp_dept", tt.KeyName) + assert.True(t, tt.KeyUnique) + }, + }, + { + name: "before image pk-only", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2",` + + `"rule-action":"add-before-image-columns","rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee"}, + "before-image-def":{"column-prefix":"BI_","column-suffix":"","column-filter":"pk-only"}}`, + wantCols: []string{"empid", "dept", "salary", "started", "notes", "BI_empid"}, wantPK: []string{"empid"}, + check: func(t *testing.T, tt dmsengine.TargetTable) { + t.Helper() + assert.True(t, tt.HasBeforeImage) + }, + }, + { + name: "before image non-lob defaults", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2",` + + `"rule-action":"add-before-image-columns","rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee"}, + "before-image-def":{"column-suffix":"_old","column-prefix":"","column-filter":"non-lob"}}`, + wantCols: []string{ + "empid", + "dept", + "salary", + "started", + "notes", + "empid_old", + "dept_old", + "salary_old", + "started_old", + }, + wantPK: []string{"empid"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(transformDoc(tt.rule)) + require.NoError(t, err) + + plan, err := m.Plan(employeeTable(), dmsengine.FamilyMySQL, "") + require.NoError(t, err) + + names := []string{} + for _, c := range plan.Columns { + names = append(names, c.Name) + } + + assert.Equal(t, tt.wantCols, names) + assert.Equal(t, tt.wantPK, plan.PK) + + if tt.check != nil { + tt.check(t, plan) + } + }) + } +} + +func TestBeforeImageProjection(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(transformDoc(`{"rule-type":"transformation","rule-id":"2","rule-name":"2", + "rule-action":"add-before-image-columns","rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee"}}`)) + require.NoError(t, err) + + plan, err := m.Plan(employeeTable(), dmsengine.FamilyMySQL, "") + require.NoError(t, err) + + before := dmsengine.Row{"1", "a", "1", "2020-01-01", nil} + after := dmsengine.Row{"3", "a", "1", "2020-01-01", nil} + + assert.Equal(t, dmsengine.Row{"3", "a", "1", "2020-01-01", nil, "1"}, + plan.ProjectWith(after, dmsengine.ProjectCtx{Op: dmsengine.OpUpdate, Before: before})) + assert.Equal(t, dmsengine.Row{"3", "a", "1", "2020-01-01", nil, nil}, plan.Project(after)) +} + +func TestDMSTypeSQL(t *testing.T) { + t.Parallel() + + tests := []struct { + pg string + my string + wantKindName string + dt dmsengine.DMSType + }{ + {dt: dmsengine.DMSType{Name: "int8"}, pg: "bigint", my: "bigint", wantKindName: "int8"}, + {dt: dmsengine.DMSType{Name: "uint4"}, pg: "bigint", my: "int unsigned", wantKindName: "uint4"}, + { + dt: dmsengine.DMSType{Name: "string", Length: 30}, + pg: "varchar(30)", + my: "varchar(30)", + wantKindName: "string", + }, + {dt: dmsengine.DMSType{Name: "string"}, pg: "text", my: "longtext", wantKindName: "string"}, + { + dt: dmsengine.DMSType{Name: "numeric", Precision: 5, Scale: 2}, + pg: "numeric(5,2)", + my: "numeric(5,2)", + wantKindName: "numeric", + }, + { + dt: dmsengine.DMSType{Name: "datetime", Scale: 3}, + pg: "timestamp(3)", + my: "datetime(3)", + wantKindName: "datetime", + }, + {dt: dmsengine.DMSType{Name: "boolean"}, pg: "boolean", my: "tinyint(1)", wantKindName: "boolean"}, + {dt: dmsengine.DMSType{Name: "bytes", Length: 16}, pg: "bytea", my: "varbinary(16)", wantKindName: "bytes"}, + } + + for _, tt := range tests { + t.Run(tt.wantKindName, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.pg, tt.dt.SQLType(dmsengine.FamilyPostgres)) + assert.Equal(t, tt.my, tt.dt.SQLType(dmsengine.FamilyMySQL)) + }) + } +} diff --git a/pkgs/dmsengine/typemap.go b/pkgs/dmsengine/typemap.go index 24c879ca9f..5cad051043 100644 --- a/pkgs/dmsengine/typemap.go +++ b/pkgs/dmsengine/typemap.go @@ -57,6 +57,26 @@ const ( tLongblob = "longblob" tJSONB = "jsonb" tDatetime6 = "datetime(6)" + tBlob = "blob" + tBinary = "binary" + tVarbinary = "varbinary" + tTinyblob = "tinyblob" + tMediumblob = "mediumblob" + tSerial = "serial" + tBigserial = "bigserial" + tInt1 = "int1" + tInt2 = "int2" + tInt4 = "int4" + tInt8 = "int8" + tReal4 = "real4" + tFloat4 = "float4" + tFloat8 = "float8" + tBool = "bool" + tClob = "clob" + tBytes = "bytes" + tReal8 = "real8" + tNclob = "nclob" + tTinyint1 = "tinyint(1)" ) func kindByBase() map[string]Kind { @@ -68,12 +88,12 @@ func kindByBase() map[string]Kind { tInteger: KindInt, tBigint: KindInt, "year": KindInt, - "serial": KindInt, - "bigserial": KindInt, + tSerial: KindInt, + tBigserial: KindInt, "smallserial": KindInt, - "int2": KindInt, - "int4": KindInt, - "int8": KindInt, + tInt2: KindInt, + tInt4: KindInt, + tInt8: KindInt, tDecimal: KindDecimal, tNumeric: KindDecimal, "dec": KindDecimal, @@ -83,10 +103,10 @@ func kindByBase() map[string]Kind { tDouble: KindFloat, tDoublePrec: KindFloat, tReal: KindFloat, - "float4": KindFloat, - "float8": KindFloat, + tFloat4: KindFloat, + tFloat8: KindFloat, tBoolean: KindBool, - "bool": KindBool, + tBool: KindBool, tDate: KindTime, tDatetime: KindTime, tTimestamp: KindTime, @@ -97,11 +117,11 @@ func kindByBase() map[string]Kind { "time with time zone": KindTime, "timestamptz": KindTime, "timetz": KindTime, - "binary": KindBytes, - "varbinary": KindBytes, - "tinyblob": KindBytes, - "blob": KindBytes, - "mediumblob": KindBytes, + tBinary: KindBytes, + tVarbinary: KindBytes, + tTinyblob: KindBytes, + tBlob: KindBytes, + tMediumblob: KindBytes, tLongblob: KindBytes, tBytea: KindBytes, "bit": KindBytes, @@ -120,7 +140,7 @@ func mysqlToPGBase() map[string]string { return map[string]string{ tTinyint: tSmallint, tMediumint: tInteger, "year": tSmallint, tFloat: tReal, tDouble: tDoublePrec, tDoublePrec: tDoublePrec, tDatetime: tTimestamp, tTimestamp: tTimestamp, tDate: tDate, tTime: tTime, - "binary": tBytea, "varbinary": tBytea, "tinyblob": tBytea, "blob": tBytea, "mediumblob": tBytea, + tBinary: tBytea, tVarbinary: tBytea, tTinyblob: tBytea, tBlob: tBytea, tMediumblob: tBytea, tLongblob: tBytea, "bit": tBytea, "geometry": tBytea, tJSON: tJSONB, } } @@ -172,10 +192,10 @@ func numericWithArgs(args string) string { func pgToMySQLBase() map[string]string { return map[string]string{ - tSmallint: tSmallint, "int2": tSmallint, "smallserial": tSmallint, tInteger: tInt, tInt: tInt, "int4": tInt, - "serial": tInt, tBigint: tBigint, "int8": tBigint, "bigserial": tBigint, tBoolean: "tinyint(1)", - "bool": "tinyint(1)", "money": "decimal(19,2)", tReal: tFloat, "float4": tFloat, tDoublePrec: tDouble, - "float8": tDouble, tTimestampN: tDatetime6, tTimestampZ: tDatetime6, tTimestamp: tDatetime6, + tSmallint: tSmallint, tInt2: tSmallint, "smallserial": tSmallint, tInteger: tInt, tInt: tInt, tInt4: tInt, + tSerial: tInt, tBigint: tBigint, tInt8: tBigint, tBigserial: tBigint, tBoolean: tTinyint1, + tBool: tTinyint1, "money": "decimal(19,2)", tReal: tFloat, tFloat4: tFloat, tDoublePrec: tDouble, + tFloat8: tDouble, tTimestampN: tDatetime6, tTimestampZ: tDatetime6, tTimestamp: tDatetime6, "timestamptz": tDatetime6, tDate: tDate, "time without time zone": tTime, "time with time zone": tTime, tTime: tTime, "timetz": tTime, tBytea: tLongblob, tJSON: tJSON, tJSONB: tJSON, "uuid": "char(36)", } diff --git a/pkgs/dns/dns.go b/pkgs/dns/dns.go index a5fa26b878..4f49cfba02 100644 --- a/pkgs/dns/dns.go +++ b/pkgs/dns/dns.go @@ -27,6 +27,7 @@ import ( "github.com/miekg/dns" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/logger" ) // DefaultListenAddr is the default UDP/TCP address the DNS server binds to. @@ -360,6 +361,12 @@ func (s *Server) Stop() error { // handleQuery is the DNS handler that answers A/CNAME/AAAA queries for registered names. func (s *Server) handleQuery(w dns.ResponseWriter, r *dns.Msg) { + defer logger.Recover(context.Background(), "dns.query", func(any) { + fail := new(dns.Msg) + fail.SetRcode(r, dns.RcodeServerFailure) + _ = w.WriteMsg(fail) + }) + msg := s.respond(w, r) if werr := w.WriteMsg(msg); werr != nil { diff --git a/pkgs/gcpaudit/audit.go b/pkgs/gcpaudit/audit.go new file mode 100644 index 0000000000..dd67cfefc0 --- /dev/null +++ b/pkgs/gcpaudit/audit.go @@ -0,0 +1,163 @@ +// Package gcpaudit lets GCP emulators record Cloud Audit Logs entries (protoPayload AuditLog) in the +// Logging emulator, which Eventarc triggers on google.cloud.audit.log.v1.written then match. +package gcpaudit + +import ( + "context" + "fmt" + "net/url" + "strings" + "sync/atomic" + "time" + + "cloud.google.com/go/logging/apiv2/loggingpb" + "github.com/google/uuid" + "google.golang.org/genproto/googleapis/api/monitoredres" + "google.golang.org/genproto/googleapis/cloud/audit" + ltype "google.golang.org/genproto/googleapis/logging/type" + "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Kind selects the audit log a record is written to. +type Kind int + +const ( + // Activity is an Admin Activity record (log id cloudaudit.googleapis.com/activity). + Activity Kind = iota + // DataWrite is a Data Access record for a data write (log id cloudaudit.googleapis.com/data_access). + DataWrite + // DataRead is a Data Access record for a data read. + DataRead +) + +// DefaultPrincipal is the caller recorded when no identity is known: emulator requests are unauthenticated. +const DefaultPrincipal = "emulator@gopherstack.iam.gserviceaccount.com" + +// Sink receives log entries; *gcplogging.InMemoryBackend satisfies it. +type Sink interface { + WriteLogEntries(ctx context.Context, req *loggingpb.WriteLogEntriesRequest) error +} + +// Record is one audited API call. +type Record struct { + Time time.Time + Resource *monitoredres.MonitoredResource + Request *structpb.Struct + Response *structpb.Struct + Failure *status.Status + Project string + ServiceName string + MethodName string + ResourceName string + Principal string + Permission string + Kind Kind +} + +// Emitter writes Records to a Sink. The zero value is ready to use and drops records until a sink is set. +type Emitter struct { + sink atomic.Pointer[Sink] +} + +// SetSink directs records to s; nil stops recording. +func (e *Emitter) SetSink(s Sink) { + if s == nil { + e.sink.Store(nil) + + return + } + + e.sink.Store(&s) +} + +// Enabled reports whether a sink is set. +func (e *Emitter) Enabled() bool { return e.sink.Load() != nil } + +// LogID returns the audit log id of kind, e.g. cloudaudit.googleapis.com/activity. +func LogID(k Kind) string { + if k == Activity { + return "cloudaudit.googleapis.com/activity" + } + + return "cloudaudit.googleapis.com/data_access" +} + +// LogName returns projects/{project}/logs/{escaped id} for kind. +func LogName(project string, k Kind) string { + return "projects/" + project + "/logs/" + url.PathEscape(LogID(k)) +} + +// Entry builds the LogEntry for r. +func (r Record) Entry() (*loggingpb.LogEntry, error) { + when := r.Time + if when.IsZero() { + when = time.Now() + } + + principal := r.Principal + if principal == "" { + principal = DefaultPrincipal + } + + payload := &audit.AuditLog{ + ServiceName: r.ServiceName, + MethodName: r.MethodName, + ResourceName: r.ResourceName, + AuthenticationInfo: &audit.AuthenticationInfo{PrincipalEmail: principal}, + Status: r.Failure, + Request: r.Request, + Response: r.Response, + RequestMetadata: &audit.RequestMetadata{CallerIp: "127.0.0.1"}, + } + + if r.Permission != "" { + payload.AuthorizationInfo = []*audit.AuthorizationInfo{{ + Resource: r.ResourceName, Permission: r.Permission, Granted: true, + }} + } + + packed, err := anypb.New(payload) + if err != nil { + return nil, fmt.Errorf("pack audit log: %w", err) + } + + res := r.Resource + if res == nil { + res = &monitoredres.MonitoredResource{ + Type: "audited_resource", + Labels: map[string]string{"service": r.ServiceName, "method": r.MethodName}, + } + } + + sev := ltype.LogSeverity_NOTICE + if r.Kind != Activity { + sev = ltype.LogSeverity_INFO + } + + return &loggingpb.LogEntry{ + LogName: LogName(r.Project, r.Kind), + Resource: res, + Timestamp: timestamppb.New(when), + Severity: sev, + InsertId: strings.ReplaceAll(uuid.NewString(), "-", "")[:16], + Payload: &loggingpb.LogEntry_ProtoPayload{ProtoPayload: packed}, + }, nil +} + +// Emit records r; it is a no-op without a sink or a project. Call it outside backend locks. +func (e *Emitter) Emit(ctx context.Context, r Record) { + sink := e.sink.Load() + if sink == nil || r.Project == "" { + return + } + + entry, err := r.Entry() + if err != nil { + return + } + + _ = (*sink).WriteLogEntries(ctx, &loggingpb.WriteLogEntriesRequest{Entries: []*loggingpb.LogEntry{entry}}) +} diff --git a/pkgs/gcpaudit/audit_test.go b/pkgs/gcpaudit/audit_test.go new file mode 100644 index 0000000000..a9bac0bb0a --- /dev/null +++ b/pkgs/gcpaudit/audit_test.go @@ -0,0 +1,168 @@ +package gcpaudit_test + +import ( + "context" + "sync" + "testing" + "time" + + "cloud.google.com/go/logging/apiv2/loggingpb" + "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/genproto/googleapis/cloud/audit" + "google.golang.org/grpc" + + "github.com/blackbirdworks/gopherstack/pkgs/gcpaudit" +) + +type sink struct { + entries []*loggingpb.LogEntry + mu sync.Mutex +} + +func (s *sink) WriteLogEntries(_ context.Context, req *loggingpb.WriteLogEntriesRequest) error { + s.mu.Lock() + defer s.mu.Unlock() + + s.entries = append(s.entries, req.GetEntries()...) + + return nil +} + +func TestRecordEntry(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + wantLog string + wantResource string + rec gcpaudit.Record + }{ + { + name: "admin activity", wantLog: "projects/p/logs/cloudaudit.googleapis.com%2Factivity", + rec: gcpaudit.Record{ + Project: "p", ServiceName: "s.googleapis.com", MethodName: "s.Create", ResourceName: "projects/p/x/1", + Kind: gcpaudit.Activity, Permission: "s.x.create", + }, + wantResource: "audited_resource", + }, + { + name: "data access", wantLog: "projects/p/logs/cloudaudit.googleapis.com%2Fdata_access", + rec: gcpaudit.Record{Project: "p", ServiceName: "s", MethodName: "m", Kind: gcpaudit.DataRead}, + wantResource: "audited_resource", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tt.rec.Time = time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC) + + e, err := tt.rec.Entry() + require.NoError(t, err) + assert.Equal(t, tt.wantLog, e.GetLogName()) + assert.Equal(t, tt.wantResource, e.GetResource().GetType()) + assert.Equal(t, tt.rec.Time, e.GetTimestamp().AsTime()) + assert.Len(t, e.GetInsertId(), 16) + + var al audit.AuditLog + require.NoError(t, e.GetProtoPayload().UnmarshalTo(&al)) + assert.Equal(t, tt.rec.MethodName, al.GetMethodName()) + assert.Equal(t, gcpaudit.DefaultPrincipal, al.GetAuthenticationInfo().GetPrincipalEmail()) + assert.Equal(t, tt.rec.Permission != "", len(al.GetAuthorizationInfo()) == 1) + }) + } +} + +func TestEmitterWithoutSinkOrProject(t *testing.T) { + t.Parallel() + + var em gcpaudit.Emitter + + em.Emit(t.Context(), gcpaudit.Record{Project: "p"}) + assert.False(t, em.Enabled()) + + s := &sink{} + em.SetSink(s) + assert.True(t, em.Enabled()) + + em.Emit(t.Context(), gcpaudit.Record{}) + assert.Empty(t, s.entries, "a record without a project has no log to go to") + + em.Emit(t.Context(), gcpaudit.Record{Project: "p", ServiceName: "s", MethodName: "m"}) + assert.Len(t, s.entries, 1) + + em.SetSink(nil) + assert.False(t, em.Enabled()) +} + +func TestUnaryInterceptorClassification(t *testing.T) { + t.Parallel() + + policy := gcpaudit.Policy{ + ServiceName: "secretmanager.googleapis.com", + AdminPrefixes: []string{"Create", "Delete"}, + Data: map[string]gcpaudit.Kind{"AccessSecretVersion": gcpaudit.DataRead}, + } + + tests := []struct { + name string + method string + wantLog string + wantLogged bool + }{ + { + name: "admin prefix", + method: "/google.cloud.secretmanager.v1.SecretManagerService/CreateSecret", + wantLogged: true, + wantLog: "activity", + }, + { + name: "data method", method: "/google.cloud.secretmanager.v1.SecretManagerService/AccessSecretVersion", + wantLogged: true, wantLog: "data_access", + }, + {name: "read is not audited", method: "/google.cloud.secretmanager.v1.SecretManagerService/GetSecret"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var em gcpaudit.Emitter + + s := &sink{} + em.SetSink(s) + + icpt := em.UnaryInterceptor(policy, time.Now) + req := &secretmanagerpb.CreateSecretRequest{Parent: "projects/p"} + _, err := icpt(t.Context(), req, &grpc.UnaryServerInfo{FullMethod: tt.method}, + func(context.Context, any) (any, error) { + return &secretmanagerpb.Secret{Name: "projects/p/secrets/x"}, nil + }) + require.NoError(t, err) + + if !tt.wantLogged { + assert.Empty(t, s.entries) + + return + } + + require.Len(t, s.entries, 1) + assert.Contains(t, s.entries[0].GetLogName(), tt.wantLog) + + var al audit.AuditLog + require.NoError(t, s.entries[0].GetProtoPayload().UnmarshalTo(&al)) + assert.Equal(t, "projects/p/secrets/x", al.GetResourceName()) + assert.Equal(t, gcpaudit.MethodName(tt.method), al.GetMethodName()) + }) + } +} + +func TestProjectOf(t *testing.T) { + t.Parallel() + + assert.Equal(t, "p", gcpaudit.ProjectOf("projects/p/secrets/s")) + assert.Empty(t, gcpaudit.ProjectOf("folders/1")) +} diff --git a/pkgs/gcpaudit/grpc.go b/pkgs/gcpaudit/grpc.go new file mode 100644 index 0000000000..9d20a6404f --- /dev/null +++ b/pkgs/gcpaudit/grpc.go @@ -0,0 +1,136 @@ +package gcpaudit + +import ( + "context" + "strings" + "time" + + "google.golang.org/genproto/googleapis/api/monitoredres" + "google.golang.org/grpc" + grpcstatus "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +// Policy says which RPCs of one gRPC API are audited. +type Policy struct { + // NameFunc overrides ResourceName for methods whose resource is not in a top-level name field. + NameFunc func(req, resp proto.Message) string + // Data names methods (short, e.g. "Commit") that are Data Access records. + Data map[string]Kind + // ResourceType is the monitored resource type recorded; empty uses audited_resource. + ResourceType string + // ServiceName is the audit serviceName, e.g. secretmanager.googleapis.com. + ServiceName string + // AdminPrefixes are method-name prefixes (Create, Delete, ...) of mutating RPCs. + AdminPrefixes []string +} + +func (p Policy) classify(short string) (Kind, bool) { + if k, ok := p.Data[short]; ok { + return k, true + } + + for _, pre := range p.AdminPrefixes { + if strings.HasPrefix(short, pre) { + return Activity, true + } + } + + return 0, false +} + +// MethodName turns a gRPC full method (/pkg.Service/Method) into the audit methodName (pkg.Service.Method). +func MethodName(fullMethod string) string { + return strings.ReplaceAll(strings.TrimPrefix(fullMethod, "/"), "/", ".") +} + +// UnaryInterceptor records the RPCs selected by p after they run, with their status. Bodies are left out +// so that secret payloads never reach the log. +func (e *Emitter) UnaryInterceptor(p Policy, now func() time.Time) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) { + resp, err := next(ctx, req) + + kind, ok := p.classify(info.FullMethod[strings.LastIndex(info.FullMethod, "/")+1:]) + if !ok || !e.Enabled() { + return resp, err + } + + rm, _ := req.(proto.Message) + pm, _ := resp.(proto.Message) + name := "" + if p.NameFunc != nil { + name = p.NameFunc(rm, pm) + } + + if name == "" { + name = ResourceName(rm, pm) + } + + rec := Record{ + Project: ProjectOf(name), ServiceName: p.ServiceName, MethodName: MethodName(info.FullMethod), + ResourceName: name, Kind: kind, Time: now(), + } + + if p.ResourceType != "" { + rec.Resource = &monitoredres.MonitoredResource{ + Type: p.ResourceType, Labels: map[string]string{"project_id": rec.Project}, + } + } + + if err != nil { + s := grpcstatus.Convert(err) + rec.Failure = s.Proto() + } + + e.Emit(ctx, rec) + + return resp, err + } +} + +// ResourceName picks the audited resource: the response's name, else the request's name, database or parent. +func ResourceName(req, resp proto.Message) string { + if n := stringField(resp, "name"); n != "" && !strings.HasPrefix(n, "operations/") && + !strings.Contains(n, "/operations/") { + return n + } + + for _, f := range []protoreflect.Name{"name", "database", "parent"} { + if n := stringField(req, f); n != "" { + return n + } + } + + return "" +} + +func stringField(m proto.Message, name protoreflect.Name) string { + if m == nil { + return "" + } + + r := m.ProtoReflect() + if !r.IsValid() { + return "" + } + + fd := r.Descriptor().Fields().ByName(name) + if fd == nil || fd.Kind() != protoreflect.StringKind || fd.IsList() { + return "" + } + + return r.Get(fd).String() +} + +// ProjectOf returns the project of a projects/{p}/... resource name, or "". +func ProjectOf(name string) string { + rest, ok := strings.CutPrefix(name, "projects/") + if !ok { + return "" + } + + p, _, _ := strings.Cut(rest, "/") + + return p +} diff --git a/pkgs/gcpenforce/grpc.go b/pkgs/gcpenforce/grpc.go index 36211ab5f3..5b09393bb5 100644 --- a/pkgs/gcpenforce/grpc.go +++ b/pkgs/gcpenforce/grpc.go @@ -37,6 +37,8 @@ func NewServer(ctx context.Context, host string, opts ...grpc.ServerOption) *grp // NewServerFor is NewServer with per-method project overrides. func NewServerFor(ctx context.Context, t Target, opts ...grpc.ServerOption) *grpc.Server { + opts = append(opts, RecoverOptions()...) + return grpc.NewServer(append(opts, ServerOptions(ctx, t)...)...) } diff --git a/pkgs/gcpenforce/recover.go b/pkgs/gcpenforce/recover.go new file mode 100644 index 0000000000..f84fc7ad0d --- /dev/null +++ b/pkgs/gcpenforce/recover.go @@ -0,0 +1,52 @@ +package gcpenforce + +import ( + "context" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" +) + +// RecoverOptions turns a handler panic into an Internal status for that one RPC; grpc-go does not recover. +func RecoverOptions() []grpc.ServerOption { + return []grpc.ServerOption{ + grpc.ChainUnaryInterceptor(recoverUnary), + grpc.ChainStreamInterceptor(recoverStream), + } +} + +func recoverUnary(ctx context.Context, req any, info *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) { + var ( + resp any + err error + ) + + func() { + defer logger.Recover(ctx, "grpc.unary "+info.FullMethod, func(any) { + resp, err = nil, status.Error(codes.Internal, errInternal) + }) + + resp, err = next(ctx, req) + }() + + return resp, err +} + +func recoverStream(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, next grpc.StreamHandler) error { + var err error + + func() { + defer logger.Recover(ss.Context(), "grpc.stream "+info.FullMethod, func(any) { + err = status.Error(codes.Internal, errInternal) + }) + + err = next(srv, ss) + }() + + return err +} + +const errInternal = "internal server error" diff --git a/pkgs/gcpenforce/recover_test.go b/pkgs/gcpenforce/recover_test.go new file mode 100644 index 0000000000..5bd2ea0aaa --- /dev/null +++ b/pkgs/gcpenforce/recover_test.go @@ -0,0 +1,60 @@ +package gcpenforce_test + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/blackbirdworks/gopherstack/pkgs/gcpenforce" +) + +func TestStreamHandlerPanicReturnsInternal(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + want codes.Code + }{ + {name: "panic", want: codes.Internal}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + lis := bufconn.Listen(1 << 16) + srv := grpc.NewServer(append(gcpenforce.RecoverOptions(), + grpc.UnknownServiceHandler(func(any, grpc.ServerStream) error { panic("injected rpc panic") }))...) + + go func() { _ = srv.Serve(lis) }() + + t.Cleanup(srv.Stop) + + dialer := func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) } + cc, err := grpc.NewClient("passthrough:///bufnet", + grpc.WithContextDialer(dialer), + grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + t.Cleanup(func() { _ = cc.Close() }) + + for range 2 { + st, serr := cc.NewStream(t.Context(), &grpc.StreamDesc{}, "/x.Y/Z") + require.NoError(t, serr) + require.NoError(t, st.SendMsg(&emptypb.Empty{})) + require.NoError(t, st.CloseSend()) + + rerr := st.RecvMsg(&emptypb.Empty{}) + assert.Equal(t, tt.want, status.Code(rerr)) + } + }) + } +} diff --git a/pkgs/httpclient/httpclient.go b/pkgs/httpclient/httpclient.go index 7697196221..56b2208e51 100644 --- a/pkgs/httpclient/httpclient.go +++ b/pkgs/httpclient/httpclient.go @@ -3,6 +3,7 @@ package httpclient import ( "net/http" + "net/netip" "time" ) @@ -22,3 +23,11 @@ func NewTransport() *http.Transport { func New(timeout time.Duration) *http.Client { return &http.Client{Timeout: timeout, Transport: NewTransport()} } + +// IsLoopbackHost reports whether host is a loopback IP literal; the only targets where TLS verification may be +// skipped (emulator-run backends; policy 2026-10-31). +func IsLoopbackHost(host string) bool { + ip, err := netip.ParseAddr(host) + + return err == nil && ip.IsLoopback() +} diff --git a/pkgs/httpclient/httpclient_test.go b/pkgs/httpclient/httpclient_test.go index 98ad7a4112..a62ffa9557 100644 --- a/pkgs/httpclient/httpclient_test.go +++ b/pkgs/httpclient/httpclient_test.go @@ -82,3 +82,21 @@ func TestNewTransportDistinct(t *testing.T) { assert.NotSame(t, a, b) assert.NotNil(t, a.Proxy) } + +func TestIsLoopbackHost(t *testing.T) { + t.Parallel() + + tests := []struct { + host string + want bool + }{ + {"127.0.0.1", true}, {"::1", true}, {"localhost", false}, {"10.0.0.1", false}, {"", false}, + } + + for _, tt := range tests { + t.Run(tt.host, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, httpclient.IsLoopbackHost(tt.host)) + }) + } +} diff --git a/pkgs/iampolicystream/stream.go b/pkgs/iampolicystream/stream.go index 8d56505c98..a72b1ad32f 100644 --- a/pkgs/iampolicystream/stream.go +++ b/pkgs/iampolicystream/stream.go @@ -31,6 +31,7 @@ const ( ReasonImplicitDeny = "implicit-deny" ReasonUnresolvedCaller = "unresolved-principal" ReasonServiceControl = "service-control-policy" + ReasonResourceControl = "resource-control-policy" ) // Deny-kind values recorded on a denied Event. @@ -47,6 +48,7 @@ const ( PolicyBoundary = "permission-boundary" PolicyIdentity = "identity" PolicySCP = "service-control-policy" + PolicyRCP = "resource-control-policy" ) // PolicyRef names the policy that produced the deciding statement. Owner is the diff --git a/pkgs/iceberg/append.go b/pkgs/iceberg/append.go index 2107ba1db9..3f6ee2d2e8 100644 --- a/pkgs/iceberg/append.go +++ b/pkgs/iceberg/append.go @@ -2,9 +2,7 @@ package iceberg import ( "encoding/binary" - "fmt" "strconv" - "strings" "time" "github.com/google/uuid" @@ -56,88 +54,7 @@ func NewSnapshotID() int64 { func (m *TableMetadata) PlanAppend( currentLocation string, prev []ManifestFile, files []DataFile, now time.Time, ) (*AppendPlan, error) { - spec := m.DefaultSpec() - snapID := NewSnapshotID() - seq := m.LastSequenceNumber + 1 - tsMS := now.UnixMilli() - - var parent *int64 - - var prevSummary map[string]string - - if cur, ok := m.CurrentSnapshot(); ok { - id := cur.SnapshotID - parent, prevSummary = &id, cur.Summary - tsMS = max(tsMS, cur.TimestampMS) - } - - base := strings.TrimSuffix(m.Location, "/") + "/metadata/" - - entries := make([]ManifestEntry, 0, len(files)) - - var rows, size int64 - - for _, f := range files { - f.Content = ContentData - entries = append(entries, ManifestEntry{ - Status: StatusAdded, SnapshotID: &snapID, SequenceNumber: &seq, DataFile: f, - }) - rows += f.RecordCount - size += f.FileSize - } - - manifestBody, err := EncodeManifest(m, spec, ContentData, entries) - if err != nil { - return nil, err - } - - manifestLoc := base + uuid.NewString() + "-m0.avro" - list := make([]ManifestFile, 0, 1+len(prev)) - list = append(list, ManifestFile{ - Path: manifestLoc, Length: int64(len(manifestBody)), SpecID: spec.SpecID, Content: ContentData, - SequenceNumber: seq, MinSequenceNumber: seq, AddedSnapshotID: snapID, - AddedFiles: len(files), AddedRows: rows, - }) - list = append(list, prev...) - - listBody, err := EncodeManifestList(snapID, parent, seq, list) - if err != nil { - return nil, err - } - - listLoc := fmt.Sprintf("%ssnap-%d-1-%s.avro", base, snapID, uuid.NewString()) - schemaID := m.CurrentSchemaID - snap := Snapshot{ - SnapshotID: snapID, ParentSnapshotID: parent, SequenceNumber: seq, TimestampMS: tsMS, - ManifestList: listLoc, SchemaID: &schemaID, - Summary: map[string]string{ - summaryOp: "append", summaryAdded: strconv.Itoa(len(files)), - summaryRecords: strconv.FormatInt(rows, 10), summarySize: strconv.FormatInt(size, 10), - totalRecords: strconv.FormatInt(summaryInt(prevSummary, totalRecords)+rows, 10), - totalFiles: strconv.FormatInt(summaryInt(prevSummary, totalFiles)+int64(len(files)), 10), - totalSize: strconv.FormatInt(summaryInt(prevSummary, totalSize)+size, 10), - totalDeletes: strconv.FormatInt(summaryInt(prevSummary, totalDeletes), 10), - totalPosDeletes: strconv.FormatInt(summaryInt(prevSummary, totalPosDeletes), 10), - totalEqDeletes: strconv.FormatInt(summaryInt(prevSummary, totalEqDeletes), 10), - }, - } - - next := m.AppendSnapshot(snap, currentLocation, now) - metaLoc := next.NextMetadataLocation() - - body, err := marshalMetadata(next) - if err != nil { - return nil, err - } - - return &AppendPlan{ - Metadata: next, MetadataLocation: metaLoc, - Objects: []Object{ - {Location: manifestLoc, Body: manifestBody}, - {Location: listLoc, Body: listBody}, - {Location: metaLoc, Body: body}, - }, - }, nil + return m.PlanCommit(currentLocation, prev, nil, Change{Operation: OpAppend, AddData: files}, now) } func marshalMetadata(m *TableMetadata) ([]byte, error) { diff --git a/pkgs/iceberg/commit.go b/pkgs/iceberg/commit.go new file mode 100644 index 0000000000..0c9bfebc29 --- /dev/null +++ b/pkgs/iceberg/commit.go @@ -0,0 +1,334 @@ +package iceberg + +import ( + "fmt" + "maps" + "slices" + "strconv" + "strings" + "time" + + "github.com/google/uuid" +) + +// Snapshot summary operations. +const ( + OpAppend = "append" + OpDelete = "delete" + OpOverwrite = "overwrite" + OpReplace = "replace" +) + +// Change is one snapshot's worth of file changes. +type Change struct { + RemoveData map[string]bool + RemoveDeletes map[string]bool + Operation string + AddData []DataFile + AddDeletes []DataFile +} + +// ManifestReader loads the entries of the manifest stored at path. +type ManifestReader func(path string) ([]ManifestEntry, error) + +type commitTally struct { + addedFiles, addedRows, addedSize int64 + addedDeleteFiles, addedDeletes int64 + removedFiles, removedRows, removedSize int64 + removedDeleteFiles, removedDeletes int64 +} + +// PlanCommit prepares a snapshot applying ch to the table's current manifests +// prev. Manifests holding removed files are rewritten (removed entries become +// DELETED, the rest EXISTING); read is only needed when ch removes files. +func (m *TableMetadata) PlanCommit( + currentLocation string, prev []ManifestFile, read ManifestReader, ch Change, now time.Time, +) (*AppendPlan, error) { + spec := m.DefaultSpec() + snapID := NewSnapshotID() + seq := m.LastSequenceNumber + 1 + tsMS := now.UnixMilli() + + var parent *int64 + + var prevSummary map[string]string + + if cur, ok := m.CurrentSnapshot(); ok { + id := cur.SnapshotID + parent, prevSummary = &id, cur.Summary + tsMS = max(tsMS, cur.TimestampMS) + } + + base := strings.TrimSuffix(m.Location, "/") + "/metadata/" + pc := &planCtx{m: m, base: base, snapID: snapID, seq: seq, read: read, ch: ch} + + var list []ManifestFile + + if len(ch.AddData) > 0 { + mf, body, err := pc.addedManifest(spec, ContentData, ch.AddData) + if err != nil { + return nil, err + } + + list = append(list, mf) + pc.objects = append(pc.objects, Object{Location: mf.Path, Body: body}) + } + + if len(ch.AddDeletes) > 0 { + mf, body, err := pc.addedManifest(spec, ContentPositionDeletes, ch.AddDeletes) + if err != nil { + return nil, err + } + + list = append(list, mf) + pc.objects = append(pc.objects, Object{Location: mf.Path, Body: body}) + } + + for _, pm := range prev { + kept, err := pc.carry(pm) + if err != nil { + return nil, err + } + + list = append(list, kept...) + } + + listBody, err := EncodeManifestList(snapID, parent, seq, list) + if err != nil { + return nil, err + } + + listLoc := fmt.Sprintf("%ssnap-%d-1-%s.avro", base, snapID, uuid.NewString()) + schemaID := m.CurrentSchemaID + snap := Snapshot{ + SnapshotID: snapID, ParentSnapshotID: parent, SequenceNumber: seq, TimestampMS: tsMS, + ManifestList: listLoc, SchemaID: &schemaID, Summary: pc.summary(prevSummary), + } + + next := m.AppendSnapshot(snap, currentLocation, now) + metaLoc := next.NextMetadataLocation() + + body, err := marshalMetadata(next) + if err != nil { + return nil, err + } + + pc.objects = append(pc.objects, Object{Location: listLoc, Body: listBody}, Object{Location: metaLoc, Body: body}) + + return &AppendPlan{Metadata: next, MetadataLocation: metaLoc, Objects: pc.objects}, nil +} + +type planCtx struct { + read ManifestReader + m *TableMetadata + ch Change + base string + objects []Object + tally commitTally + snapID int64 + seq int64 +} + +func (pc *planCtx) addedManifest(spec PartitionSpec, content int, files []DataFile) (ManifestFile, []byte, error) { + entries := make([]ManifestEntry, 0, len(files)) + + var rows int64 + + for _, f := range files { + f.Content = content + entries = append(entries, ManifestEntry{ + Status: StatusAdded, SnapshotID: &pc.snapID, SequenceNumber: &pc.seq, DataFile: f, + }) + + rows += f.RecordCount + + if content == ContentData { + pc.tally.addedFiles++ + pc.tally.addedRows += f.RecordCount + pc.tally.addedSize += f.FileSize + } else { + pc.tally.addedDeleteFiles++ + pc.tally.addedDeletes += f.RecordCount + pc.tally.addedSize += f.FileSize + } + } + + body, err := EncodeManifest(pc.m, spec, content, entries) + if err != nil { + return ManifestFile{}, nil, err + } + + return ManifestFile{ + Path: pc.base + uuid.NewString() + "-m0.avro", Length: int64(len(body)), SpecID: spec.SpecID, + Content: content, SequenceNumber: pc.seq, MinSequenceNumber: pc.seq, AddedSnapshotID: pc.snapID, + AddedFiles: len(files), AddedRows: rows, + }, body, nil +} + +func (pc *planCtx) removals(content int) map[string]bool { + if content == ContentData { + return pc.ch.RemoveData + } + + return pc.ch.RemoveDeletes +} + +// carry keeps a manifest of the previous snapshot, rewriting it when it lists a removed file. +func (pc *planCtx) carry(pm ManifestFile) ([]ManifestFile, error) { + rm := pc.removals(pm.Content) + if len(rm) == 0 || pc.read == nil { + return []ManifestFile{pm}, nil + } + + entries, err := pc.read(pm.Path) + if err != nil { + return nil, err + } + + hit := false + + for _, e := range entries { + if e.Status != StatusDeleted && rm[e.DataFile.Path] { + hit = true + } + } + + if !hit { + return []ManifestFile{pm}, nil + } + + spec, ok := pc.m.SpecByID(pm.SpecID) + if !ok { + return nil, fmt.Errorf("%w: manifest %s uses unknown partition spec %d", ErrInvalidManifest, pm.Path, pm.SpecID) + } + + out, mf := make([]ManifestEntry, 0, len(entries)), pm + mf.Path, mf.SequenceNumber, mf.AddedSnapshotID = pc.base+uuid.NewString()+"-m0.avro", pc.seq, pc.snapID + mf.AddedFiles, mf.AddedRows, mf.ExistingFiles, mf.ExistingRows, mf.DeletedFiles, mf.DeletedRows = 0, 0, 0, 0, 0, 0 + mf.MinSequenceNumber = pc.seq + + for _, e := range entries { + if e.Status == StatusDeleted { + continue + } + + if e.SequenceNumber == nil { + s := pm.SequenceNumber + e.SequenceNumber = &s + } + + if e.SnapshotID == nil { + s := pm.AddedSnapshotID + e.SnapshotID = &s + } + + if rm[e.DataFile.Path] { + e.Status, e.SnapshotID = StatusDeleted, &pc.snapID + mf.DeletedFiles++ + mf.DeletedRows += e.DataFile.RecordCount + pc.tallyRemoved(e.DataFile) + } else { + e.Status = StatusExisting + mf.ExistingFiles++ + mf.ExistingRows += e.DataFile.RecordCount + mf.MinSequenceNumber = min(mf.MinSequenceNumber, *e.SequenceNumber) + } + + out = append(out, e) + } + + body, err := EncodeManifest(pc.m, spec, pm.Content, out) + if err != nil { + return nil, err + } + + mf.Length = int64(len(body)) + pc.objects = append(pc.objects, Object{Location: mf.Path, Body: body}) + + return []ManifestFile{mf}, nil +} + +func (pc *planCtx) tallyRemoved(f DataFile) { + if f.Content == ContentData { + pc.tally.removedFiles++ + pc.tally.removedRows += f.RecordCount + } else { + pc.tally.removedDeleteFiles++ + pc.tally.removedDeletes += f.RecordCount + } + + pc.tally.removedSize += f.FileSize +} + +func (pc *planCtx) summary(prev map[string]string) map[string]string { + t := pc.tally + itoa := func(n int64) string { return strconv.FormatInt(n, 10) } + s := map[string]string{summaryOp: pc.ch.Operation} + + set := func(k string, n int64) { + if n > 0 { + s[k] = itoa(n) + } + } + + set(summaryAdded, t.addedFiles) + set(summaryRecords, t.addedRows) + set("added-delete-files", t.addedDeleteFiles) + set("added-position-delete-files", t.addedDeleteFiles) + set("added-position-deletes", t.addedDeletes) + set("deleted-data-files", t.removedFiles) + set("deleted-records", t.removedRows) + set("removed-delete-files", t.removedDeleteFiles) + set("removed-position-delete-files", t.removedDeleteFiles) + set("removed-position-deletes", t.removedDeletes) + set(summarySize, t.addedSize) + + s[totalRecords] = itoa(summaryInt(prev, totalRecords) + t.addedRows - t.removedRows) + s[totalFiles] = itoa(summaryInt(prev, totalFiles) + t.addedFiles - t.removedFiles) + s[totalSize] = itoa(max(0, summaryInt(prev, totalSize)+t.addedSize-t.removedSize)) + s[totalDeletes] = itoa(summaryInt(prev, totalDeletes) + t.addedDeleteFiles - t.removedDeleteFiles) + s[totalPosDeletes] = itoa(summaryInt(prev, totalPosDeletes) + t.addedDeletes - t.removedDeletes) + s[totalEqDeletes] = itoa(summaryInt(prev, totalEqDeletes)) + + return s +} + +// clone copies the mutable members so a metadata update never aliases the source. +func (m *TableMetadata) clone() *TableMetadata { + next := *m + next.Properties = maps.Clone(m.Properties) + next.Refs = maps.Clone(m.Refs) + next.Schemas = slices.Clone(m.Schemas) + next.PartitionSpecs = slices.Clone(m.PartitionSpecs) + next.Snapshots = slices.Clone(m.Snapshots) + next.SnapshotLog = slices.Clone(m.SnapshotLog) + next.MetadataLog = slices.Clone(m.MetadataLog) + + return &next +} + +// PlanMetadataUpdate prepares a commit that changes only the metadata +// (schema, properties, retained snapshots): mutate edits a copy of m. +func (m *TableMetadata) PlanMetadataUpdate( + currentLocation string, mutate func(next *TableMetadata) error, now time.Time, +) (*AppendPlan, error) { + next := m.clone() + if err := mutate(next); err != nil { + return nil, err + } + + if currentLocation != "" { + next.MetadataLog = append(next.MetadataLog, + MetadataLogEntry{MetadataFile: currentLocation, TimestampMS: m.LastUpdatedMS}) + } + + next.LastUpdatedMS = max(now.UnixMilli(), m.LastUpdatedMS) + loc := next.NextMetadataLocation() + + body, err := marshalMetadata(next) + if err != nil { + return nil, err + } + + return &AppendPlan{Metadata: next, MetadataLocation: loc, Objects: []Object{{Location: loc, Body: body}}}, nil +} diff --git a/pkgs/iceberg/commit_test.go b/pkgs/iceberg/commit_test.go new file mode 100644 index 0000000000..e7438b3e94 --- /dev/null +++ b/pkgs/iceberg/commit_test.go @@ -0,0 +1,185 @@ +package iceberg_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/iceberg" +) + +func TestPlanCommitRemovesAndDeletes(t *testing.T) { + t.Parallel() + + now := time.UnixMilli(1_700_000_000_000) + meta := iceberg.NewTableMetadata(iceberg.NewParams{ + Location: "s3://b/tbl", Schema: testSchema(t), + Spec: iceberg.PartitionSpec{Fields: []iceberg.PartitionField{}}, Now: now, + }) + + store := map[string][]byte{} + read := func(path string) ([]iceberg.ManifestEntry, error) { return iceberg.ReadManifest(store[path]) } + file := func(name string) iceberg.DataFile { + return iceberg.DataFile{ + Path: "s3://b/tbl/data/" + name, + Format: "PARQUET", + RecordCount: 4, + FileSize: 40, + Partition: map[string]any{}, + } + } + + cur := meta + loc := "" + + var list []iceberg.ManifestFile + + step := func(ch iceberg.Change, at time.Duration) *iceberg.AppendPlan { + plan, err := cur.PlanCommit(loc, list, read, ch, now.Add(at)) + require.NoError(t, err) + + for _, o := range plan.Objects { + store[o.Location] = o.Body + } + + snap, _ := plan.Metadata.CurrentSnapshot() + list, err = iceberg.ReadManifestList(store[snap.ManifestList]) + require.NoError(t, err) + + cur, loc = plan.Metadata, plan.MetadataLocation + + return plan + } + + step(iceberg.Change{Operation: iceberg.OpAppend, AddData: []iceberg.DataFile{file("a"), file("b")}}, time.Second) + + del := iceberg.DataFile{ + Path: "s3://b/tbl/data/d1", + Format: "PARQUET", + RecordCount: 2, + FileSize: 10, + Partition: map[string]any{}, + } + step(iceberg.Change{Operation: iceberg.OpDelete, AddDeletes: []iceberg.DataFile{del}}, 2*time.Second) + + snap, _ := cur.CurrentSnapshot() + assert.Equal(t, "1", snap.Summary["total-delete-files"]) + assert.Equal(t, "2", snap.Summary["total-position-deletes"]) + assert.Equal(t, "8", snap.Summary["total-records"]) + + step(iceberg.Change{ + Operation: iceberg.OpReplace, AddData: []iceberg.DataFile{file("c")}, + RemoveData: map[string]bool{"s3://b/tbl/data/a": true}, + RemoveDeletes: map[string]bool{del.Path: true}, + }, 3*time.Second) + + snap, _ = cur.CurrentSnapshot() + assert.Equal(t, "8", snap.Summary["total-records"]) + assert.Equal(t, "0", snap.Summary["total-delete-files"]) + assert.Equal(t, "2", snap.Summary["total-data-files"]) + assert.Equal(t, "1", snap.Summary["deleted-data-files"]) + assert.Equal(t, "replace", snap.Summary["operation"]) + + live := map[string]int{} + + for _, mf := range list { + entries, err := iceberg.ReadManifest(store[mf.Path]) + require.NoError(t, err) + + for _, e := range entries { + weight := map[int]int{iceberg.StatusDeleted: 100, iceberg.StatusExisting: 1, iceberg.StatusAdded: 1} + live[e.DataFile.Path] += weight[e.Status] + } + } + + assert.Equal(t, 100, live["s3://b/tbl/data/a"]) + assert.Equal(t, 1, live["s3://b/tbl/data/b"]) + assert.Equal(t, 1, live["s3://b/tbl/data/c"]) + assert.Equal(t, 100, live[del.Path]) +} + +func TestSchemaEvolution(t *testing.T) { + t.Parallel() + + meta := iceberg.NewTableMetadata(iceberg.NewParams{ + Location: "s3://b/tbl", + Schema: testSchema(t), + Spec: iceberg.PartitionSpec{ + Fields: []iceberg.PartitionField{{Name: "category", Transform: "identity", SourceID: 2, FieldID: 1000}}, + }, + Now: time.UnixMilli(1), + }) + s := meta.CurrentSchema() + + tests := []struct { + err error + do func() (iceberg.Schema, error) + name string + want []string + }{ + {name: "add", want: []string{"id", "category", "ts", "x"}, do: func() (iceberg.Schema, error) { + out, _, err := meta.WithColumn(s, "x", "int", "") + + return out, err + }}, + {name: "add dup", err: iceberg.ErrColumnExists, do: func() (iceberg.Schema, error) { + out, _, err := meta.WithColumn(s, "ID", "int", "") + + return out, err + }}, + { + name: "drop", + want: []string{"id", "category"}, + do: func() (iceberg.Schema, error) { return meta.WithoutColumn(s, "ts") }, + }, + { + name: "drop partition source", + err: iceberg.ErrColumnInUse, + do: func() (iceberg.Schema, error) { return meta.WithoutColumn(s, "category") }, + }, + { + name: "promote", + want: []string{"id", "category", "ts"}, + do: func() (iceberg.Schema, error) { return meta.WithRenamedColumn(s, "id", "id", "bigint") }, + }, + { + name: "bad promote", + err: iceberg.ErrBadPromotion, + do: func() (iceberg.Schema, error) { return meta.WithRenamedColumn(s, "id", "id", "int") }, + }, + { + name: "move first", + want: []string{"ts", "id", "category"}, + do: func() (iceberg.Schema, error) { return meta.WithMovedColumn(s, "ts", true, "") }, + }, + { + name: "move after", + want: []string{"id", "ts", "category"}, + do: func() (iceberg.Schema, error) { return meta.WithMovedColumn(s, "ts", false, "id") }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + out, err := tc.do() + if tc.err != nil { + require.ErrorIs(t, err, tc.err) + + return + } + + require.NoError(t, err) + + var names []string + for _, f := range out.Fields { + names = append(names, f.Name) + } + + assert.Equal(t, tc.want, names) + }) + } +} diff --git a/pkgs/iceberg/evolve.go b/pkgs/iceberg/evolve.go new file mode 100644 index 0000000000..9d38dab6a3 --- /dev/null +++ b/pkgs/iceberg/evolve.go @@ -0,0 +1,221 @@ +package iceberg + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "strings" +) + +// Schema evolution errors. +var ( + ErrColumnExists = errors.New("column already exists") + ErrColumnNotFound = errors.New("column not found") + ErrColumnInUse = errors.New("column is used by the partition spec") + ErrBadPromotion = errors.New("unsupported column type change") + ErrLastColumn = errors.New("cannot drop the last column") + errNestedAddColumn = errors.New("only primitive column types can be added") +) + +func (s Schema) index(name string) int { + return slices.IndexFunc(s.Fields, func(f Field) bool { return strings.EqualFold(f.Name, name) }) +} + +// WithColumn returns s with a new optional column using the m field id. +func (m *TableMetadata) WithColumn(s Schema, name, hiveType, doc string) (Schema, int, error) { + if s.index(name) >= 0 { + return s, 0, fmt.Errorf("%w: %s", ErrColumnExists, name) + } + + t, err := FromHive(hiveType) + if err != nil { + return s, 0, fmt.Errorf("%w: %w", errNestedAddColumn, err) + } + + raw, _ := json.Marshal(t) + id := m.LastColumnID + 1 + out := s + out.Fields = append(slices.Clone(s.Fields), Field{ID: id, Name: name, Doc: doc, Type: raw}) + + return out, id, nil +} + +// WithoutColumn returns s without the named column; partition source columns cannot be dropped. +func (m *TableMetadata) WithoutColumn(s Schema, name string) (Schema, error) { + i := s.index(name) + if i < 0 { + return s, fmt.Errorf("%w: %s", ErrColumnNotFound, name) + } + + for _, spec := range m.PartitionSpecs { + for _, pf := range spec.Fields { + if pf.SourceID == s.Fields[i].ID { + return s, fmt.Errorf("%w: %s", ErrColumnInUse, name) + } + } + } + + if len(s.Fields) == 1 { + return s, ErrLastColumn + } + + out := s + out.Fields = slices.Delete(slices.Clone(s.Fields), i, i+1) + + return out, nil +} + +// WithRenamedColumn returns s with the column renamed; a non-empty hiveType also promotes its type. +func (m *TableMetadata) WithRenamedColumn(s Schema, from, to, hiveType string) (Schema, error) { + i := s.index(from) + if i < 0 { + return s, fmt.Errorf("%w: %s", ErrColumnNotFound, from) + } + + if !strings.EqualFold(from, to) && s.index(to) >= 0 { + return s, fmt.Errorf("%w: %s", ErrColumnExists, to) + } + + out := s + out.Fields = slices.Clone(s.Fields) + f := out.Fields[i] + f.Name = to + + if hiveType != "" { + nt, err := FromHive(hiveType) + if err != nil { + return s, fmt.Errorf("%w: %w", ErrBadPromotion, err) + } + + if old := PrimitiveName(f.Type); old != nt { + if !canPromote(old, nt) { + return s, fmt.Errorf("%w: %s to %s", ErrBadPromotion, old, nt) + } + + f.Type, _ = json.Marshal(nt) + } + } + + out.Fields[i] = f + + return out, nil +} + +// canPromote reports the type changes the table spec allows: int to long, +// float to double, and widening decimal precision at a fixed scale. +func canPromote(from, to string) bool { + switch { + case from == typeInt && to == typeLong, from == typeFloat && to == typeDouble: + return true + case strings.HasPrefix(from, "decimal(") && strings.HasPrefix(to, "decimal("): + fp, fs, ok1 := decimalParts(from) + tp, ts, ok2 := decimalParts(to) + + return ok1 && ok2 && fs == ts && tp >= fp + default: + return false + } +} + +func decimalParts(t string) (int, int, bool) { + var p, s int + + _, err := fmt.Sscanf(strings.ReplaceAll(t, " ", ""), "decimal(%d,%d)", &p, &s) + + return p, s, err == nil +} + +// SetSchema makes s a new current schema of m (an update copy). +func (m *TableMetadata) SetSchema(s Schema) { + id := 0 + for _, e := range m.Schemas { + id = max(id, e.SchemaID+1) + } + + s.SchemaID, s.Type = id, typeStruct + + for _, f := range s.Fields { + m.LastColumnID = max(m.LastColumnID, f.ID) + } + + m.Schemas = append(m.Schemas, s) + m.CurrentSchemaID = id +} + +// ExpireSnapshots drops the snapshots keep rejects (never the current one) and +// returns them; log entries and refs that pointed at them go too. +func (m *TableMetadata) ExpireSnapshots(keep func(Snapshot) bool) []Snapshot { + var gone []Snapshot + + kept := make([]Snapshot, 0, len(m.Snapshots)) + + for _, s := range m.Snapshots { + if s.SnapshotID == m.CurrentSnapshotID || keep(s) { + kept = append(kept, s) + } else { + gone = append(gone, s) + } + } + + if len(gone) == 0 { + return nil + } + + alive := map[int64]bool{} + for _, s := range kept { + alive[s.SnapshotID] = true + } + + m.Snapshots = kept + m.SnapshotLog = slices.DeleteFunc(slices.Clone(m.SnapshotLog), func(e SnapshotLogEntry) bool { + return !alive[e.SnapshotID] + }) + + for name, r := range m.Refs { + if !alive[r.SnapshotID] { + delete(m.Refs, name) + } + } + + return gone +} + +// TrimMetadataLog keeps the newest max previous-metadata entries and returns the dropped ones. +func (m *TableMetadata) TrimMetadataLog(maxKeep int) []MetadataLogEntry { + if maxKeep < 0 || len(m.MetadataLog) <= maxKeep { + return nil + } + + cut := len(m.MetadataLog) - maxKeep + dropped := slices.Clone(m.MetadataLog[:cut]) + m.MetadataLog = slices.Clone(m.MetadataLog[cut:]) + + return dropped +} + +// WithMovedColumn returns s with the column moved first or after another column. +func (m *TableMetadata) WithMovedColumn(s Schema, name string, first bool, after string) (Schema, error) { + i := s.index(name) + if i < 0 { + return s, fmt.Errorf("%w: %s", ErrColumnNotFound, name) + } + + f := s.Fields[i] + rest := slices.Delete(slices.Clone(s.Fields), i, i+1) + at := 0 + + if !first { + j := slices.IndexFunc(rest, func(c Field) bool { return strings.EqualFold(c.Name, after) }) + if j < 0 { + return s, fmt.Errorf("%w: %s", ErrColumnNotFound, after) + } + + at = j + 1 + } + + out := s + out.Fields = slices.Insert(rest, at, f) + + return out, nil +} diff --git a/pkgs/k3sengine/image.go b/pkgs/k3sengine/image.go index 583f24f5b2..cd811cee27 100644 --- a/pkgs/k3sengine/image.go +++ b/pkgs/k3sengine/image.go @@ -38,7 +38,7 @@ const scriptHead = `printf '%s,admin,admin,"system:masters"\n' "$` + envAdminAut ` && { [ -s ` + tlsDir + `/server-ca.crt ] && [ -s ` + tlsDir + `/server-ca.key ] || {` + ` printf '%s\n' "$` + envCACert + `" > ` + tlsDir + `/server-ca.crt` + ` && printf '%s\n' "$` + envCAKey + `" > ` + tlsDir + `/server-ca.key; }; }` + - ` && unset ` + envCAKey + ` && exec /bin/k3s server --tls-san "$` + envHost + `"` + + ` && unset ` + envCAKey + ` && umask 022 && exec /bin/k3s server --tls-san "$` + envHost + `"` + ` --disable traefik${` + envNodeName + `:+ --node-name "$` + envNodeName + `"}` const scriptTail = ` --disable metrics-server --kube-apiserver-arg=token-auth-file=` + tokenFileName diff --git a/pkgs/k3sengine/k3sengine_test.go b/pkgs/k3sengine/k3sengine_test.go index cb238fd772..14351e67d8 100644 --- a/pkgs/k3sengine/k3sengine_test.go +++ b/pkgs/k3sengine/k3sengine_test.go @@ -26,6 +26,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/container" "github.com/blackbirdworks/gopherstack/pkgs/k3sengine" "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" ) const token = "unit-test-cluster-token-0123" @@ -95,6 +96,13 @@ func TestScript(t *testing.T) { assert.Contains(t, k3sengine.Script(false), "--disable servicelb") assert.NotContains(t, k3sengine.Script(true), "servicelb") + // umask 077 for the CA key must not leak into k3s/containerd: it would make every image root 0700. + for _, lb := range []bool{false, true} { + sc := k3sengine.Script(lb) + assert.Less(t, strings.Index(sc, "umask 077"), strings.Index(sc, "umask 022")) + assert.Less(t, strings.Index(sc, "umask 022"), strings.Index(sc, "exec /bin/k3s")) + } + for _, lb := range []bool{false, true} { s := k3sengine.Script(lb) assert.Contains(t, s, "token-auth-file=") @@ -198,8 +206,7 @@ func (r *runtime) StopAndRemove(_ context.Context, id string) error { func newEngine(t *testing.T, rt *runtime, mod func(*k3sengine.Config)) (*k3sengine.Engine, *portalloc.Allocator) { t.Helper() - ports, err := portalloc.New(42000, 42050) - require.NoError(t, err) + ports := portalloctest.New(t, 50) cfg := k3sengine.Config{Runtime: rt, Ports: ports, Prefix: "test", Token: token} if mod != nil { diff --git a/pkgs/k3sengine/node_state_test.go b/pkgs/k3sengine/node_state_test.go index 7c618f7c79..97e88e552a 100644 --- a/pkgs/k3sengine/node_state_test.go +++ b/pkgs/k3sengine/node_state_test.go @@ -176,15 +176,18 @@ func TestEngineVolumes(t *testing.T) { func TestAdopt(t *testing.T) { t.Parallel() + const outsidePool = 50000 + tests := []struct { - ports map[int]int name string - wantPort int + inPool bool + noAPI bool wantErr bool + wantPort int }{ - {name: "running node", ports: map[int]int{k3sengine.APIPort: 42010}, wantPort: 42010}, - {name: "no api port published", ports: map[int]int{80: 42011}, wantErr: true}, - {name: "port outside the pool", ports: map[int]int{k3sengine.APIPort: 50000}, wantPort: 50000}, + {name: "running node", inPool: true}, + {name: "no api port published", noAPI: true, wantErr: true}, + {name: "port outside the pool", wantPort: outsidePool}, } for _, tt := range tests { @@ -193,7 +196,21 @@ func TestAdopt(t *testing.T) { e, ports := newEngine(t, &runtime{}, nil) - inst, err := e.Adopt(container.ManagedContainer{ID: "old", Ports: tt.ports}) + port := tt.wantPort + if tt.inPool { + var err error + + port, err = ports.Acquire("probe") + require.NoError(t, err) + require.NoError(t, ports.Release(port)) + } + + published := map[int]int{k3sengine.APIPort: port} + if tt.noAPI { + published = map[int]int{80: port} + } + + inst, err := e.Adopt(container.ManagedContainer{ID: "old", Ports: published}) if tt.wantErr { require.ErrorIs(t, err, k3sengine.ErrNotAdoptable) @@ -202,16 +219,20 @@ func TestAdopt(t *testing.T) { require.NoError(t, err) assert.Equal(t, "old", inst.ContainerID) - assert.Equal(t, tt.wantPort, inst.Port) - assert.Contains(t, ports.Reserved(), tt.wantPort, "the adopted port must not be handed out again") + assert.Equal(t, port, inst.Port) + assert.Contains(t, ports.Reserved(), port, "the adopted port must not be handed out again") }) } - e, _ := newEngine(t, &runtime{}, nil) - _, err := e.Adopt(container.ManagedContainer{ID: "a", Ports: map[int]int{k3sengine.APIPort: 42020}}) + e, ports := newEngine(t, &runtime{}, nil) + dup, err := ports.Acquire("probe") + require.NoError(t, err) + require.NoError(t, ports.Release(dup)) + + _, err = e.Adopt(container.ManagedContainer{ID: "a", Ports: map[int]int{k3sengine.APIPort: dup}}) require.NoError(t, err) - _, err = e.Adopt(container.ManagedContainer{ID: "b", Ports: map[int]int{k3sengine.APIPort: 42020}}) + _, err = e.Adopt(container.ManagedContainer{ID: "b", Ports: map[int]int{k3sengine.APIPort: dup}}) require.ErrorIs(t, err, k3sengine.ErrNotAdoptable, "a port already in use cannot be adopted twice") } diff --git a/pkgs/logger/recover.go b/pkgs/logger/recover.go new file mode 100644 index 0000000000..d08c6a948e --- /dev/null +++ b/pkgs/logger/recover.go @@ -0,0 +1,26 @@ +package logger + +import ( + "context" + "runtime/debug" +) + +// Recover is deferred directly (`defer logger.Recover(ctx, "x", fn)`) at the top of a goroutine that serves a +// client outside net/http's handler recovery. It logs the panic with its stack and then runs onPanic (may be +// nil) so the caller can send a protocol-level error or close that one connection. +func Recover(ctx context.Context, component string, onPanic func(r any)) { + r := recover() + if r == nil { + return + } + + Load(ctx).ErrorContext(ctx, "recovered from panic serving connection", + "component", component, + "panic", r, + "stack", string(debug.Stack()), + ) + + if onPanic != nil { + onPanic(r) + } +} diff --git a/pkgs/logger/recover_test.go b/pkgs/logger/recover_test.go new file mode 100644 index 0000000000..7027b8ca38 --- /dev/null +++ b/pkgs/logger/recover_test.go @@ -0,0 +1,57 @@ +package logger_test + +import ( + "bytes" + "context" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" +) + +func TestRecover(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + panic bool + }{ + {name: "panics", panic: true}, + {name: "quiet", panic: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + ctx := logger.Save(context.Background(), slog.New(slog.NewTextHandler(&buf, nil))) + + var got any + + func() { + defer logger.Recover(ctx, "unit", func(r any) { got = r }) + + if tt.panic { + panic("boom") + } + }() + + if !tt.panic { + assert.Nil(t, got) + assert.Empty(t, buf.String()) + + return + } + + require.Equal(t, "boom", got) + assert.Contains(t, buf.String(), "component=unit") + assert.Contains(t, buf.String(), "boom") + assert.Contains(t, buf.String(), "recover_test.go") + }) + } +} diff --git a/pkgs/mysqlengine/mysqlengine_test.go b/pkgs/mysqlengine/mysqlengine_test.go index e0fac3ab42..bcdcee4f40 100644 --- a/pkgs/mysqlengine/mysqlengine_test.go +++ b/pkgs/mysqlengine/mysqlengine_test.go @@ -11,7 +11,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/container" "github.com/blackbirdworks/gopherstack/pkgs/mysqlengine" - "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" ) var errBoom = errors.New("boom") @@ -66,8 +66,7 @@ func TestVolumeName(t *testing.T) { func TestLaunchFailureReleasesPorts(t *testing.T) { t.Parallel() - alloc, err := portalloc.New(40100, 40110) - require.NoError(t, err) + alloc := portalloctest.New(t, 10) rt := &failingRuntime{} e, err := mysqlengine.New(mysqlengine.Config{Runtime: rt, Ports: alloc, Prefix: "t"}) diff --git a/pkgs/persistence/testdata/snapshot_inventory.json b/pkgs/persistence/testdata/snapshot_inventory.json index 298ca89f5d..6c60f2d06a 100644 --- a/pkgs/persistence/testdata/snapshot_inventory.json +++ b/pkgs/persistence/testdata/snapshot_inventory.json @@ -393,6 +393,9 @@ "Doc.Data []byte `json:\"data\"`", "Doc.ModifiedAt time.Time `json:\"modifiedAt\"`", "Doc.Pending Pending `json:\"pending\"`", + "Exemplar.Labels promql.Labels `json:\"labels\"`", + "Exemplar.T int64 `json:\"t\"`", + "Exemplar.V float64 `json:\"v\"`", "IgnoreNear.Amount *float64 `json:\"amount,omitempty\"`", "IgnoreNear.Ratio *float64 `json:\"ratio,omitempty\"`", "LabelSetLimit.LabelSet map[string]string `json:\"labelSet\"`", @@ -446,6 +449,7 @@ "ScraperLogging.LogGroupARN string `json:\"logGroupArn\"`", "ScraperLogging.ModifiedAt time.Time `json:\"modifiedAt\"`", "ScraperLogging.Pending Pending `json:\"pending\"`", + "SeriesRecord.Exemplars []Exemplar `json:\"exemplars,omitempty\"`", "SeriesRecord.Key string `json:\"key\"`", "SeriesRecord.Labels promql.Labels `json:\"labels\"`", "SeriesRecord.Samples []promql.Sample `json:\"samples\"`", @@ -3250,6 +3254,7 @@ "armSnapshot.EventGrid *eventGridSnapshot `json:\"eventGrid,omitempty\"`", "armSnapshot.EventHub *eventHubSnapshot `json:\"eventHub,omitempty\"`", "armSnapshot.KeyVault *keyVaultSnapshot `json:\"keyVault,omitempty\"`", + "armSnapshot.Logic *logicSnapshot `json:\"logic,omitempty\"`", "armSnapshot.ManagedIdentity *managedIdentitySnapshot `json:\"managedIdentity,omitempty\"`", "armSnapshot.MySQLFlex *flexSnapshot `json:\"mysqlFlex,omitempty\"`", "armSnapshot.Network *networkSnapshot `json:\"network,omitempty\"`", @@ -3259,6 +3264,7 @@ "armSnapshot.ResourceGroups map[string]*ResourceGroup `json:\"resourceGroups\"`", "armSnapshot.Resources map[string]*Resource `json:\"resources\"`", "armSnapshot.SQL *sqlSnapshot `json:\"sql,omitempty\"`", + "armSnapshot.Search *searchSnapshot `json:\"search,omitempty\"`", "armSnapshot.Web *webSnapshot `json:\"web,omitempty\"`", "authPermission.Actions []string `json:\"actions\"`", "authPermission.DataActions []string `json:\"dataActions\"`", @@ -3412,6 +3418,82 @@ "keyVaultRecord.Tags map[string]string `json:\"tags,omitempty\"`", "keyVaultSnapshot.Deleted map[string]deletedKeyVaultRecord `json:\"deleted,omitempty\"`", "keyVaultSnapshot.Vaults map[string]keyVaultRecord `json:\"vaults,omitempty\"`", + "logicActionRec.Code string `json:\"code,omitempty\"`", + "logicActionRec.End time.Time `json:\"end\"`", + "logicActionRec.Error any `json:\"error,omitempty\"`", + "logicActionRec.Inputs logicContent `json:\"inputs\"`", + "logicActionRec.Name string `json:\"name\"`", + "logicActionRec.Outputs logicContent `json:\"outputs\"`", + "logicActionRec.Start time.Time `json:\"start\"`", + "logicActionRec.Status string `json:\"status\"`", + "logicActionRec.Tracking string `json:\"tracking,omitempty\"`", + "logicActionRec.Type string `json:\"type\"`", + "logicContent.Set bool `json:\"set,omitempty\"`", + "logicContent.Value any `json:\"value,omitempty\"`", + "logicHistory.Code string `json:\"code,omitempty\"`", + "logicHistory.End time.Time `json:\"end\"`", + "logicHistory.Error any `json:\"error,omitempty\"`", + "logicHistory.Fired bool `json:\"fired\"`", + "logicHistory.Inputs logicContent `json:\"inputs\"`", + "logicHistory.Name string `json:\"name\"`", + "logicHistory.Outputs logicContent `json:\"outputs\"`", + "logicHistory.Run string `json:\"run,omitempty\"`", + "logicHistory.Scheduled time.Time `json:\"scheduled\"`", + "logicHistory.Start time.Time `json:\"start\"`", + "logicHistory.Status string `json:\"status\"`", + "logicHistory.Tracking string `json:\"tracking\"`", + "logicHistory.Trigger string `json:\"trigger\"`", + "logicRunRec.Actions map[string]*logicActionRec `json:\"actions\"`", + "logicRunRec.ClientID string `json:\"clientTrackingId,omitempty\"`", + "logicRunRec.Code string `json:\"code,omitempty\"`", + "logicRunRec.Correlation string `json:\"correlation\"`", + "logicRunRec.End time.Time `json:\"end\"`", + "logicRunRec.Error any `json:\"error,omitempty\"`", + "logicRunRec.Name string `json:\"name\"`", + "logicRunRec.Order []string `json:\"order\"`", + "logicRunRec.Outputs map[string]any `json:\"outputs,omitempty\"`", + "logicRunRec.Start time.Time `json:\"start\"`", + "logicRunRec.Status string `json:\"status\"`", + "logicRunRec.Trigger logicRunTrigger `json:\"trigger\"`", + "logicRunTrigger.Code string `json:\"code,omitempty\"`", + "logicRunTrigger.End time.Time `json:\"end\"`", + "logicRunTrigger.Error any `json:\"error,omitempty\"`", + "logicRunTrigger.Inputs logicContent `json:\"inputs\"`", + "logicRunTrigger.Name string `json:\"name\"`", + "logicRunTrigger.Outputs logicContent `json:\"outputs\"`", + "logicRunTrigger.Scheduled time.Time `json:\"scheduled\"`", + "logicRunTrigger.Start time.Time `json:\"start\"`", + "logicRunTrigger.Status string `json:\"status\"`", + "logicRunTrigger.Tracking string `json:\"tracking,omitempty\"`", + "logicSnapshot.Workflows map[string]*logicWorkflow `json:\"workflows,omitempty\"`", + "logicTriggerState.Anchor time.Time `json:\"anchor\"`", + "logicTriggerState.Changed time.Time `json:\"changed\"`", + "logicTriggerState.Created time.Time `json:\"created\"`", + "logicTriggerState.Last *time.Time `json:\"last,omitempty\"`", + "logicTriggerState.Next *time.Time `json:\"next,omitempty\"`", + "logicTriggerState.State string `json:\"state\"`", + "logicVersion.Created time.Time `json:\"created\"`", + "logicVersion.Definition map[string]any `json:\"definition\"`", + "logicVersion.Parameters map[string]any `json:\"parameters,omitempty\"`", + "logicWorkflow.Changed time.Time `json:\"changed\"`", + "logicWorkflow.Created time.Time `json:\"created\"`", + "logicWorkflow.Definition map[string]any `json:\"definition\"`", + "logicWorkflow.Extra map[string]any `json:\"extra,omitempty\"`", + "logicWorkflow.FlowID string `json:\"flowId\"`", + "logicWorkflow.Histories []*logicHistory `json:\"histories\"`", + "logicWorkflow.Identity map[string]any `json:\"identity,omitempty\"`", + "logicWorkflow.KeyPrimary string `json:\"keyPrimary\"`", + "logicWorkflow.KeySecondary string `json:\"keySecondary\"`", + "logicWorkflow.Location string `json:\"location\"`", + "logicWorkflow.Name string `json:\"name\"`", + "logicWorkflow.Parameters map[string]any `json:\"parameters,omitempty\"`", + "logicWorkflow.ResourceGrp string `json:\"resourceGroup\"`", + "logicWorkflow.Runs []*logicRunRec `json:\"runs\"`", + "logicWorkflow.State string `json:\"state\"`", + "logicWorkflow.Subscription string `json:\"subscription\"`", + "logicWorkflow.Tags map[string]string `json:\"tags,omitempty\"`", + "logicWorkflow.Triggers map[string]*logicTriggerState `json:\"triggers\"`", + "logicWorkflow.Versions []*logicVersion `json:\"versions\"`", "managedIdentitySnapshot.Identities map[string]*storedIdentity `json:\"identities\"`", "netObj.Aux map[string]string `json:\"aux,omitempty\"`", "netObj.Etag string `json:\"etag\"`", @@ -3461,6 +3543,22 @@ "resourceIdentity.TenantID string `json:\"tenantId,omitempty\"`", "resourceIdentity.Type string `json:\"type\"`", "resourceIdentity.UserAssigned []string `json:\"userAssigned,omitempty\"`", + "searchRecord.AuthOptions map[string]any `json:\"authOptions,omitempty\"`", + "searchRecord.CreatedAt time.Time `json:\"createdAt\"`", + "searchRecord.DisableLocal bool `json:\"disableLocalAuth,omitempty\"`", + "searchRecord.Extra map[string]any `json:\"extra,omitempty\"`", + "searchRecord.Host string `json:\"host,omitempty\"`", + "searchRecord.HostingMode string `json:\"hostingMode\"`", + "searchRecord.Identity map[string]any `json:\"identity,omitempty\"`", + "searchRecord.Location string `json:\"location\"`", + "searchRecord.Name string `json:\"name\"`", + "searchRecord.Partitions int `json:\"partitions\"`", + "searchRecord.PublicAccess string `json:\"publicAccess\"`", + "searchRecord.Replicas int `json:\"replicas\"`", + "searchRecord.ResourceGroup string `json:\"resourceGroup\"`", + "searchRecord.SKU string `json:\"sku\"`", + "searchRecord.Tags map[string]string `json:\"tags,omitempty\"`", + "searchSnapshot.Services map[string]searchRecord `json:\"services,omitempty\"`", "sqlAdmin.Login string `json:\"login\"`", "sqlAdmin.SID string `json:\"sid\"`", "sqlAdmin.TenantID string `json:\"tenantId\"`", @@ -3528,18 +3626,25 @@ "storedACAEnv.Storages map[string]*storedACAStorage `json:\"storages,omitempty\"`", "storedACAEnv.Sub string `json:\"sub\"`", "storedACAEnv.Tags map[string]string `json:\"tags,omitempty\"`", + "storedACAExecution.Attempts []int `json:\"attempts,omitempty\"`", + "storedACAExecution.Done bool `json:\"done,omitempty\"`", "storedACAExecution.End *time.Time `json:\"end,omitempty\"`", "storedACAExecution.Name string `json:\"name\"`", + "storedACAExecution.Need int `json:\"need,omitempty\"`", "storedACAExecution.Replicas int `json:\"replicas\"`", + "storedACAExecution.RetryLimit int `json:\"retryLimit,omitempty\"`", + "storedACAExecution.Source string `json:\"source,omitempty\"`", "storedACAExecution.Start time.Time `json:\"start\"`", "storedACAExecution.Status string `json:\"status,omitempty\"`", "storedACAExecution.Stopped bool `json:\"stopped,omitempty\"`", "storedACAExecution.Template map[string]any `json:\"template\"`", + "storedACAExecution.TimeoutSec int `json:\"timeoutSec,omitempty\"`", "storedACAJob.Config map[string]any `json:\"config\"`", "storedACAJob.Created time.Time `json:\"created\"`", "storedACAJob.EnvironmentID string `json:\"environmentId\"`", "storedACAJob.Executions []*storedACAExecution `json:\"executions,omitempty\"`", "storedACAJob.Identity resourceIdentity `json:\"identity\"`", + "storedACAJob.LastFire time.Time `json:\"lastFire,omitzero\"`", "storedACAJob.Location string `json:\"location\"`", "storedACAJob.Modified time.Time `json:\"modified\"`", "storedACAJob.Name string `json:\"name\"`", @@ -3937,6 +4042,12 @@ "metrics.Enabled bool `xml:\"Enabled\"`", "metrics.IncludeAPIs *bool `xml:\"IncludeAPIs,omitempty\"`", "metrics.Retention *retentionPolicy `xml:\"RetentionPolicy,omitempty\"`", + "nfsProps.Group string `json:\"group\"`", + "nfsProps.LinkText string `json:\"linkText,omitempty\"`", + "nfsProps.Links int `json:\"links\"`", + "nfsProps.Mode string `json:\"mode\"`", + "nfsProps.Owner string `json:\"owner\"`", + "nfsProps.Type string `json:\"type\"`", "node.Attributes string `json:\"attributes\"`", "node.CacheControl string `json:\"cacheControl,omitempty\"`", "node.Changed time.Time `json:\"changed\"`", @@ -3956,6 +4067,7 @@ "node.Lease leaseInfo `json:\"lease\"`", "node.MD5 []byte `json:\"md5,omitempty\"`", "node.Metadata map[string]string `json:\"metadata,omitempty\"`", + "node.NFS *nfsProps `json:\"nfs,omitempty\"`", "node.Name string `json:\"name\"`", "node.ParentID string `json:\"parentId\"`", "node.PermKey string `json:\"permKey\"`", @@ -4019,6 +4131,7 @@ "appState.AppCommandLine string `json:\"appCommandLine,omitempty\"`", "appState.Created time.Time `json:\"created\"`", "appState.Deployments []Deployment `json:\"deployments\"`", + "appState.Flex bool `json:\"flex,omitempty\"`", "appState.FunctionKeys map[string]map[string]string `json:\"functionKeys\"`", "appState.HealthCheckPath string `json:\"healthCheckPath,omitempty\"`", "appState.HostKeys map[string]string `json:\"hostKeys\"`", @@ -4094,6 +4207,12 @@ "Application.UniqueName string `json:\"uniqueName,omitempty\"`", "Application.VerifiedPublisher json.RawMessage `json:\"verifiedPublisher,omitempty\"`", "Application.Web WebApplication `json:\"web\"`", + "FederatedCredential.Audiences []string `json:\"audiences\"`", + "FederatedCredential.Description *string `json:\"description\"`", + "FederatedCredential.ID string `json:\"id\"`", + "FederatedCredential.Issuer string `json:\"issuer\"`", + "FederatedCredential.Name string `json:\"name\"`", + "FederatedCredential.Subject string `json:\"subject\"`", "Group.Classification string `json:\"classification,omitempty\"`", "Group.CreatedDateTime time.Time `json:\"createdDateTime\"`", "Group.Description string `json:\"description,omitempty\"`", @@ -4152,6 +4271,12 @@ "RequiredResourceAccess.ResourceAppID string `json:\"resourceAppId\"`", "ResourceAccess.ID string `json:\"id\"`", "ResourceAccess.Type string `json:\"type\"`", + "RoleAssignment.AppScopeID *string `json:\"appScopeId\"`", + "RoleAssignment.Condition *string `json:\"condition\"`", + "RoleAssignment.DirectoryScopeID string `json:\"directoryScopeId\"`", + "RoleAssignment.ID string `json:\"id\"`", + "RoleAssignment.PrincipalID string `json:\"principalId\"`", + "RoleAssignment.RoleDefinitionID string `json:\"roleDefinitionId\"`", "ServicePrincipal.AccountEnabled bool `json:\"accountEnabled\"`", "ServicePrincipal.AddIns json.RawMessage `json:\"addIns,omitempty\"`", "ServicePrincipal.AlternativeNames []string `json:\"alternativeNames\"`", @@ -4225,13 +4350,27 @@ "WebApplication.RedirectURIs []string `json:\"redirectUris\"`", "backendSnapshot.Applications []Application `json:\"applications\"`", "backendSnapshot.Assignments []AppRoleAssignment `json:\"appRoleAssignments\"`", + "backendSnapshot.Deleted []deletedItem `json:\"deletedItems\"`", + "backendSnapshot.Federated map[string][]FederatedCredential `json:\"federatedIdentityCredentials\"`", "backendSnapshot.Grants []OAuth2PermissionGrant `json:\"oauth2PermissionGrants\"`", "backendSnapshot.Groups []Group `json:\"groups\"`", + "backendSnapshot.KeyMaterial map[string]string `json:\"keyMaterial\"`", "backendSnapshot.Members map[string][]string `json:\"members\"`", "backendSnapshot.Owners map[string][]string `json:\"owners\"`", "backendSnapshot.Principals []ServicePrincipal `json:\"servicePrincipals\"`", + "backendSnapshot.RoleAssigns []RoleAssignment `json:\"roleAssignments\"`", "backendSnapshot.Secrets map[string]string `json:\"secrets\"`", - "backendSnapshot.Users []User `json:\"users\"`" + "backendSnapshot.Users []User `json:\"users\"`", + "deletedItem.Application *Application `json:\"application,omitempty\"`", + "deletedItem.DeletedDateTime time.Time `json:\"deletedDateTime\"`", + "deletedItem.Federated []FederatedCredential `json:\"federatedIdentityCredentials,omitempty\"`", + "deletedItem.Group *Group `json:\"group,omitempty\"`", + "deletedItem.KeyMaterial map[string]string `json:\"keyMaterial,omitempty\"`", + "deletedItem.Members []string `json:\"members,omitempty\"`", + "deletedItem.Owners []string `json:\"owners,omitempty\"`", + "deletedItem.Secrets map[string]string `json:\"secrets,omitempty\"`", + "deletedItem.ServicePrincipal *ServicePrincipal `json:\"servicePrincipal,omitempty\"`", + "deletedItem.User *User `json:\"user,omitempty\"`" ], "version": 1 }, @@ -4418,6 +4557,110 @@ ], "version": 1 }, + "azuresearch": { + "fields": [ + "Doc.Fields map[string]any `json:\"fields\"`", + "Doc.Seq int64 `json:\"seq\"`", + "Doc.tok map[string]*fieldTokens", + "Index.Def map[string]any `json:\"def\"`", + "Index.Docs map[string]*Doc `json:\"docs\"`", + "Index.NextSeq int64 `json:\"nextSeq\"`", + "Index.comp *compiled", + "Index.stats *termStats", + "IndexerRuntime.History []*RunResult `json:\"history\"`", + "IndexerRuntime.Last *RunResult `json:\"last,omitempty\"`", + "IndexerRuntime.NextDue time.Time `json:\"nextDue\"`", + "IndexerRuntime.Seen map[string]string `json:\"seen\"`", + "QueryKey.Key string `json:\"key\"`", + "QueryKey.Name string `json:\"name\"`", + "RunResult.EndTime time.Time `json:\"endTime\"`", + "RunResult.ErrorMessage string `json:\"errorMessage,omitempty\"`", + "RunResult.Errors []map[string]any `json:\"errors\"`", + "RunResult.ItemsFailed int `json:\"itemsFailed\"`", + "RunResult.ItemsProcessed int `json:\"itemsProcessed\"`", + "RunResult.StartTime time.Time `json:\"startTime\"`", + "RunResult.Status string `json:\"status\"`", + "RunResult.Warnings []map[string]any `json:\"warnings\"`", + "Service.Config ServiceConfig `json:\"config\"`", + "Service.Created time.Time `json:\"created\"`", + "Service.Indexes map[string]*Index `json:\"indexes\"`", + "Service.Name string `json:\"name\"`", + "Service.NextETagSeq int64 `json:\"nextETagSeq\"`", + "Service.Objects map[string]map[string]Object `json:\"objects\"`", + "Service.Primary string `json:\"primary\"`", + "Service.QueryKeys []QueryKey `json:\"queryKeys\"`", + "Service.Runtime map[string]*IndexerRuntime `json:\"runtime\"`", + "Service.Secondary string `json:\"secondary\"`", + "ServiceConfig.AADFailureMode string `json:\"aadFailureMode,omitempty\"`", + "ServiceConfig.AuthMode string `json:\"authMode\"`", + "ServiceConfig.DisableLocalAuth bool `json:\"disableLocalAuth,omitempty\"`", + "ServiceConfig.HostingMode string `json:\"hostingMode\"`", + "ServiceConfig.PartitionCount int `json:\"partitionCount\"`", + "ServiceConfig.PublicNetworkAccess string `json:\"publicNetworkAccess\"`", + "ServiceConfig.ReplicaCount int `json:\"replicaCount\"`", + "ServiceConfig.SKU string `json:\"sku\"`", + "Token.End int", + "Token.Pos int", + "Token.Start int", + "Token.Term string", + "backendSnapshot.Services map[string]*Service `json:\"services\"`", + "compiled.analyzers map[string]Analyzer", + "compiled.byPath map[string]*fieldDef", + "compiled.defProfile string", + "compiled.fields []*fieldDef", + "compiled.key *fieldDef", + "compiled.profiles map[string]*scoringProfile", + "compiled.searchable []*fieldDef", + "compiled.suggesters map[string]*suggester", + "compiled.vecProfile map[string]vectorProfile", + "fieldDef.Analyzer string", + "fieldDef.Base string", + "fieldDef.Children []*fieldDef", + "fieldDef.Collection bool", + "fieldDef.Dimensions int", + "fieldDef.Facetable bool", + "fieldDef.Filterable bool", + "fieldDef.IndexAnalyzer string", + "fieldDef.Key bool", + "fieldDef.Name string", + "fieldDef.Path string", + "fieldDef.Profile string", + "fieldDef.Retrievable bool", + "fieldDef.SearchAnalyzer string", + "fieldDef.Searchable bool", + "fieldDef.Sortable bool", + "fieldDef.SynonymMaps []string", + "fieldDef.Type string", + "fieldTokens.length int", + "fieldTokens.terms map[string][]int", + "fieldTokens.toks []Token", + "scoreFunc.Boost float64", + "scoreFunc.ConstBeyond bool", + "scoreFunc.Distance float64", + "scoreFunc.Duration time.Duration", + "scoreFunc.Field string", + "scoreFunc.Interpolation string", + "scoreFunc.RangeEnd float64", + "scoreFunc.RangeStart float64", + "scoreFunc.RefParam string", + "scoreFunc.TagsParam string", + "scoreFunc.Type string", + "scoringProfile.Aggregation string", + "scoringProfile.Funcs []scoreFunc", + "scoringProfile.Name string", + "scoringProfile.Weights map[string]float64", + "suggester.Name string", + "suggester.SourceFields []string", + "termStats.df map[string]map[string]int", + "termStats.docs map[string]int", + "termStats.lenSum map[string]int", + "termStats.n int", + "vectorProfile.Algorithm string", + "vectorProfile.Kind string", + "vectorProfile.Metric string" + ], + "version": 1 + }, "azureservicebus": { "fields": [ "EntityConfig.DefaultMessageTTL time.Duration", @@ -6684,6 +6927,7 @@ "StackEvent.StackName string `xml:\"StackName\" json:\"stackName\"`", "StackEvent.Timestamp time.Time `xml:\"Timestamp\" json:\"timestamp\"`", "StackInstance.Account string `xml:\"Account,omitempty\" json:\"account,omitempty\"`", + "StackInstance.DetailedStatus string `xml:\"-\" json:\"detailedStatus,omitempty\"`", "StackInstance.DriftStatus string `xml:\"DriftStatus,omitempty\" json:\"driftStatus,omitempty\"`", "StackInstance.LastDriftCheckTimestamp *time.Time `xml:\"LastDriftCheckTimestamp,omitempty\" json:\"lastDriftCheckTimestamp,omitempty\"`", "StackInstance.LastOperationID string `xml:\"LastOperationId,omitempty\" json:\"lastOperationID,omitempty\"`", @@ -6774,13 +7018,18 @@ "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`", "backendSnapshot.TypeConfigs map[string]string `json:\"typeConfigs\"`", "backendSnapshot.TypeVersions map[string][]*RegisteredTypeVersion `json:\"typeVersions\"`", + "pendingStackOp.Announced []string `json:\"announced,omitempty\"`", + "pendingStackOp.Created []string `json:\"created,omitempty\"`", "pendingStackOp.DisableValidation bool `json:\"disableValidation,omitempty\"`", + "pendingStackOp.ForceDelete bool `json:\"forceDelete,omitempty\"`", "pendingStackOp.Kind string `json:\"kind\"`", "pendingStackOp.OldParams []Parameter `json:\"oldParams,omitempty\"`", "pendingStackOp.OldTemplateBody string `json:\"oldTemplateBody,omitempty\"`", "pendingStackOp.OnFailure string `json:\"onFailure,omitempty\"`", + "pendingStackOp.PrevResources map[string]*StackResource `json:\"prevResources,omitempty\"`", "pendingStackOp.ResourceTypes []string `json:\"resourceTypes,omitempty\"`", - "pendingStackOp.RetainExceptOnCreate bool `json:\"retainExceptOnCreate,omitempty\"`" + "pendingStackOp.RetainExceptOnCreate bool `json:\"retainExceptOnCreate,omitempty\"`", + "pendingStackOp.RetainResources []string `json:\"retainResources,omitempty\"`" ], "version": 1 }, @@ -7969,6 +8218,7 @@ "Build.EncryptionKey string `json:\"encryptionKey,omitempty\"`", "Build.EndTime float64 `json:\"endTime,omitempty\"`", "Build.Environment *ProjectEnvironment `json:\"environment,omitempty\"`", + "Build.ExportedEnvironmentVariables []ExportedEnvironmentVariable `json:\"exportedEnvironmentVariables,omitempty\"`", "Build.FileSystemLocations []FileSystemLocation `json:\"fileSystemLocations,omitempty\"`", "Build.ID string `json:\"id\"`", "Build.Initiator string `json:\"initiator,omitempty\"`", @@ -8101,6 +8351,8 @@ "EnvironmentVariable.Name string `json:\"name\"`", "EnvironmentVariable.Type string `json:\"type,omitempty\"`", "EnvironmentVariable.Value string `json:\"value\"`", + "ExportedEnvironmentVariable.Name string `json:\"name\"`", + "ExportedEnvironmentVariable.Value string `json:\"value\"`", "FileSystemLocation.Identifier string `json:\"identifier,omitempty\"`", "FileSystemLocation.Location string `json:\"location,omitempty\"`", "FileSystemLocation.MountOptions string `json:\"mountOptions,omitempty\"`", @@ -8173,6 +8425,7 @@ "ProjectArtifacts.Type string `json:\"type\"`", "ProjectBadge.BadgeEnabled bool `json:\"badgeEnabled,omitempty\"`", "ProjectBadge.BadgeRequestURL string `json:\"badgeRequestUrl,omitempty\"`", + "ProjectCache.CacheNamespace string `json:\"cacheNamespace,omitempty\"`", "ProjectCache.Location string `json:\"location,omitempty\"`", "ProjectCache.Modes []string `json:\"modes,omitempty\"`", "ProjectCache.Type string `json:\"type\"`", @@ -9371,6 +9624,7 @@ "WebAuthnCredential.AuthenticatorAttachment string `json:\"authenticatorAttachment,omitempty\"`", "WebAuthnCredential.AuthenticatorTransports []string `json:\"authenticatorTransports,omitempty\"`", "WebAuthnCredential.CreatedAt time.Time `json:\"createdAt\"`", + "WebAuthnCredential.Credential *webauthn.Credential `json:\"credential,omitempty\"`", "WebAuthnCredential.CredentialID string `json:\"credentialID,omitempty\"`", "WebAuthnCredential.FriendlyName string `json:\"friendlyName,omitempty\"`", "WebAuthnCredential.RelyingPartyID string `json:\"relyingPartyID,omitempty\"`", @@ -10794,18 +11048,29 @@ "ReplicationConfig.CdcStartTime *time.Time `json:\",omitempty\"`", "ReplicationConfig.CdcStopPosition string `json:\",omitempty\"`", "ReplicationConfig.ComputeConfig *ComputeConfig", + "ReplicationConfig.CreateTime time.Time `json:\",omitzero\"`", + "ReplicationConfig.ElapsedTimeMillis int64 `json:\"-\"`", + "ReplicationConfig.FailureMessage string `json:\",omitempty\"`", + "ReplicationConfig.FullLoadFinishDate time.Time `json:\",omitzero\"`", + "ReplicationConfig.FullLoadStartDate time.Time `json:\",omitzero\"`", + "ReplicationConfig.RecoveryCheckpoint string `json:\",omitempty\"`", "ReplicationConfig.Region string", "ReplicationConfig.ReplicationConfigArn string", "ReplicationConfig.ReplicationConfigIdentifier string", "ReplicationConfig.ReplicationSettings string `json:\",omitempty\"`", "ReplicationConfig.ReplicationType string", "ReplicationConfig.SourceEndpointArn string", + "ReplicationConfig.StartDate time.Time `json:\",omitzero\"`", "ReplicationConfig.StartReplicationType string", "ReplicationConfig.Status string", + "ReplicationConfig.StopDate time.Time `json:\",omitzero\"`", + "ReplicationConfig.StopReason string `json:\",omitempty\"`", "ReplicationConfig.SupplementalSettings string `json:\",omitempty\"`", "ReplicationConfig.TableMappings string", + "ReplicationConfig.TableStats []TableStatistic `json:\",omitempty\"`", "ReplicationConfig.Tags *tags.Tags `json:\"-\"`", "ReplicationConfig.TargetEndpointArn string", + "ReplicationConfig.UpdateTime time.Time `json:\",omitzero\"`", "ReplicationInstance.AccountID string `json:\"accountId\"`", "ReplicationInstance.AllocatedStorage int32 `json:\"allocatedStorage\"`", "ReplicationInstance.AutoMinorVersionUpgrade bool `json:\"autoMinorVersionUpgrade\"`", @@ -13504,6 +13769,7 @@ "Deployment.Status string `json:\"status\"`", "Deployment.TaskDefinition string `json:\"taskDefinition\"`", "Deployment.UpdatedAt *float64 `json:\"updatedAt,omitempty\"`", + "Deployment.VolumeConfigurations []VolumeConfiguration `json:\"volumeConfigurations,omitempty\"`", "Deployment.completionEmitted bool", "DeploymentAlarms.AlarmNames []string `json:\"alarmNames\"`", "DeploymentAlarms.Enable bool `json:\"enable\"`", @@ -13536,6 +13802,10 @@ "DockerVolumeConfig.DriverOpts map[string]string `json:\"driverOpts,omitempty\"`", "DockerVolumeConfig.Labels map[string]string `json:\"labels,omitempty\"`", "DockerVolumeConfig.Scope string `json:\"scope,omitempty\"`", + "EBSTagSpecification.PropagateTags string `json:\"propagateTags,omitempty\"`", + "EBSTagSpecification.ResourceType string `json:\"resourceType\"`", + "EBSTagSpecification.Tags []Tag `json:\"tags,omitempty\"`", + "EBSTerminationPolicy.DeleteOnTermination bool `json:\"deleteOnTermination\"`", "EFSAuthorizationConfig.AccessPointID string `json:\"accessPointId,omitempty\"`", "EFSAuthorizationConfig.IAM string `json:\"iam,omitempty\"`", "EFSVolumeConfiguration.AuthorizationConfig *EFSAuthorizationConfig `json:\"authorizationConfig,omitempty\"`", @@ -13633,6 +13903,18 @@ "LogConfiguration.LogDriver string `json:\"logDriver\"`", "LogConfiguration.Options map[string]string `json:\"options,omitempty\"`", "LogConfiguration.SecretOptions []SecretOption `json:\"secretOptions,omitempty\"`", + "ManagedEBSVolume.Encrypted *bool `json:\"encrypted,omitempty\"`", + "ManagedEBSVolume.FilesystemType string `json:\"filesystemType,omitempty\"`", + "ManagedEBSVolume.Iops *int `json:\"iops,omitempty\"`", + "ManagedEBSVolume.KMSKeyID string `json:\"kmsKeyId,omitempty\"`", + "ManagedEBSVolume.RoleArn string `json:\"roleArn,omitempty\"`", + "ManagedEBSVolume.SizeInGiB *int `json:\"sizeInGiB,omitempty\"`", + "ManagedEBSVolume.SnapshotID string `json:\"snapshotId,omitempty\"`", + "ManagedEBSVolume.TagSpecifications []EBSTagSpecification `json:\"tagSpecifications,omitempty\"`", + "ManagedEBSVolume.TerminationPolicy *EBSTerminationPolicy `json:\"terminationPolicy,omitempty\"`", + "ManagedEBSVolume.Throughput *int `json:\"throughput,omitempty\"`", + "ManagedEBSVolume.VolumeInitializationRate *int `json:\"volumeInitializationRate,omitempty\"`", + "ManagedEBSVolume.VolumeType string `json:\"volumeType,omitempty\"`", "ManagedScaling.InstanceWarmupPeriod int `json:\"instanceWarmupPeriod,omitempty\"`", "ManagedScaling.MaximumScalingStepSize int `json:\"maximumScalingStepSize,omitempty\"`", "ManagedScaling.MinimumScalingStepSize int `json:\"minimumScalingStepSize,omitempty\"`", @@ -13719,6 +14001,7 @@ "Service.Status string `json:\"status\"`", "Service.Tags []Tag `json:\"tags,omitempty\"`", "Service.TaskDefinition string `json:\"taskDefinition\"`", + "Service.VolumeConfigurations []VolumeConfiguration `json:\"volumeConfigurations,omitempty\"`", "Service.steadyReported bool", "ServiceConnectClientAlias.DNSName string `json:\"dnsName,omitempty\"`", "ServiceConnectClientAlias.Port int `json:\"port\"`", @@ -13776,6 +14059,7 @@ "Tag.Value string `json:\"value\"`", "Task.Attachments []TaskAttachment `json:\"attachments,omitempty\"`", "Task.AwaitingCapacity bool `json:\"awaitingCapacity,omitempty\"`", + "Task.CPU string `json:\"cpu,omitempty\"`", "Task.CapacityProviderName string `json:\"capacityProviderName,omitempty\"`", "Task.ClusterArn string `json:\"clusterArn\"`", "Task.Connectivity string `json:\"connectivity,omitempty\"`", @@ -13785,10 +14069,12 @@ "Task.CreatedAt *time.Time `json:\"createdAt,omitempty\"`", "Task.DesiredStatus string `json:\"desiredStatus\"`", "Task.EnableExecuteCommand bool `json:\"enableExecuteCommand,omitempty\"`", + "Task.EphemeralStorage *EphemeralStorage `json:\"ephemeralStorage,omitempty\"`", "Task.Group string `json:\"group,omitempty\"`", "Task.HealthStatus string `json:\"healthStatus,omitempty\"`", "Task.LastStatus string `json:\"lastStatus\"`", "Task.LaunchType string `json:\"launchType,omitempty\"`", + "Task.Memory string `json:\"memory,omitempty\"`", "Task.NetworkConfiguration *NetworkConfiguration `json:\"networkConfiguration,omitempty\"`", "Task.Overrides *TaskOverride `json:\"overrides,omitempty\"`", "Task.PlatformFamily string `json:\"platformFamily,omitempty\"`", @@ -13806,6 +14092,7 @@ "Task.TaskArn string `json:\"taskArn\"`", "Task.TaskDefinitionArn string `json:\"taskDefinitionArn\"`", "Task.TaskRoleArn string `json:\"taskRoleArn,omitempty\"`", + "Task.ebsVolumes map[string]ebsMount", "Task.runtimeHealth bool", "Task.version int64", "TaskAttachment.Details []KeyValuePair `json:\"details,omitempty\"`", @@ -13836,6 +14123,7 @@ "TaskDefinition.Volumes []Volume `json:\"volumes,omitempty\"`", "TaskOverride.CPU string `json:\"cpu,omitempty\"`", "TaskOverride.ContainerOverrides []ContainerOverride `json:\"containerOverrides,omitempty\"`", + "TaskOverride.EphemeralStorage *EphemeralStorage `json:\"ephemeralStorage,omitempty\"`", "TaskOverride.ExecutionRoleArn string `json:\"executionRoleArn,omitempty\"`", "TaskOverride.Memory string `json:\"memory,omitempty\"`", "TaskOverride.TaskRoleArn string `json:\"taskRoleArn,omitempty\"`", @@ -13876,6 +14164,8 @@ "Volume.EFSVolumeConfiguration *EFSVolumeConfiguration `json:\"efsVolumeConfiguration,omitempty\"`", "Volume.Host *HostVolumeProperties `json:\"host,omitempty\"`", "Volume.Name string `json:\"name\"`", + "VolumeConfiguration.ManagedEBSVolume *ManagedEBSVolume `json:\"managedEBSVolume,omitempty\"`", + "VolumeConfiguration.Name string `json:\"name\"`", "VolumeFrom.ReadOnly bool `json:\"readOnly,omitempty\"`", "VolumeFrom.SourceContainer string `json:\"sourceContainer,omitempty\"`", "backendSnapshot.Attributes map[string]map[string]*Attribute `json:\"attributes\"`", @@ -13883,6 +14173,8 @@ "backendSnapshot.ResourceTags map[string][]Tag `json:\"resourceTags\"`", "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`", "backendSnapshot.TaskDefinitions map[string][]*TaskDefinition `json:\"taskDefinitions\"`", + "ebsMount.deleteOnTermination bool", + "ebsMount.volumeID string", "rollingPlan.currentLive int", "rollingPlan.runningTotal int", "rollingPlan.staleLive int", @@ -14345,6 +14637,7 @@ "Cluster.Tags *tags.Tags", "Cluster.TransitEncryptionEnabled bool", "Cluster.TransitEncryptionMode string", + "Cluster.dock *dockRef", "Cluster.front *sgfilter.Proxy", "Cluster.gate *redisengine.Gate", "Cluster.mini *miniredis.Miniredis", @@ -14443,6 +14736,7 @@ "ReplicationGroup.TransitEncryptionEnabled bool `json:\"transitEncryptionEnabled,omitempty\"`", "ReplicationGroup.TransitEncryptionMode string `json:\"transitEncryptionMode,omitempty\"`", "ReplicationGroup.UserGroupIDs []string `json:\"userGroupIds,omitempty\"`", + "ReplicationGroup.dock *redisdocker.Group", "ReservedCacheNode.ARN string `json:\"arn\"`", "ReservedCacheNode.CacheNodeCount int32 `json:\"cacheNodeCount\"`", "ReservedCacheNode.CacheNodeType string `json:\"cacheNodeType\"`", @@ -14500,6 +14794,7 @@ "User.CreatedAt time.Time `json:\"createdAt\"`", "User.Engine string `json:\"engine\"`", "User.PasswordCount int `json:\"passwordCount\"`", + "User.Passwords []string `json:\"passwords,omitempty\"`", "User.Status string `json:\"status\"`", "User.Tags *tags.Tags `json:\"tags,omitempty\"`", "User.UserGroupIDs []string `json:\"userGroupIds,omitempty\"`", @@ -14567,7 +14862,12 @@ "clusterSnapshot.SecurityGroupIDs []string `json:\"securityGroupIds,omitempty\"`", "clusterSnapshot.SnapshotWindow string `json:\"snapshotWindow,omitempty\"`", "clusterSnapshot.Status string `json:\"status\"`", - "clusterSnapshot.Tags *tags.Tags `json:\"tags,omitempty\"`" + "clusterSnapshot.Tags *tags.Tags `json:\"tags,omitempty\"`", + "dockRef.group *redisdocker.Group", + "dockRef.node int", + "dockRef.seeded bool", + "dockRef.shard int", + "dockRef.standalone bool" ], "version": 1 }, @@ -16126,6 +16426,7 @@ }, "gcpartifactregistry": { "fields": [ + "backendSnapshot.Attachments map[string]json.RawMessage `json:\"attachments,omitempty\"`", "backendSnapshot.GenericFiles map[string]*genFile `json:\"genericFiles,omitempty\"`", "backendSnapshot.GenericPkgs map[string]*resMeta `json:\"genericPackages,omitempty\"`", "backendSnapshot.GenericVers map[string]*resMeta `json:\"genericVersions,omitempty\"`", @@ -16136,6 +16437,7 @@ "backendSnapshot.Repositories map[string]repoSnap `json:\"repositories\"`", "backendSnapshot.Rules map[string]json.RawMessage `json:\"rules,omitempty\"`", "backendSnapshot.Settings map[string]json.RawMessage `json:\"projectSettings,omitempty\"`", + "backendSnapshot.SigningKeys map[string][]byte `json:\"signingKeys,omitempty\"`", "backendSnapshot.VPCSC map[string]json.RawMessage `json:\"vpcscConfigs,omitempty\"`", "genFile.Annotations map[string]string `json:\"annotations,omitempty\"`", "genFile.Created time.Time `json:\"created\"`", @@ -16297,14 +16599,19 @@ "backendSnapshot.BackendBuckets map[string]*computeapi.BackendBucket `json:\"backendBuckets\"`", "backendSnapshot.BackendServices map[string]*computeapi.BackendService `json:\"backendServices\"`", "backendSnapshot.Disks map[string]*computeapi.Disk `json:\"disks\"`", + "backendSnapshot.ExternalVPNs map[string]*computeapi.ExternalVpnGateway `json:\"externalVpnGateways\"`", "backendSnapshot.Firewalls map[string]*computeapi.Firewall `json:\"firewalls\"`", "backendSnapshot.FwdRules map[string]*computeapi.ForwardingRule `json:\"forwardingRules\"`", "backendSnapshot.Groups map[string]*groupRec `json:\"instanceGroups\"`", + "backendSnapshot.HTTPHealths map[string]*computeapi.HttpHealthCheck `json:\"httpHealthChecks\"`", "backendSnapshot.HTTPProxies map[string]*computeapi.TargetHttpProxy `json:\"targetHttpProxies\"`", + "backendSnapshot.HTTPSHealths map[string]*computeapi.HttpsHealthCheck `json:\"httpsHealthChecks\"`", "backendSnapshot.HTTPSProxies map[string]*computeapi.TargetHttpsProxy `json:\"targetHttpsProxies\"`", "backendSnapshot.Healths map[string]*computeapi.HealthCheck `json:\"healthChecks\"`", "backendSnapshot.Images map[string]*computeapi.Image `json:\"images\"`", "backendSnapshot.Instances map[string]*instRec `json:\"instances\"`", + "backendSnapshot.InstantSnaps map[string]*computeapi.InstantSnapshot `json:\"instantSnapshots\"`", + "backendSnapshot.MachineImages map[string]*computeapi.MachineImage `json:\"machineImages\"`", "backendSnapshot.Migs map[string]*migRec `json:\"instanceGroupManagers\"`", "backendSnapshot.NEGEndpoints map[string][]*computeapi.NetworkEndpoint `json:\"networkEndpoints\"`", "backendSnapshot.NEGs map[string]*computeapi.NetworkEndpointGroup `json:\"networkEndpointGroups\"`", @@ -16312,6 +16619,7 @@ "backendSnapshot.Ops map[string]*opRec `json:\"ops\"`", "backendSnapshot.Projects map[string]*computeapi.Metadata `json:\"projects\"`", "backendSnapshot.ResPolicies map[string]*computeapi.ResourcePolicy `json:\"resourcePolicies\"`", + "backendSnapshot.Reservations map[string]*computeapi.Reservation `json:\"reservations\"`", "backendSnapshot.Routers map[string]*routerRec `json:\"routers\"`", "backendSnapshot.Routes map[string]*computeapi.Route `json:\"routes\"`", "backendSnapshot.SSLCerts map[string]*computeapi.SslCertificate `json:\"sslCertificates\"`", @@ -16322,8 +16630,13 @@ "backendSnapshot.Snapshots map[string]*computeapi.Snapshot `json:\"snapshots\"`", "backendSnapshot.Subnets map[string]*computeapi.Subnetwork `json:\"subnets\"`", "backendSnapshot.TCPProxies map[string]*computeapi.TargetTcpProxy `json:\"targetTcpProxies\"`", + "backendSnapshot.TargetInstances map[string]*computeapi.TargetInstance `json:\"targetInstances\"`", + "backendSnapshot.TargetPools map[string]*computeapi.TargetPool `json:\"targetPools\"`", + "backendSnapshot.TargetVPNs map[string]*computeapi.TargetVpnGateway `json:\"targetVpnGateways\"`", "backendSnapshot.Templates map[string]*computeapi.InstanceTemplate `json:\"templates\"`", "backendSnapshot.URLMaps map[string]*computeapi.UrlMap `json:\"urlMaps\"`", + "backendSnapshot.VPNGateways map[string]*computeapi.VpnGateway `json:\"vpnGateways\"`", + "backendSnapshot.VPNTunnels map[string]*computeapi.VpnTunnel `json:\"vpnTunnels\"`", "groupRec.Group *computeapi.InstanceGroup `json:\"group\"`", "groupRec.Members []string `json:\"members\"`", "instRec.Inst *computeapi.Instance `json:\"inst\"`", @@ -16331,18 +16644,37 @@ "instRec.Since time.Time `json:\"since\"`", "memberRec.DeleteAt time.Time `json:\"deleteAt\"`", "memberRec.Deleting bool `json:\"deleting\"`", + "memberRec.Health string `json:\"health,omitempty\"`", + "memberRec.HealthyRun int `json:\"healthyRun,omitempty\"`", + "memberRec.InstanceID uint64 `json:\"instanceId,omitempty\"`", + "memberRec.KeptExt map[string]string `json:\"keptExternal,omitempty\"`", + "memberRec.KeptInt map[string]string `json:\"keptInternal,omitempty\"`", + "memberRec.LastError string `json:\"lastError,omitempty\"`", + "memberRec.LastProbe time.Time `json:\"lastProbe,omitzero\"`", "memberRec.Name string `json:\"name\"`", + "memberRec.Parked string `json:\"parked,omitempty\"`", "memberRec.Recreating bool `json:\"recreating\"`", + "memberRec.Request string `json:\"request,omitempty\"`", + "memberRec.RunUntil time.Time `json:\"runUntil,omitzero\"`", + "memberRec.Startup string `json:\"startup,omitempty\"`", "memberRec.Template string `json:\"template\"`", + "memberRec.UnhealthyRun int `json:\"unhealthyRun,omitempty\"`", + "memberRec.Verified bool `json:\"verified,omitempty\"`", "memberRec.Zone string `json:\"zone\"`", + "migRec.Configs []*picRec `json:\"configs,omitempty\"`", "migRec.Members []*memberRec `json:\"members\"`", "migRec.Mgr *computeapi.InstanceGroupManager `json:\"mgr\"`", "migRec.Project string `json:\"project\"`", + "migRec.Resizes []*resizeRec `json:\"resizes,omitempty\"`", "opRec.Created time.Time `json:\"created\"`", "opRec.Done time.Duration `json:\"done\"`", "opRec.Op *computeapi.Operation `json:\"op\"`", "opRec.Pending time.Duration `json:\"pending\"`", "opRec.Project string `json:\"project\"`", + "picRec.Config *computeapi.PerInstanceConfig `json:\"config\"`", + "picRec.Since time.Time `json:\"since\"`", + "resizeRec.Created time.Time `json:\"created\"`", + "resizeRec.Req *computeapi.InstanceGroupManagerResizeRequest `json:\"req\"`", "routerRec.AutoIPs map[string][]string `json:\"autoIps\"`", "routerRec.Router *computeapi.Router `json:\"router\"`" ], @@ -16410,6 +16742,7 @@ }, "gcpeventarc": { "fields": [ + "backendSnapshot.Advanced map[string]map[string]json.RawMessage `json:\"advanced,omitempty\"`", "backendSnapshot.Channels map[string]channelSnap `json:\"channels\"`", "backendSnapshot.Configs map[string]json.RawMessage `json:\"configs\"`", "backendSnapshot.Connections map[string]json.RawMessage `json:\"connections\"`", @@ -16561,10 +16894,17 @@ "bucketSnap.Entries []int `json:\"entries,omitempty\"`", "metricSnap.Metric json.RawMessage `json:\"metric\"`", "metricSnap.Series []seriesSnap `json:\"series,omitempty\"`", + "minuteSnap.Buckets []int64 `json:\"buckets,omitempty\"`", + "minuteSnap.Count int64 `json:\"count\"`", + "minuteSnap.Mean float64 `json:\"mean\"`", + "minuteSnap.SSD float64 `json:\"ssd\"`", + "minuteSnap.Start int64 `json:\"start\"`", + "minuteSnap.Sum float64 `json:\"sum\"`", "seriesSnap.Buckets []int64 `json:\"buckets,omitempty\"`", "seriesSnap.Count int64 `json:\"count\"`", "seriesSnap.Labels map[string]string `json:\"labels,omitempty\"`", "seriesSnap.Mean float64 `json:\"mean\"`", + "seriesSnap.Minutes []minuteSnap `json:\"minutes,omitempty\"`", "seriesSnap.SSD float64 `json:\"ssd\"`", "seriesSnap.Sum float64 `json:\"sum\"`" ], @@ -16577,8 +16917,10 @@ "Incident.EndedAt time.Time `json:\"endedAt\"`", "Incident.ID string `json:\"id\"`", "Incident.Key string `json:\"key\"`", + "Incident.LastSeen time.Time `json:\"lastSeen\"`", "Incident.MetricLabels map[string]string `json:\"metricLabels,omitempty\"`", "Incident.MetricType string `json:\"metricType\"`", + "Incident.Notified map[string]time.Time `json:\"notified,omitempty\"`", "Incident.Observed float64 `json:\"observed\"`", "Incident.Policy string `json:\"policy\"`", "Incident.PolicyDisplayName string `json:\"policyDisplayName\"`", @@ -16590,6 +16932,7 @@ "Incident.Threshold float64 `json:\"threshold\"`", "backendSnapshot.Channels map[string]channelSnap `json:\"channels\"`", "backendSnapshot.Descriptors map[string]json.RawMessage `json:\"descriptors\"`", + "backendSnapshot.Explicit []string `json:\"explicit_descriptors,omitempty\"`", "backendSnapshot.Groups map[string]json.RawMessage `json:\"groups\"`", "backendSnapshot.Incidents []Incident `json:\"incidents\"`", "backendSnapshot.Policies map[string]json.RawMessage `json:\"policies\"`", @@ -16910,20 +17253,37 @@ "Bucket.Location string `json:\"location\"`", "Bucket.Metageneration int64 `json:\"metageneration\"`", "Bucket.Name string `json:\"name\"`", + "Bucket.NotifSeq int64 `json:\"notifSeq,omitempty\"`", + "Bucket.Notifications []Notification `json:\"notifications,omitempty\"`", "Bucket.Objects map[string][]*Object `json:\"objects\"`", "Bucket.Project string `json:\"project\"`", + "Bucket.SoftDeleted []*Object `json:\"softDeleted,omitempty\"`", "Bucket.StorageClass string `json:\"storageClass\"`", "Bucket.Updated time.Time `json:\"updated\"`", "Bucket.Versioning bool `json:\"versioning,omitempty\"`", + "HMACKey.AccessID string `json:\"accessId\"`", + "HMACKey.Created time.Time `json:\"created\"`", + "HMACKey.Project string `json:\"project\"`", + "HMACKey.Secret string `json:\"secret\"`", + "HMACKey.ServiceAccountEmail string `json:\"serviceAccountEmail\"`", + "HMACKey.State string `json:\"state\"`", + "HMACKey.Updated time.Time `json:\"updated\"`", "IAMBinding.Condition json.RawMessage `json:\"condition,omitempty\"`", "IAMBinding.Members []string `json:\"members\"`", "IAMBinding.Role string `json:\"role\"`", "IAMPolicy.Bindings []IAMBinding `json:\"bindings\"`", "IAMPolicy.Etag string `json:\"etag\"`", + "Notification.CustomAttributes map[string]string `json:\"customAttributes,omitempty\"`", + "Notification.EventTypes []string `json:\"eventTypes,omitempty\"`", + "Notification.ID string `json:\"id\"`", + "Notification.ObjectNamePrefix string `json:\"objectNamePrefix,omitempty\"`", + "Notification.PayloadFormat string `json:\"payloadFormat\"`", + "Notification.Topic string `json:\"topic\"`", "Object.ACL []ACLEntry `json:\"acl,omitempty\"`", "Object.Bucket string `json:\"bucket\"`", "Object.CRC32C string `json:\"crc32c\"`", "Object.CacheControl string `json:\"cacheControl,omitempty\"`", + "Object.ClassUpdated time.Time `json:\"classUpdated\"`", "Object.ComponentCount int64 `json:\"componentCount,omitempty\"`", "Object.ContentDisposition string `json:\"contentDisposition,omitempty\"`", "Object.ContentEncoding string `json:\"contentEncoding,omitempty\"`", @@ -16935,15 +17295,20 @@ "Object.Deleted time.Time `json:\"deleted\"`", "Object.EventBasedHold bool `json:\"eventBasedHold,omitempty\"`", "Object.Generation int64 `json:\"generation\"`", + "Object.HardDeleteTime time.Time `json:\"hardDeleteTime\"`", + "Object.HoldReleased time.Time `json:\"holdReleased\"`", "Object.MD5 string `json:\"md5,omitempty\"`", "Object.Metadata map[string]string `json:\"metadata,omitempty\"`", "Object.Metageneration int64 `json:\"metageneration\"`", "Object.Name string `json:\"name\"`", + "Object.SoftDeleteTime time.Time `json:\"softDeleteTime\"`", "Object.StorageClass string `json:\"storageClass\"`", "Object.TemporaryHold bool `json:\"temporaryHold,omitempty\"`", "Object.Updated time.Time `json:\"updated\"`", "Object.hideACL bool", - "backendSnapshot.Buckets map[string]*Bucket `json:\"buckets\"`" + "Object.retention time.Time", + "backendSnapshot.Buckets map[string]*Bucket `json:\"buckets\"`", + "backendSnapshot.HMACKeys map[string]*HMACKey `json:\"hmacKeys,omitempty\"`" ], "version": 1 }, @@ -21334,6 +21699,79 @@ ], "version": 1 }, + "lambdaweb": { + "fields": [ + "Endpoint.ARN string", + "Endpoint.AccountID string", + "Endpoint.AuthType string", + "Endpoint.AutoDeploymentMode string", + "Endpoint.CreatedAt time.Time", + "Endpoint.Description string", + "Endpoint.DomainName string", + "Endpoint.FunctionName string", + "Endpoint.MaxEnvironments *int32", + "Endpoint.Name string", + "Endpoint.PendingUntil time.Time", + "Endpoint.RateLimit *int32", + "Endpoint.Region string", + "Endpoint.RegionDomains map[string]string", + "Endpoint.Regions []string", + "Endpoint.State string", + "Endpoint.StateReason string", + "Endpoint.Type string", + "Endpoint.UpdatePendingUntil time.Time", + "Endpoint.UpdateReason string", + "Endpoint.UpdateStatus string", + "Endpoint.UpdatedAt time.Time", + "Endpoint.Weights []RevisionWeight", + "Function.ARN string", + "Function.AccountID string", + "Function.CreatedAt time.Time", + "Function.Name string", + "Function.PendingUntil time.Time", + "Function.Policy string", + "Function.PolicyRevID string", + "Function.Region string", + "Function.State string", + "Function.StateReason string", + "Function.Tags map[string]string", + "Function.UpdatedAt time.Time", + "LoggingConfig.ApplicationLogLevel string", + "LoggingConfig.LogGroup string", + "LoggingConfig.SystemLogLevel string", + "Revision.ARN string", + "Revision.AccountID string", + "Revision.CreatedAt time.Time", + "Revision.Description string", + "Revision.Errors []RevisionError", + "Revision.FunctionName string", + "Revision.ID string", + "Revision.KmsKeyArn string", + "Revision.PendingUntil time.Time", + "Revision.Region string", + "Revision.Runtime string", + "Revision.S3 S3Object", + "Revision.ServiceConfig ServiceConfig", + "Revision.State string", + "Revision.StateReason string", + "RevisionError.Attribute string", + "RevisionError.ErrorCode string", + "RevisionError.ErrorMessage string", + "RevisionWeight.RevisionID string", + "RevisionWeight.Weight int32", + "S3Object.Bucket string", + "S3Object.Key string", + "S3Object.VersionID string", + "ServiceConfig.EgressNetworkConnectorArn string", + "ServiceConfig.EnvironmentVariables map[string]string", + "ServiceConfig.ExecutionRoleArn string", + "ServiceConfig.Logging *LoggingConfig", + "ServiceConfig.MaxConcurrencyPerEnvironment int32", + "ServiceConfig.TimeoutSeconds int32", + "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`" + ], + "version": 1 + }, "lightsail": { "fields": [ "AccessKey.AccessKeyID string", @@ -23728,6 +24166,8 @@ "Cluster.TLSEnabled bool `json:\"tlsEnabled\"`", "Cluster.Tags map[string]string `json:\"tags\"`", "Cluster.authUsers []string", + "Cluster.dock *dockState", + "Cluster.endpoints [][]redisdocker.Endpoint", "Cluster.engine *redisengine.Engine", "Event.Date time.Time `json:\"date\"`", "Event.Message string `json:\"message\"`", @@ -23817,6 +24257,8 @@ "backendSnapshot.SubnetGroups map[string][]*SubnetGroup `json:\"subnetGroups\"`", "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`", "backendSnapshot.Users map[string][]*User `json:\"users\"`", + "dockState.cancel context.CancelFunc", + "dockState.group *redisdocker.Group", "recurringChargeObject.RecurringChargeAmount float64 `json:\"RecurringChargeAmount,omitempty\"`", "recurringChargeObject.RecurringChargeFrequency string `json:\"RecurringChargeFrequency,omitempty\"`", "resourceRef.Kind string `json:\"kind\"`", @@ -24603,6 +25045,7 @@ "backendSnapshot.ParameterOverrides map[string]map[string]ParameterValue `json:\"parameterOverrides\"`", "backendSnapshot.PendingMaintenanceActions map[string]map[string]PendingMaintenanceAction `json:\"pendingMaintenanceActions\"`", "backendSnapshot.Region string `json:\"region\"`", + "backendSnapshot.SparqlStores map[string]string `json:\"sparqlStores,omitempty\"`", "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`", "backendSnapshot.Tags map[string]map[string][]Tag `json:\"tags\"`", "xmlDBClusterSnapshot.AllocatedStorage int `xml:\"AllocatedStorage,omitempty\"`", @@ -25173,13 +25616,25 @@ }, "ociapigateway": { "fields": [ + "API.Compartment string `json:\"compartmentId\"`", + "API.Content string `json:\"content,omitempty\"`", + "API.DisplayName string `json:\"displayName\"`", + "API.Etag string `json:\"etag\"`", + "API.ID string `json:\"id\"`", + "API.Locks []Lock `json:\"locks,omitempty\"`", + "API.State string `json:\"state\"`", + "API.Tags Tags `json:\"tags\"`", + "API.TimeCreated time.Time `json:\"timeCreated\"`", + "API.TimeUpdated time.Time `json:\"timeUpdated\"`", + "API.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", "Certificate.CertPEM string `json:\"certificate\"`", "Certificate.CompartmentID string `json:\"compartmentId\"`", "Certificate.DisplayName string `json:\"displayName\"`", "Certificate.Etag string `json:\"etag\"`", "Certificate.ID string `json:\"id\"`", "Certificate.Intermediate string `json:\"intermediateCertificates,omitempty\"`", - "Certificate.Locks json.RawMessage `json:\"locks,omitempty\"`", + "Certificate.Locks []Lock `json:\"locks,omitempty\"`", + "Certificate.PrivateKey string `json:\"privateKey,omitempty\"`", "Certificate.State string `json:\"state\"`", "Certificate.SubjectNames []string `json:\"subjectNames\"`", "Certificate.Tags Tags `json:\"tags\"`", @@ -25187,12 +25642,14 @@ "Certificate.TimeNotValidAfter time.Time `json:\"timeNotValidAfter\"`", "Certificate.TimeUpdated time.Time `json:\"timeUpdated\"`", "Certificate.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", + "Client.Name string `json:\"name\"`", + "Client.Token string `json:\"token\"`", "Deployment.CompartmentID string `json:\"compartmentId\"`", "Deployment.DisplayName string `json:\"displayName\"`", "Deployment.Etag string `json:\"etag\"`", "Deployment.GatewayID string `json:\"gatewayId\"`", "Deployment.ID string `json:\"id\"`", - "Deployment.Locks json.RawMessage `json:\"locks,omitempty\"`", + "Deployment.Locks []Lock `json:\"locks,omitempty\"`", "Deployment.PathPrefix string `json:\"pathPrefix\"`", "Deployment.Spec json.RawMessage `json:\"specification\"`", "Deployment.State string `json:\"state\"`", @@ -25200,6 +25657,12 @@ "Deployment.TimeCreated time.Time `json:\"timeCreated\"`", "Deployment.TimeUpdated time.Time `json:\"timeUpdated\"`", "Deployment.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", + "Entitlement.Description string `json:\"description,omitempty\"`", + "Entitlement.Name string `json:\"name\"`", + "Entitlement.Quota *Quota `json:\"quota,omitempty\"`", + "Entitlement.RateLimit *RateLimit `json:\"rateLimit,omitempty\"`", + "Entitlement.Targets []EntitlementTarget `json:\"targets,omitempty\"`", + "EntitlementTarget.DeploymentID string `json:\"deploymentId\"`", "Gateway.CaBundles json.RawMessage `json:\"caBundles,omitempty\"`", "Gateway.CertificateID string `json:\"certificateId,omitempty\"`", "Gateway.CompartmentID string `json:\"compartmentId\"`", @@ -25209,7 +25672,7 @@ "Gateway.Hostname string `json:\"hostname\"`", "Gateway.ID string `json:\"id\"`", "Gateway.IPMode string `json:\"ipMode,omitempty\"`", - "Gateway.Locks json.RawMessage `json:\"locks,omitempty\"`", + "Gateway.Locks []Lock `json:\"locks,omitempty\"`", "Gateway.NSGs []string `json:\"networkSecurityGroupIds,omitempty\"`", "Gateway.ResponseCache json.RawMessage `json:\"responseCache,omitempty\"`", "Gateway.State string `json:\"state\"`", @@ -25218,10 +25681,57 @@ "Gateway.TimeCreated time.Time `json:\"timeCreated\"`", "Gateway.TimeUpdated time.Time `json:\"timeUpdated\"`", "Gateway.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", + "Lock.Message string `json:\"message,omitempty\"`", + "Lock.RelatedResourceID string `json:\"relatedResourceId,omitempty\"`", + "Lock.TimeCreated time.Time `json:\"timeCreated\"`", + "Lock.Type string `json:\"type\"`", + "Quota.OperationOnBreach string `json:\"operationOnBreach\"`", + "Quota.ResetPolicy string `json:\"resetPolicy\"`", + "Quota.Unit string `json:\"unit\"`", + "Quota.Value int `json:\"value\"`", + "RateLimit.Unit string `json:\"unit\"`", + "RateLimit.Value int `json:\"value\"`", + "SDK.APIID string `json:\"apiId\"`", + "SDK.Compartment string `json:\"compartmentId\"`", + "SDK.DisplayName string `json:\"displayName\"`", + "SDK.Etag string `json:\"etag\"`", + "SDK.ID string `json:\"id\"`", + "SDK.Locks []Lock `json:\"locks,omitempty\"`", + "SDK.Parameters map[string]string `json:\"parameters,omitempty\"`", + "SDK.State string `json:\"state\"`", + "SDK.Tags Tags `json:\"tags\"`", + "SDK.TargetLanguage string `json:\"targetLanguage\"`", + "SDK.TimeCreated time.Time `json:\"timeCreated\"`", + "SDK.TimeUpdated time.Time `json:\"timeUpdated\"`", + "SDK.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", + "Subscriber.Clients []Client `json:\"clients\"`", + "Subscriber.Compartment string `json:\"compartmentId\"`", + "Subscriber.DisplayName string `json:\"displayName\"`", + "Subscriber.Etag string `json:\"etag\"`", + "Subscriber.ID string `json:\"id\"`", + "Subscriber.Locks []Lock `json:\"locks,omitempty\"`", + "Subscriber.State string `json:\"state\"`", + "Subscriber.Tags Tags `json:\"tags\"`", + "Subscriber.TimeCreated time.Time `json:\"timeCreated\"`", + "Subscriber.TimeUpdated time.Time `json:\"timeUpdated\"`", + "Subscriber.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", + "Subscriber.UsagePlans []string `json:\"usagePlans\"`", "Tags.DefinedTags map[string]map[string]any `json:\"definedTags,omitempty\"`", "Tags.FreeformTags map[string]string `json:\"freeformTags,omitempty\"`", + "UsagePlan.Compartment string `json:\"compartmentId\"`", + "UsagePlan.DisplayName string `json:\"displayName\"`", + "UsagePlan.Entitlements []Entitlement `json:\"entitlements\"`", + "UsagePlan.Etag string `json:\"etag\"`", + "UsagePlan.ID string `json:\"id\"`", + "UsagePlan.Locks []Lock `json:\"locks,omitempty\"`", + "UsagePlan.State string `json:\"state\"`", + "UsagePlan.Tags Tags `json:\"tags\"`", + "UsagePlan.TimeCreated time.Time `json:\"timeCreated\"`", + "UsagePlan.TimeUpdated time.Time `json:\"timeUpdated\"`", + "UsagePlan.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", "WorkRequest.Accepted time.Time `json:\"accepted\"`", "WorkRequest.Action string `json:\"action\"`", + "WorkRequest.Canceled bool `json:\"canceled,omitempty\"`", "WorkRequest.CompartmentID string `json:\"compartmentId\"`", "WorkRequest.EntityID string `json:\"entityId\"`", "WorkRequest.EntityType string `json:\"entityType\"`", @@ -25229,11 +25739,16 @@ "WorkRequest.ID string `json:\"id\"`", "WorkRequest.Operation string `json:\"operation\"`", "WorkRequest.Seq int64 `json:\"seq\"`", + "backendSnapshot.APIs map[string]*API `json:\"apis\"`", "backendSnapshot.Auth []ociauth.KeyRecord `json:\"authKeys\"`", + "backendSnapshot.CaPEMs map[string]string `json:\"caBundlePems\"`", "backendSnapshot.Certificates map[string]*Certificate `json:\"certificates\"`", "backendSnapshot.Deployments map[string]*Deployment `json:\"deployments\"`", "backendSnapshot.Gateways map[string]*Gateway `json:\"gateways\"`", + "backendSnapshot.Sdks map[string]*SDK `json:\"sdks\"`", "backendSnapshot.Seq int64 `json:\"seq\"`", + "backendSnapshot.Subscribers map[string]*Subscriber `json:\"subscribers\"`", + "backendSnapshot.UsagePlans map[string]*UsagePlan `json:\"usagePlans\"`", "backendSnapshot.WorkReqs map[string]*WorkRequest `json:\"workRequests\"`" ], "version": 1 @@ -25584,6 +26099,16 @@ "Base.Internal *Internal `json:\"internal,omitempty\"`", "Base.LifecycleState string `json:\"lifecycleState\"`", "Base.TimeCreated time.Time `json:\"timeCreated\"`", + "BlockVolumeReplica.AvailabilityDomain string `json:\"availabilityDomain\"`", + "BlockVolumeReplica.BlockVolumeID string `json:\"blockVolumeId\"`", + "BlockVolumeReplica.KmsKeyID string `json:\"kmsKeyId,omitempty\"`", + "BlockVolumeReplica.SizeInGBs int64 `json:\"sizeInGBs\"`", + "BlockVolumeReplica.TimeLastSynced time.Time `json:\"timeLastSynced\"`", + "BlockVolumeReplica.TotalDataTransferredInGBs int64 `json:\"totalDataTransferredInGBs\"`", + "BlockVolumeReplicaInfo.AvailabilityDomain string `json:\"availabilityDomain\"`", + "BlockVolumeReplicaInfo.BlockVolumeReplicaID string `json:\"blockVolumeReplicaId\"`", + "BlockVolumeReplicaInfo.DisplayName string `json:\"displayName\"`", + "BlockVolumeReplicaInfo.KmsKeyID string `json:\"kmsKeyId,omitempty\"`", "BootVolume.AvailabilityDomain string `json:\"availabilityDomain\"`", "BootVolume.ImageID string `json:\"imageId,omitempty\"`", "BootVolume.IsAutoTuneEnabled bool `json:\"isAutoTuneEnabled\"`", @@ -25592,10 +26117,26 @@ "BootVolume.SizeInGBs int64 `json:\"sizeInGBs\"`", "BootVolume.SizeInMBs int64 `json:\"sizeInMBs\"`", "BootVolume.SourceDetails *VolumeSource `json:\"sourceDetails,omitempty\"`", + "BootVolume.VolumeGroupID string `json:\"volumeGroupId,omitempty\"`", "BootVolume.VpusPerGB int64 `json:\"vpusPerGB\"`", "BootVolumeAttachment.AvailabilityDomain string `json:\"availabilityDomain\"`", "BootVolumeAttachment.BootVolumeID string `json:\"bootVolumeId\"`", "BootVolumeAttachment.InstanceID string `json:\"instanceId\"`", + "BootVolumeBackup.BootVolumeID string `json:\"bootVolumeId\"`", + "BootVolumeBackup.ImageID string `json:\"imageId,omitempty\"`", + "BootVolumeBackup.KmsKeyID string `json:\"kmsKeyId,omitempty\"`", + "BootVolumeBackup.Region string `json:\"region,omitempty\"`", + "BootVolumeBackup.SizeInGBs int64 `json:\"sizeInGBs\"`", + "BootVolumeBackup.SourceBootVolumeBackupID string `json:\"sourceBootVolumeBackupId,omitempty\"`", + "BootVolumeBackup.SourceType string `json:\"sourceType\"`", + "BootVolumeBackup.TimeRequestReceived time.Time `json:\"timeRequestReceived\"`", + "BootVolumeBackup.Type string `json:\"type\"`", + "BootVolumeBackup.UniqueSizeInGBs int64 `json:\"uniqueSizeInGBs\"`", + "BootVolumeBackup.VolumeGroupBackupID string `json:\"volumeGroupBackupId,omitempty\"`", + "DefaultDrgRouteTables.IpsecTunnel string `json:\"ipsecTunnel,omitempty\"`", + "DefaultDrgRouteTables.RemotePeeringConnection string `json:\"remotePeeringConnection,omitempty\"`", + "DefaultDrgRouteTables.Vcn string `json:\"vcn,omitempty\"`", + "DefaultDrgRouteTables.VirtualCircuit string `json:\"virtualCircuit,omitempty\"`", "DhcpOption.CustomDNSServers []string `json:\"customDnsServers,omitempty\"`", "DhcpOption.SearchDomainNames []string `json:\"searchDomainNames,omitempty\"`", "DhcpOption.ServerType string `json:\"serverType,omitempty\"`", @@ -25603,6 +26144,41 @@ "DhcpOptions.DomainNameType string `json:\"domainNameType,omitempty\"`", "DhcpOptions.Options []DhcpOption `json:\"options\"`", "DhcpOptions.VcnID string `json:\"vcnId\"`", + "Drg.DefaultDrgRouteTables *DefaultDrgRouteTables `json:\"defaultDrgRouteTables,omitempty\"`", + "Drg.DefaultExportDrgRouteDistributionID string `json:\"defaultExportDrgRouteDistributionId,omitempty\"`", + "DrgAttachment.DrgID string `json:\"drgId\"`", + "DrgAttachment.DrgRouteTableID string `json:\"drgRouteTableId,omitempty\"`", + "DrgAttachment.ExportDrgRouteDistributionID string `json:\"exportDrgRouteDistributionId,omitempty\"`", + "DrgAttachment.IsCrossTenancy bool `json:\"isCrossTenancy\"`", + "DrgAttachment.NetworkDetails *DrgNetworkDetails `json:\"networkDetails,omitempty\"`", + "DrgAttachment.RouteTableID string `json:\"routeTableId,omitempty\"`", + "DrgAttachment.VcnID string `json:\"vcnId,omitempty\"`", + "DrgMatchCriteria.AttachmentType string `json:\"attachmentType,omitempty\"`", + "DrgMatchCriteria.DrgAttachmentID string `json:\"drgAttachmentId,omitempty\"`", + "DrgMatchCriteria.MatchType string `json:\"matchType\"`", + "DrgNetworkDetails.ID string `json:\"id,omitempty\"`", + "DrgNetworkDetails.IDs []string `json:\"ids,omitempty\"`", + "DrgNetworkDetails.IpsecConnectionID string `json:\"ipsecConnectionId,omitempty\"`", + "DrgNetworkDetails.RouteTableID string `json:\"routeTableId,omitempty\"`", + "DrgNetworkDetails.TransportAttachmentID string `json:\"transportAttachmentId,omitempty\"`", + "DrgNetworkDetails.TransportOnlyMode *bool `json:\"transportOnlyMode,omitempty\"`", + "DrgNetworkDetails.Type string `json:\"type\"`", + "DrgNetworkDetails.VcnRouteType string `json:\"vcnRouteType,omitempty\"`", + "DrgRouteDistribution.DistributionType string `json:\"distributionType\"`", + "DrgRouteDistribution.DrgID string `json:\"drgId\"`", + "DrgRouteDistribution.Statements []DrgStatement `json:\"statements,omitempty\"`", + "DrgRouteTable.DrgID string `json:\"drgId\"`", + "DrgRouteTable.ImportDrgRouteDistributionID string `json:\"importDrgRouteDistributionId,omitempty\"`", + "DrgRouteTable.IsEcmpEnabled bool `json:\"isEcmpEnabled\"`", + "DrgRouteTable.Rules []DrgStaticRule `json:\"staticRules,omitempty\"`", + "DrgStatement.Action string `json:\"action\"`", + "DrgStatement.ID string `json:\"id\"`", + "DrgStatement.MatchCriteria []DrgMatchCriteria `json:\"matchCriteria\"`", + "DrgStatement.Priority int `json:\"priority\"`", + "DrgStaticRule.Destination string `json:\"destination\"`", + "DrgStaticRule.DestinationType string `json:\"destinationType\"`", + "DrgStaticRule.ID string `json:\"id\"`", + "DrgStaticRule.NextHopDrgAttachmentID string `json:\"nextHopDrgAttachmentId\"`", "EgressRule.Description string `json:\"description,omitempty\"`", "EgressRule.Destination string `json:\"destination\"`", "EgressRule.DestinationType string `json:\"destinationType,omitempty\"`", @@ -25611,6 +26187,8 @@ "EgressRule.Protocol string `json:\"protocol\"`", "EgressRule.TCPOptions *PortOptions `json:\"tcpOptions,omitempty\"`", "EgressRule.UDPOptions *PortOptions `json:\"udpOptions,omitempty\"`", + "IPInventoryRequest.Details InventoryDetails `json:\"details\"`", + "IPInventoryRequest.Seq int64 `json:\"seq\"`", "IcmpOptions.Code *int `json:\"code,omitempty\"`", "IcmpOptions.Type int `json:\"type\"`", "Image.BaseImageID string `json:\"baseImageId,omitempty\"`", @@ -25658,6 +26236,26 @@ "InternetGateway.IsEnabled bool `json:\"isEnabled\"`", "InternetGateway.RouteTableID string `json:\"routeTableId,omitempty\"`", "InternetGateway.VcnID string `json:\"vcnId\"`", + "InventoryDetails.AddressTypeList []string `json:\"addressTypeList\"`", + "InventoryDetails.CompartmentList []string `json:\"compartmentList\"`", + "InventoryDetails.OverlappingVcnsOnly *bool `json:\"overlappingVcnsOnly\"`", + "InventoryDetails.OverrideFilters *bool `json:\"overrideFilters\"`", + "InventoryDetails.PaginationLimit *int `json:\"paginationLimit\"`", + "InventoryDetails.PaginationOffset *int `json:\"paginationOffset\"`", + "InventoryDetails.RegionList []string `json:\"regionList\"`", + "InventoryDetails.ResourceTypeList []string `json:\"resourceTypeList\"`", + "InventoryDetails.SearchKeyword string `json:\"searchKeyword\"`", + "InventoryDetails.SortBy string `json:\"sortBy\"`", + "InventoryDetails.SortOrder string `json:\"sortOrder\"`", + "InventoryDetails.Utilization *float64 `json:\"utilization\"`", + "Ipv6.CidrPrefixLength *int `json:\"cidrPrefixLength,omitempty\"`", + "Ipv6.Hostname string `json:\"hostname,omitempty\"`", + "Ipv6.IPAddress string `json:\"ipAddress\"`", + "Ipv6.IPState string `json:\"ipState\"`", + "Ipv6.Lifetime string `json:\"lifetime\"`", + "Ipv6.RouteTableID string `json:\"routeTableId,omitempty\"`", + "Ipv6.SubnetID string `json:\"subnetId\"`", + "Ipv6.VnicID string `json:\"vnicId,omitempty\"`", "LocalPeeringGateway.IsCrossTenancyPeering bool `json:\"isCrossTenancyPeering\"`", "LocalPeeringGateway.PeerAdvertisedCidr string `json:\"peerAdvertisedCidr,omitempty\"`", "LocalPeeringGateway.PeerAdvertisedDetails []string `json:\"peerAdvertisedCidrDetails,omitempty\"`", @@ -25694,6 +26292,13 @@ "PublicIP.PrivateIPID string `json:\"privateIpId,omitempty\"`", "PublicIP.PublicIPPoolID string `json:\"publicIpPoolId,omitempty\"`", "PublicIP.Scope string `json:\"scope\"`", + "Retention.IsIndefiniteRetentionEnabled bool `json:\"isIndefiniteRetentionEnabled\"`", + "Retention.IsPreventDeletionEnabled bool `json:\"isPreventDeletionEnabled\"`", + "Retention.IsRetentionLockEnabled bool `json:\"isRetentionLockEnabled\"`", + "Retention.RetentionPeriod *RetentionDuration `json:\"retentionPeriod,omitempty\"`", + "Retention.TimeRetentionExpiresAt *time.Time `json:\"timeRetentionExpiresAt,omitempty\"`", + "RetentionDuration.RetentionTimeAmount int `json:\"retentionTimeAmount\"`", + "RetentionDuration.RetentionTimeUnit string `json:\"retentionTimeUnit\"`", "RouteRule.CidrBlock string `json:\"cidrBlock,omitempty\"`", "RouteRule.Description string `json:\"description,omitempty\"`", "RouteRule.Destination string `json:\"destination,omitempty\"`", @@ -25758,6 +26363,7 @@ "Vcn.VcnDomainName string `json:\"vcnDomainName,omitempty\"`", "Vnic.AvailabilityDomain string `json:\"availabilityDomain\"`", "Vnic.HostnameLabel string `json:\"hostnameLabel,omitempty\"`", + "Vnic.Ipv6Addresses []string `json:\"ipv6Addresses,omitempty\"`", "Vnic.IsPrimary bool `json:\"isPrimary\"`", "Vnic.MacAddress string `json:\"macAddress\"`", "Vnic.NsgIDs []string `json:\"nsgIds\"`", @@ -25772,12 +26378,14 @@ "VnicAttachment.VlanTag int `json:\"vlanTag\"`", "VnicAttachment.VnicID string `json:\"vnicId\"`", "Volume.AvailabilityDomain string `json:\"availabilityDomain\"`", + "Volume.BlockVolumeReplicas []BlockVolumeReplicaInfo `json:\"blockVolumeReplicas,omitempty\"`", "Volume.IsAutoTuneEnabled bool `json:\"isAutoTuneEnabled\"`", "Volume.IsHydrated bool `json:\"isHydrated\"`", "Volume.KmsKeyID string `json:\"kmsKeyId,omitempty\"`", "Volume.SizeInGBs int64 `json:\"sizeInGBs\"`", "Volume.SizeInMBs int64 `json:\"sizeInMBs\"`", "Volume.SourceDetails *VolumeSource `json:\"sourceDetails,omitempty\"`", + "Volume.VolumeGroupID string `json:\"volumeGroupId,omitempty\"`", "Volume.VpusPerGB int64 `json:\"vpusPerGB\"`", "VolumeAttachment.AttachmentType string `json:\"attachmentType\"`", "VolumeAttachment.AvailabilityDomain string `json:\"availabilityDomain\"`", @@ -25797,29 +26405,73 @@ "VolumeAttachment.Port int `json:\"port,omitempty\"`", "VolumeAttachment.VolumeID string `json:\"volumeId\"`", "VolumeBackup.KmsKeyID string `json:\"kmsKeyId,omitempty\"`", + "VolumeBackup.Region string `json:\"region,omitempty\"`", "VolumeBackup.SizeInGBs int64 `json:\"sizeInGBs\"`", "VolumeBackup.SizeInMBs int64 `json:\"sizeInMBs\"`", "VolumeBackup.SourceType string `json:\"sourceType\"`", + "VolumeBackup.SourceVolumeBackupID string `json:\"sourceVolumeBackupId,omitempty\"`", "VolumeBackup.TimeRequestReceived time.Time `json:\"timeRequestReceived\"`", "VolumeBackup.Type string `json:\"type\"`", "VolumeBackup.UniqueSizeInGBs int64 `json:\"uniqueSizeInGBs\"`", "VolumeBackup.UniqueSizeInMbs int64 `json:\"uniqueSizeInMbs\"`", + "VolumeBackup.VolumeGroupBackupID string `json:\"volumeGroupBackupId,omitempty\"`", "VolumeBackup.VolumeID string `json:\"volumeId\"`", "VolumeBackupPolicy.DestinationRegion string `json:\"destinationRegion,omitempty\"`", "VolumeBackupPolicy.Schedules []BackupSchedule `json:\"schedules\"`", + "VolumeGroup.AvailabilityDomain string `json:\"availabilityDomain\"`", + "VolumeGroup.IsHydrated bool `json:\"isHydrated\"`", + "VolumeGroup.SizeInGBs int64 `json:\"sizeInGBs\"`", + "VolumeGroup.SizeInMBs int64 `json:\"sizeInMBs\"`", + "VolumeGroup.SourceDetails *VolumeGroupSource `json:\"sourceDetails,omitempty\"`", + "VolumeGroup.VolumeIDs []string `json:\"volumeIds\"`", + "VolumeGroupBackup.Region string `json:\"region,omitempty\"`", + "VolumeGroupBackup.SizeInGBs int64 `json:\"sizeInGBs\"`", + "VolumeGroupBackup.SizeInMBs int64 `json:\"sizeInMBs\"`", + "VolumeGroupBackup.SourceType string `json:\"sourceType\"`", + "VolumeGroupBackup.SourceVolumeGroupBackupID string `json:\"sourceVolumeGroupBackupId,omitempty\"`", + "VolumeGroupBackup.TimeRequestReceived time.Time `json:\"timeRequestReceived\"`", + "VolumeGroupBackup.Type string `json:\"type\"`", + "VolumeGroupBackup.UniqueSizeInGbs int64 `json:\"uniqueSizeInGbs\"`", + "VolumeGroupBackup.UniqueSizeInMbs int64 `json:\"uniqueSizeInMbs\"`", + "VolumeGroupBackup.VolumeBackupIDs []string `json:\"volumeBackupIds\"`", + "VolumeGroupBackup.VolumeGroupID string `json:\"volumeGroupId,omitempty\"`", + "VolumeGroupSource.Type string `json:\"type\"`", + "VolumeGroupSource.VolumeGroupBackupID string `json:\"volumeGroupBackupId,omitempty\"`", + "VolumeGroupSource.VolumeGroupID string `json:\"volumeGroupId,omitempty\"`", + "VolumeGroupSource.VolumeGroupReplicaID string `json:\"volumeGroupReplicaId,omitempty\"`", + "VolumeGroupSource.VolumeIDs []string `json:\"volumeIds,omitempty\"`", "VolumeSource.ID string `json:\"id\"`", "VolumeSource.Type string `json:\"type\"`", + "WorkRequest.CompartmentID string `json:\"compartmentId\"`", + "WorkRequest.FinishAt time.Time `json:\"finishAt\"`", + "WorkRequest.ID string `json:\"id\"`", + "WorkRequest.OperationType string `json:\"operationType\"`", + "WorkRequest.Resources []WorkResource `json:\"resources\"`", + "WorkRequest.Seq int64 `json:\"seq\"`", + "WorkRequest.TimeAccepted time.Time `json:\"timeAccepted\"`", + "WorkResource.ActionType string `json:\"actionType\"`", + "WorkResource.EntityType string `json:\"entityType\"`", + "WorkResource.EntityURI string `json:\"entityUri,omitempty\"`", + "WorkResource.Identifier string `json:\"identifier\"`", "backendSnapshot.Assigns map[string]*BackupPolicyAssignment `json:\"volumeBackupPolicyAssignments\"`", "backendSnapshot.Auth []ociauth.KeyRecord `json:\"authKeys\"`", "backendSnapshot.Backups map[string]*VolumeBackup `json:\"volumeBackups\"`", "backendSnapshot.BootAtts map[string]*BootVolumeAttachment `json:\"bootVolumeAttachments\"`", + "backendSnapshot.BootBackups map[string]*BootVolumeBackup `json:\"bootVolumeBackups\"`", "backendSnapshot.BootVols map[string]*BootVolume `json:\"bootVolumes\"`", "backendSnapshot.Dhcp map[string]*DhcpOptions `json:\"dhcpOptions\"`", + "backendSnapshot.DrgAtts map[string]*DrgAttachment `json:\"drgAttachments\"`", + "backendSnapshot.DrgDists map[string]*DrgRouteDistribution `json:\"drgRouteDistributions\"`", + "backendSnapshot.DrgTables map[string]*DrgRouteTable `json:\"drgRouteTables\"`", + "backendSnapshot.Drgs map[string]*Drg `json:\"drgs\"`", + "backendSnapshot.GroupBackups map[string]*VolumeGroupBackup `json:\"volumeGroupBackups\"`", "backendSnapshot.IGWs map[string]*InternetGateway `json:\"internetGateways\"`", + "backendSnapshot.IPInvReqs map[string]*IPInventoryRequest `json:\"ipInventoryRequests\"`", "backendSnapshot.IPSeq int64 `json:\"ipSeq\"`", "backendSnapshot.IPv6Seq int64 `json:\"ipv6Seq\"`", "backendSnapshot.Images map[string]*Image `json:\"images\"`", "backendSnapshot.Instances map[string]*Instance `json:\"instances\"`", + "backendSnapshot.Ipv6s map[string]*Ipv6 `json:\"ipv6s\"`", "backendSnapshot.LPGs map[string]*LocalPeeringGateway `json:\"localPeeringGateways\"`", "backendSnapshot.NATs map[string]*NatGateway `json:\"natGateways\"`", "backendSnapshot.NSGRules map[string]*NSGRule `json:\"securityRules\"`", @@ -25827,6 +26479,7 @@ "backendSnapshot.Policies map[string]*VolumeBackupPolicy `json:\"volumeBackupPolicies\"`", "backendSnapshot.PrivateIPs map[string]*PrivateIP `json:\"privateIps\"`", "backendSnapshot.PublicIPs map[string]*PublicIP `json:\"publicIps\"`", + "backendSnapshot.Replicas map[string]*BlockVolumeReplica `json:\"blockVolumeReplicas\"`", "backendSnapshot.RouteTabs map[string]*RouteTable `json:\"routeTables\"`", "backendSnapshot.SGWs map[string]*ServiceGateway `json:\"serviceGateways\"`", "backendSnapshot.SecLists map[string]*SecurityList `json:\"securityLists\"`", @@ -25836,7 +26489,183 @@ "backendSnapshot.VnicAtts map[string]*VnicAttachment `json:\"vnicAttachments\"`", "backendSnapshot.Vnics map[string]*Vnic `json:\"vnics\"`", "backendSnapshot.VolAtts map[string]*VolumeAttachment `json:\"volumeAttachments\"`", - "backendSnapshot.Volumes map[string]*Volume `json:\"volumes\"`" + "backendSnapshot.VolGroups map[string]*VolumeGroup `json:\"volumeGroups\"`", + "backendSnapshot.Volumes map[string]*Volume `json:\"volumes\"`", + "backendSnapshot.WorkReqs map[string]*WorkRequest `json:\"workRequests\"`" + ], + "version": 1 + }, + "ocidatabase": { + "fields": [ + "ADB.AccessControl bool `json:\"accessControl\"`", + "ADB.AutoScaling bool `json:\"autoScaling\"`", + "ADB.BackupDays int `json:\"backupRetentionDays\"`", + "ADB.CharacterSet string `json:\"characterSet\"`", + "ADB.CompartmentID string `json:\"compartmentId\"`", + "ADB.ComputeCount float32 `json:\"computeCount\"`", + "ADB.ComputeModel string `json:\"computeModel\"`", + "ADB.Created time.Time `json:\"created\"`", + "ADB.DBName string `json:\"dbName\"`", + "ADB.DBVersion string `json:\"dbVersion\"`", + "ADB.DefinedTags map[string]map[string]any `json:\"definedTags,omitempty\"`", + "ADB.Details string `json:\"details,omitempty\"`", + "ADB.DisplayName string `json:\"displayName\"`", + "ADB.EndpointIP string `json:\"privateEndpointIp,omitempty\"`", + "ADB.EndpointLabel string `json:\"privateEndpointLabel,omitempty\"`", + "ADB.Engine bool `json:\"engine,omitempty\"`", + "ADB.Etag string `json:\"etag\"`", + "ADB.FreeformTags map[string]string `json:\"freeformTags,omitempty\"`", + "ADB.ID string `json:\"id\"`", + "ADB.IsFreeTier bool `json:\"isFreeTier\"`", + "ADB.LicenseModel string `json:\"licenseModel\"`", + "ADB.Mtls bool `json:\"mtlsRequired\"`", + "ADB.NCharacterSet string `json:\"ncharacterSet\"`", + "ADB.NsgIDs []string `json:\"nsgIds,omitempty\"`", + "ADB.Seq int64 `json:\"seq\"`", + "ADB.State string `json:\"state\"`", + "ADB.StorageGBs int `json:\"storageGBs\"`", + "ADB.StorageScales bool `json:\"storageAutoScaling\"`", + "ADB.SubnetID string `json:\"subnetId,omitempty\"`", + "ADB.Target string `json:\"target,omitempty\"`", + "ADB.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", + "ADB.UniqueID string `json:\"uniqueId\"`", + "ADB.WalletRotated *time.Time `json:\"walletRotated,omitempty\"`", + "ADB.Whitelisted []string `json:\"whitelistedIps,omitempty\"`", + "ADB.Workload string `json:\"workload\"`", + "Backup.AvailableTill *time.Time `json:\"availableTill,omitempty\"`", + "Backup.CompartmentID string `json:\"compartmentId\"`", + "Backup.DBID string `json:\"autonomousDatabaseId\"`", + "Backup.DBVersion string `json:\"dbVersion\"`", + "Backup.Details string `json:\"details,omitempty\"`", + "Backup.DisplayName string `json:\"displayName\"`", + "Backup.Ended *time.Time `json:\"ended,omitempty\"`", + "Backup.ID string `json:\"id\"`", + "Backup.IsAutomatic bool `json:\"isAutomatic\"`", + "Backup.RetentionDays int `json:\"retentionDays\"`", + "Backup.Seq int64 `json:\"seq\"`", + "Backup.SizeTBs float64 `json:\"sizeInTBs\"`", + "Backup.Started time.Time `json:\"started\"`", + "Backup.State string `json:\"state\"`", + "Backup.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", + "Backup.Type string `json:\"type\"`", + "WorkRequest.Accepted time.Time `json:\"accepted\"`", + "WorkRequest.Action string `json:\"action\"`", + "WorkRequest.Compartment string `json:\"compartmentId\"`", + "WorkRequest.ErrCode string `json:\"errCode,omitempty\"`", + "WorkRequest.ErrMsg string `json:\"errMsg,omitempty\"`", + "WorkRequest.FinishAt *time.Time `json:\"finishAt,omitempty\"`", + "WorkRequest.FinishedAt *time.Time `json:\"finishedAt,omitempty\"`", + "WorkRequest.ID string `json:\"id\"`", + "WorkRequest.ResourceID string `json:\"resourceId\"`", + "WorkRequest.ResourceType string `json:\"resourceType\"`", + "WorkRequest.Seq int64 `json:\"seq\"`", + "WorkRequest.Status string `json:\"status\"`", + "WorkRequest.Type string `json:\"type\"`", + "backendSnapshot.Auth []ociauth.KeyRecord `json:\"authKeys\"`", + "backendSnapshot.Backups map[string]*Backup `json:\"backups\"`", + "backendSnapshot.Databases map[string]*ADB `json:\"databases\"`", + "backendSnapshot.Seq int64 `json:\"seq\"`", + "backendSnapshot.Tokens map[string]string `json:\"retryTokens\"`", + "backendSnapshot.WorkReqs map[string]*WorkRequest `json:\"workRequests\"`" + ], + "version": 1 + }, + "ocidns": { + "fields": [ + "Attachment.CompartmentID string `json:\"compartmentId\"`", + "Attachment.Created time.Time `json:\"created\"`", + "Attachment.DisplayName string `json:\"displayName\"`", + "Attachment.Domain string `json:\"domain\"`", + "Attachment.ID string `json:\"id\"`", + "Attachment.Rtypes []string `json:\"rtypes\"`", + "Attachment.Seq int64 `json:\"seq\"`", + "Attachment.SteeringPolicyID string `json:\"steeringPolicyId\"`", + "Attachment.ZoneID string `json:\"zoneId\"`", + "Endpoint.CompartmentID string `json:\"compartmentId\"`", + "Endpoint.Created time.Time `json:\"created\"`", + "Endpoint.ForwardingAddress string `json:\"forwardingAddress,omitempty\"`", + "Endpoint.ID string `json:\"id\"`", + "Endpoint.IsForwarding bool `json:\"isForwarding\"`", + "Endpoint.IsListening bool `json:\"isListening\"`", + "Endpoint.ListeningAddress string `json:\"listeningAddress,omitempty\"`", + "Endpoint.Name string `json:\"name\"`", + "Endpoint.NsgIDs []string `json:\"nsgIds\"`", + "Endpoint.ResolverID string `json:\"resolverId\"`", + "Endpoint.Seq int64 `json:\"seq\"`", + "Endpoint.SubnetID string `json:\"subnetId\"`", + "Endpoint.Updated time.Time `json:\"updated\"`", + "ExtServer.Address string `json:\"address\"`", + "ExtServer.Port *int `json:\"port,omitempty\"`", + "ExtServer.TsigKeyID string `json:\"tsigKeyId,omitempty\"`", + "RRSet.Domain string `json:\"domain\"`", + "RRSet.Rdata []string `json:\"rdata\"`", + "RRSet.Rtype string `json:\"rtype\"`", + "RRSet.TTL int `json:\"ttl\"`", + "Resolver.AttachedViews []string `json:\"attachedViews\"`", + "Resolver.CompartmentID string `json:\"compartmentId\"`", + "Resolver.Created time.Time `json:\"created\"`", + "Resolver.DefaultViewID string `json:\"defaultViewId\"`", + "Resolver.DisplayName string `json:\"displayName\"`", + "Resolver.ID string `json:\"id\"`", + "Resolver.Rules []ResolverRule `json:\"rules\"`", + "Resolver.Seq int64 `json:\"seq\"`", + "Resolver.Updated time.Time `json:\"updated\"`", + "Resolver.VcnID string `json:\"vcnId\"`", + "ResolverRule.ClientAddressConditions []string `json:\"clientAddressConditions\"`", + "ResolverRule.DestinationAddresses []string `json:\"destinationAddresses\"`", + "ResolverRule.QnameCoverConditions []string `json:\"qnameCoverConditions\"`", + "ResolverRule.SourceEndpointName string `json:\"sourceEndpointName\"`", + "SteeringPolicy.Answers []map[string]any `json:\"answers\"`", + "SteeringPolicy.CompartmentID string `json:\"compartmentId\"`", + "SteeringPolicy.Created time.Time `json:\"created\"`", + "SteeringPolicy.DisplayName string `json:\"displayName\"`", + "SteeringPolicy.HealthCheckMonitorID string `json:\"healthCheckMonitorId,omitempty\"`", + "SteeringPolicy.ID string `json:\"id\"`", + "SteeringPolicy.Rules []map[string]any `json:\"rules\"`", + "SteeringPolicy.Seq int64 `json:\"seq\"`", + "SteeringPolicy.TTL int `json:\"ttl\"`", + "SteeringPolicy.Template string `json:\"template\"`", + "Tags.DefinedTags map[string]map[string]any `json:\"definedTags,omitempty\"`", + "Tags.FreeformTags map[string]string `json:\"freeformTags,omitempty\"`", + "TsigKey.Algorithm string `json:\"algorithm\"`", + "TsigKey.CompartmentID string `json:\"compartmentId\"`", + "TsigKey.Created time.Time `json:\"created\"`", + "TsigKey.ID string `json:\"id\"`", + "TsigKey.Name string `json:\"name\"`", + "TsigKey.Secret string `json:\"secret\"`", + "TsigKey.Seq int64 `json:\"seq\"`", + "TsigKey.Updated time.Time `json:\"updated\"`", + "View.CompartmentID string `json:\"compartmentId\"`", + "View.Created time.Time `json:\"created\"`", + "View.DisplayName string `json:\"displayName\"`", + "View.ID string `json:\"id\"`", + "View.Protected bool `json:\"protected\"`", + "View.Seq int64 `json:\"seq\"`", + "View.Updated time.Time `json:\"updated\"`", + "Zone.CompartmentID string `json:\"compartmentId\"`", + "Zone.Created time.Time `json:\"created\"`", + "Zone.DnssecState string `json:\"dnssecState\"`", + "Zone.ExternalDownstreams []ExtServer `json:\"externalDownstreams\"`", + "Zone.ExternalMasters []ExtServer `json:\"externalMasters\"`", + "Zone.ID string `json:\"id\"`", + "Zone.Name string `json:\"name\"`", + "Zone.Records map[string]*RRSet `json:\"records\"`", + "Zone.ResolutionMode string `json:\"resolutionMode\"`", + "Zone.Scope string `json:\"scope\"`", + "Zone.Seq int64 `json:\"seq\"`", + "Zone.ViewID string `json:\"viewId,omitempty\"`", + "Zone.ZoneType string `json:\"zoneType\"`", + "backendSnapshot.Attachments map[string]*Attachment `json:\"steeringPolicyAttachments\"`", + "backendSnapshot.Auth []ociauth.KeyRecord `json:\"authKeys\"`", + "backendSnapshot.Endpoints map[string]*Endpoint `json:\"endpoints\"`", + "backendSnapshot.Policies map[string]*SteeringPolicy `json:\"steeringPolicies\"`", + "backendSnapshot.Resolvers map[string]*Resolver `json:\"resolvers\"`", + "backendSnapshot.Seq int64 `json:\"seq\"`", + "backendSnapshot.TsigKeys map[string]*TsigKey `json:\"tsigKeys\"`", + "backendSnapshot.Views map[string]*View `json:\"views\"`", + "backendSnapshot.Zones map[string]*Zone `json:\"zones\"`", + "meta.ETagSeq int64 `json:\"etagSeq\"`", + "meta.Modified time.Time `json:\"modified\"`" ], "version": 1 }, @@ -25871,6 +26700,50 @@ ], "version": 1 }, + "ocifilestorage": { + "fields": [ + "Snapshot.ExclusiveBytes int64 `json:\"exclusiveBytes\"`", + "Snapshot.ExpirationTime *time.Time `json:\"expirationTime,omitempty\"`", + "Snapshot.FileSystemID string `json:\"fileSystemId\"`", + "Snapshot.FilesystemSnapshotPolicyID string `json:\"filesystemSnapshotPolicyId,omitempty\"`", + "Snapshot.IsCloneSource bool `json:\"isCloneSource\"`", + "Snapshot.LifecycleDetails string `json:\"lifecycleDetails\"`", + "Snapshot.LockDurationDetails *lockDuration `json:\"lockDurationDetails,omitempty\"`", + "Snapshot.Name string `json:\"name\"`", + "Snapshot.ProvenanceID string `json:\"provenanceId\"`", + "Snapshot.SnapshotTime time.Time `json:\"snapshotTime\"`", + "Snapshot.SnapshotType string `json:\"snapshotType\"`", + "Snapshot.TimeLocked *time.Time `json:\"timeLocked,omitempty\"`", + "backendSnapshot.Auth []ociauth.KeyRecord `json:\"authKeys\"`", + "backendSnapshot.Connectors table[*Connector] `json:\"connectors\"`", + "backendSnapshot.ExportSets table[*ExportSet] `json:\"exportSets\"`", + "backendSnapshot.Exports table[*Export] `json:\"exports\"`", + "backendSnapshot.FileSystems table[*FileSystem] `json:\"fileSystems\"`", + "backendSnapshot.MountTargets table[*MountTarget] `json:\"mountTargets\"`", + "backendSnapshot.Policies table[*Policy] `json:\"policies\"`", + "backendSnapshot.Quotas table[*QuotaRule] `json:\"quotaRules\"`", + "backendSnapshot.Replications table[*Replication] `json:\"replications\"`", + "backendSnapshot.Seq int64 `json:\"seq\"`", + "backendSnapshot.Snapshots table[*Snapshot] `json:\"snapshots\"`", + "backendSnapshot.Targets table[*ReplicationTarget] `json:\"replicationTargets\"`", + "backendSnapshot.Tokens map[string]string `json:\"retryTokens\"`", + "base.DefinedTags map[string]map[string]any `json:\"definedTags\"`", + "base.FreeformTags map[string]string `json:\"freeformTags\"`", + "base.ID string `json:\"id\"`", + "base.Locks []lock `json:\"locks\"`", + "base.State string `json:\"lifecycleState\"`", + "base.SystemTags map[string]map[string]any `json:\"systemTags,omitempty\"`", + "base.TimeCreated time.Time `json:\"timeCreated\"`", + "lock.Message string `json:\"message,omitempty\"`", + "lock.RelatedResourceID string `json:\"relatedResourceId,omitempty\"`", + "lock.TimeCreated *time.Time `json:\"timeCreated,omitempty\"`", + "lock.Type string `json:\"type\"`", + "lockDuration.CoolOffDuration *int `json:\"coolOffDuration,omitempty\"`", + "lockDuration.LockDuration *int `json:\"lockDuration,omitempty\"`", + "lockDuration.LockMode string `json:\"lockMode\"`" + ], + "version": 1 + }, "ocifunctions": { "fields": [ "Application.CompartmentID string `json:\"compartmentId\"`", @@ -26618,11 +27491,28 @@ "Bucket.Objects map[string][]*Object `json:\"objects\"`", "Bucket.PARs map[string]*PAR `json:\"pars\"`", "Bucket.PublicAccessType string `json:\"publicAccessType\"`", + "Bucket.ReplQueue []ReplTask `json:\"replQueue,omitempty\"`", + "Bucket.Replication *ReplicationPolicy `json:\"replication,omitempty\"`", + "Bucket.ReplicationSource *ReplicationSource `json:\"replicationSource,omitempty\"`", "Bucket.Retention map[string]*Retention `json:\"retention\"`", "Bucket.StorageTier string `json:\"storageTier\"`", "Bucket.TimeCreated time.Time `json:\"timeCreated\"`", "Bucket.Uploads map[string]*Upload `json:\"uploads\"`", "Bucket.Versioning string `json:\"versioning\"`", + "Checksum.Algorithm string `json:\"algorithm,omitempty\"`", + "Checksum.Content string `json:\"content,omitempty\"`", + "Checksum.Multipart string `json:\"multipart,omitempty\"`", + "CopyInput.DestIfMatch string `json:\"destinationObjectIfMatchETag\"`", + "CopyInput.DestIfNoneMatch string `json:\"destinationObjectIfNoneMatchETag\"`", + "CopyInput.DestinationBucket string `json:\"destinationBucket\"`", + "CopyInput.DestinationMetadata map[string]string `json:\"destinationObjectMetadata\"`", + "CopyInput.DestinationNS string `json:\"destinationNamespace\"`", + "CopyInput.DestinationObject string `json:\"destinationObjectName\"`", + "CopyInput.DestinationRegion string `json:\"destinationRegion\"`", + "CopyInput.DestinationTier string `json:\"destinationObjectStorageTier\"`", + "CopyInput.SourceIfMatch string `json:\"sourceObjectIfMatchETag\"`", + "CopyInput.SourceObjectName string `json:\"sourceObjectName\"`", + "CopyInput.SourceVersionID string `json:\"sourceVersionId\"`", "Lifecycle.Items []LifecycleRule `json:\"items\"`", "Lifecycle.TimeCreated time.Time `json:\"timeCreated\"`", "LifecycleRule.Action string `json:\"action\"`", @@ -26635,15 +27525,19 @@ "NameFilter.ExclusionPatterns []string `json:\"exclusionPatterns,omitempty\"`", "NameFilter.InclusionPatterns []string `json:\"inclusionPatterns,omitempty\"`", "NameFilter.InclusionPrefixes []string `json:\"inclusionPrefixes,omitempty\"`", + "Object.Checksum Checksum `json:\"checksum,omitzero\"`", "Object.Data []byte `json:\"data\"`", "Object.Etag string `json:\"etag\"`", "Object.IsDeleteMarker bool `json:\"isDeleteMarker\"`", + "Object.KmsKeyID string `json:\"kmsKeyId,omitempty\"`", "Object.MD5 string `json:\"md5\"`", "Object.Meta ObjectMeta `json:\"meta\"`", "Object.MultipartMD5 string `json:\"multipartMd5,omitempty\"`", "Object.Name string `json:\"name\"`", "Object.RestoreExpiry time.Time `json:\"restoreExpiry\"`", "Object.RestoreReadyAt time.Time `json:\"restoreReadyAt\"`", + "Object.SseKeySha256 string `json:\"sseKeySha256,omitempty\"`", + "Object.TimeArchived time.Time `json:\"timeArchived,omitzero\"`", "Object.TimeCreated time.Time `json:\"timeCreated\"`", "Object.TimeModified time.Time `json:\"timeModified\"`", "Object.VersionID string `json:\"versionId\"`", @@ -26662,9 +27556,32 @@ "PAR.TimeCreated time.Time `json:\"timeCreated\"`", "PAR.TimeExpires time.Time `json:\"timeExpires\"`", "PAR.Token string `json:\"token\"`", + "Part.Checksum string `json:\"checksum,omitempty\"`", + "Part.ChecksumAlg string `json:\"checksumAlg,omitempty\"`", "Part.Data []byte `json:\"data\"`", "Part.Etag string `json:\"etag\"`", "Part.MD5 string `json:\"md5\"`", + "PendingCopy.DueAt time.Time `json:\"dueAt\"`", + "PendingCopy.Input CopyInput `json:\"input\"`", + "PendingCopy.KmsKeyID string `json:\"kmsKeyId,omitempty\"`", + "PendingCopy.SourceBucket string `json:\"sourceBucket\"`", + "PendingCopy.SourceNS string `json:\"sourceNs\"`", + "PendingCopy.SseKeySha256 string `json:\"sseKeySha256,omitempty\"`", + "PendingCopy.VersionID string `json:\"versionId\"`", + "ReplTask.Delete bool `json:\"delete,omitempty\"`", + "ReplTask.DueAt time.Time `json:\"dueAt\"`", + "ReplTask.Name string `json:\"name\"`", + "ReplicationPolicy.DestinationBucketName string `json:\"destinationBucketName\"`", + "ReplicationPolicy.DestinationRegionName string `json:\"destinationRegionName\"`", + "ReplicationPolicy.ID string `json:\"id\"`", + "ReplicationPolicy.Name string `json:\"name\"`", + "ReplicationPolicy.Status string `json:\"status\"`", + "ReplicationPolicy.StatusMessage string `json:\"statusMessage\"`", + "ReplicationPolicy.TimeCreated time.Time `json:\"timeCreated\"`", + "ReplicationPolicy.TimeLastSync time.Time `json:\"timeLastSync\"`", + "ReplicationSource.PolicyName string `json:\"policyName\"`", + "ReplicationSource.SourceBucketName string `json:\"sourceBucketName\"`", + "ReplicationSource.SourceRegionName string `json:\"sourceRegionName\"`", "Retention.Amount int64 `json:\"amount\"`", "Retention.DisplayName string `json:\"displayName\"`", "Retention.Etag string `json:\"etag\"`", @@ -26674,15 +27591,20 @@ "Retention.TimeModified time.Time `json:\"timeModified\"`", "Retention.TimeRuleLocked *time.Time `json:\"timeRuleLocked,omitempty\"`", "Retention.Unit string `json:\"unit,omitempty\"`", + "Upload.ChecksumAlg string `json:\"checksumAlg,omitempty\"`", "Upload.ID string `json:\"id\"`", + "Upload.KmsKeyID string `json:\"kmsKeyId,omitempty\"`", "Upload.Meta ObjectMeta `json:\"meta\"`", "Upload.Object string `json:\"object\"`", "Upload.Parts map[int]Part `json:\"parts\"`", + "Upload.SseKeySha string `json:\"sseKeySha256,omitempty\"`", "Upload.TimeCreated time.Time `json:\"timeCreated\"`", "WorkRequest.CompartmentID string `json:\"compartmentId\"`", "WorkRequest.EntityURI string `json:\"entityUri\"`", "WorkRequest.ID string `json:\"id\"`", + "WorkRequest.Message string `json:\"message,omitempty\"`", "WorkRequest.OperationType string `json:\"operationType\"`", + "WorkRequest.Pending *PendingCopy `json:\"pending,omitempty\"`", "WorkRequest.Status string `json:\"status\"`", "WorkRequest.TimeAccepted time.Time `json:\"timeAccepted\"`", "WorkRequest.TimeFinished time.Time `json:\"timeFinished\"`", @@ -26831,6 +27753,10 @@ }, "ocivault": { "fields": [ + "GenerationContext.GenerationTemplate string `json:\"generationTemplate\"`", + "GenerationContext.GenerationType string `json:\"generationType\"`", + "GenerationContext.PassphraseLength int `json:\"passphraseLength,omitempty\"`", + "GenerationContext.SecretTemplate string `json:\"secretTemplate,omitempty\"`", "Key.AutoRotation bool `json:\"autoRotation\"`", "Key.CompartmentID string `json:\"compartmentId\"`", "Key.CurrentVersion string `json:\"currentVersion\"`", @@ -26841,6 +27767,8 @@ "Key.ID string `json:\"id\"`", "Key.PrevState string `json:\"prevState,omitempty\"`", "Key.ProtectionMode string `json:\"protectionMode\"`", + "Key.ReplicationID string `json:\"replicationId,omitempty\"`", + "Key.RestoredFrom string `json:\"restoredFrom,omitempty\"`", "Key.Rotation *RotationDetails `json:\"rotation,omitempty\"`", "Key.Seq int64 `json:\"seq\"`", "Key.Shape KeyShape `json:\"shape\"`", @@ -26859,24 +27787,30 @@ "KeyVersion.State string `json:\"state\"`", "KeyVersion.TimeCreated time.Time `json:\"timeCreated\"`", "KeyVersion.TimeOfDeletion *time.Time `json:\"timeOfDeletion,omitempty\"`", + "RotationConfig.IsScheduledRotationEnabled bool `json:\"isScheduledRotationEnabled\"`", + "RotationConfig.RotationInterval string `json:\"rotationInterval,omitempty\"`", + "RotationConfig.TargetSystemDetails TargetSystemDetails `json:\"targetSystemDetails\"`", "RotationDetails.IntervalDays int `json:\"rotationIntervalInDays,omitempty\"`", "RotationDetails.LastMessage string `json:\"lastRotationMessage,omitempty\"`", "RotationDetails.LastRotation *time.Time `json:\"timeOfLastRotation,omitempty\"`", "RotationDetails.LastStatus string `json:\"lastRotationStatus,omitempty\"`", "RotationDetails.NextRotation *time.Time `json:\"timeOfNextRotation,omitempty\"`", "RotationDetails.ScheduleStart *time.Time `json:\"timeOfScheduleStart,omitempty\"`", + "Secret.AutoGen bool `json:\"autoGen,omitempty\"`", "Secret.CompartmentID string `json:\"compartmentId\"`", "Secret.Current int64 `json:\"current\"`", "Secret.DefinedTags map[string]map[string]any `json:\"definedTags,omitempty\"`", "Secret.Description string `json:\"description,omitempty\"`", "Secret.Etag string `json:\"etag\"`", "Secret.FreeformTags map[string]string `json:\"freeformTags,omitempty\"`", + "Secret.GenContext *GenerationContext `json:\"genContext,omitempty\"`", "Secret.ID string `json:\"id\"`", "Secret.KeyID string `json:\"keyId\"`", "Secret.Metadata map[string]any `json:\"metadata,omitempty\"`", "Secret.Name string `json:\"name\"`", "Secret.Pending int64 `json:\"pending\"`", "Secret.Previous int64 `json:\"previous\"`", + "Secret.Rotation *SecretRotation `json:\"rotation,omitempty\"`", "Secret.Rules []json.RawMessage `json:\"rules,omitempty\"`", "Secret.Seq int64 `json:\"seq\"`", "Secret.State string `json:\"state\"`", @@ -26884,12 +27818,19 @@ "Secret.TimeOfDeletion *time.Time `json:\"timeOfDeletion,omitempty\"`", "Secret.VaultID string `json:\"vaultId\"`", "Secret.Versions []*SecretVersion `json:\"versions\"`", + "SecretRotation.Config RotationConfig `json:\"config\"`", + "SecretRotation.LastRotation *time.Time `json:\"lastRotation,omitempty\"`", + "SecretRotation.NextRotation *time.Time `json:\"nextRotation,omitempty\"`", + "SecretRotation.Status string `json:\"status\"`", "SecretVersion.Ciphertext string `json:\"ciphertext\"`", "SecretVersion.Name string `json:\"name,omitempty\"`", "SecretVersion.Number int64 `json:\"number\"`", "SecretVersion.TimeCreated time.Time `json:\"timeCreated\"`", "SecretVersion.TimeOfDeletion *time.Time `json:\"timeOfDeletion,omitempty\"`", "SecretVersion.TimeOfExpiry *time.Time `json:\"timeOfExpiry,omitempty\"`", + "TargetSystemDetails.AdbID string `json:\"adbId,omitempty\"`", + "TargetSystemDetails.FunctionID string `json:\"functionId,omitempty\"`", + "TargetSystemDetails.TargetSystemType string `json:\"targetSystemType\"`", "Vault.CompartmentID string `json:\"compartmentId\"`", "Vault.DefinedTags map[string]map[string]any `json:\"definedTags,omitempty\"`", "Vault.DisplayName string `json:\"displayName\"`", @@ -26897,6 +27838,9 @@ "Vault.FreeformTags map[string]string `json:\"freeformTags,omitempty\"`", "Vault.ID string `json:\"id\"`", "Vault.Prefix string `json:\"prefix\"`", + "Vault.Replicas []*VaultReplica `json:\"replicas,omitempty\"`", + "Vault.ReplicationID string `json:\"replicationId,omitempty\"`", + "Vault.RestoredFrom string `json:\"restoredFrom,omitempty\"`", "Vault.Seq int64 `json:\"seq\"`", "Vault.State string `json:\"state\"`", "Vault.TimeCreated time.Time `json:\"timeCreated\"`", @@ -26905,11 +27849,21 @@ "Vault.WrappingKeyCreated *time.Time `json:\"wrappingKeyCreated,omitempty\"`", "Vault.WrappingKeyDER []byte `json:\"wrappingKeyDer,omitempty\"`", "Vault.WrappingKeyID string `json:\"wrappingKeyId\"`", + "VaultReplica.Region string `json:\"region\"`", + "VaultReplica.ReplicationID string `json:\"replicationId\"`", + "VaultReplica.Status string `json:\"status\"`", + "VaultReplica.TimeCreated time.Time `json:\"timeCreated\"`", "backendSnapshot.Auth []ociauth.KeyRecord `json:\"authKeys\"`", + "backendSnapshot.BackupKey []byte `json:\"backupKey,omitempty\"`", "backendSnapshot.Keys map[string]*Key `json:\"keys\"`", + "backendSnapshot.Repl map[string]*replOp `json:\"replication,omitempty\"`", "backendSnapshot.Secrets map[string]*Secret `json:\"secrets\"`", "backendSnapshot.Seq int64 `json:\"seq\"`", - "backendSnapshot.Vaults map[string]*Vault `json:\"vaults\"`" + "backendSnapshot.Vaults map[string]*Vault `json:\"vaults\"`", + "replOp.ReadyAt time.Time `json:\"readyAt\"`", + "replOp.Regions []string `json:\"regions\"`", + "transition.SettleAt *time.Time `json:\"settleAt,omitempty\"`", + "transition.SettleTo string `json:\"settleTo,omitempty\"`" ], "version": 1 }, @@ -27279,13 +28233,17 @@ }, "opensearch": { "fields": [ + "AdvancedSecurityOptions.AnonymousAuthDisableDate time.Time `json:\"anonymousAuthDisableDate,omitzero\"`", "AdvancedSecurityOptions.AnonymousAuthEnabled bool `json:\"anonymousAuthEnabled,omitempty\"`", "AdvancedSecurityOptions.Enabled bool `json:\"enabled\"`", + "AdvancedSecurityOptions.IAMFederationOptions *IAMFederationOptions `json:\"iamFederationOptions,omitempty\"`", "AdvancedSecurityOptions.InternalUserDatabaseEnabled bool `json:\"internalUserDatabaseEnabled,omitempty\"`", + "AdvancedSecurityOptions.JWTOptions *JWTOptions `json:\"jwtOptions,omitempty\"`", "AdvancedSecurityOptions.MasterUserARN string `json:\"masterUserArn,omitempty\"`", "AdvancedSecurityOptions.MasterUserName string `json:\"masterUserName,omitempty\"`", "AdvancedSecurityOptions.MasterUserPasswordHash string `json:\"masterUserPasswordHash,omitempty\"`", "AdvancedSecurityOptions.SAMLOptions *SAMLOptionsInput `json:\"samlOptions,omitempty\"`", + "AdvancedSecurityOptions.fieldsSet advancedSecurityPresence", "AppConfig.Key string `json:\"key\"`", "AppConfig.Value string `json:\"value\"`", "AppDataSource.DataSourceArn string `json:\"dataSourceArn\"`", @@ -27428,6 +28386,9 @@ "EnableSoftwareUpdateOptions.AutoSoftwareUpdateEnabled bool `json:\"autoSoftwareUpdateEnabled\"`", "EncryptionAtRestOptions.Enabled bool `json:\"enabled\"`", "EncryptionAtRestOptions.KMSKeyID string `json:\"kmsKeyId,omitempty\"`", + "IAMFederationOptions.Enabled bool `json:\"enabled,omitempty\"`", + "IAMFederationOptions.RolesKey string `json:\"rolesKey,omitempty\"`", + "IAMFederationOptions.SubjectKey string `json:\"subjectKey,omitempty\"`", "IdentityCenterOptions.EnabledAPIAccess bool `json:\"enabledAPIAccess\"`", "IdentityCenterOptions.IdentityCenterApplicationARN string `json:\"identityCenterApplicationARN,omitempty\"`", "IdentityCenterOptions.IdentityCenterInstanceARN string `json:\"identityCenterInstanceARN,omitempty\"`", @@ -27441,6 +28402,17 @@ "InboundConnection.Status string `json:\"status\"`", "InboundConnection.StatusMessage string `json:\"statusMessage,omitempty\"`", "InboundConnection.StatusUntil time.Time `json:\"statusUntil,omitzero\"`", + "InternalUser.Attributes map[string]string `json:\"attributes\"`", + "InternalUser.BackendRoles []string `json:\"backend_roles\"`", + "InternalUser.Description string `json:\"description\"`", + "InternalUser.Hash string `json:\"hash\"`", + "InternalUser.Hidden bool `json:\"hidden\"`", + "InternalUser.Reserved bool `json:\"reserved\"`", + "JWTOptions.Enabled bool `json:\"enabled,omitempty\"`", + "JWTOptions.JwksURL string `json:\"jwksUrl,omitempty\"`", + "JWTOptions.PublicKey string `json:\"publicKey,omitempty\"`", + "JWTOptions.RolesKey string `json:\"rolesKey,omitempty\"`", + "JWTOptions.SubjectKey string `json:\"subjectKey,omitempty\"`", "LogPublishingOption.CloudWatchLogsLogGroupARN string `json:\"cloudWatchLogsLogGroupArn,omitempty\"`", "LogPublishingOption.Enabled bool `json:\"enabled\"`", "Migration.ApplicationID string `json:\"applicationId\"`", @@ -27495,9 +28467,25 @@ "ReservedInstance.StartTime float64 `json:\"StartTime\"`", "ReservedInstance.State string `json:\"State\"`", "ReservedInstance.UsagePrice float64 `json:\"UsagePrice\"`", + "RoleIndexPermission.AllowedActions []string `json:\"allowed_actions\"`", + "RoleIndexPermission.DLS string `json:\"dls\"`", + "RoleIndexPermission.FLS []string `json:\"fls\"`", + "RoleIndexPermission.IndexPatterns []string `json:\"index_patterns\"`", + "RoleIndexPermission.MaskedFields []string `json:\"masked_fields\"`", + "RoleMapping.AndBackendRoles []string `json:\"and_backend_roles\"`", + "RoleMapping.BackendRoles []string `json:\"backend_roles\"`", + "RoleMapping.Description string `json:\"description\"`", + "RoleMapping.Hidden bool `json:\"hidden\"`", + "RoleMapping.Hosts []string `json:\"hosts\"`", + "RoleMapping.Reserved bool `json:\"reserved\"`", + "RoleMapping.Users []string `json:\"users\"`", + "RoleTenantPermission.AllowedActions []string `json:\"allowed_actions\"`", + "RoleTenantPermission.TenantPatterns []string `json:\"tenant_patterns\"`", "SAMLOptionsInput.Enabled bool `json:\"enabled,omitempty\"`", "SAMLOptionsInput.IDPEntityID string `json:\"idpEntityId,omitempty\"`", "SAMLOptionsInput.IDPMetadataContent string `json:\"idpMetadataContent,omitempty\"`", + "SAMLOptionsInput.MasterBackendRole string `json:\"masterBackendRole,omitempty\"`", + "SAMLOptionsInput.MasterUserName string `json:\"masterUserName,omitempty\"`", "SAMLOptionsInput.RolesKey string `json:\"rolesKey,omitempty\"`", "SAMLOptionsInput.SessionTimeoutMinutes int `json:\"sessionTimeoutMinutes,omitempty\"`", "SAMLOptionsInput.SubjectKey string `json:\"subjectKey,omitempty\"`", @@ -27510,6 +28498,28 @@ "ScheduledAction.Severity string `json:\"Severity\"`", "ScheduledAction.Status string `json:\"Status\"`", "ScheduledAction.Type string `json:\"Type\"`", + "SecurityActionGroup.AllowedActions []string `json:\"allowed_actions\"`", + "SecurityActionGroup.Description string `json:\"description\"`", + "SecurityActionGroup.Hidden bool `json:\"hidden\"`", + "SecurityActionGroup.Reserved bool `json:\"reserved\"`", + "SecurityActionGroup.Static bool `json:\"static\"`", + "SecurityActionGroup.Type string `json:\"type\"`", + "SecurityConfig.ActionGroups map[string]*SecurityActionGroup `json:\"actionGroups\"`", + "SecurityConfig.Domain string `json:\"domain\"`", + "SecurityConfig.Roles map[string]*SecurityRole `json:\"roles\"`", + "SecurityConfig.RolesMapping map[string]*RoleMapping `json:\"rolesMapping\"`", + "SecurityConfig.Tenants map[string]*SecurityTenant `json:\"tenants\"`", + "SecurityConfig.Users map[string]*InternalUser `json:\"users\"`", + "SecurityRole.ClusterPermissions []string `json:\"cluster_permissions\"`", + "SecurityRole.Description string `json:\"description\"`", + "SecurityRole.Hidden bool `json:\"hidden\"`", + "SecurityRole.IndexPermissions []RoleIndexPermission `json:\"index_permissions\"`", + "SecurityRole.Reserved bool `json:\"reserved\"`", + "SecurityRole.Static bool `json:\"static\"`", + "SecurityRole.TenantPermissions []RoleTenantPermission `json:\"tenant_permissions\"`", + "SecurityTenant.Description string `json:\"description\"`", + "SecurityTenant.Hidden bool `json:\"hidden\"`", + "SecurityTenant.Reserved bool `json:\"reserved\"`", "ServerlessAccessPolicy.CreatedDate float64 `json:\"createdDate\"`", "ServerlessAccessPolicy.Description string `json:\"description,omitempty\"`", "ServerlessAccessPolicy.LastModifiedDate float64 `json:\"lastModifiedDate\"`", @@ -27629,6 +28639,9 @@ "Workspace.Type string `json:\"type\"`", "Workspace.WorkspaceID string `json:\"workspaceId\"`", "ZoneAwarenessConfig.AvailabilityZoneCount int `json:\"availabilityZoneCount\"`", + "advancedSecurityPresence.anonymous bool", + "advancedSecurityPresence.enabled bool", + "advancedSecurityPresence.internal bool", "backendSnapshot.AccountCapacityLimits ServerlessCapacityLimits `json:\"accountCapacityLimits\"`", "backendSnapshot.AccountID string `json:\"accountID\"`", "backendSnapshot.AppIDCounter int `json:\"appIDCounter\"`", @@ -35771,6 +36784,12 @@ "ExecutionStats.ExecutionTimeInMillis int64 `json:\"ExecutionTimeInMillis,omitempty\"`", "ExecutionStats.QueryResultRows int64 `json:\"QueryResultRows,omitempty\"`", "ExecutionStats.RecordsIngested int64 `json:\"RecordsIngested,omitempty\"`", + "FailedRun.ErrorReport *S3ReportLocation `json:\"error_report,omitempty\"`", + "FailedRun.Failure string `json:\"failure\"`", + "FailedRun.InvocationTime time.Time `json:\"invocation_time\"`", + "FailedRun.Stats ExecutionStats `json:\"stats\"`", + "FailedRun.Status string `json:\"status\"`", + "FailedRun.TriggerTime time.Time `json:\"trigger_time\"`", "MixedMeasureMapping.MeasureName string `json:\"MeasureName,omitempty\"`", "MixedMeasureMapping.MeasureValueType string `json:\"MeasureValueType\"`", "MixedMeasureMapping.MultiMeasureAttributeMappings []MultiMeasureAttributeMapping `json:\"MultiMeasureAttributeMappings,omitempty\"`", @@ -35797,6 +36816,8 @@ "QueryStatusDetail.CumulativeBytesScanned int64 `json:\"CumulativeBytesScanned\"`", "QueryStatusDetail.ProgressPercentage float64 `json:\"ProgressPercentage\"`", "Row.Data []Datum `json:\"Data\"`", + "S3ReportLocation.BucketName string `json:\"BucketName,omitempty\"`", + "S3ReportLocation.ObjectKey string `json:\"ObjectKey,omitempty\"`", "ScheduledQuery.Arn string `json:\"arn\"`", "ScheduledQuery.CreationTime time.Time `json:\"creation_time\"`", "ScheduledQuery.ErrorReportEncryptionOption string `json:\"error_report_encryption_option,omitempty\"`", @@ -35804,13 +36825,17 @@ "ScheduledQuery.ErrorReportS3BucketName string `json:\"error_report_s3_bucket_name\"`", "ScheduledQuery.ExecutionRoleArn string `json:\"execution_role_arn\"`", "ScheduledQuery.KmsKeyID string `json:\"kms_key_id,omitempty\"`", + "ScheduledQuery.LastRunErrorReport *S3ReportLocation `json:\"last_run_error_report,omitempty\"`", "ScheduledQuery.LastRunFailure string `json:\"last_run_failure,omitempty\"`", "ScheduledQuery.LastRunStats *ExecutionStats `json:\"last_run_stats,omitempty\"`", "ScheduledQuery.LastRunStatus string `json:\"last_run_status,omitempty\"`", "ScheduledQuery.LastRunTime time.Time `json:\"last_run_time\"`", + "ScheduledQuery.LastRunTriggerTime time.Time `json:\"last_run_trigger_time,omitzero\"`", "ScheduledQuery.Name string `json:\"name\"`", + "ScheduledQuery.NextRun time.Time `json:\"next_run,omitzero\"`", "ScheduledQuery.NotificationTopicArn string `json:\"notification_topic_arn\"`", "ScheduledQuery.QueryString string `json:\"query_string\"`", + "ScheduledQuery.RecentlyFailedRuns []FailedRun `json:\"recently_failed_runs,omitempty\"`", "ScheduledQuery.ScheduleExpression string `json:\"schedule_expression\"`", "ScheduledQuery.State string `json:\"state\"`", "ScheduledQuery.Tags map[string]string `json:\"tags\"`", @@ -35911,6 +36936,10 @@ "Record.MeasureValues []MeasureValue `json:\"measure_values,omitempty\"`", "Record.Time string `json:\"time\"`", "Record.TimeUnit string `json:\"time_unit\"`", + "RecordShape.Attributes []ShapeMeasure `json:\"attributes,omitempty\"`", + "RecordShape.Dimensions []string `json:\"dimensions,omitempty\"`", + "RecordShape.MeasureName string `json:\"measure_name\"`", + "RecordShape.MeasureValueType string `json:\"measure_value_type\"`", "ReportConfiguration.ReportS3Configuration *DataSourceS3Configuration `json:\"ReportS3Configuration,omitempty\"`", "RetentionProperties.MagneticStoreRetentionPeriodInDays int64 `json:\"MagneticStoreRetentionPeriodInDays,omitempty\"`", "RetentionProperties.MemoryStoreRetentionPeriodInHours int64 `json:\"MemoryStoreRetentionPeriodInHours,omitempty\"`", @@ -35919,6 +36948,8 @@ "S3Configuration.KmsKeyID string `json:\"kms_key_id,omitempty\"`", "S3Configuration.ObjectKeyPrefix string `json:\"object_key_prefix,omitempty\"`", "Schema.CompositePartitionKey []PartitionKey `json:\"composite_partition_key,omitempty\"`", + "ShapeMeasure.Name string `json:\"name\"`", + "ShapeMeasure.Type string `json:\"type\"`", "Table.ARN string `json:\"arn\"`", "Table.CreationTime time.Time `json:\"creation_time\"`", "Table.DatabaseName string `json:\"database_name\"`", @@ -35930,6 +36961,7 @@ "Table.TableStatus string `json:\"table_status\"`", "backendSnapshot.NextTaskID int `json:\"next_task_id\"`", "backendSnapshot.Records map[string]map[string][]Record `json:\"records\"`", + "backendSnapshot.Shapes map[string]map[string][]RecordShape `json:\"shapes,omitempty\"`", "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`", "backendSnapshot.Tags map[string]map[string]string `json:\"tags\"`" ], @@ -35956,14 +36988,18 @@ "CallAnalyticsJob.FailureReason string `json:\"failureReason,omitempty\"`", "CallAnalyticsJob.IdentifiedLanguageScore float32 `json:\"identifiedLanguageScore,omitempty\"`", "CallAnalyticsJob.LanguageCode string `json:\"languageCode\"`", + "CallAnalyticsJob.ManagedOutputs map[string][]byte `json:\"managedOutputs,omitempty\"`", + "CallAnalyticsJob.ManagedToken string `json:\"managedToken,omitempty\"`", "CallAnalyticsJob.Media Media `json:\"media\"`", "CallAnalyticsJob.MediaFormat string `json:\"mediaFormat,omitempty\"`", "CallAnalyticsJob.MediaSampleRateHertz int32 `json:\"mediaSampleRateHertz,omitempty\"`", "CallAnalyticsJob.OutputLocation string `json:\"outputLocation,omitempty\"`", + "CallAnalyticsJob.RedactedTranscriptURI string `json:\"redactedTranscriptFileUri,omitempty\"`", "CallAnalyticsJob.Settings *CallAnalyticsSettings `json:\"settings,omitempty\"`", "CallAnalyticsJob.StartTime time.Time `json:\"startTime\"`", "CallAnalyticsJob.Tags map[string]string `json:\"tags,omitempty\"`", - "CallAnalyticsJob.TranscriptJSON []byte `json:\"-\"`", + "CallAnalyticsJob.TranscriptFileURI string `json:\"transcriptFileUri,omitempty\"`", + "CallAnalyticsJob.TranscriptJSON []byte `json:\"transcriptJson,omitempty\"`", "CallAnalyticsRule.InterruptionFilter *InterruptionFilter `json:\"InterruptionFilter,omitempty\"`", "CallAnalyticsRule.NonTalkTimeFilter *NonTalkTimeFilter `json:\"NonTalkTimeFilter,omitempty\"`", "CallAnalyticsRule.SentimentFilter *SentimentFilter `json:\"SentimentFilter,omitempty\"`", @@ -36039,6 +37075,8 @@ "MedicalTranscriptionJob.CreationTime time.Time `json:\"creationTime\"`", "MedicalTranscriptionJob.FailureReason string `json:\"failureReason,omitempty\"`", "MedicalTranscriptionJob.LanguageCode string `json:\"languageCode\"`", + "MedicalTranscriptionJob.ManagedOutputs map[string][]byte `json:\"managedOutputs,omitempty\"`", + "MedicalTranscriptionJob.ManagedToken string `json:\"managedToken,omitempty\"`", "MedicalTranscriptionJob.Media Media `json:\"media\"`", "MedicalTranscriptionJob.MediaFormat string `json:\"mediaFormat,omitempty\"`", "MedicalTranscriptionJob.MediaSampleRateHertz int32 `json:\"mediaSampleRateHertz,omitempty\"`", @@ -36050,7 +37088,8 @@ "MedicalTranscriptionJob.Specialty string `json:\"specialty,omitempty\"`", "MedicalTranscriptionJob.StartTime time.Time `json:\"startTime\"`", "MedicalTranscriptionJob.Tags map[string]string `json:\"tags,omitempty\"`", - "MedicalTranscriptionJob.TranscriptJSON []byte `json:\"-\"`", + "MedicalTranscriptionJob.TranscriptFileURI string `json:\"transcriptFileUri,omitempty\"`", + "MedicalTranscriptionJob.TranscriptJSON []byte `json:\"transcriptJson,omitempty\"`", "MedicalTranscriptionJob.TranscriptionJobStatus string `json:\"transcriptionJobStatus\"`", "MedicalTranscriptionJob.Type string `json:\"type,omitempty\"`", "MedicalTranscriptionSettings.ChannelIdentification bool `json:\"ChannelIdentification,omitempty\"`", @@ -36106,6 +37145,8 @@ "TranscriptionJob.LanguageCodes []LanguageCodeItem `json:\"languageCodes,omitempty\"`", "TranscriptionJob.LanguageIDSettings map[string]LanguageIDSettings `json:\"languageIdSettings,omitempty\"`", "TranscriptionJob.LanguageOptions []string `json:\"languageOptions,omitempty\"`", + "TranscriptionJob.ManagedOutputs map[string][]byte `json:\"managedOutputs,omitempty\"`", + "TranscriptionJob.ManagedToken string `json:\"managedToken,omitempty\"`", "TranscriptionJob.Media Media `json:\"media\"`", "TranscriptionJob.MediaFormat string `json:\"mediaFormat,omitempty\"`", "TranscriptionJob.MediaSampleRateHertz int32 `json:\"mediaSampleRateHertz,omitempty\"`", @@ -36113,13 +37154,14 @@ "TranscriptionJob.OutputBucketName string `json:\"outputBucketName,omitempty\"`", "TranscriptionJob.OutputEncryptionKMSKeyID string `json:\"outputEncryptionKMSKeyId,omitempty\"`", "TranscriptionJob.OutputKey string `json:\"outputKey,omitempty\"`", + "TranscriptionJob.RedactedTranscriptURI string `json:\"redactedTranscriptFileUri,omitempty\"`", "TranscriptionJob.Settings *TranscriptionSettings `json:\"settings,omitempty\"`", "TranscriptionJob.StartTime time.Time `json:\"startTime\"`", "TranscriptionJob.Subtitles *SubtitlesOutput `json:\"subtitles,omitempty\"`", "TranscriptionJob.Tags map[string]string `json:\"tags,omitempty\"`", "TranscriptionJob.ToxicityDetection []ToxicityDetectionSettings `json:\"toxicityDetection,omitempty\"`", - "TranscriptionJob.TranscriptJSON []byte `json:\"-\"`", - "TranscriptionJob.TranscriptText string `json:\"transcriptText\"`", + "TranscriptionJob.TranscriptFileURI string `json:\"transcriptFileUri,omitempty\"`", + "TranscriptionJob.TranscriptJSON []byte `json:\"transcriptJson,omitempty\"`", "TranscriptionSettings.ChannelIdentification bool `json:\"ChannelIdentification,omitempty\"`", "TranscriptionSettings.MaxAlternatives int32 `json:\"MaxAlternatives,omitempty\"`", "TranscriptionSettings.MaxSpeakerLabels int32 `json:\"MaxSpeakerLabels,omitempty\"`", diff --git a/pkgs/redisdocker/acl.go b/pkgs/redisdocker/acl.go new file mode 100644 index 0000000000..b06336abe8 --- /dev/null +++ b/pkgs/redisdocker/acl.go @@ -0,0 +1,172 @@ +package redisdocker + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" +) + +// AdminUser is the internal account the emulator uses for control traffic; it is never exposed to API users. +const AdminUser = "gs-admin" + +const defaultUser = "default" + +// ErrACL is returned for an unusable ACL definition. +var ErrACL = errors.New("invalid ACL user") + +// ACLUser is one Redis ACL account derived from an API user. +type ACLUser struct { + Name string + AccessString string + Passwords []string + // NoPassword lets the user in without a password. + NoPassword bool +} + +// ACLPolicy is the full set of accounts a group serves. +type ACLPolicy struct { + // DefaultPassword protects the "default" account when no listed user is named "default". + DefaultPassword string + Users []ACLUser + // OpenDefault leaves the "default" account passwordless and unrestricted when no listed user is named "default". + OpenDefault bool +} + +// SetUserArgs translates u into an ACL SETUSER command. The user is reset first so repeated syncs converge, then +// gets its passwords and the rules of its access string ("on ~app:* +@all"). +func SetUserArgs(u ACLUser) ([]any, error) { + if u.Name == "" || strings.ContainsAny(u.Name, " \t\r\n") || u.Name == AdminUser { + return nil, fmt.Errorf("%w: bad name %q", ErrACL, u.Name) + } + + args := []any{"ACL", "SETUSER", u.Name, "reset"} + + switch { + case u.NoPassword: + args = append(args, "nopass") + default: + for _, p := range u.Passwords { + args = append(args, ">"+p) + } + } + + for rule := range strings.FieldsSeq(u.AccessString) { + args = append(args, rule) + } + + return args, nil +} + +// commands returns the SETUSER commands for p; accounts are listed before "default" so a failure leaves it intact. +func (p ACLPolicy) commands() ([][]any, error) { + cmds := make([][]any, 0, len(p.Users)+1) + + for _, u := range p.Users { + if u.Name == defaultUser { + continue + } + + args, err := SetUserArgs(u) + if err != nil { + return nil, err + } + + cmds = append(cmds, args) + } + + if def, ok := p.defaultCommand(); ok { + cmds = append(cmds, def) + } + + return cmds, nil +} + +func (p ACLPolicy) defaultCommand() ([]any, bool) { + if i := slices.IndexFunc(p.Users, func(u ACLUser) bool { return u.Name == defaultUser }); i >= 0 { + args, err := SetUserArgs(p.Users[i]) + + return args, err == nil + } + + base := []any{"ACL", "SETUSER", defaultUser, "reset"} + + switch { + case p.DefaultPassword != "": + return append(base, "on", ">"+p.DefaultPassword, "~*", "&*", "+@all"), true + case p.OpenDefault: + return append(base, "on", "nopass", "~*", "&*", "+@all"), true + default: + return base, true + } +} + +// names lists the accounts p defines, excluding "default". +func (p ACLPolicy) names() []string { + var out []string + + for _, u := range p.Users { + if u.Name != defaultUser { + out = append(out, u.Name) + } + } + + return out +} + +// ApplyACL syncs policy onto every node; accounts dropped since the previous sync are deleted. +func (g *Group) ApplyACL(ctx context.Context, policy ACLPolicy) error { + cmds, err := policy.commands() + if err != nil { + return err + } + + g.mu.Lock() + defer g.mu.Unlock() + + wanted := policy.names() + stale := slices.DeleteFunc(slices.Clone(g.aclUsers), func(n string) bool { return slices.Contains(wanted, n) }) + + for _, n := range g.nodes { + if err = g.syncNodeACL(ctx, n, cmds, stale); err != nil { + return err + } + } + + g.aclUsers = wanted + + return nil +} + +func (g *Group) syncNodeACL(ctx context.Context, n *node, cmds [][]any, stale []string) error { + adm := g.dial(n) + defer func() { _ = adm.Close() }() + + for _, name := range stale { + if _, err := adm.Do(ctx, "ACL", "DELUSER", name); err != nil { + return fmt.Errorf("delete acl user: %w", err) + } + } + + for _, cmd := range cmds { + if _, err := adm.Do(ctx, cmd...); err != nil { + return fmt.Errorf("set acl user %v: %w", cmd[2], redactErr(err, cmd)) + } + } + + return nil +} + +// redactErr drops any password-bearing text from err. +func redactErr(err error, cmd []any) error { + msg := err.Error() + + for _, a := range cmd { + if s, ok := a.(string); ok && strings.HasPrefix(s, ">") { + msg = strings.ReplaceAll(msg, s[1:], "***") + } + } + + return fmt.Errorf("%w: %s", ErrACL, msg) +} diff --git a/pkgs/redisdocker/acl_test.go b/pkgs/redisdocker/acl_test.go new file mode 100644 index 0000000000..6c9d5beaed --- /dev/null +++ b/pkgs/redisdocker/acl_test.go @@ -0,0 +1,56 @@ +package redisdocker_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/redisdocker" +) + +func TestSetUserArgs(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + want []any + user redisdocker.ACLUser + wantErr bool + }{ + { + name: "restricted keys", + user: redisdocker.ACLUser{Name: "app", AccessString: "on ~app:* +@all", Passwords: []string{"pw1", "pw2"}}, + want: []any{"ACL", "SETUSER", "app", "reset", ">pw1", ">pw2", "on", "~app:*", "+@all"}, + }, + { + name: "nopass", + user: redisdocker.ACLUser{Name: "open", AccessString: "on ~* &* +@all", NoPassword: true}, + want: []any{"ACL", "SETUSER", "open", "reset", "nopass", "on", "~*", "&*", "+@all"}, + }, + { + name: "no access string stays off", + user: redisdocker.ACLUser{Name: "bare", Passwords: []string{"p"}}, + want: []any{"ACL", "SETUSER", "bare", "reset", ">p"}, + }, + {name: "empty name", user: redisdocker.ACLUser{}, wantErr: true}, + {name: "space in name", user: redisdocker.ACLUser{Name: "a b"}, wantErr: true}, + {name: "reserved admin", user: redisdocker.ACLUser{Name: redisdocker.AdminUser}, wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := redisdocker.SetUserArgs(tc.user) + if tc.wantErr { + require.ErrorIs(t, err, redisdocker.ErrACL) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/pkgs/redisdocker/admin.go b/pkgs/redisdocker/admin.go new file mode 100644 index 0000000000..17cfe8045f --- /dev/null +++ b/pkgs/redisdocker/admin.go @@ -0,0 +1,72 @@ +package redisdocker + +import ( + "context" + "crypto/tls" + "fmt" + "strings" + + "github.com/redis/go-redis/v9" +) + +const respVersion = 2 + +// Target addresses one node's control endpoint. +type Target struct { + TLS *tls.Config + Addr string + Username string + Password string +} + +// Admin sends commands to one node. +type Admin interface { + Do(ctx context.Context, args ...any) (any, error) + Close() error +} + +// Dialer opens an Admin for t; it does not connect until the first command. +type Dialer func(t Target) Admin + +type goRedisAdmin struct{ c *redis.Client } + +func dialRedis(t Target) Admin { + return &goRedisAdmin{c: redis.NewClient(&redis.Options{ + Addr: t.Addr, Username: t.Username, Password: t.Password, TLSConfig: t.TLS, + Protocol: respVersion, DisableIdentity: true, MaxRetries: -1, + })} +} + +func (a *goRedisAdmin) Do(ctx context.Context, args ...any) (any, error) { + res, err := a.c.Do(ctx, args...).Result() + if err != nil { + return nil, fmt.Errorf("redis %v: %w", args[0], err) + } + + return res, nil +} + +func (a *goRedisAdmin) Close() error { return a.c.Close() } + +func asString(v any) string { + switch s := v.(type) { + case string: + return s + case []byte: + return string(s) + default: + return fmt.Sprint(v) + } +} + +// infoField returns the value of key in INFO/CLUSTER INFO style "key:value" lines. +func infoField(text, key string) string { + for line := range strings.SplitSeq(text, "\n") { + k, v, ok := strings.Cut(strings.TrimSpace(line), ":") + if ok && k == key { + return v + } + } + + return "" +} diff --git a/pkgs/redisdocker/docker_test.go b/pkgs/redisdocker/docker_test.go new file mode 100644 index 0000000000..8cbc1fd280 --- /dev/null +++ b/pkgs/redisdocker/docker_test.go @@ -0,0 +1,112 @@ +package redisdocker_test + +import ( + "crypto/tls" + "crypto/x509" + "os/exec" + "testing" + "time" + + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/container" + "github.com/blackbirdworks/gopherstack/pkgs/devtls" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc/portalloctest" + "github.com/blackbirdworks/gopherstack/pkgs/redisdocker" +) + +func dockerEngine(t *testing.T) *redisdocker.Engine { + t.Helper() + + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker CLI not available") + } + + rt, err := container.NewRuntime(container.Config{}) + if err != nil { + t.Skipf("container runtime unavailable: %v", err) + } + + t.Cleanup(func() { _ = rt.Close() }) + + if err = rt.Ping(t.Context()); err != nil { + t.Skipf("container runtime unreachable: %v", err) + } + + ca, err := devtls.NewCA() + require.NoError(t, err) + + alloc := portalloctest.New(t, 100) + + return redisdocker.New(redisdocker.Config{ + Runtime: rt, Ports: alloc, CA: ca, Label: "redisdocker-test", TempDir: t.TempDir(), + StartTimeout: 5 * time.Minute, + }) +} + +func TestDockerClusterTLSACLSnapshot(t *testing.T) { + t.Parallel() + + eng := dockerEngine(t) + + g, err := eng.Launch(t.Context(), redisdocker.Spec{ + Version: "7.1", ClusterMode: true, TLS: true, + Shards: []redisdocker.ShardSpec{{Replicas: 1}, {Replicas: 1}}, + }) + require.NoError(t, err) + + t.Cleanup(g.Close) + + require.NoError(t, g.ApplyACL(t.Context(), redisdocker.ACLPolicy{Users: []redisdocker.ACLUser{ + {Name: "app", AccessString: "on ~app:* +@all", Passwords: []string{"app-password-1"}}, + }})) + + pool := x509.NewCertPool() + require.True(t, pool.AppendCertsFromPEM(eng.CAPEM())) + eps := g.Endpoints() + cc := redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: []string{eps[0][0].Addr()}, Username: "app", Password: "app-password-1", ReadOnly: true, + TLSConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + }) + + t.Cleanup(func() { _ = cc.Close() }) + + for _, k := range []string{"app:a", "app:b", "app:c", "app:d", "app:e", "app:f"} { + require.NoError(t, cc.Set(t.Context(), k, "v-"+k, 0).Err(), k) + } + + got, err := cc.Get(t.Context(), "app:a").Result() + require.NoError(t, err) + assert.Equal(t, "v-app:a", got) + + err = cc.Set(t.Context(), "other:x", "1", 0).Err() + require.Error(t, err) + assert.Contains(t, err.Error(), "NOPERM") + + var shards []redis.ClusterShard + + require.Eventually(t, func() bool { + var serr error + + shards, serr = cc.ClusterShards(t.Context()).Result() + + return serr == nil && len(shards) == 2 && len(shards[0].Nodes) == 2 && len(shards[1].Nodes) == 2 + }, time.Minute, 500*time.Millisecond, "replicas listed in CLUSTER SHARDS") + + covered := 0 + + for _, sh := range shards { + for _, r := range sh.Slots { + covered += int(r.End - r.Start + 1) + } + } + + assert.Equal(t, redisdocker.TotalSlots, covered) + + snap, err := g.Snapshot(t.Context()) + require.NoError(t, err) + require.Len(t, snap, 2) + assert.True(t, redisdocker.IsRDB(snap[0])) +} diff --git a/pkgs/redisdocker/engine.go b/pkgs/redisdocker/engine.go new file mode 100644 index 0000000000..b054a89092 --- /dev/null +++ b/pkgs/redisdocker/engine.go @@ -0,0 +1,289 @@ +package redisdocker + +import ( + "context" + "crypto/rand" + "crypto/tls" + "encoding/hex" + "errors" + "fmt" + "net" + "os" + "strconv" + "sync" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/container" + "github.com/blackbirdworks/gopherstack/pkgs/devtls" + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc" +) + +const ( + defaultHost = "127.0.0.1" + defaultStartTimeout = 3 * time.Minute + defaultPollInterval = 250 * time.Millisecond + stopTimeout = 30 * time.Second + adminPasswordBytes = 16 + containerNameBytes = 4 + busPort = 16379 + mountDir = "/gs" + dirMode = 0o755 +) + +// ErrNoCA is returned when TLS is requested but the engine has no certificate authority. +var ErrNoCA = errors.New("redis group needs a TLS certificate authority") + +// Config configures an Engine. +type Config struct { + Runtime Runtime + Ports *portalloc.Allocator + // CA signs node certificates; TLS groups cannot launch without it. + CA *devtls.CA + // Dial opens a control connection; defaults to go-redis. + Dial Dialer + // FetchRDB reads a node's dataset; defaults to a SYNC handshake. + FetchRDB func(ctx context.Context, t Target) ([]byte, error) + // Host is the address clients are told to use; defaults to 127.0.0.1. + Host string + // Label names the port reservations, e.g. "memorydb". + Label string + // TempDir holds per-group mount directories; defaults to the OS temp dir. + TempDir string + StartTimeout time.Duration + PollInterval time.Duration +} + +// Engine launches groups of Redis containers. +type Engine struct { + cfg Config +} + +// New returns an Engine; zero Config fields take their defaults. +func New(cfg Config) *Engine { + if cfg.Host == "" { + cfg.Host = defaultHost + } + + if cfg.StartTimeout <= 0 { + cfg.StartTimeout = defaultStartTimeout + } + + if cfg.PollInterval <= 0 { + cfg.PollInterval = defaultPollInterval + } + + if cfg.Dial == nil { + cfg.Dial = dialRedis + } + + if cfg.FetchRDB == nil { + cfg.FetchRDB = fetchRDB + } + + if cfg.Label == "" { + cfg.Label = "redis" + } + + return &Engine{cfg: cfg} +} + +// Host is the address nodes are advertised on. +func (e *Engine) Host() string { return e.cfg.Host } + +// CAPEM returns the PEM of the CA that signs node certificates, or nil without one. +func (e *Engine) CAPEM() []byte { + if e.cfg.CA == nil { + return nil + } + + return e.cfg.CA.CertPEM() +} + +type node struct { + containerID string + ip string + port int + shard int + replica int +} + +func (n *node) primary() bool { return n.replica == 0 } + +// Group is a launched set of containers. +type Group struct { + engine *Engine + admin string + dir string + aclUsers []string + nodes []*node + spec Spec + mu sync.Mutex + closed bool +} + +// Endpoint is a client-reachable node address. +type Endpoint struct { + Host string + Port int +} + +// Addr formats the endpoint as host:port. +func (e Endpoint) Addr() string { return net.JoinHostPort(e.Host, strconv.Itoa(e.Port)) } + +// Endpoints lists each shard's nodes, primary first. +func (g *Group) Endpoints() [][]Endpoint { + out := make([][]Endpoint, len(g.spec.Shards)) + + for _, n := range g.nodes { + out[n.shard] = append(out[n.shard], Endpoint{Host: g.engine.cfg.Host, Port: n.port}) + } + + return out +} + +// ClusterMode reports whether the group runs Redis Cluster. +func (g *Group) ClusterMode() bool { return g.spec.ClusterMode } + +// TLS reports whether nodes serve TLS. +func (g *Group) TLS() bool { return g.spec.TLS } + +func randomHex(n int) string { + buf := make([]byte, n) + _, _ = rand.Read(buf) + + return hex.EncodeToString(buf) +} + +// Launch starts the containers of spec, wires replication or the cluster, and returns once the group serves traffic. +func (e *Engine) Launch(ctx context.Context, spec Spec) (*Group, error) { + spec = spec.withDefaultSlots() + if err := spec.validate(); err != nil { + return nil, err + } + + if spec.TLS && e.cfg.CA == nil { + return nil, ErrNoCA + } + + g := &Group{engine: e, spec: spec, admin: randomHex(adminPasswordBytes)} + + lctx, cancel := context.WithTimeout(ctx, e.cfg.StartTimeout) + defer cancel() + + if err := g.start(lctx); err != nil { + g.Close() + + return nil, err + } + + return g, nil +} + +func (g *Group) start(ctx context.Context) error { + dir, err := os.MkdirTemp(g.engine.cfg.TempDir, "gopherstack-redis-") + if err != nil { + return fmt.Errorf("group directory: %w", err) + } + + g.dir = dir + + if err = os.Chmod(dir, dirMode); err != nil { + return fmt.Errorf("group directory mode: %w", err) + } + + if err = g.writeFiles(); err != nil { + return err + } + + for si, sh := range g.spec.Shards { + for r := 0; r <= sh.Replicas; r++ { + if err = g.startNode(ctx, &node{shard: si, replica: r}); err != nil { + return err + } + } + } + + if err = g.awaitNodes(ctx); err != nil { + return err + } + + return g.wire(ctx) +} + +func (g *Group) startNode(ctx context.Context, n *node) error { + cfg := g.engine.cfg + + id, err := container.CreateAndStartRetry(ctx, cfg.Runtime, + func() (container.Spec, error) { + port, perr := portalloc.AcquireFrom(cfg.Ports, cfg.Label) + if perr != nil { + return container.Spec{}, fmt.Errorf("allocate redis port: %w", perr) + } + + n.port = port + + return g.containerSpec(n), nil + }, + func() { portalloc.ReleaseTo(cfg.Ports, n.port); n.port = 0 }) + if err != nil { + return fmt.Errorf("start redis node: %w", err) + } + + n.containerID = id + + g.mu.Lock() + g.nodes = append(g.nodes, n) + g.mu.Unlock() + + return nil +} + +// Close removes the containers, frees their ports and deletes the mount directory. It is idempotent. +func (g *Group) Close() { + g.mu.Lock() + if g.closed { + g.mu.Unlock() + + return + } + + g.closed = true + nodes := append([]*node(nil), g.nodes...) + g.mu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), stopTimeout) + defer cancel() + + for _, n := range nodes { + if err := g.engine.cfg.Runtime.StopAndRemove(ctx, n.containerID); err != nil { + logger.Load(ctx).WarnContext(ctx, "redis group: container removal failed", + "container", n.containerID, "error", err) + } + + portalloc.ReleaseTo(g.engine.cfg.Ports, n.port) + } + + if g.dir != "" { + _ = os.RemoveAll(g.dir) + } +} + +func (g *Group) tlsConfig() *tls.Config { + if !g.spec.TLS { + return nil + } + + return &tls.Config{RootCAs: g.engine.cfg.CA.Pool(), ServerName: defaultHost, MinVersion: tls.VersionTLS12} +} + +func (g *Group) target(n *node) Target { + return Target{ + Addr: net.JoinHostPort(defaultHost, strconv.Itoa(n.port)), + Username: AdminUser, + Password: g.admin, + TLS: g.tlsConfig(), + } +} + +func (g *Group) dial(n *node) Admin { return g.engine.cfg.Dial(g.target(n)) } diff --git a/pkgs/redisdocker/files.go b/pkgs/redisdocker/files.go new file mode 100644 index 0000000000..2a2510b2e6 --- /dev/null +++ b/pkgs/redisdocker/files.go @@ -0,0 +1,72 @@ +package redisdocker + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "os" + "path/filepath" +) + +const ( + subdirMode = 0o755 + fileMode = 0o644 +) + +// writeFiles lays out the read-only directory every node mounts: shard seeds and, for TLS, the node certificate. +func (g *Group) writeFiles() error { + for i, sh := range g.spec.Shards { + if len(sh.Seed) == 0 { + continue + } + + if err := os.WriteFile(filepath.Join(g.dir, seedName(i)), sh.Seed, fileMode); err != nil { + return fmt.Errorf("write seed: %w", err) + } + } + + if !g.spec.TLS { + return nil + } + + return g.writeTLS() +} + +func (g *Group) writeTLS() error { + ca := g.engine.cfg.CA + hosts := append([]string{g.engine.cfg.Host, "localhost", defaultHost}, g.spec.TLSNames...) + + cert, err := ca.IssueServerClient(hosts...) + if err != nil { + return fmt.Errorf("issue redis certificate: %w", err) + } + + keyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey) + if err != nil { + return fmt.Errorf("marshal redis key: %w", err) + } + + var certPEM []byte + for _, der := range cert.Certificate { + certPEM = append(certPEM, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...) + } + + dir := filepath.Join(g.dir, tlsDir) + if err = os.Mkdir(dir, subdirMode); err != nil { + return fmt.Errorf("tls directory: %w", err) + } + + files := map[string][]byte{ + certFile: certPEM, + keyFile: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), + caFile: ca.CertPEM(), + } + + for name, data := range files { + if err = os.WriteFile(filepath.Join(dir, name), data, fileMode); err != nil { + return fmt.Errorf("write %s: %w", name, err) + } + } + + return nil +} diff --git a/pkgs/redisdocker/launch_test.go b/pkgs/redisdocker/launch_test.go new file mode 100644 index 0000000000..4cf563356c --- /dev/null +++ b/pkgs/redisdocker/launch_test.go @@ -0,0 +1,312 @@ +package redisdocker_test + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/container" + "github.com/blackbirdworks/gopherstack/pkgs/redisdocker" + "github.com/blackbirdworks/gopherstack/pkgs/redisdocker/redisdockertest" +) + +func newTestEngine(t *testing.T) (*redisdocker.Engine, *redisdockertest.Fake) { + t.Helper() + + f, eng := redisdockertest.New(t, redisdockertest.Ports(41000, 41100)) + + return eng, f +} + +func joined(s container.Spec) string { return strings.Join(s.Cmd, " ") } + +func TestLaunchClusterTopology(t *testing.T) { + t.Parallel() + + eng, fk := newTestEngine(t) + + g, err := eng.Launch(t.Context(), redisdocker.Spec{ + ClusterMode: true, + Shards: []redisdocker.ShardSpec{ + {Slots: []redisdocker.SlotRange{{Start: 0, End: 99}, {Start: 5000, End: 16383}}, Replicas: 1}, + {Slots: []redisdocker.SlotRange{{Start: 100, End: 4999}}, Replicas: 1}, + }, + }) + require.NoError(t, err) + + t.Cleanup(g.Close) + + require.Len(t, fk.Specs(), 4) + + eps := g.Endpoints() + require.Len(t, eps, 2) + seen := map[string]bool{} + + for _, shard := range eps { + require.Len(t, shard, 2) + + for _, ep := range shard { + assert.False(t, seen[ep.Addr()], "endpoint %s repeated", ep.Addr()) + seen[ep.Addr()] = true + } + } + + for i, spec := range fk.Specs() { + port := strconv.Itoa(eps[i/2][i%2].Port) + assert.Contains(t, joined(spec), "--cluster-enabled yes") + assert.Contains(t, joined(spec), "--cluster-announce-hostname 127.0.0.1") + assert.Contains(t, joined(spec), "--cluster-preferred-endpoint-type hostname") + assert.Contains(t, joined(spec), "--port "+port) + assert.Equal(t, []string{"127.0.0.1:" + port + ":" + port}, spec.Ports) + assert.NotContains(t, strings.Join(spec.Env, " "), "GS_SEED") + } + + meets := fk.Cmds("CLUSTER", "MEET") + assert.Len(t, meets, 3) + + slots := fk.Cmds("CLUSTER", "ADDSLOTSRANGE") + require.Len(t, slots, 2) + assert.Equal(t, []any{"CLUSTER", "ADDSLOTSRANGE", "0", "99", "5000", "16383"}, slots[0].Args) + assert.Equal(t, []any{"CLUSTER", "ADDSLOTSRANGE", "100", "4999"}, slots[1].Args) + assert.Equal(t, eps[0][0].Addr(), slots[0].Addr) + assert.Equal(t, eps[1][0].Addr(), slots[1].Addr) + + repl := fk.Cmds("CLUSTER", "REPLICATE") + require.Len(t, repl, 2) + assert.Equal(t, []any{"CLUSTER", "REPLICATE", "id-" + eps[0][0].Addr()}, repl[0].Args) + assert.Equal(t, eps[0][1].Addr(), repl[0].Addr) + assert.Equal(t, []any{"CLUSTER", "REPLICATE", "id-" + eps[1][0].Addr()}, repl[1].Args) + assert.Equal(t, eps[1][1].Addr(), repl[1].Addr) +} + +func TestLaunchReplicationWithoutClusterMode(t *testing.T) { + t.Parallel() + + eng, fk := newTestEngine(t) + + g, err := eng.Launch(t.Context(), redisdocker.Spec{Shards: []redisdocker.ShardSpec{{Replicas: 2}}}) + require.NoError(t, err) + + t.Cleanup(g.Close) + + require.Len(t, fk.Specs(), 3) + assert.NotContains(t, joined(fk.Specs()[0]), "cluster-enabled") + assert.False(t, g.ClusterMode()) + + eps := g.Endpoints()[0] + of := fk.Cmds("REPLICAOF") + require.Len(t, of, 2) + assert.Equal(t, []any{"REPLICAOF", "10.0.0.1", strconv.Itoa(eps[0].Port)}, of[0].Args) + assert.Equal(t, eps[1].Addr(), of[0].Addr) + assert.Equal(t, eps[2].Addr(), of[1].Addr) + assert.Empty(t, fk.Cmds("CLUSTER", "MEET")) +} + +func TestLaunchTLSIssuesCertsAndDisablesPlaintext(t *testing.T) { + t.Parallel() + + eng, fk := newTestEngine(t) + + g, err := eng.Launch(t.Context(), redisdocker.Spec{ + TLS: true, ClusterMode: true, + Shards: []redisdocker.ShardSpec{{Replicas: 0}}, + }) + require.NoError(t, err) + + t.Cleanup(g.Close) + + require.Len(t, fk.Specs(), 1) + cmd := joined(fk.Specs()[0]) + port := strconv.Itoa(g.Endpoints()[0][0].Port) + + assert.Contains(t, cmd, "--port 0 --tls-port "+port) + assert.Contains(t, cmd, "--tls-cluster yes") + assert.Contains(t, cmd, "--tls-replication yes") + assert.True(t, fk.CertsPresentAtStart(fk.Specs()[0].Name), "certificate files must exist when the container starts") + assert.NotEmpty(t, eng.CAPEM()) +} + +func TestLaunchTLSWithoutCAFails(t *testing.T) { + t.Parallel() + + fk := &redisdockertest.Fake{} + eng := redisdocker.New(redisdocker.Config{Runtime: fk, TempDir: t.TempDir()}) + + _, err := eng.Launch(t.Context(), redisdocker.Spec{TLS: true, Shards: []redisdocker.ShardSpec{{}}}) + require.ErrorIs(t, err, redisdocker.ErrNoCA) + assert.Empty(t, fk.Specs()) +} + +func TestLaunchValidatesSpec(t *testing.T) { + t.Parallel() + + tests := []struct { + err error + name string + spec redisdocker.Spec + }{ + {name: "no shards", spec: redisdocker.Spec{}, err: redisdocker.ErrSpec}, + { + name: "two shards without cluster mode", + spec: redisdocker.Spec{Shards: []redisdocker.ShardSpec{{}, {}}}, + err: redisdocker.ErrSpec, + }, + { + name: "too many replicas", + spec: redisdocker.Spec{Shards: []redisdocker.ShardSpec{{Replicas: 6}}}, + err: redisdocker.ErrSpec, + }, + { + name: "slot gap", + spec: redisdocker.Spec{ClusterMode: true, Shards: []redisdocker.ShardSpec{ + {Slots: []redisdocker.SlotRange{{Start: 0, End: 10}}}, + {Slots: []redisdocker.SlotRange{{Start: 12, End: 16383}}}, + }}, + err: redisdocker.ErrSlots, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + eng, fk := newTestEngine(t) + _, err := eng.Launch(t.Context(), tc.spec) + require.ErrorIs(t, err, tc.err) + assert.Empty(t, fk.Specs()) + }) + } +} + +func TestLaunchDefaultsToEvenSlots(t *testing.T) { + t.Parallel() + + eng, fk := newTestEngine(t) + + g, err := eng.Launch(t.Context(), redisdocker.Spec{ + ClusterMode: true, Shards: []redisdocker.ShardSpec{{}, {}}, + }) + require.NoError(t, err) + + t.Cleanup(g.Close) + + slots := fk.Cmds("CLUSTER", "ADDSLOTSRANGE") + require.Len(t, slots, 2) + assert.Equal(t, []any{"CLUSTER", "ADDSLOTSRANGE", "0", "8191"}, slots[0].Args) + assert.Equal(t, []any{"CLUSTER", "ADDSLOTSRANGE", "8192", "16383"}, slots[1].Args) +} + +func TestLaunchSeedsPrimariesOnly(t *testing.T) { + t.Parallel() + + eng, fk := newTestEngine(t) + + g, err := eng.Launch(t.Context(), redisdocker.Spec{ + Shards: []redisdocker.ShardSpec{{Replicas: 1, Seed: []byte("REDIS0011")}}, + }) + require.NoError(t, err) + + t.Cleanup(g.Close) + + assert.Contains(t, strings.Join(fk.Specs()[0].Env, " "), "GS_SEED=/gs/seed-0.rdb") + assert.NotContains(t, strings.Join(fk.Specs()[1].Env, " "), "GS_SEED") +} + +func TestCloseRemovesContainersAndReleasesPorts(t *testing.T) { + t.Parallel() + + eng, fk := newTestEngine(t) + + g, err := eng.Launch(t.Context(), redisdocker.Spec{Shards: []redisdocker.ShardSpec{{Replicas: 1}}}) + require.NoError(t, err) + + g.Close() + g.Close() + + assert.ElementsMatch(t, []string{"c1", "c2"}, fk.Removed()) +} + +func TestSnapshotReadsEachShardPrimary(t *testing.T) { + t.Parallel() + + eng, _ := newTestEngine(t) + + g, err := eng.Launch(t.Context(), redisdocker.Spec{ + ClusterMode: true, Shards: []redisdocker.ShardSpec{{Replicas: 1}, {}}, + }) + require.NoError(t, err) + + t.Cleanup(g.Close) + + got, err := g.Snapshot(t.Context()) + require.NoError(t, err) + + eps := g.Endpoints() + require.Len(t, got, 2) + assert.Equal(t, "REDIS0011"+eps[0][0].Addr(), string(got[0])) + assert.Equal(t, "REDIS0011"+eps[1][0].Addr(), string(got[1])) +} + +func TestApplyACLSyncsEveryNodeAndDropsStaleUsers(t *testing.T) { + t.Parallel() + + eng, fk := newTestEngine(t) + + g, err := eng.Launch(t.Context(), redisdocker.Spec{Shards: []redisdocker.ShardSpec{{Replicas: 1}}}) + require.NoError(t, err) + + t.Cleanup(g.Close) + + first := redisdocker.ACLPolicy{Users: []redisdocker.ACLUser{ + {Name: "app", AccessString: "on ~app:* +@all", Passwords: []string{"secret"}}, + {Name: "old", AccessString: "on ~* +@all", NoPassword: true}, + }} + require.NoError(t, g.ApplyACL(t.Context(), first)) + + setuser := fk.Cmds("ACL", "SETUSER") + require.Len(t, setuser, 6, "two users plus a locked-down default on each of two nodes") + assert.Equal(t, []any{"ACL", "SETUSER", "default", "reset"}, setuser[2].Args) + + second := redisdocker.ACLPolicy{OpenDefault: true, Users: []redisdocker.ACLUser{first.Users[0]}} + require.NoError(t, g.ApplyACL(t.Context(), second)) + + del := fk.Cmds("ACL", "DELUSER") + require.Len(t, del, 2) + assert.Equal(t, []any{"ACL", "DELUSER", "old"}, del[0].Args) + + opened := false + + for _, c := range fk.Cmds("ACL", "SETUSER", "default") { + opened = opened || slicesContain(c.Args, "nopass") + } + + assert.True(t, opened, "OpenDefault leaves default passwordless") +} + +func TestApplyACLRejectsBadUsersAndNeverEchoesPasswords(t *testing.T) { + t.Parallel() + + eng, _ := newTestEngine(t) + + g, err := eng.Launch(t.Context(), redisdocker.Spec{Shards: []redisdocker.ShardSpec{{}}}) + require.NoError(t, err) + + t.Cleanup(g.Close) + + err = g.ApplyACL(t.Context(), redisdocker.ACLPolicy{Users: []redisdocker.ACLUser{{Name: "a b"}}}) + require.ErrorIs(t, err, redisdocker.ErrACL) +} + +func slicesContain(args []any, want string) bool { + for _, a := range args { + if fmt.Sprint(a) == want { + return true + } + } + + return false +} diff --git a/pkgs/redisdocker/nodespec.go b/pkgs/redisdocker/nodespec.go new file mode 100644 index 0000000000..5ff39caab7 --- /dev/null +++ b/pkgs/redisdocker/nodespec.go @@ -0,0 +1,78 @@ +package redisdocker + +import ( + "path" + "strconv" + + "github.com/blackbirdworks/gopherstack/pkgs/container" +) + +const ( + tlsDir = "tls" + certFile = "tls.crt" + keyFile = "tls.key" + caFile = "ca.crt" + seedPrefix = "seed-" + seedSuffix = ".rdb" +) + +// entryScript copies the shard's seed RDB into the data directory, then starts the server with the admin account +// and replication credentials taken from the environment so they stay out of the argument list. +const entryScript = `if [ -n "$GS_SEED" ] && [ -f "$GS_SEED" ]; then cp "$GS_SEED" /data/dump.rdb; fi +exec "$@" --masteruser ` + AdminUser + ` --masterauth "$GS_ADMIN_PW" --user ` + AdminUser + + ` on ">$GS_ADMIN_PW" '~*' '&*' '+@all'` + +func seedName(shard int) string { return seedPrefix + strconv.Itoa(shard) + seedSuffix } + +func (g *Group) containerSpec(n *node) container.Spec { + cfg := g.engine.cfg + image, binary := ImageFor(g.spec.Engine, g.spec.Version) + env := []string{"GS_ADMIN_PW=" + g.admin} + + if n.primary() && len(g.spec.Shards[n.shard].Seed) > 0 { + env = append(env, "GS_SEED="+path.Join(mountDir, seedName(n.shard))) + } + + port := strconv.Itoa(n.port) + + return container.Spec{ + Name: "gopherstack-" + cfg.Label + "-" + randomHex(containerNameBytes), + Image: image, + Env: env, + Entrypoint: []string{"sh", "-c", entryScript, "gs-redis"}, + Cmd: append([]string{binary}, g.serverArgs(n)...), + Mounts: []string{g.dir + ":" + mountDir + ":ro"}, + Ports: []string{container.PortSpec(container.BindHostFor(cfg.Host), port, port)}, + } +} + +// serverArgs are the server flags for node n. The server listens on its published port number inside the +// container too, so the port a cluster gossips equals the one clients use. +func (g *Group) serverArgs(n *node) []string { + port := strconv.Itoa(n.port) + args := []string{"--dir", "/data", "--save", "", "--appendonly", "no", "--protected-mode", "no"} + + if g.spec.TLS { + args = append(args, "--port", "0", "--tls-port", port, + "--tls-cert-file", path.Join(mountDir, tlsDir, certFile), + "--tls-key-file", path.Join(mountDir, tlsDir, keyFile), + "--tls-ca-cert-file", path.Join(mountDir, tlsDir, caFile), + "--tls-auth-clients", "no", "--tls-replication", "yes") + } else { + args = append(args, "--port", port) + } + + if !g.spec.ClusterMode { + return args + } + + args = append(args, "--cluster-enabled", "yes", "--cluster-config-file", "/data/nodes.conf", + "--cluster-port", strconv.Itoa(busPort), "--cluster-node-timeout", "5000", + "--cluster-announce-hostname", g.engine.cfg.Host, "--cluster-preferred-endpoint-type", "hostname") + + if g.spec.TLS { + args = append(args, "--tls-cluster", "yes") + } + + return args +} diff --git a/pkgs/redisdocker/redisdockertest/fake.go b/pkgs/redisdocker/redisdockertest/fake.go new file mode 100644 index 0000000000..909328100a --- /dev/null +++ b/pkgs/redisdocker/redisdockertest/fake.go @@ -0,0 +1,267 @@ +// Package redisdockertest provides an in-memory container runtime and control connection for exercising +// redisdocker groups without Docker. +package redisdockertest + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/container" + "github.com/blackbirdworks/gopherstack/pkgs/devtls" + "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/redisdocker" +) + +// Cmd is one control command a group sent to a node. +type Cmd struct { + Addr string + Args []any +} + +// Fake is a runtime and control plane that records everything and answers like a healthy cluster. +type Fake struct { + tlsSeen map[string]bool + specs []container.Spec + removed []string + cmds []Cmd + mu sync.Mutex +} + +// New returns a Fake and an Engine wired to it. +func New(t *testing.T, base portRange) (*Fake, *redisdocker.Engine) { + t.Helper() + + alloc, err := portalloc.New(base.Start, base.End) + require.NoError(t, err) + + ca, err := devtls.NewCA() + require.NoError(t, err) + + f := &Fake{tlsSeen: map[string]bool{}} + eng := redisdocker.New(redisdocker.Config{ + Runtime: f, Ports: alloc, CA: ca, Host: "127.0.0.1", Label: "fake", TempDir: t.TempDir(), + Dial: func(tg redisdocker.Target) redisdocker.Admin { return &admin{target: tg, f: f} }, + FetchRDB: func(_ context.Context, tg redisdocker.Target) ([]byte, error) { + return []byte("REDIS0011" + tg.Addr), nil + }, + }) + + return f, eng +} + +type portRange struct{ Start, End int } + +// Ports returns a port range for the fake's allocator. +func Ports(start, end int) portRange { return portRange{Start: start, End: end} } + +// CreateAndStart implements redisdocker.Runtime. +func (f *Fake) CreateAndStart(_ context.Context, spec container.Spec) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.specs = append(f.specs, spec) + + if f.tlsSeen == nil { + f.tlsSeen = map[string]bool{} + } + + for _, m := range spec.Mounts { + dir, _, _ := strings.Cut(m, ":") + if _, err := os.Stat(filepath.Join(dir, "tls", "tls.key")); err == nil { + f.tlsSeen[spec.Name] = true + } + } + + return "c" + strconv.Itoa(len(f.specs)), nil +} + +// StopAndRemove implements redisdocker.Runtime. +func (f *Fake) StopAndRemove(_ context.Context, id string) error { + f.mu.Lock() + defer f.mu.Unlock() + + f.removed = append(f.removed, id) + + return nil +} + +// Exec implements redisdocker.Runtime; it answers `hostname -i`. +func (f *Fake) Exec(_ context.Context, id string, _ []string) (container.ExecResult, error) { + return container.ExecResult{Stdout: "10.0.0." + strings.TrimPrefix(id, "c") + " \n"}, nil +} + +// Specs returns the containers started so far. +func (f *Fake) Specs() []container.Spec { + f.mu.Lock() + defer f.mu.Unlock() + + return append([]container.Spec(nil), f.specs...) +} + +// Removed returns the IDs of removed containers. +func (f *Fake) Removed() []string { + f.mu.Lock() + defer f.mu.Unlock() + + return append([]string(nil), f.removed...) +} + +// CertsPresentAtStart reports whether the named container's TLS files existed when it started. +func (f *Fake) CertsPresentAtStart(name string) bool { + f.mu.Lock() + defer f.mu.Unlock() + + return f.tlsSeen[name] +} + +// Cmds returns the control commands whose leading arguments equal prefix. +func (f *Fake) Cmds(prefix ...string) []Cmd { + f.mu.Lock() + defer f.mu.Unlock() + + var out []Cmd + + for _, c := range f.cmds { + if len(c.Args) < len(prefix) { + continue + } + + match := true + + for i, p := range prefix { + if fmt.Sprint(c.Args[i]) != p { + match = false + } + } + + if match { + out = append(out, c) + } + } + + return out +} + +// indexOf maps a published host:port to the 1-based container index. +func (f *Fake) indexOf(addr string) int { + f.mu.Lock() + defer f.mu.Unlock() + + _, port, _ := net.SplitHostPort(addr) + + for i, sp := range f.specs { + if len(sp.Ports) == 1 && strings.HasSuffix(sp.Ports[0], ":"+port+":"+port) { + return i + 1 + } + } + + return 0 +} + +// components groups container indexes joined by CLUSTER MEET into gossip components. +func (f *Fake) components() func(int) int { + parent := map[int]int{} + + var find func(int) int + + find = func(x int) int { + if p, ok := parent[x]; ok && p != x { + parent[x] = find(p) + + return parent[x] + } + + parent[x] = x + + return x + } + + for _, m := range f.Cmds("CLUSTER", "MEET") { + ip := fmt.Sprint(m.Args[2]) + oct, _ := strconv.Atoi(ip[strings.LastIndex(ip, ".")+1:]) + parent[find(f.indexOf(m.Addr))] = find(oct) + } + + return find +} + +// knownBy counts the nodes in addr's gossip component. +func (f *Fake) knownBy(addr string) int { + find := f.components() + me := find(f.indexOf(addr)) + n := 0 + + for i := 1; i <= len(f.Specs()); i++ { + if find(i) == me { + n++ + } + } + + return n +} + +// replicasIn counts the CLUSTER REPLICATE commands issued inside addr's gossip component. +func (f *Fake) replicasIn(addr string) int { + find := f.components() + me := find(f.indexOf(addr)) + n := 0 + + for _, r := range f.Cmds("CLUSTER", "REPLICATE") { + if find(f.indexOf(r.Addr)) == me { + n++ + } + } + + return n +} + +func (f *Fake) record(c Cmd) { + f.mu.Lock() + defer f.mu.Unlock() + + f.cmds = append(f.cmds, c) +} + +type admin struct { + f *Fake + target redisdocker.Target +} + +func (a *admin) Do(_ context.Context, args ...any) (any, error) { + a.f.record(Cmd{Addr: a.target.Addr, Args: args}) + + switch strings.ToUpper(fmt.Sprint(args[0])) { + case "PING": + return "PONG", nil + case "INFO": + return "role:slave\r\nmaster_link_status:up\r\n", nil + case "CLUSTER": + return a.cluster(args), nil + } + + return "OK", nil +} + +func (a *admin) cluster(args []any) any { + switch fmt.Sprint(args[1]) { + case "INFO": + return fmt.Sprintf("cluster_state:ok\r\ncluster_known_nodes:%d\r\n", a.f.knownBy(a.target.Addr)) + case "NODES": + return strings.Repeat("n 1.2.3.4:1 myself,slave p 0 0 1 connected\n", a.f.replicasIn(a.target.Addr)) + case "MYID": + return "id-" + a.target.Addr + } + + return "OK" +} + +func (a *admin) Close() error { return nil } diff --git a/pkgs/redisdocker/slots.go b/pkgs/redisdocker/slots.go new file mode 100644 index 0000000000..67e9c61f9b --- /dev/null +++ b/pkgs/redisdocker/slots.go @@ -0,0 +1,112 @@ +// Package redisdocker runs real Redis or Valkey containers as a replication group or a sharded cluster and manages +// their ACLs, TLS and RDB snapshots. +package redisdocker + +import ( + "errors" + "fmt" + "sort" + "strconv" + "strings" +) + +// TotalSlots is the size of the Redis cluster hash-slot space. +const TotalSlots = 16384 + +// ErrSlots is returned when a slot assignment is malformed or does not cover the hash-slot space exactly once. +var ErrSlots = errors.New("invalid slot assignment") + +// SlotRange is an inclusive range of hash slots. +type SlotRange struct { + Start int + End int +} + +// String formats r as "start-end". +func (r SlotRange) String() string { return fmt.Sprintf("%d-%d", r.Start, r.End) } + +// ParseSlots parses "0-5461" or a comma or space separated list such as "0-100,200-300,500". +func ParseSlots(s string) ([]SlotRange, error) { + fields := strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == ' ' }) + out := make([]SlotRange, 0, len(fields)) + + for _, f := range fields { + lo, hi, isRange := strings.Cut(f, "-") + + start, err := strconv.Atoi(lo) + if err != nil { + return nil, fmt.Errorf("%w: %q", ErrSlots, f) + } + + end := start + if isRange { + if end, err = strconv.Atoi(hi); err != nil { + return nil, fmt.Errorf("%w: %q", ErrSlots, f) + } + } + + if start < 0 || end < start || end >= TotalSlots { + return nil, fmt.Errorf("%w: %q is outside 0-%d", ErrSlots, f, TotalSlots-1) + } + + out = append(out, SlotRange{Start: start, End: end}) + } + + return out, nil +} + +// EvenSlots splits the hash-slot space across n shards; the last shard takes the remainder. +func EvenSlots(n int) [][]SlotRange { + if n < 1 { + return nil + } + + size := TotalSlots / n + out := make([][]SlotRange, n) + + for i := range n { + end := i*size + size - 1 + if i == n-1 { + end = TotalSlots - 1 + } + + out[i] = []SlotRange{{Start: i * size, End: end}} + } + + return out +} + +// ValidateSlotMap checks that the shards together cover every hash slot exactly once. +func ValidateSlotMap(shards [][]SlotRange) error { + var all []SlotRange + + for _, s := range shards { + all = append(all, s...) + } + + sort.Slice(all, func(i, j int) bool { return all[i].Start < all[j].Start }) + + next := 0 + + for _, r := range all { + if r.Start != next { + return fmt.Errorf("%w: slot %d is %s", ErrSlots, next, gapOrOverlap(r.Start, next)) + } + + next = r.End + 1 + } + + if next != TotalSlots { + return fmt.Errorf("%w: slots %d-%d are unassigned", ErrSlots, next, TotalSlots-1) + } + + return nil +} + +func gapOrOverlap(start, next int) string { + if start > next { + return "unassigned" + } + + return "assigned twice" +} diff --git a/pkgs/redisdocker/snapshot.go b/pkgs/redisdocker/snapshot.go new file mode 100644 index 0000000000..d863b78324 --- /dev/null +++ b/pkgs/redisdocker/snapshot.go @@ -0,0 +1,257 @@ +package redisdocker + +import ( + "bufio" + "bytes" + "context" + "crypto/tls" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "strconv" + "strings" + "time" + + "github.com/alicebob/miniredis/v2" + + "github.com/blackbirdworks/gopherstack/pkgs/redisengine" +) + +const ( + rdbMagic = "REDIS" + bundleMagic = "GSRDBS1\n" + maxRDBBytes = 1 << 30 + syncDeadline = 2 * time.Minute +) + +// ErrBadSnapshot is returned when snapshot data cannot be read. +var ErrBadSnapshot = errors.New("invalid redis snapshot data") + +// ErrShape is returned when snapshot data cannot be spread over the requested shards. +var ErrShape = errors.New("snapshot shards do not match the cluster") + +// Snapshot reads each shard primary's dataset as an RDB file. +func (g *Group) Snapshot(ctx context.Context) ([][]byte, error) { + out := make([][]byte, len(g.spec.Shards)) + + for i := range g.spec.Shards { + primary := g.shardNodes(i)[0] + + rdb, err := g.engine.cfg.FetchRDB(ctx, g.target(primary)) + if err != nil { + return nil, fmt.Errorf("snapshot shard %d: %w", i, err) + } + + out[i] = rdb + } + + return out, nil +} + +// fetchRDB performs a SYNC handshake, which makes the server hand over its dataset as one RDB payload. +func fetchRDB(ctx context.Context, t Target) ([]byte, error) { + dialer := &net.Dialer{Timeout: syncDeadline} + + conn, err := dialer.DialContext(ctx, "tcp", t.Addr) + if err != nil { + return nil, fmt.Errorf("dial: %w", err) + } + + if t.TLS != nil { + tc := tls.Client(conn, t.TLS) + if err = tc.HandshakeContext(ctx); err != nil { + _ = conn.Close() + + return nil, fmt.Errorf("tls: %w", err) + } + + conn = tc + } + + defer func() { _ = conn.Close() }() + + _ = conn.SetDeadline(time.Now().Add(syncDeadline)) + + r := bufio.NewReader(conn) + + if err = command(conn, r, "AUTH", t.Username, t.Password); err != nil { + return nil, fmt.Errorf("auth: %w", redactErr(err, []any{">" + t.Password})) + } + + return syncPayload(conn, r) +} + +func command(w io.Writer, r *bufio.Reader, args ...string) error { + var b strings.Builder + + b.WriteString("*" + strconv.Itoa(len(args)) + "\r\n") + + for _, a := range args { + b.WriteString("$" + strconv.Itoa(len(a)) + "\r\n" + a + "\r\n") + } + + if _, err := io.WriteString(w, b.String()); err != nil { + return fmt.Errorf("write: %w", err) + } + + line, err := r.ReadString('\n') + if err != nil { + return fmt.Errorf("read: %w", err) + } + + if !strings.HasPrefix(line, "+") { + return fmt.Errorf("%w: %s", ErrBadSnapshot, strings.TrimSpace(line)) + } + + return nil +} + +func syncPayload(w io.Writer, r *bufio.Reader) ([]byte, error) { + if _, err := io.WriteString(w, "SYNC\r\n"); err != nil { + return nil, fmt.Errorf("write sync: %w", err) + } + + for { + line, rerr := r.ReadString('\n') + if rerr != nil { + return nil, fmt.Errorf("read sync: %w", rerr) + } + + line = strings.TrimSpace(line) + if line == "" { + continue + } + + if !strings.HasPrefix(line, "$") { + return nil, fmt.Errorf("%w: %s", ErrBadSnapshot, line) + } + + size, perr := strconv.Atoi(line[1:]) + if perr != nil || size < 0 || size > maxRDBBytes { + return nil, fmt.Errorf("%w: payload size %q", ErrBadSnapshot, line) + } + + rdb := make([]byte, size) + if _, err := io.ReadFull(r, rdb); err != nil { + return nil, fmt.Errorf("read rdb: %w", err) + } + + return rdb, nil + } +} + +// EncodeBundle packs one RDB per shard into a single blob. +func EncodeBundle(shards [][]byte) []byte { + out := []byte(bundleMagic) + out = binary.AppendUvarint(out, uint64(len(shards))) + + for _, s := range shards { + out = binary.AppendUvarint(out, uint64(len(s))) + out = append(out, s...) + } + + return out +} + +// DecodeBundle unpacks EncodeBundle output; ok is false when blob is not a bundle. +func DecodeBundle(blob []byte) ([][]byte, bool) { + rest, found := bytes.CutPrefix(blob, []byte(bundleMagic)) + if !found { + return nil, false + } + + count, n := binary.Uvarint(rest) + if n <= 0 || count > uint64(len(rest)) { + return nil, false + } + + rest = rest[n:] + + var shards [][]byte + + for range count { + size, m := binary.Uvarint(rest) + if m <= 0 || size > uint64(len(rest)-m) { //nolint:gosec // len(rest) > m is checked by the loop invariant + return nil, false + } + + end := m + int(size) //nolint:gosec // size <= len(rest) + shards = append(shards, rest[m:end]) + rest = rest[end:] + } + + return shards, true +} + +// IsRDB reports whether blob is a raw RDB file. +func IsRDB(blob []byte) bool { return bytes.HasPrefix(blob, []byte(rdbMagic)) } + +// SeedFromBlob converts any snapshot payload this module produces (a raw RDB or an embedded keyspace dump) to an RDB. +func SeedFromBlob(blob []byte) ([]byte, error) { + if len(blob) == 0 || IsRDB(blob) { + return blob, nil + } + + mr := miniredis.NewMiniRedis() + if err := redisengine.LoadKeyspace(mr, blob); err != nil { + return nil, fmt.Errorf("%w: %w", ErrBadSnapshot, err) + } + + return redisengine.EncodeRDB(mr) +} + +// SeedsFor turns a snapshot blob (bundle, raw RDB or embedded dump) into per-shard RDB seeds for shards shards. +func SeedsFor(blob []byte, shards int) ([][]byte, error) { + if len(blob) == 0 { + return make([][]byte, shards), nil + } + + parts, isBundle := DecodeBundle(blob) + if !isBundle { + parts = [][]byte{blob} + } + + if len(parts) != shards { + return nil, fmt.Errorf("%w: snapshot holds %d shard(s), cluster has %d", ErrShape, len(parts), shards) + } + + out := make([][]byte, len(parts)) + + for i, p := range parts { + seed, err := SeedFromBlob(p) + if err != nil { + return nil, err + } + + out[i] = seed + } + + return out, nil +} + +// LoadEmbedded loads a snapshot blob produced by either engine mode into an embedded server. +func LoadEmbedded(mr *miniredis.Miniredis, blob []byte) error { + if mr == nil && len(blob) > 0 { + return fmt.Errorf("%w: engine not running", ErrBadSnapshot) + } + + if parts, ok := DecodeBundle(blob); ok { + if len(parts) != 1 { + return fmt.Errorf("%w: snapshot holds %d shards", ErrShape, len(parts)) + } + + blob = parts[0] + } + + if IsRDB(blob) { + if err := redisengine.DecodeRDB(mr, blob); err != nil { + return fmt.Errorf("%w: %w", ErrBadSnapshot, err) + } + + return nil + } + + return redisengine.LoadKeyspace(mr, blob) +} diff --git a/pkgs/redisdocker/spec.go b/pkgs/redisdocker/spec.go new file mode 100644 index 0000000000..3f5bcb0899 --- /dev/null +++ b/pkgs/redisdocker/spec.go @@ -0,0 +1,146 @@ +package redisdocker + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/blackbirdworks/gopherstack/pkgs/container" +) + +// Engine names. +const ( + EngineRedis = "redis" + EngineValkey = "valkey" +) + +const ( + minModernMajor = 7 + versionParts = 2 + + maxReplicas = 5 +) + +// ErrSpec is returned when a launch request is inconsistent. +var ErrSpec = errors.New("invalid redis group spec") + +// Runtime is the slice of the container runtime a group needs. +type Runtime interface { + CreateAndStart(ctx context.Context, spec container.Spec) (string, error) + StopAndRemove(ctx context.Context, containerID string) error + Exec(ctx context.Context, containerID string, cmd []string) (container.ExecResult, error) +} + +// ShardSpec describes one shard: its slots, replica count and optional RDB to start from. +type ShardSpec struct { + // Seed is an RDB file the primary starts with; empty starts blank. + Seed []byte + // Slots are the hash slots served by the shard; ignored unless the group is in cluster mode. + Slots []SlotRange + // Replicas is the number of replicas of the shard's primary. + Replicas int +} + +// Spec describes a group to launch. +type Spec struct { + // Engine is "redis" (default) or "valkey". + Engine string + // Version is the engine version ("7.1", "7.2.4"); only major.minor selects the image. + Version string + // TLSNames are extra certificate names beyond localhost and the advertised host. + TLSNames []string + Shards []ShardSpec + // ClusterMode runs Redis Cluster; without it the group is one primary and its replicas. + ClusterMode bool + // TLS serves TLS only (plaintext port disabled). + TLS bool +} + +func (s Spec) validate() error { + if len(s.Shards) == 0 { + return fmt.Errorf("%w: no shards", ErrSpec) + } + + if !s.ClusterMode && len(s.Shards) != 1 { + return fmt.Errorf("%w: a group without cluster mode has exactly one shard", ErrSpec) + } + + for i, sh := range s.Shards { + if sh.Replicas < 0 || sh.Replicas > maxReplicas { + return fmt.Errorf("%w: shard %d replicas must be 0-%d", ErrSpec, i, maxReplicas) + } + } + + if !s.ClusterMode { + return nil + } + + slots := make([][]SlotRange, len(s.Shards)) + for i, sh := range s.Shards { + slots[i] = sh.Slots + } + + return ValidateSlotMap(slots) +} + +// withDefaultSlots fills empty slot assignments of a cluster-mode spec with an even split. +func (s Spec) withDefaultSlots() Spec { + if !s.ClusterMode { + return s + } + + for _, sh := range s.Shards { + if len(sh.Slots) > 0 { + return s + } + } + + even := EvenSlots(len(s.Shards)) + out := s + out.Shards = make([]ShardSpec, len(s.Shards)) + + for i, sh := range s.Shards { + sh.Slots = even[i] + out.Shards[i] = sh + } + + return out +} + +// ImageFor maps an engine and API version to a container image. Versions below 7 run on 7.0, whose cluster +// hostname support the group relies on; odd minors map to the next published image. +func ImageFor(engine, version string) (string, string) { + major, minor := parseVersion(version) + + if engine == EngineValkey { + if major < 1 { + major, minor = 8, 0 + } + + return "valkey/valkey:" + strconv.Itoa(major) + "." + strconv.Itoa(minor) + "-alpine", "valkey-server" + } + + if major < minModernMajor { + major, minor = minModernMajor, 0 + } + + if minor%2 == 1 { + minor++ + } + + return "redis:" + strconv.Itoa(major) + "." + strconv.Itoa(minor) + "-alpine", "redis-server" +} + +func parseVersion(v string) (int, int) { + parts := strings.SplitN(v, ".", versionParts+1) + major, _ := strconv.Atoi(parts[0]) + + minor := 0 + if len(parts) >= versionParts { + minor, _ = strconv.Atoi(parts[1]) + } + + return major, minor +} diff --git a/pkgs/redisdocker/spec_test.go b/pkgs/redisdocker/spec_test.go new file mode 100644 index 0000000000..4a145be535 --- /dev/null +++ b/pkgs/redisdocker/spec_test.go @@ -0,0 +1,165 @@ +package redisdocker_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/redisdocker" +) + +func TestParseSlots(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want []redisdocker.SlotRange + wantErr bool + }{ + {name: "range", in: "0-5461", want: []redisdocker.SlotRange{{Start: 0, End: 5461}}}, + {name: "list", in: "0-100,200-300 500", want: []redisdocker.SlotRange{{0, 100}, {200, 300}, {500, 500}}}, + {name: "empty", in: "", want: []redisdocker.SlotRange{}}, + {name: "reversed", in: "10-5", wantErr: true}, + {name: "too big", in: "0-16384", wantErr: true}, + {name: "junk", in: "a-b", wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := redisdocker.ParseSlots(tc.in) + if tc.wantErr { + require.ErrorIs(t, err, redisdocker.ErrSlots) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestEvenSlotsCoverAllSlots(t *testing.T) { + t.Parallel() + + for _, n := range []int{1, 2, 3, 5, 7, 500} { + require.NoError(t, redisdocker.ValidateSlotMap(redisdocker.EvenSlots(n)), "shards=%d", n) + } + + assert.Equal(t, [][]redisdocker.SlotRange{{{Start: 0, End: 8191}}, {{Start: 8192, End: 16383}}}, + redisdocker.EvenSlots(2)) +} + +func TestValidateSlotMap(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + shards [][]redisdocker.SlotRange + ok bool + }{ + {name: "full split", shards: [][]redisdocker.SlotRange{{{0, 99}}, {{100, 16383}}}, ok: true}, + {name: "interleaved", shards: [][]redisdocker.SlotRange{{{0, 99}, {200, 16383}}, {{100, 199}}}, ok: true}, + {name: "gap", shards: [][]redisdocker.SlotRange{{{0, 99}}, {{101, 16383}}}}, + {name: "overlap", shards: [][]redisdocker.SlotRange{{{0, 100}}, {{100, 16383}}}}, + {name: "short", shards: [][]redisdocker.SlotRange{{{0, 16382}}}}, + {name: "none", shards: nil}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + err := redisdocker.ValidateSlotMap(tc.shards) + if tc.ok { + require.NoError(t, err) + + return + } + + require.ErrorIs(t, err, redisdocker.ErrSlots) + }) + } +} + +func TestImageFor(t *testing.T) { + t.Parallel() + + tests := []struct { + engine, version, image, binary string + }{ + {"redis", "7.1", "redis:7.2-alpine", "redis-server"}, + {"redis", "7.0", "redis:7.0-alpine", "redis-server"}, + {"redis", "6.2", "redis:7.0-alpine", "redis-server"}, + {"redis", "7.2.4", "redis:7.2-alpine", "redis-server"}, + {"redis", "", "redis:7.0-alpine", "redis-server"}, + {"valkey", "8.0", "valkey/valkey:8.0-alpine", "valkey-server"}, + {"valkey", "", "valkey/valkey:8.0-alpine", "valkey-server"}, + } + + for _, tc := range tests { + t.Run(tc.engine+"-"+tc.version, func(t *testing.T) { + t.Parallel() + + image, binary := redisdocker.ImageFor(tc.engine, tc.version) + assert.Equal(t, tc.image, image) + assert.Equal(t, tc.binary, binary) + }) + } +} + +func TestBundleRoundTrip(t *testing.T) { + t.Parallel() + + in := [][]byte{[]byte("REDIS0011a"), {}, []byte("REDIS0011bcd")} + got, ok := redisdocker.DecodeBundle(redisdocker.EncodeBundle(in)) + require.True(t, ok) + assert.Equal(t, in, got) + + _, ok = redisdocker.DecodeBundle([]byte("REDIS0011")) + assert.False(t, ok) + + trunc := redisdocker.EncodeBundle(in) + _, ok = redisdocker.DecodeBundle(trunc[:len(trunc)-3]) + assert.False(t, ok) +} + +func TestSeedsFor(t *testing.T) { + t.Parallel() + + rdb := []byte("REDIS0011xyz") + + tests := []struct { + name string + blob []byte + want [][]byte + shards int + wantErr bool + }{ + {name: "empty", blob: nil, shards: 2, want: [][]byte{nil, nil}}, + {name: "raw single", blob: rdb, shards: 1, want: [][]byte{rdb}}, + {name: "bundle", blob: redisdocker.EncodeBundle([][]byte{rdb, rdb}), shards: 2, want: [][]byte{rdb, rdb}}, + {name: "shape mismatch", blob: redisdocker.EncodeBundle([][]byte{rdb}), shards: 2, wantErr: true}, + {name: "raw for many", blob: rdb, shards: 3, wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := redisdocker.SeedsFor(tc.blob, tc.shards) + if tc.wantErr { + require.ErrorIs(t, err, redisdocker.ErrShape) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/pkgs/redisdocker/wire.go b/pkgs/redisdocker/wire.go new file mode 100644 index 0000000000..bd99bb0e8c --- /dev/null +++ b/pkgs/redisdocker/wire.go @@ -0,0 +1,406 @@ +package redisdocker + +import ( + "context" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" +) + +// ErrNotConverged is returned when a group does not reach its target state before the start timeout. +var ErrNotConverged = errors.New("redis group did not converge") + +const ( + nodeFlagsField = 2 + nodeSlotsField = 8 +) + +var errNoIP = errors.New("container reported no address") + +// poll runs cond every PollInterval until it reports done, fails, or ctx ends; then the last error is returned. +func (g *Group) poll(ctx context.Context, what string, cond func(ctx context.Context) (bool, error)) error { + t := time.NewTicker(g.engine.cfg.PollInterval) + defer t.Stop() + + var last error + + for { + done, err := cond(ctx) + if done { + return nil + } + + if err != nil { + last = err + } + + select { + case <-ctx.Done(): + return fmt.Errorf("%w: %s: %w", ErrNotConverged, what, errors.Join(ctx.Err(), last)) + case <-t.C: + } + } +} + +// onNode runs fn with a connection to n. +func (g *Group) onNode(n *node, fn func(Admin) (bool, error)) (bool, error) { + adm := g.dial(n) + defer func() { _ = adm.Close() }() + + return fn(adm) +} + +func (g *Group) awaitNodes(ctx context.Context) error { + for _, n := range g.nodes { + err := g.poll(ctx, "node "+strconv.Itoa(n.port)+" accepting commands", func(ctx context.Context) (bool, error) { + return g.onNode(n, func(a Admin) (bool, error) { + res, err := a.Do(ctx, "PING") + + return err == nil && asString(res) == "PONG", err + }) + }) + if err != nil { + return err + } + } + + return nil +} + +func (g *Group) shardNodes(shard int) []*node { + var out []*node + + for _, n := range g.nodes { + if n.shard == shard { + out = append(out, n) + } + } + + return out +} + +func (g *Group) resolveIPs(ctx context.Context) error { + for _, n := range g.nodes { + res, err := g.engine.cfg.Runtime.Exec(ctx, n.containerID, []string{"hostname", "-i"}) + if err != nil { + return fmt.Errorf("resolve node address: %w", err) + } + + for f := range strings.FieldsSeq(res.Stdout) { + if ip := net.ParseIP(f); ip != nil && ip.To4() != nil { + n.ip = f + + break + } + } + + if n.ip == "" { + return errNoIP + } + } + + return nil +} + +func (g *Group) wire(ctx context.Context) error { + if err := g.resolveIPs(ctx); err != nil { + return err + } + + if g.spec.ClusterMode { + return g.formCluster(ctx) + } + + return g.replicate(ctx) +} + +// replicate points each replica at the primary and waits for the link. +func (g *Group) replicate(ctx context.Context) error { + nodes := g.shardNodes(0) + primary := nodes[0] + + for _, r := range nodes[1:] { + if err := g.follow(ctx, r, primary); err != nil { + return err + } + } + + return nil +} + +func (g *Group) follow(ctx context.Context, replica, primary *node) error { + return g.poll(ctx, "replica link", func(ctx context.Context) (bool, error) { + return g.onNode(replica, func(a Admin) (bool, error) { + if _, err := a.Do(ctx, "REPLICAOF", primary.ip, strconv.Itoa(primary.port)); err != nil { + return false, err + } + + res, err := a.Do(ctx, "INFO", "replication") + if err != nil { + return false, err + } + + return infoField(asString(res), "master_link_status") == "up", nil + }) + }) +} + +func (g *Group) formCluster(ctx context.Context) error { + first := g.nodes[0] + + if err := g.meetAll(ctx, first); err != nil { + return err + } + + if err := g.assignSlots(ctx); err != nil { + return err + } + + if err := g.awaitKnown(ctx); err != nil { + return err + } + + if err := g.attachReplicas(ctx); err != nil { + return err + } + + return g.awaitClusterOK(ctx) +} + +func (g *Group) meetAll(ctx context.Context, first *node) error { + _, err := g.onNode(first, func(a Admin) (bool, error) { + for _, n := range g.nodes[1:] { + if _, err := a.Do(ctx, "CLUSTER", "MEET", n.ip, strconv.Itoa(n.port), strconv.Itoa(busPort)); err != nil { + return false, err + } + } + + return true, nil + }) + + return err +} + +func (g *Group) assignSlots(ctx context.Context) error { + for i, sh := range g.spec.Shards { + primary := g.shardNodes(i)[0] + + _, err := g.onNode(primary, func(a Admin) (bool, error) { + missing := sh.Slots + + if len(sh.Seed) > 0 { + // a node that loaded keys already claimed the slots they hash to + var perr error + if missing, perr = g.unclaimed(ctx, a, sh.Slots); perr != nil { + return false, perr + } + } + + if len(missing) == 0 { + return true, nil + } + + args := []any{"CLUSTER", "ADDSLOTSRANGE"} + for _, r := range missing { + args = append(args, strconv.Itoa(r.Start), strconv.Itoa(r.End)) + } + + _, derr := a.Do(ctx, args...) + + return derr == nil, derr + }) + if err != nil { + return err + } + } + + return nil +} + +func (g *Group) unclaimed(ctx context.Context, a Admin, want []SlotRange) ([]SlotRange, error) { + res, err := a.Do(ctx, "CLUSTER", "NODES") + if err != nil { + return nil, err + } + + return subtractSlots(want, ownedSlots(asString(res))) +} + +// ownedSlots lists the slots the "myself" line of CLUSTER NODES output serves; migrating markers are skipped. +func ownedSlots(nodes string) []SlotRange { + var out []SlotRange + + for line := range strings.SplitSeq(nodes, "\n") { + fields := strings.Fields(line) + if len(fields) <= nodeSlotsField || !strings.Contains(fields[nodeFlagsField], "myself") { + continue + } + + for _, tok := range fields[nodeSlotsField:] { + if strings.HasPrefix(tok, "[") { + continue + } + + if r, err := ParseSlots(tok); err == nil { + out = append(out, r...) + } + } + } + + return out +} + +// subtractSlots returns the slots of want not in owned; owned slots outside want are an error. +func subtractSlots(want, owned []SlotRange) ([]SlotRange, error) { + var set [TotalSlots]bool + + for _, r := range want { + for s := r.Start; s <= r.End; s++ { + set[s] = true + } + } + + for _, r := range owned { + for s := r.Start; s <= r.End; s++ { + if !set[s] { + return nil, fmt.Errorf("%w: snapshot holds keys for slot %d outside the shard's slots", ErrSlots, s) + } + + set[s] = false + } + } + + var out []SlotRange + + for s := range TotalSlots { + switch { + case !set[s]: + case len(out) > 0 && out[len(out)-1].End == s-1: + out[len(out)-1].End = s + default: + out = append(out, SlotRange{Start: s, End: s}) + } + } + + return out, nil +} + +// awaitKnown waits until every node has met every other node. +func (g *Group) awaitKnown(ctx context.Context) error { + want := strconv.Itoa(len(g.nodes)) + + for _, n := range g.nodes { + err := g.poll(ctx, "cluster gossip", func(ctx context.Context) (bool, error) { + return g.onNode(n, func(a Admin) (bool, error) { + res, err := a.Do(ctx, "CLUSTER", "INFO") + + return err == nil && infoField(asString(res), "cluster_known_nodes") == want, err + }) + }) + if err != nil { + return err + } + } + + return nil +} + +func (g *Group) attachReplicas(ctx context.Context) error { + for i := range g.spec.Shards { + nodes := g.shardNodes(i) + + id, err := g.nodeID(ctx, nodes[0]) + if err != nil { + return err + } + + for _, r := range nodes[1:] { + err = g.poll(ctx, "cluster replicate", func(ctx context.Context) (bool, error) { + return g.onNode(r, func(a Admin) (bool, error) { + _, derr := a.Do(ctx, "CLUSTER", "REPLICATE", id) + + return derr == nil, derr + }) + }) + if err != nil { + return err + } + } + } + + return nil +} + +func (g *Group) nodeID(ctx context.Context, n *node) (string, error) { + var id string + + _, err := g.onNode(n, func(a Admin) (bool, error) { + res, derr := a.Do(ctx, "CLUSTER", "MYID") + id = asString(res) + + return derr == nil, derr + }) + + return id, err +} + +func (g *Group) awaitClusterOK(ctx context.Context) error { + for _, n := range g.nodes { + err := g.poll(ctx, "cluster state", func(ctx context.Context) (bool, error) { + return g.onNode(n, func(a Admin) (bool, error) { + res, err := a.Do(ctx, "CLUSTER", "INFO") + if err != nil || infoField(asString(res), "cluster_state") != "ok" { + return false, err + } + + if ok, nerr := g.replicasVisible(ctx, a); !ok { + return false, nerr + } + + if n.primary() { + return true, nil + } + + repl, rerr := a.Do(ctx, "INFO", "replication") + + return rerr == nil && infoField(asString(repl), "master_link_status") == "up", rerr + }) + }) + if err != nil { + return err + } + } + + return nil +} + +// replicasVisible reports whether the node's view lists every replica as a healthy slave. +func (g *Group) replicasVisible(ctx context.Context, a Admin) (bool, error) { + res, err := a.Do(ctx, "CLUSTER", "NODES") + if err != nil { + return false, err + } + + want, slaves := len(g.nodes)-len(g.spec.Shards), 0 + + for line := range strings.SplitSeq(asString(res), "\n") { + fields := strings.Fields(line) + if len(fields) < nodeFlagsField+1 { + continue + } + + flags := fields[nodeFlagsField] + if strings.Contains(flags, "fail") || strings.Contains(flags, "handshake") { + return false, nil + } + + if strings.Contains(flags, "slave") { + slaves++ + } + } + + return slaves == want, nil +} diff --git a/pkgs/redisdocker/wire_internal_test.go b/pkgs/redisdocker/wire_internal_test.go new file mode 100644 index 0000000000..59a2274e6f --- /dev/null +++ b/pkgs/redisdocker/wire_internal_test.go @@ -0,0 +1,54 @@ +package redisdocker + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOwnedSlotsParsesMyselfLine(t *testing.T) { + t.Parallel() + + nodes := "aaa 1.1.1.1:1@2 master - 0 0 1 connected 100-200\n" + + "bbb :6379@16379,127.0.0.1 myself,master - 0 0 0 connected 0-9 20 [93->-ccc] 30-31\n" + + assert.Equal(t, []SlotRange{{0, 9}, {20, 20}, {30, 31}}, ownedSlots(nodes)) + assert.Empty(t, ownedSlots("aaa 1.1.1.1:1@2 master - 0 0 1 connected 1-2\n")) +} + +func TestSubtractSlots(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + want []SlotRange + owned []SlotRange + missing []SlotRange + wantErr bool + }{ + {name: "nothing owned", want: []SlotRange{{0, 9}}, missing: []SlotRange{{0, 9}}}, + { + name: "partly owned", want: []SlotRange{{0, 9}}, owned: []SlotRange{{3, 4}, {9, 9}}, + missing: []SlotRange{{0, 2}, {5, 8}}, + }, + {name: "all owned", want: []SlotRange{{0, 9}}, owned: []SlotRange{{0, 9}}}, + {name: "owned outside shard", want: []SlotRange{{0, 9}}, owned: []SlotRange{{10, 10}}, wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, err := subtractSlots(tc.want, tc.owned) + if tc.wantErr { + require.ErrorIs(t, err, ErrSlots) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.missing, got) + }) + } +} diff --git a/pkgs/sgfilter/proxy.go b/pkgs/sgfilter/proxy.go index 26cc3f197f..9d592a149f 100644 --- a/pkgs/sgfilter/proxy.go +++ b/pkgs/sgfilter/proxy.go @@ -7,6 +7,8 @@ import ( "net" "sync" "time" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" ) // DialTimeout bounds each upstream dial. @@ -57,6 +59,7 @@ func (p *Proxy) acceptLoop() { p.conns.Store(conn, struct{}{}) p.wg.Go(func() { defer p.conns.Delete(conn) + defer logger.Recover(context.Background(), "sgfilter.proxy", func(any) { _ = conn.Close() }) p.serve(conn) }) diff --git a/pkgs/sgfilter/recover_test.go b/pkgs/sgfilter/recover_test.go new file mode 100644 index 0000000000..774cf9e120 --- /dev/null +++ b/pkgs/sgfilter/recover_test.go @@ -0,0 +1,44 @@ +package sgfilter_test + +import ( + "context" + "net" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" +) + +type panicOnceDialer struct { + echoDialer + calls atomic.Int32 +} + +func (d *panicOnceDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + if d.calls.Add(1) == 1 { + panic("injected dial panic") + } + + return d.echoDialer.DialContext(ctx, network, addr) +} + +func TestConnectionPanicClosesOnlyThatClient(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + }{ + {name: "dial panic"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + addr := startProxy(t, nil, &panicOnceDialer{}) + + assert.False(t, roundTrip(t, addr), "panicking connection must be closed without an echo") + assert.True(t, roundTrip(t, addr), "proxy must keep serving other clients") + }) + } +} diff --git a/pkgs/testsnap/testsnap.go b/pkgs/testsnap/testsnap.go new file mode 100644 index 0000000000..f2a3306323 --- /dev/null +++ b/pkgs/testsnap/testsnap.go @@ -0,0 +1,88 @@ +// Package testsnap asserts that a value handed out by a backend is not +// mutated by later backend transitions (async completion, janitor sweeps). +package testsnap + +import ( + "reflect" + + "github.com/stretchr/testify/assert" +) + +// TB is the subset of testing.TB that Unchanged needs. +type TB interface { + assert.TestingT + Helper() +} + +// Unchanged deep-copies *held, runs transition, then asserts *held still +// equals the copy. Unexported fields are shared with the copy, so they are +// not checked. +func Unchanged[T any](t TB, held *T, transition func()) { + t.Helper() + + snap := new(T) + reflect.ValueOf(snap).Elem().Set(deepCopy(reflect.ValueOf(held)).Elem()) + + transition() + + assert.Equal(t, snap, held, "handed-out value was mutated by a backend transition") +} + +func deepCopy(v reflect.Value) reflect.Value { + out := reflect.New(v.Type()).Elem() + copyInto(out, v) + + return out +} + +func copyInto(dst, src reflect.Value) { + switch src.Kind() { + case reflect.Pointer: + if src.IsNil() { + return + } + + dst.Set(reflect.New(src.Type().Elem())) + copyInto(dst.Elem(), src.Elem()) + case reflect.Slice: + if src.IsNil() { + return + } + + dst.Set(reflect.MakeSlice(src.Type(), src.Len(), src.Len())) + + for i := range src.Len() { + copyInto(dst.Index(i), src.Index(i)) + } + case reflect.Map: + if src.IsNil() { + return + } + + dst.Set(reflect.MakeMapWithSize(src.Type(), src.Len())) + + for it := src.MapRange(); it.Next(); { + val := reflect.New(src.Type().Elem()).Elem() + copyInto(val, it.Value()) + dst.SetMapIndex(it.Key(), val) + } + case reflect.Struct: + dst.Set(src) + + for i := range src.NumField() { + if src.Type().Field(i).IsExported() { + copyInto(dst.Field(i), src.Field(i)) + } + } + case reflect.Interface: + if src.IsNil() { + return + } + + inner := reflect.New(src.Elem().Type()).Elem() + copyInto(inner, src.Elem()) + dst.Set(inner) + default: + dst.Set(src) + } +} diff --git a/pkgs/testsnap/testsnap_test.go b/pkgs/testsnap/testsnap_test.go new file mode 100644 index 0000000000..078bf493ac --- /dev/null +++ b/pkgs/testsnap/testsnap_test.go @@ -0,0 +1,54 @@ +package testsnap_test + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/blackbirdworks/gopherstack/pkgs/testsnap" +) + +type sample struct { + M map[string]string + P *int + S []string +} + +type recorder struct { + errs int +} + +func (r *recorder) Helper() {} + +func (r *recorder) Errorf(string, ...any) { r.errs++ } + +func TestUnchanged(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(*sample) + name string + wantErr bool + }{ + {name: "noop", mutate: func(*sample) {}}, + {name: "replace_map", mutate: func(s *sample) { s.M = map[string]string{"x": "y"} }, wantErr: true}, + {name: "mutate_map", mutate: func(s *sample) { s.M["k"] = "changed" }, wantErr: true}, + {name: "mutate_slice", mutate: func(s *sample) { s.S[0] = "changed" }, wantErr: true}, + {name: "mutate_ptr", mutate: func(s *sample) { *s.P = 9 }, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + n := 1 + held := &sample{M: map[string]string{"k": "v"}, P: &n, S: []string{"a"}} + rec := &recorder{} + + testsnap.Unchanged(rec, held, func() { tt.mutate(held) }) + + assert.Equal(t, tt.wantErr, rec.errs > 0, strconv.Itoa(rec.errs)) + }) + } +} diff --git a/podman-compose.yml b/podman-compose.yml index 9a8bd74722..9b0a99c4a3 100644 --- a/podman-compose.yml +++ b/podman-compose.yml @@ -9,6 +9,10 @@ services: container_name: gopherstack volumes: - gopherstack-data:/data + # Docker-backed engines (Lambda, ECS, ...): the image runs as UID 65532, so also mount the socket and + # join its group, e.g. `DOCKER_GID=$(stat -c %g /var/run/docker.sock) docker compose up`. + # - /var/run/docker.sock:/var/run/docker.sock + # group_add: ["${DOCKER_GID}"] volumes: gopherstack-data: diff --git a/scripts/lint-changed.sh b/scripts/lint-changed.sh index 41d5b7cac9..a5135727d1 100755 --- a/scripts/lint-changed.sh +++ b/scripts/lint-changed.sh @@ -21,8 +21,13 @@ # scripts/lint-changed.sh --base # diff against instead of origin/main # scripts/lint-changed.sh FILE... # lint only these files' packages # +# When go.mod differs from the base, packages that directly import a module +# whose requirement/replace changed (scripts/lint-modimpact.sh) are added to +# the changed-package set. +# # Env: LINT_BATCH (15) max packages per golangci-lint/go vet invocation, # LINT_TIMEOUT (30m) per-batch limit, LINT_CONCURRENCY (from GOFLAGS -p, else nproc/2), +# LINT_LIST_ONLY=1 prints the resolved package list and exits without linting, # LINT_LOCK_RETRIES (20) x 15s waits when another golangci-lint holds the lock. set -euo pipefail @@ -82,6 +87,7 @@ for bin in golangci-lint go; do } done +mod_dirs=() if [[ ${#args[@]} -gt 0 ]]; then changed_files=("${args[@]}") else @@ -107,9 +113,15 @@ else git ls-files --others --exclude-standard -- '*.go' } | sort -u ) + if [[ -n "$merge_base" ]]; then + while IFS= read -r d; do + [[ -n "$d" ]] && mod_dirs+=("$d") + done < <(bash "$(dirname -- "${BASH_SOURCE[0]}")/lint-modimpact.sh" "$merge_base") + [[ ${#mod_dirs[@]} -gt 0 ]] && echo "lint-changed: go.mod changed -- adding ${#mod_dirs[@]} package(s) importing a changed module" + fi fi -if [[ ${#changed_files[@]} -eq 0 ]]; then +if [[ ${#changed_files[@]} -eq 0 && ${#mod_dirs[@]} -eq 0 ]]; then echo "lint-changed: no changed Go files -- nothing to check." exit 0 fi @@ -135,6 +147,12 @@ for f in "${changed_files[@]}"; do fi done +for dir in "${mod_dirs[@]}"; do + [[ -n "${seen[$dir]:-}" ]] && continue + seen["$dir"]=1 + [[ -d "$dir" ]] && pkgs+=("$dir") +done + if [[ ${#pkgs[@]} -eq 0 ]]; then echo "lint-changed: ${#changed_files[@]} changed Go file(s), but every containing package was deleted or excluded -- nothing to check." if [[ ${#skipped[@]} -gt 0 ]]; then @@ -147,7 +165,7 @@ patterns=() for d in "${pkgs[@]}"; do [[ "$d" == "." ]] && patterns+=(".") || patterns+=("./$d") done -IFS=$'\n' sorted_patterns=($(sort <<<"${patterns[*]}")); unset IFS +mapfile -t sorted_patterns < <(printf '%s\n' "${patterns[@]}" | sort) echo "lint-changed: checking ${#sorted_patterns[@]} package(s):" printf ' %s\n' "${sorted_patterns[@]}" @@ -156,6 +174,10 @@ if [[ ${#skipped[@]} -gt 0 ]]; then printf ' %s\n' "${skipped[@]}" fi +if [[ "${LINT_LIST_ONLY:-0}" == 1 ]]; then + exit 0 +fi + # Packages whose files all carry a //go:build constraint (test/e2e) are empty # under default tags and make go vet/golangci-lint fail with "build # constraints exclude all Go files". Detect the tags and lint those packages diff --git a/scripts/lint-modimpact.sh b/scripts/lint-modimpact.sh new file mode 100755 index 0000000000..a14b5ba86f --- /dev/null +++ b/scripts/lint-modimpact.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# lint-modimpact.sh BASE_REV -- print repo package dirs (one per line) that +# directly import (non-test, test or external-test) any module whose go.mod +# requirement or replace changed between BASE_REV and the working tree. +# +# Direct importers only: an API break in a bumped module surfaces where the +# module's packages are called; a package that merely depends on it through +# another repo package is covered when that package is linted. Transitive +# closure would re-select the whole repo for any x/sys bump. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" +base="${1:?usage: lint-modimpact.sh BASE_REV}" + +command -v jq >/dev/null 2>&1 || { + echo "lint-modimpact: jq not found on PATH" >&2 + exit 1 +} + +git diff --quiet "$base" -- go.mod 2>/dev/null && exit 0 + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +git show "$base:go.mod" >"$tmp/old.mod" 2>/dev/null || : >"$tmp/old.mod" + +mods() { + go mod edit -json "$1" | jq -r ' + ((.Require // [])[] | "r \(.Path) \(.Version)"), + ((.Replace // [])[] | "p \(.Old.Path) \(.Old.Version // "") \(.New.Path) \(.New.Version // "")")' +} + +mods "$tmp/old.mod" | LC_ALL=C sort -u >"$tmp/old.txt" +mods go.mod | LC_ALL=C sort -u >"$tmp/new.txt" + +# Modules added, removed or re-versioned (either side of the diff). +{ LC_ALL=C comm -3 "$tmp/old.txt" "$tmp/new.txt"; } | + sed -E 's/^[[:space:]]+//' | awk '{print $2}' | sort -u >"$tmp/changed.txt" + +[[ -s "$tmp/changed.txt" ]] || exit 0 + +go list -e -f '{{.Dir}}|{{join .Imports " "}} {{join .TestImports " "}} {{join .XTestImports " "}}' ./... | + awk -F'|' -v root="$PWD/" -v mods="$tmp/changed.txt" ' + BEGIN { while ((getline m < mods) > 0) mod[m] = 1 } + { + n = split($2, imp, " ") + for (i = 1; i <= n; i++) { + p = imp[i] + while (p != "") { + if (p in mod) { d = $1; if (index(d, root) == 1) d = substr(d, length(root) + 1); else d = "."; print d; next } + if (!sub("/[^/]*$", "", p)) break + } + } + }' | sort -u diff --git a/scripts/lint-shard.sh b/scripts/lint-shard.sh new file mode 100755 index 0000000000..80f5539342 --- /dev/null +++ b/scripts/lint-shard.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# lint-shard.sh SHARD -- full-repo golangci-lint, split by package path so each +# CI shard finishes inside the timeout. Shard "rest" takes every package that no +# named shard matches, so new top-level dirs and services are never skipped. +# +# Usage: scripts/lint-shard.sh {svc-a|svc-b-c|...|rest} (--list prints shard names) +# Env: LINT_TIMEOUT (55m), LINT_LIST_ONLY=1 prints the package list and exits. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +declare -A shard_re=( + [svc-a]='^services/a' + [svc-b-c]='^services/[bc]' + [svc-d-e]='^services/[de]' + [svc-f-h]='^services/[f-h]' + [svc-i-m]='^services/[i-m]' + [svc-n-o]='^services/[no]' + [svc-p-r]='^services/[p-r]' + [svc-s]='^services/s' + [svc-t-z]='^services/[t-z_]' + [pkgs]='^pkgs/' +) +order=(svc-a svc-b-c svc-d-e svc-f-h svc-i-m svc-n-o svc-p-r svc-s svc-t-z pkgs rest) + +if [[ "${1:-}" == "--list" ]]; then + printf '%s\n' "${order[@]}" + exit 0 +fi +shard="${1:?usage: lint-shard.sh SHARD|--list}" + +module=$(go list -m) +all_re=$(printf '%s|' "${shard_re[@]}") +all_re="(${all_re%|})" + +mapfile -t pkgs < <( + go list -f '{{.ImportPath}}' ./... | sed -e "s|^$module/\\{0,1\\}||" -e 's|^$|.|' | + while IFS= read -r rel; do + if [[ "$shard" == rest ]]; then + [[ "$rel" =~ $all_re ]] || echo "$rel" + else + [[ -n "${shard_re[$shard]:-}" ]] || { + echo "lint-shard: unknown shard $shard" >&2 + exit 2 + } + [[ "$rel" =~ ${shard_re[$shard]} ]] && echo "$rel" + fi + done +) + +if [[ ${#pkgs[@]} -eq 0 ]]; then + echo "lint-shard: shard $shard matched no packages" >&2 + exit 1 +fi +echo "lint-shard: $shard -> ${#pkgs[@]} package(s)" +[[ "${LINT_LIST_ONLY:-0}" == 1 ]] && { printf ' %s\n' "${pkgs[@]}"; exit 0; } + +args=() +for p in "${pkgs[@]}"; do + [[ "$p" == . ]] && args+=(.) || args+=("./$p") +done +exec golangci-lint run --timeout "${LINT_TIMEOUT:-55m}" "${args[@]}" diff --git a/services/amp/PARITY.md b/services/amp/PARITY.md index 3f1803e9cb..a8280d63cb 100644 --- a/services/amp/PARITY.md +++ b/services/amp/PARITY.md @@ -48,28 +48,28 @@ ops: UpdateScraper: {wire: ok, errors: ok, state: ok, persist: ok} UpdateScraperLoggingConfiguration: {wire: ok, errors: ok, state: ok, persist: ok, note: "upsert: there is no Create operation for scraper logging in the SDK"} UpdateWorkspaceAlias: {wire: ok, errors: ok, state: ok, persist: ok, note: "204 with no body (the one bodyless success)"} - UpdateWorkspaceConfiguration: {wire: ok, errors: ok, state: ok, persist: ok, note: "PATCH; retentionPeriodInDays and limitsPerLabelSet are enforced on ingest and query, outOfOrderTimeWindowInSeconds is listed in items_still_open"} + UpdateWorkspaceConfiguration: {wire: ok, errors: ok, state: ok, persist: ok, note: "PATCH; retentionPeriodInDays and limitsPerLabelSet are enforced on ingest and query, outOfOrderTimeWindowInSeconds rejects samples older than the window on ingest"} families: Workspace: {status: ok, note: "Singleton sub-resources (alert manager, logging, query logging, configuration, resource policy) live inside the workspace record; mutations need an ACTIVE workspace (ConflictException otherwise). Deleting workspaces purge lazily on the next write."} RuleGroupsNamespace: {status: ok, note: "YAML parsed, every expr parsed as PromQL; ARN arn:aws:aps:::rulegroupsnamespace//. Groups are evaluated per interval (default 1m, query_offset or ruleQueryOffsetInSeconds honoured): recording rules write series, alerting rules track pending, firing and resolved with for and keep_firing_for and write ALERTS and ALERTS_FOR_STATE, ended series get stale markers. Rule and alert state is derived and not persisted: the samples are, and after a restore the first evaluation resumes the for clock from ALERTS_FOR_STATE."} Scraper: {status: ok, note: "Region and account scoped; EKS source validated through internal/app/amp_wiring.go; scraper logging configuration is a child of the scraper."} AnomalyDetector: {status: ok, note: "Random Cut Forest configuration stored and returned; no model runs."} Tags: {status: ok, note: "Workspaces, rule groups namespaces, scrapers and anomaly detectors are taggable; keys starting with aws: are rejected. Also listed, tagged and untagged through the Resource Groups Tagging API (wired in internal/app/rgtapi_extra_wiring.go)."} - RemoteWrite: {status: ok, note: "samples older than the workspace retention (default 150 days) are dropped on ingest, hidden from queries and pruned by the worker once a minute; a new series that would pass a limitsPerLabelSet maxSeries is refused and the request answers 400 after storing the rest; metric metadata (field 3) is stored per workspace and persisted; POST /workspaces/{id}/api/v1/remote_write: snappy block + protobuf WriteRequest decoded with klauspost/compress/snappy and protowire; samples kept in memory per series, ordered by time, a repeated timestamp is a duplicate; NaN and Inf values (including stale markers) are dropped; persisted in the snapshot (stale markers written by rule evaluation persist as staleAt timestamps)."} - Query: {status: ok, note: "GET and POST (form body) of /api/v1/query, query_range, series, labels and label/{name}/values return Prometheus JSON envelopes (success, bad_data 400, execution 422, timeout 503); the timeout parameter bounds evaluation (capped at 2m); GET rules (type, rule_name[], rule_group[], file[] filters), alerts, metadata (ingested from remote write) and status/buildinfo are served, query_exemplars returns an empty list."} + RemoteWrite: {status: ok, note: "samples older than the workspace retention (default 150 days) are dropped on ingest, hidden from queries and pruned by the worker once a minute; a new series that would pass a limitsPerLabelSet maxSeries is refused and the request answers 400 after storing the rest; metric metadata (field 3) is stored per workspace and persisted; POST /workspaces/{id}/api/v1/remote_write: snappy block + protobuf WriteRequest decoded with klauspost/compress/snappy and protowire; samples kept in memory per series, ordered by time, a repeated timestamp is a duplicate; NaN and Inf values (including stale markers) are dropped; native histograms (field 4, integer delta or float counts, schema -4..8) are stored as samples and exemplars (field 3) per series; a sample older than the series newest sample by more than outOfOrderTimeWindowInSeconds (default 0) is refused and the request answers 400 after storing the rest; persisted in the snapshot (stale markers written by rule evaluation persist as staleAt timestamps)."} + Query: {status: ok, note: "GET and POST (form body) of /api/v1/query, query_range, series, labels and label/{name}/values return Prometheus JSON envelopes (success, bad_data 400, execution 422, timeout 503); the timeout parameter bounds evaluation (capped at 2m); GET rules (type, rule_name[], rule_group[], file[] filters), alerts, metadata (ingested from remote write) and status/buildinfo are served, query_exemplars returns the exemplars ingested for the selectors of the query within start and end."} AlertManager: {status: ok, note: "One dispatcher per workspace built from the stored alertmanager_config: route tree with matchers, match, match_re, continue, group_by (including ...), group_wait, group_interval, repeat_interval; inhibit_rules; sns_configs delivered through the SNS seam wired in internal/app/amp_wiring.go (subject and message Go templates, attributes, send_resolved). Alerts come from rule evaluation or POST /workspaces/{id}/alertmanager/api/v2/alerts; GET lists them. Dispatcher state is derived and not persisted."} PromQL: {status: ok, note: "services/amp/promql (home-grown, not the upstream engine): subqueries, the @ modifier (timestamps plus start() and end()), histogram_quantile and histogram_fraction (classic buckets), holt_winters/double_exponential_smoothing, absent_over_time, sort_by_label(_desc), limitk and limit_ratio, label_replace, label_join, group_left and group_right, count_values, deriv, predict_linear, quantile_over_time, stddev/stdvar/present_over_time, the date and time functions and timestamp; instant and range selectors with =, !=, =~, !~ (fully anchored) and offset; rate, irate, increase, delta, idelta, changes, resets and the *_over_time family with Prometheus extrapolation; abs, ceil, floor, sqrt, exp, ln, log2, log10, round, clamp*, absent, time, vector, scalar, sort, sort_desc; sum, avg, min, max, count, group, stddev, stdvar, quantile, topk, bottomk with by and without; + - * / % ^, comparisons with bool, and, or, unless with on and ignoring."} gaps: [] items_still_open: - - "PromQL: native histograms are not ingested, so histogram_count, histogram_sum, histogram_avg, histogram_stddev and histogram_stdvar (which Prometheus defines on native histograms only) always return an empty vector for float samples; histogram_fraction works on classic le buckets only; limit_ratio samples by an FNV hash of the label set, not Prometheus' hash, so the selected series differ from Prometheus (the positive/negative ratio complement property holds)." + - "PromQL: native histograms are queryable only through a bare selector (histogram_count/sum/avg/stddev/stdvar/quantile/fraction(metric) and the selector itself); rate, increase, sum, arithmetic and subqueries over native histograms are not implemented and treat them as absent; custom-bucket (NHCB, schema -53) histograms are rejected; the quantile, stddev and fraction maths follow the Prometheus histogram documentation from memory and are unverified against a real Prometheus; limit_ratio samples by an FNV hash of the label set, not Prometheus' hash, so the selected series differ from Prometheus (the positive/negative ratio complement property holds)." - "AlertManagerDefinition: only sns_configs receivers deliver (AMP supports nothing else); silences, the v1 API and template_files are not served; sigv4 settings are accepted and ignored; the dispatcher keeps no notification log across restarts, so a restart renotifies firing alerts" - - "Scraper: stored with a real ACTIVE lifecycle but nothing scrapes the EKS or VPC source" - - "WorkspaceConfiguration: outOfOrderTimeWindowInSeconds is stored and returned but not enforced; limitsPerLabelSet does not apply to series written by rule evaluation" - - "RemoteWrite: out-of-order samples are always accepted, native histograms and exemplars in the request are ignored, per-workspace ingestion limits and throttling do not exist" - - "Query: query_exemplars always returns no exemplars (none are ingested); targets and status config, flags, runtimeinfo and tsdb endpoints are not served" - - "LoggingConfiguration: the CloudWatch Logs log group ARN is checked for shape only, never looked up, and no log is delivered" - - "Authorization: SigV4 signatures and resource policies are not evaluated on the data plane" - - "CreateWorkspace: KMS key existence is not checked" + - "Scraper: stored with a real ACTIVE lifecycle but nothing scrapes the EKS or VPC source; the EKS emulator exposes no pod or node metrics endpoints to scrape, so there is no in-process target" + - "WorkspaceConfiguration: limitsPerLabelSet does not apply to series written by rule evaluation" + - "RemoteWrite: the out-of-order window is measured against the same series' newest sample (Prometheus measures against the head's newest sample across series) and its default of 0 is unverified; exemplar limits (128 runes, newest 100 per series) follow Prometheus and are silently dropped; per-workspace ingestion limits and throttling do not exist" + - "Query: targets and status config, flags, runtimeinfo and tsdb endpoints are not served" + - "LoggingConfiguration: the delivered log line layout (JSON with level, component, ts, msg) and the stream names ruler and alertmanager are not documented by AWS and are unverified; only rule evaluation failures and alert notification failures are delivered" + - "Authorization: with --validate-sigv4 the data plane requires an aps-scoped SigV4 signature checked against --sigv4-secret and evaluates the workspace resource policy by account only (the IAM evaluator ignores Principal); the error codes for missing and mis-scoped signatures are unverified; cross-account callers resolve workspaces in their own account, so a cross-account allow still answers 404" + - "CreateWorkspace: a missing or disabled KMS key is a ValidationException; the real error code is unverified, and the key is not used to encrypt anything" structural_gaps: - "No ingestion at scale or durable storage: samples live in process memory (and the persistence snapshot), there is no compaction, replication or long-term object storage." - "The default scraper configuration document returned by GetDefaultScraperConfiguration is AWS-authored text; the emulator serves an equivalent Kubernetes service-discovery configuration." @@ -88,3 +88,7 @@ Region and account come from the request (SigV4 scope and access key), so worksp Emulator choices the SDK does not document: a rejected rules or alert manager file is an immediate ValidationException; the lifecycle dwell follows `AMP_LIFECYCLE_DELAY` (default 500ms); deleted resources disappear on the first write after their window; workspace retention defaults to 150 days; ClientToken replays CreateWorkspace, CreateRuleGroupsNamespace, CreateScraper, CreateAnomalyDetector and the alert manager and logging creates. A Prometheus server can write to `api/v1/remote_write` (the `PrometheusEndpoint` of the workspace) and Grafana can query ``. + +## Pass 2026-11-06 (gopherstack-znm9b) + +Sources: UpdateWorkspaceConfiguration API reference (outOfOrderTimeWindowInSeconds: "Out of order samples older than this window are rejected", minimum 0); user guide CW-logs page (Alert Manager and Ruler warning and error events go to the configured log group; needs logs:* delivery permissions); remote-write protobuf layout (Histogram fields 1-15, Exemplar fields 1-3) and the Prometheus HTTP API histogram and query_exemplars JSON shapes from the Prometheus documentation, recalled rather than re-read. The AMP user guide pages for out-of-order ingestion and remote write could not be fetched, so the 0 default and the 400 status are unverified. diff --git a/services/amp/README.md b/services/amp/README.md index 506112d6dc..3bba782cf3 100644 --- a/services/amp/README.md +++ b/services/amp/README.md @@ -16,15 +16,15 @@ ### Known gaps -- PromQL: native histograms are not ingested, so histogram_count, histogram_sum, histogram_avg, histogram_stddev and histogram_stdvar (which Prometheus defines on native histograms only) always return an empty vector for float samples; histogram_fraction works on classic le buckets only; limit_ratio samples by an FNV hash of the label set, not Prometheus' hash, so the selected series differ from Prometheus (the positive/negative ratio complement property holds). +- PromQL: native histograms are queryable only through a bare selector (histogram_count/sum/avg/stddev/stdvar/quantile/fraction(metric) and the selector itself); rate, increase, sum, arithmetic and subqueries over native histograms are not implemented and treat them as absent; custom-bucket (NHCB, schema -53) histograms are rejected; the quantile, stddev and fraction maths follow the Prometheus histogram documentation from memory and are unverified against a real Prometheus; limit_ratio samples by an FNV hash of the label set, not Prometheus' hash, so the selected series differ from Prometheus (the positive/negative ratio complement property holds). - AlertManagerDefinition: only sns_configs receivers deliver (AMP supports nothing else); silences, the v1 API and template_files are not served; sigv4 settings are accepted and ignored; the dispatcher keeps no notification log across restarts, so a restart renotifies firing alerts -- Scraper: stored with a real ACTIVE lifecycle but nothing scrapes the EKS or VPC source -- WorkspaceConfiguration: outOfOrderTimeWindowInSeconds is stored and returned but not enforced; limitsPerLabelSet does not apply to series written by rule evaluation -- RemoteWrite: out-of-order samples are always accepted, native histograms and exemplars in the request are ignored, per-workspace ingestion limits and throttling do not exist -- Query: query_exemplars always returns no exemplars (none are ingested); targets and status config, flags, runtimeinfo and tsdb endpoints are not served -- LoggingConfiguration: the CloudWatch Logs log group ARN is checked for shape only, never looked up, and no log is delivered -- Authorization: SigV4 signatures and resource policies are not evaluated on the data plane -- CreateWorkspace: KMS key existence is not checked +- Scraper: stored with a real ACTIVE lifecycle but nothing scrapes the EKS or VPC source; the EKS emulator exposes no pod or node metrics endpoints to scrape, so there is no in-process target +- WorkspaceConfiguration: limitsPerLabelSet does not apply to series written by rule evaluation +- RemoteWrite: the out-of-order window is measured against the same series' newest sample (Prometheus measures against the head's newest sample across series) and its default of 0 is unverified; exemplar limits (128 runes, newest 100 per series) follow Prometheus and are silently dropped; per-workspace ingestion limits and throttling do not exist +- Query: targets and status config, flags, runtimeinfo and tsdb endpoints are not served +- LoggingConfiguration: the delivered log line layout (JSON with level, component, ts, msg) and the stream names ruler and alertmanager are not documented by AWS and are unverified; only rule evaluation failures and alert notification failures are delivered +- Authorization: with --validate-sigv4 the data plane requires an aps-scoped SigV4 signature checked against --sigv4-secret and evaluates the workspace resource policy by account only (the IAM evaluator ignores Principal); the error codes for missing and mis-scoped signatures are unverified; cross-account callers resolve workspaces in their own account, so a cross-account allow still answers 404 +- CreateWorkspace: a missing or disabled KMS key is a ValidationException; the real error code is unverified, and the key is not used to encrypt anything ### Structural gaps diff --git a/services/amp/alertmanager_notify.go b/services/amp/alertmanager_notify.go index 1de91f1ddc..792aa86caf 100644 --- a/services/amp/alertmanager_notify.go +++ b/services/amp/alertmanager_notify.go @@ -264,12 +264,23 @@ func truncate(s string, limit int) string { } // deliverNotifications publishes pending notifications and reports each outcome to the dispatcher. -func deliverNotifications(ctx context.Context, pub SNSPublisher, pending []pendingNotify) { +func deliverNotifications(ctx context.Context, pub SNSPublisher, pending []pendingNotify) []notifyFailure { + var failed []notifyFailure + for _, p := range pending { - p.done(deliverOne(ctx, pub, p)) + err := deliverOne(ctx, pub, p) + p.done(err) + + if err != nil { + failed = append(failed, notifyFailure{receiver: p.receiver.name, err: err.Error()}) + } } + + return failed } +type notifyFailure struct{ receiver, err string } + func deliverOne(ctx context.Context, pub SNSPublisher, p pendingNotify) error { if pub == nil { return errNoPublisher diff --git a/services/amp/dataplane.go b/services/amp/dataplane.go index b8293c1ae6..de1d06ee36 100644 --- a/services/amp/dataplane.go +++ b/services/amp/dataplane.go @@ -21,6 +21,11 @@ type IngestResult struct { Dropped int // Expired counts samples older than the workspace retention period. Expired int + // OutOfOrder counts samples older than the series' newest sample by more than the out-of-order window. + OutOfOrder int + // Exemplars counts stored exemplars; ExemplarsDropped those that were oversized or expired. + Exemplars int + ExemplarsDropped int // LimitExceeded counts series refused by a limitsPerLabelSet entry. LimitExceeded int } @@ -98,18 +103,62 @@ func (b *InMemoryBackend) ingestSeriesLocked(w *Workspace, cutoff int64, ws writ b.series.Put(rec) } + window := int64(w.Config.OutOfOrderTimeWindowInSeconds) * msPerSecondInt + for _, s := range ws.Samples { switch { - case math.IsNaN(s.V) || math.IsInf(s.V, 0): + case s.H == nil && (math.IsNaN(s.V) || math.IsInf(s.V, 0)): res.Dropped++ case s.T < cutoff: res.Expired++ + case tooOld(rec, s.T, window): + res.OutOfOrder++ case insertSample(rec, s): res.Samples++ default: res.Duplicate++ } } + + for _, ex := range ws.Exemplars { + if ex.T < cutoff || exemplarRunes(ex) > maxExemplarRunes { + res.ExemplarsDropped++ + + continue + } + + if addExemplar(rec, ex) { + res.Exemplars++ + } + } +} + +const ( + msPerSecondInt = 1000 + maxSeriesExemplar = 100 +) + +// tooOld reports a sample behind the series' newest one by more than the out-of-order window. +func tooOld(rec *SeriesRecord, t, windowMs int64) bool { + n := len(rec.Samples) + + return n > 0 && t < rec.Samples[n-1].T && rec.Samples[n-1].T-t > windowMs +} + +// addExemplar keeps the newest maxSeriesExemplar exemplars per series, ordered by time. +func addExemplar(rec *SeriesRecord, ex Exemplar) bool { + i, found := slices.BinarySearchFunc(rec.Exemplars, ex, func(a, b Exemplar) int { return cmp.Compare(a.T, b.T) }) + if found && rec.Exemplars[i].V == ex.V && + promql.Fingerprint(rec.Exemplars[i].Labels) == promql.Fingerprint(ex.Labels) { + return false + } + + rec.Exemplars = slices.Insert(rec.Exemplars, i, ex) + if len(rec.Exemplars) > maxSeriesExemplar { + rec.Exemplars = rec.Exemplars[len(rec.Exemplars)-maxSeriesExemplar:] + } + + return true } // insertSample keeps rec.Samples ordered by time; a repeated timestamp is a duplicate. diff --git a/services/amp/dataplane_exemplars.go b/services/amp/dataplane_exemplars.go new file mode 100644 index 0000000000..e04278c00b --- /dev/null +++ b/services/amp/dataplane_exemplars.go @@ -0,0 +1,103 @@ +package amp + +import ( + "context" + "net/url" + "slices" + "strings" + "time" + + "github.com/blackbirdworks/gopherstack/services/amp/promql" +) + +type exemplarJSON struct { + Labels promql.Labels `json:"labels"` + Value string `json:"value"` + Timestamp float64 `json:"timestamp"` +} + +type exemplarSeriesJSON struct { + SeriesLabels promql.Labels `json:"seriesLabels"` + Exemplars []exemplarJSON `json:"exemplars"` +} + +// exemplarsData answers GET/POST query_exemplars: ingested exemplars of series selected by query in [start, end]. +func (h *Handler) exemplarsData(ctx context.Context, wsID string, vals url.Values) (any, error) { + q := vals.Get("query") + if q == "" { + return nil, promParamError("invalid parameter \"query\": query is required") + } + + expr, err := promql.Parse(q) + if err != nil { + return nil, promParamError("invalid parameter \"query\": " + err.Error()) + } + + start, err := parseTime(vals.Get("start"), time.Unix(0, 0).UTC()) + if err != nil { + return nil, promParamError("invalid parameter \"start\": " + err.Error()) + } + + end, err := parseTime(vals.Get("end"), h.Backend.now()) + if err != nil { + return nil, promParamError("invalid parameter \"end\": " + err.Error()) + } + + return h.Backend.queryExemplars(ctx, wsID, promql.Selectors(expr), start.UnixMilli(), end.UnixMilli()) +} + +func (b *InMemoryBackend) queryExemplars( + ctx context.Context, wsID string, selectors [][]*promql.Matcher, minT, maxT int64, +) ([]exemplarSeriesJSON, error) { + b.mu.RLock("QueryExemplars") + defer b.mu.RUnlock() + + if err := b.dataWorkspaceLocked(b.scopeOf(ctx), wsID); err != nil { + return nil, err + } + + w, _ := b.workspaces.Get(wsID) + minT = max(minT, b.retentionCutoffMs(w)) + out := []exemplarSeriesJSON{} + seen := map[string]bool{} + + for _, rec := range b.seriesByWS.Get(wsID) { + if seen[rec.Key] || !matchesAny(rec.Labels, selectors) { + continue + } + + seen[rec.Key] = true + + var exs []exemplarJSON + + for _, ex := range rec.Exemplars { + if ex.T >= minT && ex.T <= maxT { + exs = append(exs, exemplarJSON{Labels: ex.Labels, Value: formatValue(ex.V), Timestamp: seconds(ex.T)}) + } + } + + if len(exs) > 0 { + out = append(out, exemplarSeriesJSON{SeriesLabels: rec.Labels, Exemplars: exs}) + } + } + + sortExemplarSeries(out) + + return out, nil +} + +func matchesAny(l promql.Labels, selectors [][]*promql.Matcher) bool { + for _, ms := range selectors { + if promql.MatchesAll(l, ms) { + return true + } + } + + return false +} + +func sortExemplarSeries(s []exemplarSeriesJSON) { + slices.SortFunc(s, func(a, b exemplarSeriesJSON) int { + return strings.Compare(promql.Fingerprint(a.SeriesLabels), promql.Fingerprint(b.SeriesLabels)) + }) +} diff --git a/services/amp/dataplane_http.go b/services/amp/dataplane_http.go index cfeccbc841..587e067430 100644 --- a/services/amp/dataplane_http.go +++ b/services/amp/dataplane_http.go @@ -68,6 +68,38 @@ type promSeriesJSON struct { Metric promql.Labels `json:"metric"` Value *promPoint `json:"value,omitempty"` Values []promPoint `json:"values,omitempty"` + + Histogram *promPoint `json:"histogram,omitempty"` + Histograms []promPoint `json:"histograms,omitempty"` +} + +type promHistJSON struct { + Count string `json:"count"` + Sum string `json:"sum"` + Buckets [][]any `json:"buckets,omitempty"` +} + +// histPoint renders a native histogram in the Prometheus HTTP API shape; zero-count buckets are omitted. +func histPoint(t int64, h *promql.FloatHistogram) promPoint { + out := promHistJSON{Count: formatValue(h.Count), Sum: formatValue(h.Sum)} + + for _, b := range h.Buckets() { + if b.Count == 0 { + continue + } + + rule := 0 + switch { + case b.Zero: + rule = 3 + case b.Negative: + rule = 1 + } + + out.Buckets = append(out.Buckets, []any{rule, formatValue(b.Lower), formatValue(b.Upper), formatValue(b.Count)}) + } + + return promPoint{seconds(t), out} } type promData struct { @@ -83,7 +115,7 @@ type promResponse struct { } func promError(c *echo.Context, status int, errType, msg string) error { - return c.JSON(status, promResponse{Status: "error", ErrorType: errType, Error: msg}) + return c.JSON(status, promResponse{Status: fieldErr, ErrorType: errType, Error: msg}) } func formatValue(v float64) string { @@ -117,6 +149,13 @@ func resultJSON(v promql.Value) promData { out := make([]promSeriesJSON, 0, len(x)) for _, s := range x { + if s.H != nil { + p := histPoint(s.T, s.H) + out = append(out, promSeriesJSON{Metric: s.Metric, Histogram: &p}) + + continue + } + p := point(s.T, s.V) out = append(out, promSeriesJSON{Metric: s.Metric, Value: &p}) } @@ -134,11 +173,22 @@ func matrixJSON(m promql.Matrix) []promSeriesJSON { for _, s := range m { vals := make([]promPoint, 0, len(s.Samples)) + hists := []promPoint{} + for _, p := range s.Samples { - vals = append(vals, point(p.T, p.V)) + if p.H != nil { + hists = append(hists, histPoint(p.T, p.H)) + } else { + vals = append(vals, point(p.T, p.V)) + } + } + + sj := promSeriesJSON{Metric: s.Labels, Values: vals} + if len(hists) > 0 { + sj.Histograms = hists } - out = append(out, promSeriesJSON{Metric: s.Labels, Values: vals}) + out = append(out, sj) } return out @@ -205,6 +255,10 @@ func parseStep(raw string) (time.Duration, error) { func (h *Handler) serveData(c *echo.Context, op string, params map[string]string) error { wsID := params[keyWorkspaceID] + if done, aerr := h.authorizeData(c, wsID); done { + return aerr + } + if op == opRemoteWrite { return h.remoteWrite(c, wsID) } @@ -260,6 +314,12 @@ func (h *Handler) remoteWrite(c *echo.Context, wsID string) error { return h.server.WriteErr(c, ierr) } + if res.OutOfOrder > 0 { + return restroute.WriteError(c, http.StatusBadRequest, "ValidationException", + fmt.Sprintf("out of order sample: %d samples are older than the workspace out-of-order window", + res.OutOfOrder), nil) + } + if res.LimitExceeded > 0 { return restroute.WriteError(c, http.StatusBadRequest, "ValidationException", fmt.Sprintf("per-labelset series limit exceeded for %d series", res.LimitExceeded), nil) diff --git a/services/amp/dataplane_status_http.go b/services/amp/dataplane_status_http.go index 8fce8f7509..b700cf5e11 100644 --- a/services/amp/dataplane_status_http.go +++ b/services/amp/dataplane_status_http.go @@ -57,6 +57,8 @@ func (h *Handler) serveStatus(c *echo.Context, op, wsID string, vals url.Values) data, err = h.alertsData(ctx, wsID) case opMetadata: data, err = h.metadataData(ctx, wsID, vals) + case opExemplars: + data, err = h.exemplarsData(ctx, wsID, vals) case opAMAlerts: out, aerr := h.Backend.alertManagerAlerts(ctx, wsID) if aerr != nil { diff --git a/services/amp/dataplane_test.go b/services/amp/dataplane_test.go index 583ac4104c..eb14b36863 100644 --- a/services/amp/dataplane_test.go +++ b/services/amp/dataplane_test.go @@ -219,11 +219,11 @@ func TestRemoteWriteRules(t *testing.T) { run func(t *testing.T, f *fixture, ws string) name string }{ - {name: "duplicates and out of order", run: func(t *testing.T, f *fixture, ws string) { + {name: "duplicates", run: func(t *testing.T, f *fixture, ws string) { t.Helper() lbl := map[string]string{"__name__": "m"} - require.Equal(t, http.StatusOK, f.write(t, ws, amp.NewSeries(lbl, sample(2000, 2), sample(1000, 1))).status) + require.Equal(t, http.StatusOK, f.write(t, ws, amp.NewSeries(lbl, sample(1000, 1), sample(2000, 2))).status) require.Equal( t, http.StatusOK, diff --git a/services/amp/integrations.go b/services/amp/integrations.go new file mode 100644 index 0000000000..ee5f82278c --- /dev/null +++ b/services/amp/integrations.go @@ -0,0 +1,276 @@ +package amp + +import ( + "context" + "encoding/json" + "maps" + "net/http" + "regexp" + "strings" + "time" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/httputils" + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/restroute" +) + +const ( + fieldErr = "error" + logLevelError = fieldErr + logLevelWarn = "warn" + logComponentR = "ruler" + logComponentA = "alertmanager" + logStreamR = "ruler" + logStreamA = "alertmanager" + maxLogQueue = 1000 + sigV4Scheme = "AWS4-HMAC-SHA256" + accountDigits = 12 + credScopeSvc = 3 +) + +// KeyChecker looks a KMS key up in the in-process KMS. +type KeyChecker interface { + KMSKeyState(ctx context.Context, keyARN string) (exists, enabled bool) +} + +// LogSink reaches the in-process CloudWatch Logs. +type LogSink interface { + LogGroupExists(ctx context.Context, region, group string) bool + PutLogLine(ctx context.Context, region, group, stream, message string, at time.Time) error +} + +// PolicyEvaluator decides a resource policy for a caller account; denied wins over allowed. +type PolicyEvaluator func(doc, action, resource, callerAccount string) (allowed, denied bool) + +// DataPlaneAuth makes the Prometheus endpoints require a SigV4 signature scoped to aps. +type DataPlaneAuth struct { + Evaluate PolicyEvaluator + Secret string +} + +type logEvent struct { + at time.Time + fields map[string]string + wsID string + component string + stream string + level string + msg string +} + +var logGroupNameRe = regexp.MustCompile(`^arn:aws[a-z-]*:logs:([a-z0-9-]+):\d{12}:log-group:(.+?)(?::\*)?$`) + +// SetKeyChecker wires the KMS lookup behind CreateWorkspace kmsKeyArn. +func (b *InMemoryBackend) SetKeyChecker(k KeyChecker) { + b.mu.Lock("SetKeyChecker") + defer b.mu.Unlock() + + b.keys = k +} + +// SetLogSink wires CloudWatch Logs for log group checks and log delivery. +func (b *InMemoryBackend) SetLogSink(s LogSink) { + b.mu.Lock("SetLogSink") + defer b.mu.Unlock() + + b.logs = s +} + +// SetDataPlaneAuth turns on SigV4 enforcement for the data plane; nil turns it off. +func (b *InMemoryBackend) SetDataPlaneAuth(a *DataPlaneAuth) { + b.mu.Lock("SetDataPlaneAuth") + defer b.mu.Unlock() + + b.auth = a +} + +func (b *InMemoryBackend) integrations() (KeyChecker, LogSink, *DataPlaneAuth) { + b.mu.RLock("integrations") + defer b.mu.RUnlock() + + return b.keys, b.logs, b.auth +} + +func (b *InMemoryBackend) checkKMSKey(ctx context.Context, arn string) error { + keys, _, _ := b.integrations() + if arn == "" || keys == nil { + return nil + } + + exists, enabled := keys.KMSKeyState(ctx, arn) + + switch { + case !exists: + return invalid("KMS key %s does not exist", arn) + case !enabled: + return invalid("KMS key %s is not enabled", arn) + } + + return nil +} + +func (b *InMemoryBackend) checkLogGroup(ctx context.Context, arn string) error { + _, logs, _ := b.integrations() + if logs == nil { + return nil + } + + ref, ok := splitLogGroupARN(arn) + if ok && !logs.LogGroupExists(ctx, ref.region, ref.group) { + return notFound("LogGroup", arn, "Log group %s does not exist", ref.group) + } + + return nil +} + +type logGroupRef struct{ region, group string } + +func splitLogGroupARN(arn string) (logGroupRef, bool) { + m := logGroupNameRe.FindStringSubmatch(arn) + if m == nil { + return logGroupRef{}, false + } + + return logGroupRef{region: m[1], group: m[2]}, true +} + +// queueLogLocked records a ruler or alert manager warning/error for delivery after the lock drops. +func (b *InMemoryBackend) queueLogLocked(ev logEvent) { + if b.logs == nil || len(b.logQueue) >= maxLogQueue { + return + } + + b.logQueue = append(b.logQueue, ev) +} + +// flushLogs delivers queued events to each workspace's configured log group; events of workspaces without one drop. +func (b *InMemoryBackend) flushLogs(ctx context.Context) { + b.mu.Lock("flushLogs") + events, sink := b.logQueue, b.logs + b.logQueue = nil + + targets := map[string]string{} + + for _, ev := range events { + if w, ok := b.workspaces.Get(ev.wsID); ok && w.Logging != nil && !b.gone(w.Logging.Pending) { + targets[ev.wsID] = w.Logging.LogGroupARN + } + } + + b.mu.Unlock() + + if sink == nil { + return + } + + for _, ev := range events { + ref, ok := splitLogGroupARN(targets[ev.wsID]) + if !ok { + continue + } + + line := map[string]string{ + "level": ev.level, "component": ev.component, "workspaceId": ev.wsID, + "ts": ev.at.UTC().Format(time.RFC3339Nano), "msg": ev.msg, + } + maps.Copy(line, ev.fields) + + raw, _ := json.Marshal(line) + + if err := sink.PutLogLine(ctx, ref.region, ref.group, ev.stream, string(raw), ev.at); err != nil { + logger.Load(ctx).WarnContext(ctx, "amp: log delivery failed", "logGroup", ref.group, "error", err) + } + } +} + +// authorizeData enforces SigV4 (service aps) and the workspace resource policy when DataPlaneAuth is set; +// done reports that a rejection was already written. +func (h *Handler) authorizeData(c *echo.Context, wsID string) (bool, error) { + _, _, auth := h.Backend.integrations() + if auth == nil { + return false, nil + } + + r := c.Request() + hdr := httputils.HeaderValue(r.Header, "Authorization") + + if !strings.HasPrefix(hdr, sigV4Scheme) { + return true, restroute.WriteError(c, http.StatusForbidden, "MissingAuthenticationToken", + "Missing Authentication Token", nil) + } + + parts := credentialParts(hdr) + if len(parts) <= credScopeSvc || parts[credScopeSvc] != signingName { + return true, restroute.WriteError(c, http.StatusForbidden, "IncompleteSignatureException", + "Credential should be scoped to correct service: 'aps'.", nil) + } + + if serr := httputils.NewSigV4Validator(auth.Secret).Verify(r); serr != nil { + return true, restroute.WriteError(c, serr.Status, serr.Code, serr.Message, nil) + } + + if auth.Evaluate == nil { + return false, nil + } + + action := dataAction(r.Method, r.URL.Path) + + if arn, ok := h.Backend.policyDenies(wsID, action, parts[0], auth.Evaluate); ok { + return true, restroute.WriteError(c, http.StatusForbidden, "AccessDeniedException", + "User is not authorized to perform "+action+" on resource "+arn, nil) + } + + return false, nil +} + +func credentialParts(auth string) []string { + _, after, ok := strings.Cut(auth, "Credential=") + if !ok { + return nil + } + + cred, _, _ := strings.Cut(after, ",") + + return strings.Split(strings.TrimSpace(cred), "/") +} + +func dataAction(method, path string) string { + if method == http.MethodPost && strings.HasSuffix(path, "/remote_write") { + return "aps:RemoteWrite" + } + + switch { + case strings.Contains(path, "/alertmanager/"): + return "aps:GetAlertManagerStatus" + case strings.Contains(path, "/rules"), strings.Contains(path, "/alerts"): + return "aps:ListRules" + } + + return "aps:QueryMetrics" +} + +// policyDenies reports whether the workspace resource policy refuses the caller: an explicit deny, +// or a cross-account caller without an allow. +func (b *InMemoryBackend) policyDenies(wsID, action, accessKey string, eval PolicyEvaluator) (string, bool) { + b.mu.RLock("policyDenies") + defer b.mu.RUnlock() + + w, ok := b.workspaces.Get(wsID) + if !ok { + return "", false + } + + caller := w.AccountID + if len(accessKey) == accountDigits && strings.Trim(accessKey, "0123456789") == "" { + caller = accessKey + } + + var allowed, denied bool + if w.Policy != nil { + allowed, denied = eval(w.Policy.Document, action, w.ARN, caller) + } + + return w.ARN, denied || (caller != w.AccountID && !allowed) +} diff --git a/services/amp/integrations_test.go b/services/amp/integrations_test.go new file mode 100644 index 0000000000..7cf3a67332 --- /dev/null +++ b/services/amp/integrations_test.go @@ -0,0 +1,271 @@ +package amp_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + ampsdk "github.com/aws/aws-sdk-go-v2/service/amp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/amp" +) + +type fakeKeys map[string]bool + +func (k fakeKeys) KMSKeyState(_ context.Context, arn string) (bool, bool) { + enabled, ok := k[arn] + + return ok, enabled +} + +type logLine struct{ region, group, stream, msg string } + +type fakeLogs struct { + groups map[string]bool + lines []logLine + mu sync.Mutex +} + +func (l *fakeLogs) LogGroupExists(_ context.Context, _, group string) bool { return l.groups[group] } + +func (l *fakeLogs) PutLogLine(_ context.Context, region, group, stream, msg string, _ time.Time) error { + l.mu.Lock() + defer l.mu.Unlock() + + l.lines = append(l.lines, logLine{region, group, stream, msg}) + + return nil +} + +func TestCreateWorkspaceKMSKey(t *testing.T) { + t.Parallel() + + const ( + ok = "arn:aws:kms:us-east-1:123456789012:key/11111111-1111-1111-1111-111111111111" + disabled = "arn:aws:kms:us-east-1:123456789012:key/22222222-2222-2222-2222-222222222222" + missing = "arn:aws:kms:us-east-1:123456789012:key/33333333-3333-3333-3333-333333333333" + ) + + tests := []struct { + name string + key string + code string + }{ + {name: "enabled", key: ok}, + {name: "disabled", key: disabled, code: "ValidationException"}, + {name: "missing", key: missing, code: "ValidationException"}, + {name: "none", key: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := newFixture(t) + f.backend.SetKeyChecker(fakeKeys{ok: true, disabled: false}) + + in := &sdk.CreateWorkspaceInput{Alias: aws.String("w")} + if tt.key != "" { + in.KmsKeyArn = aws.String(tt.key) + } + + _, err := f.client.CreateWorkspace(t.Context(), in) + if tt.code == "" { + require.NoError(t, err) + + return + } + + assert.Equal(t, tt.code, apiCode(err)) + }) + } +} + +func TestLoggingConfigurationLogGroup(t *testing.T) { + t.Parallel() + + const ( + have = "arn:aws:logs:us-east-1:123456789012:log-group:/aws/vendedlogs/amp" + gone = "arn:aws:logs:us-east-1:123456789012:log-group:/nope" + ) + + f := newFixture(t) + ws := f.workspace(t) + f.backend.SetLogSink(&fakeLogs{groups: map[string]bool{"/aws/vendedlogs/amp": true}}) + + _, err := f.client.CreateLoggingConfiguration(t.Context(), + &sdk.CreateLoggingConfigurationInput{WorkspaceId: aws.String(ws), LogGroupArn: aws.String(gone)}) + assert.Equal(t, "ResourceNotFoundException", apiCode(err)) + + _, err = f.client.CreateLoggingConfiguration(t.Context(), + &sdk.CreateLoggingConfigurationInput{WorkspaceId: aws.String(ws), LogGroupArn: aws.String(have)}) + require.NoError(t, err) + f.settle() + + _, err = f.client.UpdateLoggingConfiguration(t.Context(), + &sdk.UpdateLoggingConfigurationInput{WorkspaceId: aws.String(ws), LogGroupArn: aws.String(gone)}) + assert.Equal(t, "ResourceNotFoundException", apiCode(err)) +} + +func TestRuleFailureLogged(t *testing.T) { + t.Parallel() + + const group = "arn:aws:logs:us-east-1:123456789012:log-group:/amp" + + tests := []struct { + name string + logging bool + want int + }{ + {name: "delivered", logging: true, want: 1}, + {name: "no logging configured", want: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := newFixture(t) + sink := &fakeLogs{groups: map[string]bool{"/amp": true}} + f.backend.SetLogSink(sink) + ws := f.workspace(t) + + if tt.logging { + _, err := f.client.CreateLoggingConfiguration( + t.Context(), + &sdk.CreateLoggingConfigurationInput{ + WorkspaceId: aws.String(ws), + LogGroupArn: aws.String(group), + }, + ) + require.NoError(t, err) + f.settle() + } + + f.putRules(t, ws, "groups:\n - name: g\n rules:\n - record: bad\n expr: '\"x\"'\n") + f.clock.advance(2 * time.Minute) + f.backend.Tick(t.Context()) + + sink.mu.Lock() + defer sink.mu.Unlock() + + require.GreaterOrEqual(t, len(sink.lines), tt.want) + + if tt.want == 0 { + assert.Empty(t, sink.lines) + + return + } + + assert.Equal(t, "/amp", sink.lines[0].group) + assert.Equal(t, "us-east-1", sink.lines[0].region) + assert.Contains(t, sink.lines[0].msg, `"component":"ruler"`) + assert.Contains(t, sink.lines[0].msg, `"level":"error"`) + }) + } +} + +func (f *fixture) signedRequest(t *testing.T, method, path, svc, secret, key string) (*http.Request, httpResp) { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), method, f.url+path, strings.NewReader("")) + require.NoError(t, err) + + sum := sha256.Sum256(nil) + err = v4.NewSigner().SignHTTP(t.Context(), + aws.Credentials{AccessKeyID: key, SecretAccessKey: secret}, req, + hex.EncodeToString(sum[:]), svc, "us-east-1", time.Now()) + require.NoError(t, err) + + res, err := f.http.Do(req) + require.NoError(t, err) + + defer res.Body.Close() + + b, err := io.ReadAll(res.Body) + require.NoError(t, err) + + return req, httpResp{status: res.StatusCode, body: b} +} + +func TestDataPlaneAuth(t *testing.T) { + t.Parallel() + + denyAll := func(string, string, string, string) (bool, bool) { return false, true } + allowAll := func(string, string, string, string) (bool, bool) { return true, false } + + tests := []struct { + eval amp.PolicyEvaluator + name string + svc string + secret string + key string + code string + want int + policy bool + sign bool + }{ + {name: "valid", sign: true, svc: "aps", secret: "s3cret", want: http.StatusOK}, + {name: "unsigned", want: http.StatusForbidden, code: "Missing Authentication Token"}, + {name: "bad secret", sign: true, svc: "aps", secret: "other", want: http.StatusForbidden, + code: "signature we calculated does not match"}, + {name: "policy denies", sign: true, svc: "aps", secret: "s3cret", eval: denyAll, policy: true, + want: http.StatusForbidden, code: "not authorized to perform aps:QueryMetrics"}, + {name: "cross account without allow", sign: true, svc: "aps", secret: "s3cret", key: "999999999999", + eval: func(string, string, string, string) (bool, bool) { return false, false }, policy: true, + want: http.StatusForbidden, code: "not authorized"}, + {name: "policy allows", sign: true, svc: "aps", secret: "s3cret", eval: allowAll, policy: true, + want: http.StatusOK}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := newFixture(t) + ws := f.workspace(t) + f.backend.SetDataPlaneAuth(&.DataPlaneAuth{Secret: "s3cret", Evaluate: tt.eval}) + + if tt.policy { + _, err := f.client.PutResourcePolicy(t.Context(), &sdk.PutResourcePolicyInput{ + WorkspaceId: aws.String(ws), + PolicyDocument: aws.String( + `{"Version":"2012-10-17","Statement":[` + + `{"Effect":"Allow","Principal":"*","Action":"aps:*","Resource":"*"}]}`, + ), + }) + require.NoError(t, err) + } + + path := "/workspaces/" + ws + "/api/v1/query?query=vector(1)&time=60" + + var r httpResp + if tt.sign { + key := tt.key + if key == "" { + key = "test" + } + + _, r = f.signedRequest(t, http.MethodGet, path, tt.svc, tt.secret, key) + } else { + r = f.do(t, http.MethodGet, path, nil, nil) + } + + assert.Equal(t, tt.want, r.status, string(r.body)) + + if tt.code != "" { + assert.Contains(t, string(r.body), tt.code) + } + }) + } +} diff --git a/services/amp/models.go b/services/amp/models.go index b8922eb855..66b306a186 100644 --- a/services/amp/models.go +++ b/services/amp/models.go @@ -183,4 +183,5 @@ type SeriesRecord struct { Key string `json:"key"` WorkspaceID string `json:"workspaceId"` Samples []promql.Sample `json:"samples"` + Exemplars []Exemplar `json:"exemplars,omitempty"` } diff --git a/services/amp/native_test.go b/services/amp/native_test.go new file mode 100644 index 0000000000..620443ba27 --- /dev/null +++ b/services/amp/native_test.go @@ -0,0 +1,420 @@ +package amp_test + +import ( + "fmt" + "math" + "net/http" + "net/url" + "strconv" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + ampsdk "github.com/aws/aws-sdk-go-v2/service/amp" + "github.com/klauspost/compress/snappy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" + + "github.com/blackbirdworks/gopherstack/services/amp" +) + +type histSpec struct { + labels map[string]string + exemplar *exemplarSpec + posSpans [][2]int64 + posDeltas []int64 + posCounts []float64 + count uint64 + zeroCount uint64 + sum float64 + t int64 + floatCount float64 + schema int32 + floatHist bool + staleSum bool +} + +type exemplarSpec struct { + labels map[string]string + v float64 + t int64 +} + +func labelBytes(k, v string) []byte { + var b []byte + + b = protowire.AppendTag(b, 1, protowire.BytesType) + b = protowire.AppendString(b, k) + b = protowire.AppendTag(b, 2, protowire.BytesType) + b = protowire.AppendString(b, v) + + return b +} + +func histogramBytes(h histSpec) []byte { + var b []byte + + if h.floatHist { + b = protowire.AppendTag(b, 2, protowire.Fixed64Type) + b = protowire.AppendFixed64(b, math.Float64bits(h.floatCount)) + } else { + b = protowire.AppendTag(b, 1, protowire.VarintType) + b = protowire.AppendVarint(b, h.count) + } + + sum := h.sum + if h.staleSum { + sum = math.Float64frombits(0x7ff0000000000002) + } + + b = protowire.AppendTag(b, 3, protowire.Fixed64Type) + b = protowire.AppendFixed64(b, math.Float64bits(sum)) + b = protowire.AppendTag(b, 4, protowire.VarintType) + b = protowire.AppendVarint(b, protowire.EncodeZigZag(int64(h.schema))) + + if !h.floatHist { + b = protowire.AppendTag(b, 6, protowire.VarintType) + b = protowire.AppendVarint(b, h.zeroCount) + } + + for _, sp := range h.posSpans { + var sb []byte + + sb = protowire.AppendTag(sb, 1, protowire.VarintType) + sb = protowire.AppendVarint(sb, protowire.EncodeZigZag(sp[0])) + sb = protowire.AppendTag(sb, 2, protowire.VarintType) + sb = protowire.AppendVarint(sb, uint64(sp[1])) + b = protowire.AppendTag(b, 11, protowire.BytesType) + b = protowire.AppendBytes(b, sb) + } + + if len(h.posDeltas) > 0 { + var pk []byte + for _, d := range h.posDeltas { + pk = protowire.AppendVarint(pk, protowire.EncodeZigZag(d)) + } + + b = protowire.AppendTag(b, 12, protowire.BytesType) + b = protowire.AppendBytes(b, pk) + } + + if len(h.posCounts) > 0 { + var pk []byte + for _, c := range h.posCounts { + pk = protowire.AppendFixed64(pk, math.Float64bits(c)) + } + + b = protowire.AppendTag(b, 13, protowire.BytesType) + b = protowire.AppendBytes(b, pk) + } + + b = protowire.AppendTag(b, 15, protowire.VarintType) + b = protowire.AppendVarint(b, uint64(h.t)) + + return b +} + +func exemplarBytes(e exemplarSpec) []byte { + var b []byte + + for k, v := range e.labels { + b = protowire.AppendTag(b, 1, protowire.BytesType) + b = protowire.AppendBytes(b, labelBytes(k, v)) + } + + b = protowire.AppendTag(b, 2, protowire.Fixed64Type) + b = protowire.AppendFixed64(b, math.Float64bits(e.v)) + b = protowire.AppendTag(b, 3, protowire.VarintType) + b = protowire.AppendVarint(b, uint64(e.t)) + + return b +} + +func (f *fixture) writeRaw(t *testing.T, ws string, hs ...histSpec) httpResp { + t.Helper() + + var raw []byte + + for _, h := range hs { + var ts []byte + + for k, v := range h.labels { + ts = protowire.AppendTag(ts, 1, protowire.BytesType) + ts = protowire.AppendBytes(ts, labelBytes(k, v)) + } + + if h.exemplar != nil { + ts = protowire.AppendTag(ts, 3, protowire.BytesType) + ts = protowire.AppendBytes(ts, exemplarBytes(*h.exemplar)) + } + + if h.count > 0 || h.floatHist || h.staleSum { + ts = protowire.AppendTag(ts, 4, protowire.BytesType) + ts = protowire.AppendBytes(ts, histogramBytes(h)) + } + + raw = protowire.AppendTag(raw, 1, protowire.BytesType) + raw = protowire.AppendBytes(raw, ts) + } + + return f.do(t, http.MethodPost, "/workspaces/"+ws+"/api/v1/remote_write", + strings.NewReader(string(snappy.Encode(nil, raw))), + map[string]string{"Content-Encoding": "snappy", "Authorization": signedAPS}) +} + +func TestNativeHistogramIngestAndQuery(t *testing.T) { + t.Parallel() + + lbl := map[string]string{"__name__": "lat", "job": "api"} + // schema 0: buckets (1,2] (2,4] (4,8] each holding 2 observations. + intHist := histSpec{ + labels: lbl, posSpans: [][2]int64{{1, 3}}, posDeltas: []int64{2, 0, 0}, count: 6, sum: 20, t: 60_000, + } + floatHist := histSpec{ + labels: lbl, floatHist: true, floatCount: 6, sum: 20, t: 60_000, + posSpans: [][2]int64{{1, 3}}, posCounts: []float64{2, 2, 2}, + } + + tests := []struct { + name string + query string + hist histSpec + want float64 + }{ + {name: "count int", query: "histogram_count(lat)", want: 6, hist: intHist}, + {name: "count float", query: "histogram_count(lat)", want: 6, hist: floatHist}, + {name: "sum", query: "histogram_sum(lat)", want: 20, hist: intHist}, + {name: "avg", query: "histogram_avg(lat)", want: 20.0 / 6, hist: intHist}, + {name: "quantile", query: "histogram_quantile(0.5, lat)", want: math.Pow(2, 1.5), hist: intHist}, + {name: "fraction", query: "histogram_fraction(0, 4, lat)", want: 4.0 / 6, hist: intHist}, + {name: "stdvar", query: "histogram_stdvar(lat)", want: stdvar(20.0/6, []float64{2, 8, 32}), hist: intHist}, + { + name: "stddev", + query: "histogram_stddev(lat)", + want: math.Sqrt(stdvar(20.0/6, []float64{2, 8, 32})), + hist: intHist, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := newFixture(t) + ws := f.workspace(t) + r := f.writeRaw(t, ws, tt.hist) + require.Equal(t, http.StatusOK, r.status, string(r.body)) + assert.Equal(t, 1, f.backend.SampleCount(ws)) + + _, pr := f.query(t, ws, "query", url.Values{"query": {tt.query}, "time": {"60"}}) + require.Equal(t, "success", pr.Status) + assert.Contains(t, string(pr.Data.Result), fmt.Sprintf(`"%s"]`, fmtFloat(tt.want))) + }) + } +} + +func fmtFloat(v float64) string { return strconv.FormatFloat(v, 'f', -1, 64) } + +// stdvar of a histogram whose three buckets each hold count/3 observations at squared midpoints sq. +func stdvar(mean float64, sq []float64) float64 { + v := 0.0 + for _, s := range sq { + d := math.Sqrt(s) - mean + v += 2 * d * d + } + + return v / 6 +} + +func TestNativeHistogramRendering(t *testing.T) { + t.Parallel() + + f := newFixture(t) + ws := f.workspace(t) + h := histSpec{ + labels: map[string]string{"__name__": "lat"}, posSpans: [][2]int64{{1, 2}}, posDeltas: []int64{1, 1}, + count: 3, sum: 7, t: 60_000, zeroCount: 0, + } + require.Equal(t, http.StatusOK, f.writeRaw(t, ws, h).status) + + _, pr := f.query(t, ws, "query", url.Values{"query": {"lat"}, "time": {"60"}}) + assert.JSONEq( + t, + `[{"metric":{"__name__":"lat"},"histogram":[60,{"count":"3","sum":"7",`+ + `"buckets":[[0,"1","2","1"],[0,"2","4","2"]]}]}]`, + string(pr.Data.Result), + ) + + _, pr = f.query(t, ws, "query_range", url.Values{"query": {"lat"}, "start": {"60"}, "end": {"60"}, "step": {"1"}}) + assert.Contains(t, string(pr.Data.Result), `"histograms":[[60,{"count":"3"`) + + _, pr = f.query(t, ws, "query", url.Values{"query": {"lat + 1"}, "time": {"60"}}) + assert.JSONEq(t, `[]`, string(pr.Data.Result), "histograms are not float operands") +} + +func TestNativeHistogramRejects(t *testing.T) { + t.Parallel() + + lbl := map[string]string{"__name__": "lat"} + tests := []struct { + name string + hist histSpec + }{ + {name: "span bucket mismatch", hist: histSpec{ + labels: lbl, posSpans: [][2]int64{{1, 3}}, posDeltas: []int64{1}, count: 1, sum: 1, t: 1, + }}, + {name: "schema out of range", hist: histSpec{ + labels: lbl, posSpans: [][2]int64{{1, 1}}, posDeltas: []int64{1}, count: 1, sum: 1, t: 1, schema: 20, + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := newFixture(t) + ws := f.workspace(t) + r := f.writeRaw(t, ws, tt.hist) + assert.Equal(t, http.StatusBadRequest, r.status) + assert.Equal(t, 0, f.backend.SampleCount(ws)) + }) + } +} + +func TestNativeHistogramStaleMarkerDropped(t *testing.T) { + t.Parallel() + + f := newFixture(t) + ws := f.workspace(t) + lbl := map[string]string{"__name__": "lat"} + require.Equal(t, http.StatusOK, f.writeRaw(t, ws, histSpec{ + labels: lbl, posSpans: [][2]int64{{1, 1}}, posDeltas: []int64{1}, count: 1, sum: 1, t: 10_000, + }).status) + require.Equal(t, http.StatusOK, f.writeRaw(t, ws, histSpec{labels: lbl, staleSum: true, t: 20_000}).status) + + assert.Equal(t, 1, f.backend.SampleCount(ws), "remote-write stale markers are dropped like float ones") +} + +func TestExemplars(t *testing.T) { + t.Parallel() + + lbl := map[string]string{"__name__": "req_seconds_bucket", "le": "1"} + ex := &exemplarSpec{labels: map[string]string{"trace_id": "abc"}, v: 0.7, t: 55_000} + long := &exemplarSpec{labels: map[string]string{"trace_id": strings.Repeat("x", 200)}, v: 1, t: 56_000} + + tests := []struct { + query url.Values + name string + want string + write []histSpec + }{ + {name: "returned", write: []histSpec{{labels: lbl, exemplar: ex}}, + query: url.Values{"query": {"req_seconds_bucket"}, "start": {"0"}, "end": {"100"}}, + want: `[{"seriesLabels":{"__name__":"req_seconds_bucket","le":"1"},` + + `"exemplars":[{"labels":{"trace_id":"abc"},"value":"0.7","timestamp":55}]}]`}, + {name: "outside range", write: []histSpec{{labels: lbl, exemplar: ex}}, + query: url.Values{"query": {"req_seconds_bucket"}, "start": {"0"}, "end": {"10"}}, want: `[]`}, + {name: "other selector", write: []histSpec{{labels: lbl, exemplar: ex}}, + query: url.Values{"query": {"other"}, "start": {"0"}, "end": {"100"}}, want: `[]`}, + {name: "oversized dropped", write: []histSpec{{labels: lbl, exemplar: long}}, + query: url.Values{"query": {"req_seconds_bucket"}, "start": {"0"}, "end": {"100"}}, want: `[]`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := newFixture(t) + ws := f.workspace(t) + require.Equal(t, http.StatusOK, f.writeRaw(t, ws, tt.write...).status) + + r := f.do(t, http.MethodGet, "/workspaces/"+ws+"/api/v1/query_exemplars?"+tt.query.Encode(), nil, nil) + require.Equal(t, http.StatusOK, r.status, string(r.body)) + assert.JSONEq(t, `{"status":"success","data":`+tt.want+`}`, string(r.body)) + }) + } +} + +func TestOutOfOrderWindow(t *testing.T) { + t.Parallel() + + lbl := map[string]string{"__name__": "m"} + tests := []struct { + window *int32 + name string + second int64 + wantStatus int + wantCount int + }{ + {name: "default rejects older", second: 59_000, wantStatus: http.StatusBadRequest, wantCount: 1}, + { + name: "zero rejects older", + window: aws.Int32(0), + second: 59_000, + wantStatus: http.StatusBadRequest, + wantCount: 1, + }, + {name: "inside window", window: aws.Int32(30), second: 40_000, wantStatus: http.StatusOK, wantCount: 2}, + {name: "on window edge", window: aws.Int32(30), second: 30_000, wantStatus: http.StatusOK, wantCount: 2}, + { + name: "outside window", + window: aws.Int32(30), + second: 29_999, + wantStatus: http.StatusBadRequest, + wantCount: 1, + }, + {name: "newer always fine", second: 61_000, wantStatus: http.StatusOK, wantCount: 2}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f := newFixture(t) + ws := f.workspace(t) + + if tt.window != nil { + _, err := f.client.UpdateWorkspaceConfiguration(t.Context(), &sdk.UpdateWorkspaceConfigurationInput{ + WorkspaceId: aws.String(ws), OutOfOrderTimeWindowInSeconds: tt.window, + }) + require.NoError(t, err) + f.settle() + } + + require.Equal(t, http.StatusOK, f.write(t, ws, newSeries(lbl, 60_000, 1)).status) + + r := f.write(t, ws, newSeries(lbl, tt.second, 2)) + assert.Equal(t, tt.wantStatus, r.status, string(r.body)) + assert.Equal(t, tt.wantCount, f.backend.SampleCount(ws)) + }) + } +} + +func newSeries(lbl map[string]string, ts int64, v float64) amp.WriteSeries { + return amp.NewSeries(lbl, sample(ts, v)) +} + +func TestNativeHistogramAndExemplarPersist(t *testing.T) { + t.Parallel() + + src := newFixture(t) + ws := src.workspace(t) + lbl := map[string]string{"__name__": "lat"} + require.Equal(t, http.StatusOK, src.writeRaw(t, ws, histSpec{ + labels: lbl, posSpans: [][2]int64{{1, 2}}, posDeltas: []int64{1, 1}, count: 3, sum: 7, t: 60_000, + exemplar: &exemplarSpec{labels: map[string]string{"trace_id": "t1"}, v: 2, t: 59_000}, + }).status) + + snap := src.backend.Snapshot(t.Context()) + + dst := newFixture(t) + require.NoError(t, dst.backend.Restore(t.Context(), snap)) + dst.settle() + + _, pr := dst.query(t, ws, "query", url.Values{"query": {"histogram_sum(lat)"}, "time": {"60"}}) + assert.Contains(t, string(pr.Data.Result), `"7"`) + + r := dst.do(t, http.MethodGet, "/workspaces/"+ws+"/api/v1/query_exemplars?query=lat&start=0&end=100", nil, nil) + assert.Contains(t, string(r.body), `"trace_id":"t1"`) +} diff --git a/services/amp/promql/eval.go b/services/amp/promql/eval.go index 845beade26..4791dde65e 100644 --- a/services/amp/promql/eval.go +++ b/services/amp/promql/eval.go @@ -66,6 +66,7 @@ type Scalar struct { // VectorSample is one instant sample of a labelled series. type VectorSample struct { Metric Labels + H *FloatHistogram T int64 V float64 } @@ -96,6 +97,7 @@ type Engine struct { Store Storage Lookback time.Duration steps int + histOK bool } func isRangeExpr(x Expr) bool { @@ -129,6 +131,7 @@ func (e *Engine) Instant(src string, ts time.Time) (Value, error) { func (e *Engine) InstantExpr(expr Expr, ts time.Time) (Value, error) { t := ts.UnixMilli() resolveAt(expr, t, t) + e.histOK = isBareSelector(expr) v, err := e.eval(expr, t) if err != nil { @@ -162,6 +165,7 @@ func (e *Engine) Range(src string, start, end time.Time, step time.Duration) (Ma } resolveAt(expr, start.UnixMilli(), end.UnixMilli()) + e.histOK = isBareSelector(expr) if isRangeExpr(expr) { return nil, evalErr( @@ -180,10 +184,10 @@ func (e *Engine) Range(src string, start, end time.Time, step time.Duration) (Ma switch x := v.(type) { case Scalar: - addPoint(acc, &order, Labels{}, Sample(x)) + addPoint(acc, &order, Labels{}, Sample{T: x.T, V: x.V}) case Vector: for _, s := range x { - addPoint(acc, &order, s.Metric, Sample{T: s.T, V: s.V}) + addPoint(acc, &order, s.Metric, Sample{T: s.T, V: s.V, H: s.H}) } default: return nil, evalErr("invalid expression type %q for range query", v.valueType()) @@ -252,9 +256,12 @@ func (e *Engine) evalVector(vs *VectorSelector, t int64) Vector { out := make(Vector, 0, len(series)) for _, s := range series { - if p, ok := lastAtOrBefore(s.Samples, maxT, minT); ok { - out = append(out, VectorSample{Metric: s.Labels, T: t, V: p.V}) + p, ok := lastAtOrBefore(s.Samples, maxT, minT) + if !ok || (p.H != nil && !e.histOK) { + continue } + + out = append(out, VectorSample{Metric: s.Labels, T: t, V: p.V, H: p.H}) } sortVector(out) @@ -288,7 +295,7 @@ func (e *Engine) evalMatrix(ms *MatrixSelector, t int64) Matrix { var win []Sample for _, p := range s.Samples { - if p.T > minT && p.T <= maxT && !IsStale(p.V) { + if p.T > minT && p.T <= maxT && !IsStale(p.V) && p.H == nil { win = append(win, p) } } diff --git a/services/amp/promql/funcs.go b/services/amp/promql/funcs.go index 0c49ea3ba2..ba758efcfd 100644 --- a/services/amp/promql/funcs.go +++ b/services/amp/promql/funcs.go @@ -21,8 +21,8 @@ var functions = map[string]funcSpec{ //nolint:gochecknoglobals // immutable look "stddev_over_time": {1, 1}, "stdvar_over_time": {1, 1}, "present_over_time": {1, 1}, "holt_winters": {3, 3}, "double_exponential_smoothing": {3, 3}, "absent_over_time": {1, 1}, "sort_by_label": {2, maxArgs}, fnSortByLabelDesc: {2, maxArgs}, "histogram_fraction": {3, 3}, - "histogram_count": {1, 1}, "histogram_sum": {1, 1}, "histogram_avg": {1, 1}, - "histogram_stddev": {1, 1}, "histogram_stdvar": {1, 1}, + fnHistCount: {1, 1}, fnHistSum: {1, 1}, fnHistAvg: {1, 1}, + fnHistStddev: {1, 1}, fnHistStdvar: {1, 1}, "minute": {0, 1}, "hour": {0, 1}, "month": {0, 1}, "year": {0, 1}, "day_of_month": {0, 1}, "day_of_week": {0, 1}, "day_of_year": {0, 1}, "days_in_month": {0, 1}, "timestamp": {1, 1}, } diff --git a/services/amp/promql/funcs_extra.go b/services/amp/promql/funcs_extra.go index 2c1187c79d..3d647ca93f 100644 --- a/services/amp/promql/funcs_extra.go +++ b/services/amp/promql/funcs_extra.go @@ -42,7 +42,7 @@ func (e *Engine) callExtra(c *Call, t int64) (Value, bool, error) { v, err = e.callHistogramQuantile(c, t) case "histogram_fraction": v, err = e.callHistogramFraction(c, t) - case "histogram_count", "histogram_sum", "histogram_avg", "histogram_stddev", "histogram_stdvar": + case fnHistCount, fnHistSum, fnHistAvg, fnHistStddev, fnHistStdvar: v, err = e.callNativeHistogram(c, t) case "holt_winters", "double_exponential_smoothing": v, err = e.callDoubleExponentialSmoothing(c, t) @@ -199,15 +199,20 @@ func (e *Engine) callHistogramQuantile(c *Call, t int64) (Value, error) { return nil, err } - vec, err := e.instantArg(c.Args[1], t, c.Func) + vec, err := e.histArg(c.Args[1], t, c.Func) if err != nil { return nil, err } + native, vec := splitNative(vec) groups, order := groupBuckets(vec) out := Vector{} + for _, s := range native { + out = append(out, VectorSample{Metric: s.Metric.without(NameLabel), T: t, V: s.H.Quantile(q)}) + } + for _, k := range order { g := groups[k] out = append(out, VectorSample{Metric: g.labels, T: t, V: bucketQuantile(q, g.buckets)}) @@ -467,10 +472,10 @@ func (e *Engine) evalSubquery(sq *Subquery, t int64) (Matrix, error) { func collectPoints(acc map[string]*Series, order *[]string, v Value) error { switch x := v.(type) { case Scalar: - addPoint(acc, order, Labels{}, Sample(x)) + addPoint(acc, order, Labels{}, Sample{T: x.T, V: x.V}) case Vector: for _, s := range x { - addPoint(acc, order, s.Metric, Sample{T: s.T, V: s.V}) + addPoint(acc, order, s.Metric, Sample{T: s.T, V: s.V, H: s.H}) } default: return evalErr("subquery must return an instant vector or scalar, got %s", v.valueType()) diff --git a/services/amp/promql/funcs_prom3.go b/services/amp/promql/funcs_prom3.go index cfe73618ea..2214694981 100644 --- a/services/amp/promql/funcs_prom3.go +++ b/services/amp/promql/funcs_prom3.go @@ -173,13 +173,23 @@ func chunk(s string) string { return s } -// callNativeHistogram applies the native-histogram accessors; Prometheus ignores float samples, the only kind stored. +// callNativeHistogram applies the native-histogram accessors; float samples are ignored like Prometheus does. func (e *Engine) callNativeHistogram(c *Call, t int64) (Value, error) { - if _, err := e.instantArg(c.Args[0], t, c.Func); err != nil { + vec, err := e.histArg(c.Args[0], t, c.Func) + if err != nil { return nil, err } - return Vector{}, nil + native, _ := splitNative(vec) + out := Vector{} + + for _, s := range native { + out = append(out, VectorSample{Metric: s.Metric.without(NameLabel), T: t, V: s.H.accessor(c.Func)}) + } + + sortVector(out) + + return out, nil } func (e *Engine) callHistogramFraction(c *Call, t int64) (Value, error) { @@ -193,15 +203,20 @@ func (e *Engine) callHistogramFraction(c *Call, t int64) (Value, error) { return nil, err } - vec, err := e.instantArg(c.Args[2], t, c.Func) + vec, err := e.histArg(c.Args[2], t, c.Func) if err != nil { return nil, err } + native, vec := splitNative(vec) groups, order := groupBuckets(vec) out := Vector{} + for _, s := range native { + out = append(out, VectorSample{Metric: s.Metric.without(NameLabel), T: t, V: s.H.Fraction(lower, upper)}) + } + for _, k := range order { g := groups[k] out = append(out, VectorSample{Metric: g.labels, T: t, V: bucketFraction(lower, upper, g.buckets)}) diff --git a/services/amp/promql/histogram.go b/services/amp/promql/histogram.go new file mode 100644 index 0000000000..2e92fad4ff --- /dev/null +++ b/services/amp/promql/histogram.go @@ -0,0 +1,332 @@ +package promql + +import ( + "math" + "slices" +) + +const ( + fnHistCount = "histogram_count" + fnHistSum = "histogram_sum" + fnHistAvg = "histogram_avg" + fnHistStddev = "histogram_stddev" + fnHistStdvar = "histogram_stdvar" +) + +// Span is a run of consecutive populated buckets starting Offset buckets after the previous span. +type Span struct { + Offset int32 `json:"offset"` + Length uint32 `json:"length"` +} + +// FloatHistogram is a native histogram with absolute (not delta-encoded) bucket counts. +type FloatHistogram struct { + PositiveSpans []Span `json:"ps,omitempty"` + NegativeSpans []Span `json:"ns,omitempty"` + PositiveBuckets []float64 `json:"pb,omitempty"` + NegativeBuckets []float64 `json:"nb,omitempty"` + ZeroThreshold float64 `json:"zt,omitempty"` + ZeroCount float64 `json:"zc,omitempty"` + Count float64 `json:"c"` + Sum float64 `json:"s"` + Schema int32 `json:"schema"` +} + +// Schema bounds accepted for exponential native histograms. +const ( + MinSchema = -4 + MaxSchema = 8 +) + +// HBucket is one materialised bucket; Lower is exclusive for positive buckets and Upper for negative ones. +type HBucket struct { + Lower, Upper, Count float64 + Zero, Negative bool +} + +// Valid reports whether the span and bucket arrays agree and the schema is supported. +func (h *FloatHistogram) Valid() bool { + return h.Schema >= MinSchema && h.Schema <= MaxSchema && + spanTotal(h.PositiveSpans) == len(h.PositiveBuckets) && + spanTotal(h.NegativeSpans) == len(h.NegativeBuckets) +} + +func spanTotal(sp []Span) int { + n := 0 + for _, s := range sp { + n += int(s.Length) + } + + return n +} + +// bound is the upper boundary of bucket index i: 2^(i * 2^-schema). +func bound(i, schema int32) float64 { + return math.Exp2(float64(i) * math.Exp2(-float64(schema))) +} + +func expand(sp []Span, counts []float64, schema int32, neg bool) []HBucket { + var ( + out []HBucket + idx int32 + k int + ) + + for _, s := range sp { + idx += s.Offset + + for range s.Length { + if k >= len(counts) { + return out + } + + lo, hi := bound(idx-1, schema), bound(idx, schema) + if neg { + lo, hi = -hi, -lo + } + + out = append(out, HBucket{Lower: lo, Upper: hi, Count: counts[k], Negative: neg}) + idx++ + k++ + } + } + + return out +} + +// Buckets lists every bucket in increasing value order, the zero bucket included. +func (h *FloatHistogram) Buckets() []HBucket { + neg := expand(h.NegativeSpans, h.NegativeBuckets, h.Schema, true) + + out := make([]HBucket, 0, len(neg)+len(h.PositiveBuckets)+1) + for _, b := range slices.Backward(neg) { + out = append(out, b) + } + + out = append(out, HBucket{Lower: -h.ZeroThreshold, Upper: h.ZeroThreshold, Count: h.ZeroCount, Zero: true}) + + return append(out, expand(h.PositiveSpans, h.PositiveBuckets, h.Schema, false)...) +} + +func (h *FloatHistogram) accessor(fn string) float64 { + switch fn { + case fnHistCount: + return h.Count + case fnHistSum: + return h.Sum + case fnHistAvg: + return h.Sum / h.Count + case fnHistStdvar: + return h.variance() + default: + return math.Sqrt(h.variance()) + } +} + +func (h *FloatHistogram) variance() float64 { + if h.Count == 0 { + return math.NaN() + } + + mean := h.Sum / h.Count + v := 0.0 + + for _, b := range h.Buckets() { + m := 0.0 + if !b.Zero { + m = math.Sqrt(b.Upper * b.Lower) + if b.Upper < 0 { + m = -m + } + } + + v += b.Count * (m - mean) * (m - mean) + } + + return v / h.Count +} + +// interp places a point at fraction f of the way through b, geometrically for regular buckets. +func interp(b HBucket, f float64) float64 { + switch { + case b.Zero || b.Lower == 0 || b.Upper == 0: + return b.Lower + (b.Upper-b.Lower)*f + case b.Lower > 0: + return math.Exp2(math.Log2(b.Lower)*(1-f) + math.Log2(b.Upper)*f) + default: + return -math.Exp2(math.Log2(-b.Upper)*f + math.Log2(-b.Lower)*(1-f)) + } +} + +// Quantile estimates the q-quantile like Prometheus' histogram_quantile on a native histogram. +func (h *FloatHistogram) Quantile(q float64) float64 { + switch { + case math.IsNaN(q) || h.Count == 0: + return math.NaN() + case q < 0: + return math.Inf(-1) + case q > 1: + return math.Inf(1) + } + + bs := h.Buckets() + rank := q * h.Count + + var cum float64 + + last := -1 + + for i, b := range bs { + if b.Count == 0 { + continue + } + + last = i + + if cum+b.Count >= rank { + return interp(clipZero(bs, i), (rank-cum)/b.Count) + } + + cum += b.Count + } + + if last < 0 { + return math.NaN() + } + + return bs[last].Upper +} + +// clipZero narrows the zero bucket to the side that holds observations when the other side is empty. +func clipZero(bs []HBucket, i int) HBucket { + b := bs[i] + if !b.Zero { + return b + } + + if allEmpty(bs[:i]) { + b.Lower = 0 + } + + if allEmpty(bs[i+1:]) { + b.Upper = 0 + } + + return b +} + +func allEmpty(bs []HBucket) bool { + for _, b := range bs { + if b.Count != 0 { + return false + } + } + + return true +} + +// below is the interpolated number of observations <= x. +func (h *FloatHistogram) below(x float64) float64 { + var cum float64 + + for _, b := range h.Buckets() { + switch { + case x >= b.Upper: + cum += b.Count + case x > b.Lower: + return cum + b.Count*fractionIn(b, x) + default: + return cum + } + } + + return cum +} + +func fractionIn(b HBucket, x float64) float64 { + if b.Zero || b.Lower <= 0 && b.Upper >= 0 { + return (x - b.Lower) / (b.Upper - b.Lower) + } + + if b.Lower > 0 { + return (math.Log2(x) - math.Log2(b.Lower)) / (math.Log2(b.Upper) - math.Log2(b.Lower)) + } + + return 1 - (math.Log2(-x)-math.Log2(-b.Upper))/(math.Log2(-b.Lower)-math.Log2(-b.Upper)) +} + +// Fraction is the share of observations in [lower, upper]. +func (h *FloatHistogram) Fraction(lower, upper float64) float64 { + if h.Count == 0 || math.IsNaN(lower) || math.IsNaN(upper) { + return math.NaN() + } + + if lower >= upper { + return 0 + } + + return (h.below(upper) - h.below(lower)) / h.Count +} + +// isBareSelector reports an expression that is a plain instant selector. +func isBareSelector(x Expr) bool { + _, ok := x.(*VectorSelector) + + return ok +} + +// histArg evaluates arg and keeps native histogram samples when arg is a plain selector. +func (e *Engine) histArg(arg Expr, t int64, fn string) (Vector, error) { + prev := e.histOK + e.histOK = prev || isBareSelector(arg) + + defer func() { e.histOK = prev }() + + return e.instantArg(arg, t, fn) +} + +func splitNative(v Vector) (Vector, Vector) { + var native, floats Vector + + for _, s := range v { + if s.H != nil { + native = append(native, s) + } else { + floats = append(floats, s) + } + } + + return native, floats +} + +// Selectors lists the matcher sets of every selector in expr. +func Selectors(expr Expr) [][]*Matcher { + switch x := expr.(type) { + case *VectorSelector: + return [][]*Matcher{x.Matchers} + case *MatrixSelector: + return Selectors(x.VS) + case *Subquery: + return Selectors(x.Expr) + case *Call: + var out [][]*Matcher + for _, a := range x.Args { + out = append(out, Selectors(a)...) + } + + return out + case *Aggregation: + out := Selectors(x.Expr) + if x.Param != nil { + out = append(out, Selectors(x.Param)...) + } + + return out + case *Binary: + return append(Selectors(x.L), Selectors(x.R)...) + case *Unary: + return Selectors(x.X) + } + + return nil +} diff --git a/services/amp/promql/histogram_test.go b/services/amp/promql/histogram_test.go new file mode 100644 index 0000000000..2b0c26303e --- /dev/null +++ b/services/amp/promql/histogram_test.go @@ -0,0 +1,81 @@ +package promql_test + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/blackbirdworks/gopherstack/services/amp/promql" +) + +func TestFloatHistogramQuantile(t *testing.T) { + t.Parallel() + + zeroPos := &promql.FloatHistogram{ + ZeroThreshold: 0.5, ZeroCount: 2, Count: 4, Sum: 3, + PositiveSpans: []promql.Span{{Offset: 1, Length: 1}}, PositiveBuckets: []float64{2}, + } + neg := &promql.FloatHistogram{ + Count: 4, Sum: -6, NegativeSpans: []promql.Span{{Offset: 1, Length: 1}}, NegativeBuckets: []float64{4}, + } + + tests := []struct { + h *promql.FloatHistogram + name string + q float64 + want float64 + }{ + {name: "zero bucket linear", h: zeroPos, q: 0.25, want: 0.25}, + {name: "positive geometric", h: zeroPos, q: 0.75, want: math.Sqrt2}, + {name: "top of last bucket", h: zeroPos, q: 1, want: 2}, + {name: "negative bucket", h: neg, q: 0.5, want: -math.Sqrt2}, + {name: "below zero", h: zeroPos, q: -1, want: math.Inf(-1)}, + {name: "above one", h: zeroPos, q: 2, want: math.Inf(1)}, + {name: "empty", h: &promql.FloatHistogram{}, q: 0.5, want: math.NaN()}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := tt.h.Quantile(tt.q) + if math.IsNaN(tt.want) { + assert.True(t, math.IsNaN(got)) + + return + } + + assert.InDelta(t, tt.want, got, 1e-9) + }) + } +} + +func TestFloatHistogramValid(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + h promql.FloatHistogram + want bool + }{ + { + name: "ok", + h: promql.FloatHistogram{PositiveSpans: []promql.Span{{Length: 2}}, PositiveBuckets: []float64{1, 1}}, + want: true, + }, + { + name: "mismatch", + h: promql.FloatHistogram{PositiveSpans: []promql.Span{{Length: 2}}, PositiveBuckets: []float64{1}}, + }, + {name: "schema high", h: promql.FloatHistogram{Schema: promql.MaxSchema + 1}}, + {name: "schema low", h: promql.FloatHistogram{Schema: promql.MinSchema - 1}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, tt.h.Valid()) + }) + } +} diff --git a/services/amp/promql/labels.go b/services/amp/promql/labels.go index 0a460a336b..4dab57f838 100644 --- a/services/amp/promql/labels.go +++ b/services/amp/promql/labels.go @@ -22,8 +22,9 @@ type Labels map[string]string // Sample is one timestamped value; T is Unix milliseconds. type Sample struct { - T int64 `json:"t"` - V float64 `json:"v"` + H *FloatHistogram `json:"h,omitempty"` + T int64 `json:"t"` + V float64 `json:"v"` } // Series is a labelled run of samples ordered by time. diff --git a/services/amp/remotewrite.go b/services/amp/remotewrite.go index 815e030614..617e6fd93e 100644 --- a/services/amp/remotewrite.go +++ b/services/amp/remotewrite.go @@ -20,6 +20,8 @@ const ( fieldLabelValue = 2 fieldSampleValue = 1 fieldSampleTimestamp = 2 + fieldSeriesExemplars = 3 + fieldSeriesHistogram = 4 maxDecodedWrite = 32 << 20 ) @@ -27,8 +29,16 @@ const ( var errBadWrite = errors.New("invalid remote write payload") type writeSeries struct { - Labels promql.Labels - Samples []promql.Sample + Labels promql.Labels + Samples []promql.Sample + Exemplars []Exemplar +} + +// Exemplar is a remote-write exemplar: a trace-style label set attached to one observation. +type Exemplar struct { + Labels promql.Labels `json:"labels"` + T int64 `json:"t"` + V float64 `json:"v"` } // decodeWriteRequest unpacks a snappy-compressed remote-write body. @@ -92,7 +102,8 @@ func decodeTimeSeries(msg []byte) (writeSeries, error) { msg = msg[n:] - if typ == protowire.BytesType && (num == fieldSeriesLabels || num == fieldSeriesSamples) { + if typ == protowire.BytesType && (num == fieldSeriesLabels || num == fieldSeriesSamples || + num == fieldSeriesExemplars || num == fieldSeriesHistogram) { sub, m := protowire.ConsumeBytes(msg) if m < 0 { return ws, fmt.Errorf("%w: bad series field", errBadWrite) @@ -130,6 +141,28 @@ func (ws *writeSeries) add(num protowire.Number, sub []byte) error { return nil } + switch num { + case fieldSeriesExemplars: + ex, err := decodeExemplar(sub) + if err != nil { + return err + } + + ws.Exemplars = append(ws.Exemplars, ex) + + return nil + case fieldSeriesHistogram: + s, err := decodeHistogram(sub) + if err != nil { + return err + } + + ws.Samples = append(ws.Samples, s) + + return nil + default: + } + s, err := decodeSample(sub) if err != nil { return err diff --git a/services/amp/remotewrite_histogram.go b/services/amp/remotewrite_histogram.go new file mode 100644 index 0000000000..a4b2d2dce4 --- /dev/null +++ b/services/amp/remotewrite_histogram.go @@ -0,0 +1,311 @@ +package amp + +import ( + "fmt" + "math" + + "google.golang.org/protobuf/encoding/protowire" + + "github.com/blackbirdworks/gopherstack/services/amp/promql" +) + +// Field numbers of the remote-write Histogram, BucketSpan and Exemplar messages. +const ( + fieldHistCountInt = 1 + fieldHistCountFloat = 2 + fieldHistSum = 3 + fieldHistSchema = 4 + fieldHistZeroThresh = 5 + fieldHistZeroInt = 6 + fieldHistZeroFloat = 7 + fieldHistNegSpans = 8 + fieldHistNegDeltas = 9 + fieldHistNegCounts = 10 + fieldHistPosSpans = 11 + fieldHistPosDeltas = 12 + fieldHistPosCounts = 13 + fieldHistTimestamp = 15 + fieldSpanOffset = 1 + fieldSpanLength = 2 + fieldExemplarLabels = 1 + fieldExemplarValue = 2 + fieldExemplarTime = 3 + maxExemplarRunes = 128 + maxExemplarsPerWrite = 10000 +) + +func (d *histDecoder) bad(what string) error { + return fmt.Errorf("%w: bad histogram %s", errBadWrite, what) +} + +type histDecoder struct { + posDeltas []int64 + negDeltas []int64 + h promql.FloatHistogram + countInt uint64 + zeroInt uint64 + t int64 + intCount bool +} + +func (d *histDecoder) field(num protowire.Number, typ protowire.Type, msg []byte) (int, error) { + switch typ { + case protowire.VarintType: + v, n := protowire.ConsumeVarint(msg) + if n < 0 { + return 0, d.bad("varint") + } + + d.varint(num, v) + + return n, nil + case protowire.Fixed64Type: + v, n := protowire.ConsumeFixed64(msg) + if n < 0 { + return 0, d.bad("double") + } + + d.double(num, math.Float64frombits(v)) + + return n, nil + case protowire.BytesType: + b, n := protowire.ConsumeBytes(msg) + if n < 0 { + return 0, d.bad("bytes") + } + + return n, d.bytes(num, b) + default: + } + + n := protowire.ConsumeFieldValue(num, typ, msg) + if n < 0 { + return 0, d.bad("field") + } + + return n, nil +} + +func (d *histDecoder) varint(num protowire.Number, v uint64) { + switch num { + case fieldHistCountInt: + d.countInt, d.intCount = v, true + case fieldHistZeroInt: + d.zeroInt = v + case fieldHistSchema: + d.h.Schema = int32(protowire.DecodeZigZag(v)) //nolint:gosec // sint32 range + case fieldHistTimestamp: + d.t = int64(v) //nolint:gosec // protobuf int64 round-trips through uint64 + default: + } +} + +func (d *histDecoder) double(num protowire.Number, v float64) { + switch num { + case fieldHistCountFloat: + d.h.Count = v + case fieldHistSum: + d.h.Sum = v + case fieldHistZeroThresh: + d.h.ZeroThreshold = v + case fieldHistZeroFloat: + d.h.ZeroCount = v + default: + } +} + +func (d *histDecoder) bytes(num protowire.Number, b []byte) error { + var err error + + switch num { + case fieldHistNegSpans: + d.h.NegativeSpans, err = appendSpan(d.h.NegativeSpans, b) + case fieldHistPosSpans: + d.h.PositiveSpans, err = appendSpan(d.h.PositiveSpans, b) + case fieldHistNegDeltas: + d.negDeltas, err = packedSint(d.negDeltas, b) + case fieldHistPosDeltas: + d.posDeltas, err = packedSint(d.posDeltas, b) + case fieldHistNegCounts: + d.h.NegativeBuckets, err = packedDouble(d.h.NegativeBuckets, b) + case fieldHistPosCounts: + d.h.PositiveBuckets, err = packedDouble(d.h.PositiveBuckets, b) + default: + } + + return err +} + +func appendSpan(out []promql.Span, b []byte) ([]promql.Span, error) { + var sp promql.Span + + for len(b) > 0 { + num, typ, n := protowire.ConsumeTag(b) + if n < 0 || typ != protowire.VarintType { + return out, fmt.Errorf("%w: bad bucket span", errBadWrite) + } + + v, m := protowire.ConsumeVarint(b[n:]) + if m < 0 { + return out, fmt.Errorf("%w: bad bucket span", errBadWrite) + } + + switch num { + case fieldSpanOffset: + sp.Offset = int32(protowire.DecodeZigZag(v)) //nolint:gosec // sint32 range + case fieldSpanLength: + sp.Length = uint32(v) //nolint:gosec // uint32 field + default: + } + + b = b[n+m:] + } + + return append(out, sp), nil +} + +func packedSint(out []int64, b []byte) ([]int64, error) { + for len(b) > 0 { + v, n := protowire.ConsumeVarint(b) + if n < 0 { + return out, fmt.Errorf("%w: bad bucket deltas", errBadWrite) + } + + out = append(out, protowire.DecodeZigZag(v)) + b = b[n:] + } + + return out, nil +} + +func packedDouble(out []float64, b []byte) ([]float64, error) { + for len(b) > 0 { + v, n := protowire.ConsumeFixed64(b) + if n < 0 { + return out, fmt.Errorf("%w: bad bucket counts", errBadWrite) + } + + out = append(out, math.Float64frombits(v)) + b = b[n:] + } + + return out, nil +} + +func absolute(deltas []int64) []float64 { + out := make([]float64, len(deltas)) + + var cur int64 + + for i, d := range deltas { + cur += d + out[i] = float64(cur) + } + + return out +} + +// decodeHistogram unpacks a remote-write Histogram into a sample carrying a float histogram. +func decodeHistogram(msg []byte) (promql.Sample, error) { + d := &histDecoder{} + + for len(msg) > 0 { + num, typ, n := protowire.ConsumeTag(msg) + if n < 0 { + return promql.Sample{}, d.bad("tag") + } + + m, err := d.field(num, typ, msg[n:]) + if err != nil { + return promql.Sample{}, err + } + + msg = msg[n+m:] + } + + h := d.h + if d.intCount { + h.Count = float64(d.countInt) + h.ZeroCount = float64(d.zeroInt) + h.PositiveBuckets = absolute(d.posDeltas) + h.NegativeBuckets = absolute(d.negDeltas) + } + + if promql.IsStale(h.Sum) { + return promql.Sample{T: d.t, V: promql.StaleNaN()}, nil + } + + if !h.Valid() || !finite(h.Sum, h.Count, h.ZeroCount, h.ZeroThreshold) { + return promql.Sample{}, fmt.Errorf("%w: invalid native histogram (schema %d, span/bucket mismatch)", + errBadWrite, h.Schema) + } + + return promql.Sample{T: d.t, H: &h}, nil +} + +func decodeExemplar(msg []byte) (Exemplar, error) { + ex := Exemplar{Labels: promql.Labels{}} + + for len(msg) > 0 { + num, typ, n := protowire.ConsumeTag(msg) + if n < 0 { + return ex, fmt.Errorf("%w: bad exemplar tag", errBadWrite) + } + + msg = msg[n:] + + var m int + + switch { + case num == fieldExemplarLabels && typ == protowire.BytesType: + sub, k := protowire.ConsumeBytes(msg) + m = k + + if k >= 0 { + name, value, err := decodeLabel(sub) + if err != nil { + return ex, err + } + + ex.Labels[name] = value + } + case num == fieldExemplarValue && typ == protowire.Fixed64Type: + v, k := protowire.ConsumeFixed64(msg) + m, ex.V = k, math.Float64frombits(v) + case num == fieldExemplarTime && typ == protowire.VarintType: + v, k := protowire.ConsumeVarint(msg) + m, ex.T = k, int64(v) //nolint:gosec // protobuf int64 round-trips through uint64 + default: + m = protowire.ConsumeFieldValue(num, typ, msg) + } + + if m < 0 { + return ex, fmt.Errorf("%w: bad exemplar field", errBadWrite) + } + + msg = msg[m:] + } + + return ex, nil +} + +// exemplarRunes is the combined rune length of an exemplar's label names and values. +func exemplarRunes(ex Exemplar) int { + n := 0 + + for k, v := range ex.Labels { + n += len([]rune(k)) + len([]rune(v)) + } + + return n +} + +func finite(vs ...float64) bool { + for _, v := range vs { + if math.IsNaN(v) || math.IsInf(v, 0) { + return false + } + } + + return true +} diff --git a/services/amp/retention.go b/services/amp/retention.go index 51230b0c9f..fc645122fb 100644 --- a/services/amp/retention.go +++ b/services/amp/retention.go @@ -36,6 +36,8 @@ func (b *InMemoryBackend) pruneExpiredLocked(now time.Time) { for _, rec := range slices.Clone(b.seriesByWS.Get(w.ID)) { rec.Samples = slices.DeleteFunc(rec.Samples, func(s promql.Sample) bool { return s.T < cutoff }) + rec.Exemplars = slices.DeleteFunc(rec.Exemplars, func(e Exemplar) bool { return e.T < cutoff }) + if len(rec.Samples) == 0 { b.series.Delete(rec.Key) } diff --git a/services/amp/rules_eval.go b/services/amp/rules_eval.go index cfaacb47fc..8cc9b63944 100644 --- a/services/amp/rules_eval.go +++ b/services/amp/rules_eval.go @@ -218,6 +218,10 @@ func (b *InMemoryBackend) evalRuleLocked(g *groupRuntime, r *ruleRuntime, now, q if err != nil { r.health, r.lastErr = healthErr, err.Error() + b.queueLogLocked(logEvent{ + at: now, wsID: g.wsID, component: logComponentR, stream: logStreamR, level: logLevelError, + msg: "rule evaluation failed", fields: map[string]string{"namespace": g.ns, fieldErr: err.Error()}, + }) } return alerts diff --git a/services/amp/store.go b/services/amp/store.go index 6b41d06e21..31372f2252 100644 --- a/services/amp/store.go +++ b/services/amp/store.go @@ -27,25 +27,29 @@ type ClusterResolver interface { // InMemoryBackend is the AMP state; one coarse lock guards every table. type InMemoryBackend struct { lastPrune time.Time - workspaces *store.Table[Workspace] - namespaces *store.Table[RuleGroupsNamespace] - scrapers *store.Table[Scraper] - detectors *store.Table[AnomalyDetector] - series *store.Table[SeriesRecord] + logs LogSink + keys KeyChecker + publisher SNSPublisher + clusters ClusterResolver seriesByWS *store.Index[SeriesRecord] + scrapers *store.Table[Scraper] metadata *store.Table[MetadataRecord] metadataByWS *store.Index[MetadataRecord] registry *store.Registry mu *lockmetrics.RWMutex now func() time.Time - clusters ClusterResolver + detectors *store.Table[AnomalyDetector] idem *idempotency.Memo - publisher SNSPublisher + series *store.Table[SeriesRecord] + namespaces *store.Table[RuleGroupsNamespace] + workspaces *store.Table[Workspace] + auth *DataPlaneAuth + ams map[string]*alertManager groups map[string]*groupRuntime parsed map[string]*parsedNamespace - ams map[string]*alertManager accountID string region string + logQueue []logEvent dwellOverride time.Duration } diff --git a/services/amp/tick.go b/services/amp/tick.go index 96b9da0839..59f8e7dbf7 100644 --- a/services/amp/tick.go +++ b/services/amp/tick.go @@ -123,9 +123,18 @@ func (b *InMemoryBackend) Tick(ctx context.Context) { } } - for _, m := range ams { - deliverNotifications(ctx, pub, m.Flush(now)) + for wsID, m := range ams { + for _, f := range deliverNotifications(ctx, pub, m.Flush(now)) { + b.mu.Lock("Tick") + b.queueLogLocked(logEvent{ + at: now, wsID: wsID, component: logComponentA, stream: logStreamA, level: logLevelError, + msg: "notification delivery failed", fields: map[string]string{"receiver": f.receiver, fieldErr: f.err}, + }) + b.mu.Unlock() + } } + + b.flushLogs(ctx) } // StartWorker evaluates rules and flushes alert managers every second until ctx ends. diff --git a/services/amp/workspace_docs.go b/services/amp/workspace_docs.go index 2637d834a6..7180cfffb1 100644 --- a/services/amp/workspace_docs.go +++ b/services/amp/workspace_docs.go @@ -146,6 +146,10 @@ func (b *InMemoryBackend) CreateLoggingConfiguration( return wStatus{}, err } + if err := b.checkLogGroup(ctx, logGroup); err != nil { + return wStatus{}, err + } + b.mu.Lock("CreateLoggingConfiguration") defer b.mu.Unlock() @@ -180,6 +184,10 @@ func (b *InMemoryBackend) UpdateLoggingConfiguration(ctx context.Context, wsID, return wStatus{}, err } + if err := b.checkLogGroup(ctx, logGroup); err != nil { + return wStatus{}, err + } + b.mu.Lock("UpdateLoggingConfiguration") defer b.mu.Unlock() diff --git a/services/amp/workspaces.go b/services/amp/workspaces.go index a1ff784f58..1928c54180 100644 --- a/services/amp/workspaces.go +++ b/services/amp/workspaces.go @@ -91,6 +91,10 @@ func (b *InMemoryBackend) CreateWorkspace(ctx context.Context, in CreateWorkspac } } + if err := b.checkKMSKey(ctx, in.KMSKeyARN); err != nil { + return wWorkspace{}, err + } + sc := b.scopeOf(ctx) b.mu.Lock("CreateWorkspace") diff --git a/services/apigateway/client_certificates.go b/services/apigateway/client_certificates.go index 18e9e2b1a4..c17fef6c89 100644 --- a/services/apigateway/client_certificates.go +++ b/services/apigateway/client_certificates.go @@ -21,7 +21,7 @@ func (b *InMemoryBackend) GenerateClientCertificate(input GenerateClientCertific Description: input.Description, PemEncodedCertificate: "-----BEGIN CERTIFICATE-----\nMIICpDCCAYwCCQDU...(mock)...\n-----END CERTIFICATE-----", CreatedDate: unixEpochTime{now}, - ExpirationDate: unixEpochTime{now.AddDate(0, 0, clientCertValidityDays)}, + ExpirationDate: unixEpochTime{now.UTC().AddDate(0, 0, clientCertValidityDays)}, } b.clientCertificates.Put(cert) diff --git a/services/apigatewayv2/handler_apis.go b/services/apigatewayv2/handler_apis.go index 40c08e7fd9..b259682c14 100644 --- a/services/apigatewayv2/handler_apis.go +++ b/services/apigatewayv2/handler_apis.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "net/http" - "net/url" - "sort" "strconv" "strings" @@ -214,32 +212,6 @@ func (h *Handler) handleUpdateAPI(c *echo.Context, apiID string) error { return c.JSON(http.StatusOK, api) } -// openAPISpec is a minimal representation of an OpenAPI 3 / Swagger 2 document. -// Only the fields needed to derive an API name, routes, and integrations are -// modeled; unknown fields are ignored so that minimal specs are tolerated. -type openAPISpec struct { - Paths map[string]map[string]openAPIOperation `json:"paths"` - Info openAPIInfo `json:"info"` - BasePath string `json:"basePath"` // Swagger 2 - Servers []openAPIServer `json:"servers"` // OpenAPI 3 -} - -type openAPIInfo struct { - Title string `json:"title"` -} - -// openAPIServer is the OpenAPI 3 servers[] entry; its URL's path component is -// the OpenAPI-3 equivalent of Swagger 2's top-level basePath. -type openAPIServer struct { - URL string `json:"url"` -} - -const ( - basepathIgnore = "ignore" - basepathPrepend = "prepend" - basepathSplit = "split" -) - // validateBasepath returns ErrBadRequest unless basepath is empty (defaults // to ignore) or one of the three values ImportApiInput/ReimportApiInput // document (api_op_ImportApi.go:37-41, aws-sdk-go-v2/service/apigatewayv2@v1.37.4: @@ -268,33 +240,6 @@ func failOnWarnings(c *echo.Context) (bool, error) { return v, nil } -// importWarnings lists the documented import warnings: HTTP APIs support only Lambda proxy and HTTP proxy -// integrations (docs: Use OpenAPI definitions for HTTP APIs). -func importWarnings(spec *openAPISpec) []string { - var warnings []string - - for path, methods := range spec.Paths { - for method, op := range methods { - if op.Integration == nil { - continue - } - - if t := strings.ToLower(op.Integration.Type); t != "aws_proxy" && t != "http_proxy" { - warnings = append(warnings, fmt.Sprintf( - "%s %s: integration type %q is not supported for HTTP APIs", - strings.ToUpper(method), - path, - op.Integration.Type, - )) - } - } - } - - sort.Strings(warnings) - - return warnings -} - // checkFailOnWarnings rejects the import with BadRequestException when failOnWarnings is set and spec warns. func checkFailOnWarnings(c *echo.Context, spec *openAPISpec) error { fail, err := failOnWarnings(c) @@ -309,147 +254,6 @@ func checkFailOnWarnings(c *echo.Context, spec *openAPISpec) error { return nil } -// splitBasePath drops the top-most segment of a base path ("/a/b/c" -> "/b/c"), per the -// split option in docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-import-api-basePath.html. -func splitBasePath(basePath string) string { - trimmed := strings.Trim(basePath, "/") - - _, rest, found := strings.Cut(trimmed, "/") - if !found { - return "" - } - - return "/" + rest -} - -// specBasePath returns the OpenAPI document's declared base path: Swagger 2's -// top-level basePath, or the path component of OpenAPI 3's first servers[] -// entry. Returns "" if neither is present or the server URL doesn't parse. -func specBasePath(spec *openAPISpec) string { - if spec.BasePath != "" { - return spec.BasePath - } - - if len(spec.Servers) == 0 { - return "" - } - - u, err := url.Parse(spec.Servers[0].URL) - if err != nil { - return "" - } - - return u.Path -} - -type openAPIOperation struct { - Integration *openAPIIntegration `json:"x-amazon-apigateway-integration"` -} - -// openAPIIntegration models the x-amazon-apigateway-integration extension. -type openAPIIntegration struct { - Type string `json:"type"` - HTTPMethod string `json:"httpMethod"` - URI string `json:"uri"` - PayloadFormatVersion string `json:"payloadFormatVersion"` - ConnectionType string `json:"connectionType"` - TimeoutInMillis int32 `json:"timeoutInMillis"` -} - -// parseOpenAPISpec decodes an OpenAPI body into an openAPISpec, tolerating -// minimal/partial documents. -func parseOpenAPISpec(body string) (*openAPISpec, error) { - var spec openAPISpec - if strings.TrimSpace(body) == "" { - return &spec, nil - } - - if err := json.Unmarshal([]byte(body), &spec); err != nil { - return nil, err - } - - return &spec, nil -} - -// applyOpenAPIToAPI creates a route (and integration, when defined) for each path+method in the -// spec. Entries that are not valid HTTP route keys are skipped. basepath "prepend" prefixes the -// document's base path onto every route path; "split" prefixes all but its top-most segment. -func (h *Handler) applyOpenAPIToAPI(apiID string, spec *openAPISpec, basepath string) { - prefix := "" - - switch basepath { - case basepathPrepend: - prefix = strings.TrimSuffix(specBasePath(spec), "/") - case basepathSplit: - prefix = splitBasePath(specBasePath(spec)) - } - - for path, methods := range spec.Paths { - if !strings.HasPrefix(path, "/") { - path = "/" + path - } - - if prefix != "" { - path = prefix + path - } - - for method, op := range methods { - h.applyOpenAPIOperation(apiID, path, method, op) - } - } -} - -// applyOpenAPIOperation creates the route (and integration, when defined) for -// a single path+method entry from an imported OpenAPI spec. Route keys the -// backend would reject (e.g. spec-level fields like "parameters" that share -// the path map) are skipped gracefully. -func (h *Handler) applyOpenAPIOperation(apiID, path, method string, op openAPIOperation) { - routeKey := strings.ToUpper(method) + " " + path - if err := validateHTTPRouteKey(routeKey); err != nil { - return - } - - var target string - - if op.Integration != nil { - integ, err := h.Backend.CreateIntegration(apiID, CreateIntegrationInput{ - IntegrationType: openAPIIntegrationType(op.Integration.Type), - IntegrationMethod: op.Integration.HTTPMethod, - IntegrationURI: op.Integration.URI, - PayloadFormatVersion: op.Integration.PayloadFormatVersion, - ConnectionType: op.Integration.ConnectionType, - TimeoutInMillis: op.Integration.TimeoutInMillis, - }) - if err == nil && integ != nil { - target = "integrations/" + integ.IntegrationID - } - } - - if _, err := h.Backend.CreateRoute(apiID, CreateRouteInput{ - RouteKey: routeKey, - Target: target, - }); err != nil { - return - } -} - -// openAPIIntegrationType maps an OpenAPI x-amazon-apigateway-integration -// "type" extension value to the API Gateway v2 IntegrationType wire value. -func openAPIIntegrationType(t string) string { - switch t { - case "http", "http_proxy": - return integrationTypeHTTPProxy - case "aws", "aws_proxy": - return IntegrationTypeAWSProxy - case "mock": - return integrationTypeMock - case "": - return IntegrationTypeAWSProxy - default: - return strings.ToUpper(t) - } -} - func (h *Handler) handleImportAPI(c *echo.Context) error { // basepath and failOnWarnings are HTTP query-string params on the real // ImportApiInput, not JSON body fields (api_op_ImportApi.go:37-45, @@ -492,7 +296,9 @@ func (h *Handler) handleImportAPI(c *echo.Context) error { return writeErr(c, http.StatusInternalServerError, err.Error()) } - h.applyOpenAPIToAPI(api.APIID, spec, basepath) + if err = applyOpenAPISpec(h.Backend, api.APIID, spec, basepath); err != nil { + return writeErr(c, http.StatusBadRequest, err.Error()) + } return c.JSON(http.StatusCreated, api) } @@ -548,7 +354,9 @@ func (h *Handler) handleReimportAPI(c *echo.Context, apiID string) error { return writeErr(c, http.StatusInternalServerError, err.Error()) } - h.applyOpenAPIToAPI(apiID, spec, basepath) + if err = applyOpenAPISpec(h.Backend, apiID, spec, basepath); err != nil { + return writeErr(c, http.StatusBadRequest, err.Error()) + } return c.JSON(http.StatusCreated, api) } diff --git a/services/apigatewayv2/openapi_import.go b/services/apigatewayv2/openapi_import.go new file mode 100644 index 0000000000..6798276ca2 --- /dev/null +++ b/services/apigatewayv2/openapi_import.go @@ -0,0 +1,496 @@ +package apigatewayv2 + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "slices" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +const ( + basepathIgnore = "ignore" + basepathPrepend = "prepend" + basepathSplit = "split" + + oaAnyMethod = "x-amazon-apigateway-any-method" + oaAuthTypeIAM = "awsSigv4" + authTypeNone = "NONE" + authTypeIAM = "AWS_IAM" + authTypeCustom = "CUSTOM" +) + +// openAPISpec is the subset of an OpenAPI 3 / Swagger 2 document an HTTP API import reads. +type openAPISpec struct { + Paths map[string]map[string]json.RawMessage `json:"paths"` + Components struct { + SecuritySchemes map[string]openAPISecurityScheme `json:"securitySchemes"` + } `json:"components"` + Cors *openAPICors `json:"x-amazon-apigateway-cors"` + Info openAPIInfo `json:"info"` + BasePath string `json:"basePath"` + Servers []openAPIServer `json:"servers"` + Tags []openAPITag `json:"tags"` +} + +type openAPIInfo struct { + Title string `json:"title"` + Description string `json:"description"` + Version string `json:"version"` +} + +type openAPIServer struct { + Endpoint *struct { + DisableExecuteAPIEndpoint *bool `json:"disableExecuteApiEndpoint"` + } `json:"x-amazon-apigateway-endpoint-configuration"` + URL string `json:"url"` +} + +type openAPITag struct { + Name string `json:"name"` + Value string `json:"x-amazon-apigateway-tag-value"` +} + +type openAPICors struct { + AllowOrigins []string `json:"allowOrigins"` + AllowMethods []string `json:"allowMethods"` + AllowHeaders []string `json:"allowHeaders"` + ExposeHeaders []string `json:"exposeHeaders"` + MaxAge int32 `json:"maxAge"` + AllowCredentials bool `json:"allowCredentials"` +} + +type openAPISecurityScheme struct { + Authorizer *openAPIAuthorizer `json:"x-amazon-apigateway-authorizer"` + Type string `json:"type"` + AuthType string `json:"x-amazon-apigateway-authtype"` +} + +type openAPIAuthorizer struct { + JwtConfiguration *JwtConfiguration `json:"jwtConfiguration"` + TTL *int32 `json:"authorizerResultTtlInSeconds"` + IdentitySource json.RawMessage `json:"identitySource"` + Type string `json:"type"` + URI string `json:"authorizerUri"` + Credentials string `json:"authorizerCredentials"` + PayloadFormat json.RawMessage `json:"authorizerPayloadFormatVersion"` + SimpleResponses bool `json:"enableSimpleResponses"` +} + +type openAPIOperation struct { + Integration *openAPIIntegration `json:"x-amazon-apigateway-integration"` + OperationID string `json:"operationId"` + Security []map[string][]string `json:"security"` + IsDefaultRoute bool `json:"isDefaultRoute"` +} + +// openAPIIntegration models the x-amazon-apigateway-integration extension. +type openAPIIntegration struct { + RequestParameters map[string]string `json:"requestParameters"` + Type string `json:"type"` + HTTPMethod string `json:"httpMethod"` + URI string `json:"uri"` + ConnectionType string `json:"connectionType"` + ConnectionID string `json:"connectionId"` + Credentials string `json:"credentials"` + Description string `json:"description"` + PayloadFormatVersion json.RawMessage `json:"payloadFormatVersion"` + TimeoutInMillis int32 `json:"timeoutInMillis"` +} + +// ErrOpenAPIInvalid is returned for an unparsable OpenAPI document. +var ErrOpenAPIInvalid = errors.New("invalid OpenAPI document") + +// parseOpenAPISpec decodes a JSON or YAML OpenAPI body, tolerating minimal/partial documents. +func parseOpenAPISpec(body string) (*openAPISpec, error) { + var spec openAPISpec + if strings.TrimSpace(body) == "" { + return &spec, nil + } + + if !strings.HasPrefix(strings.TrimSpace(body), "{") { + var doc any + if err := yaml.Unmarshal([]byte(body), &doc); err != nil { + return nil, err + } + + raw, err := json.Marshal(doc) + if err != nil { + return nil, err + } + + body = string(raw) + } + + if err := json.Unmarshal([]byte(body), &spec); err != nil { + return nil, err + } + + return &spec, nil +} + +func (s *openAPISpec) operation(path, method string) (openAPIOperation, bool) { + raw := s.Paths[path][method] + + var op openAPIOperation + + if len(raw) == 0 || raw[0] != '{' || json.Unmarshal(raw, &op) != nil { + return openAPIOperation{}, false + } + + return op, true +} + +func (s *openAPISpec) sortedPaths() []string { + paths := make([]string, 0, len(s.Paths)) + for p := range s.Paths { + paths = append(paths, p) + } + + sort.Strings(paths) + + return paths +} + +// importWarnings lists the documented import warnings: HTTP APIs support only Lambda proxy and HTTP proxy +// integrations (docs: Use OpenAPI definitions for HTTP APIs). +func importWarnings(spec *openAPISpec) []string { + var warnings []string + + for path, methods := range spec.Paths { + for method := range methods { + op, ok := spec.operation(path, method) + if !ok || op.Integration == nil { + continue + } + + if t := strings.ToLower(op.Integration.Type); t != "aws_proxy" && t != "http_proxy" { + warnings = append(warnings, fmt.Sprintf( + "%s %s: integration type %q is not supported for HTTP APIs", + strings.ToUpper(method), path, op.Integration.Type, + )) + } + } + } + + sort.Strings(warnings) + + return warnings +} + +// splitBasePath drops the top-most segment of a base path ("/a/b/c" -> "/b/c"), per the +// split option in docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-import-api-basePath.html. +func splitBasePath(basePath string) string { + trimmed := strings.Trim(basePath, "/") + + _, rest, found := strings.Cut(trimmed, "/") + if !found { + return "" + } + + return "/" + rest +} + +// specBasePath returns the document's declared base path: Swagger 2's basePath, or the path of +// OpenAPI 3's first servers[] entry. +func specBasePath(spec *openAPISpec) string { + if spec.BasePath != "" { + return spec.BasePath + } + + if len(spec.Servers) == 0 { + return "" + } + + u, err := url.Parse(spec.Servers[0].URL) + if err != nil { + return "" + } + + return u.Path +} + +// OpenAPIImportWarnings lists the warnings an import of body would report (FailOnWarnings rejects them). +func OpenAPIImportWarnings(body string) ([]string, error) { + spec, err := parseOpenAPISpec(body) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrOpenAPIInvalid, err) + } + + return importWarnings(spec), nil +} + +// ApplyOpenAPI imports an OpenAPI document into an existing HTTP API: API metadata, CORS, tags, +// authorizers, and a route (plus integration and authorization) per path and method. +func ApplyOpenAPI(b StorageBackend, apiID, body, basepath string) error { + spec, err := parseOpenAPISpec(body) + if err != nil { + return fmt.Errorf("%w: %w", ErrOpenAPIInvalid, err) + } + + return applyOpenAPISpec(b, apiID, spec, basepath) +} + +func applyOpenAPISpec(b StorageBackend, apiID string, spec *openAPISpec, basepath string) error { + if err := applyOpenAPIMetadata(b, apiID, spec); err != nil { + return err + } + + authorizers, err := importAuthorizers(b, apiID, spec) + if err != nil { + return err + } + + prefix := "" + + switch basepath { + case basepathPrepend: + prefix = strings.TrimSuffix(specBasePath(spec), "/") + case basepathSplit: + prefix = splitBasePath(specBasePath(spec)) + } + + for _, path := range spec.sortedPaths() { + methods := slices.Sorted(mapKeys(spec.Paths[path])) + for _, method := range methods { + op, ok := spec.operation(path, method) + if !ok { + continue + } + + importOperation(b, apiID, prefix, path, method, op, authorizers) + } + } + + return nil +} + +func mapKeys[V any](m map[string]V) func(yield func(string) bool) { + return func(yield func(string) bool) { + for k := range m { + if !yield(k) { + return + } + } + } +} + +func applyOpenAPIMetadata(b StorageBackend, apiID string, spec *openAPISpec) error { + update := UpdateAPIInput{ + Name: spec.Info.Title, + Description: spec.Info.Description, + Version: spec.Info.Version, + } + + if spec.Cors != nil { + update.CorsConfiguration = &CorsConfiguration{ + AllowOrigins: spec.Cors.AllowOrigins, AllowMethods: spec.Cors.AllowMethods, + AllowHeaders: spec.Cors.AllowHeaders, ExposeHeaders: spec.Cors.ExposeHeaders, + MaxAge: spec.Cors.MaxAge, AllowCredentials: spec.Cors.AllowCredentials, + } + } + + if len(spec.Tags) > 0 { + update.Tags = map[string]string{} + + for _, t := range spec.Tags { + update.Tags[t.Name] = t.Value + } + } + + for _, s := range spec.Servers { + if s.Endpoint != nil && s.Endpoint.DisableExecuteAPIEndpoint != nil { + update.DisableExecuteAPIEndpoint = s.Endpoint.DisableExecuteAPIEndpoint + } + } + + if _, err := b.UpdateAPI(apiID, update); err != nil { + return err + } + + return nil +} + +// importedAuthorizer is what a route needs to reference a security scheme. +type importedAuthorizer struct { + id string + authType string +} + +func importAuthorizers(b StorageBackend, apiID string, spec *openAPISpec) (map[string]importedAuthorizer, error) { + out := map[string]importedAuthorizer{} + + names := slices.Sorted(mapKeys(spec.Components.SecuritySchemes)) + for _, name := range names { + scheme := spec.Components.SecuritySchemes[name] + + if scheme.AuthType == oaAuthTypeIAM { + out[name] = importedAuthorizer{authType: authTypeIAM} + + continue + } + + if scheme.Authorizer == nil { + continue + } + + in := authorizerInput(name, scheme.Authorizer) + + a, err := b.CreateAuthorizer(apiID, in) + if err != nil { + return nil, err + } + + kind := authTypeCustom + if in.AuthorizerType == authorizerTypeJWT { + kind = authorizerTypeJWT + } + + out[name] = importedAuthorizer{id: a.AuthorizerID, authType: kind} + } + + return out, nil +} + +func authorizerInput(name string, az *openAPIAuthorizer) CreateAuthorizerInput { + in := CreateAuthorizerInput{Name: name, AuthorizerType: "REQUEST", AuthorizerURI: az.URI, + AuthorizerCredentialsArn: az.Credentials, EnableSimpleResponses: az.SimpleResponses} + if strings.EqualFold(az.Type, "jwt") { + in.AuthorizerType = authorizerTypeJWT + in.JwtConfiguration = az.JwtConfiguration + } + + if az.TTL != nil { + in.AuthorizerResultTTLInSeconds = *az.TTL + } + + in.AuthorizerPayloadFormatVersion = rawScalar(az.PayloadFormat) + in.IdentitySource = identitySources(az.IdentitySource) + + return in +} + +func rawScalar(raw json.RawMessage) string { + var s string + if json.Unmarshal(raw, &s) == nil { + return s + } + + var f float64 + if json.Unmarshal(raw, &f) == nil { + return fmt.Sprintf("%.1f", f) + } + + return "" +} + +func identitySources(raw json.RawMessage) []string { + var list []string + if json.Unmarshal(raw, &list) == nil { + return list + } + + var one string + if json.Unmarshal(raw, &one) == nil && one != "" { + return strings.Split(one, ",") + } + + return nil +} + +// importOperation creates the route (and integration and authorization, when defined) for one +// path+method entry. Route keys the backend rejects (e.g. path-level "parameters") are skipped. +func importOperation( + b StorageBackend, apiID, prefix, path, method string, op openAPIOperation, auths map[string]importedAuthorizer, +) { + routeKey := openAPIRouteKey(prefix, path, method, op) + if err := validateHTTPRouteKey(routeKey); err != nil { + return + } + + in := CreateRouteInput{RouteKey: routeKey, OperationName: op.OperationID} + + if op.Integration != nil { + integ, err := b.CreateIntegration(apiID, integrationInput(op.Integration)) + if err == nil && integ != nil { + in.Target = "integrations/" + integ.IntegrationID + } + } + + applyOperationSecurity(&in, op, auths) + + _, _ = b.CreateRoute(apiID, in) +} + +func openAPIRouteKey(prefix, path, method string, op openAPIOperation) string { + if op.IsDefaultRoute { + return routeKeyDefault + } + + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + verb := strings.ToUpper(method) + if method == oaAnyMethod { + verb = "ANY" + } + + return verb + " " + prefix + path +} + +func integrationInput(i *openAPIIntegration) CreateIntegrationInput { + return CreateIntegrationInput{ + IntegrationType: openAPIIntegrationType(i.Type), + IntegrationMethod: i.HTTPMethod, + IntegrationURI: i.URI, + PayloadFormatVersion: rawScalar(i.PayloadFormatVersion), + ConnectionType: i.ConnectionType, + ConnectionID: i.ConnectionID, + CredentialsArn: i.Credentials, + RequestParameters: i.RequestParameters, + Description: i.Description, + TimeoutInMillis: i.TimeoutInMillis, + } +} + +func applyOperationSecurity(in *CreateRouteInput, op openAPIOperation, auths map[string]importedAuthorizer) { + for _, req := range op.Security { + for name, scopes := range req { + if name == authTypeNone { + in.AuthorizationType = authTypeNone + + return + } + + a, ok := auths[name] + if !ok { + continue + } + + in.AuthorizationType, in.AuthorizerID, in.AuthorizationScopes = a.authType, a.id, scopes + + return + } + } +} + +// openAPIIntegrationType maps an x-amazon-apigateway-integration "type" to the API Gateway v2 IntegrationType. +func openAPIIntegrationType(t string) string { + switch t { + case "http", "http_proxy": + return integrationTypeHTTPProxy + case "aws", "aws_proxy": + return IntegrationTypeAWSProxy + case "mock": + return integrationTypeMock + case "": + return IntegrationTypeAWSProxy + default: + return strings.ToUpper(t) + } +} diff --git a/services/apigatewayv2/openapi_import_test.go b/services/apigatewayv2/openapi_import_test.go new file mode 100644 index 0000000000..cec76a3af8 --- /dev/null +++ b/services/apigatewayv2/openapi_import_test.go @@ -0,0 +1,136 @@ +package apigatewayv2_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/apigatewayv2" +) + +const openAPIAuthSpec = `{ + "openapi": "3.0.1", + "info": {"title": "Pets", "description": "pet store", "version": "7"}, + "x-amazon-apigateway-cors": {"allowOrigins": ["https://a.example"], "maxAge": 60}, + "tags": [{"name": "team", "x-amazon-apigateway-tag-value": "core"}], + "servers": [{"x-amazon-apigateway-endpoint-configuration": {"disableExecuteApiEndpoint": true}}], + "components": {"securitySchemes": { + "Jwt": {"type": "oauth2", "x-amazon-apigateway-authorizer": {"type": "jwt", + "identitySource": "$request.header.Authorization", + "jwtConfiguration": {"issuer": "https://iss.example/", "audience": ["aud"]}}}, + "Req": {"type": "apiKey", "name": "Unused", "in": "header", "x-amazon-apigateway-authorizer": { + "type": "request", + "authorizerUri": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn/invocations", + "authorizerPayloadFormatVersion": 2.0, "enableSimpleResponses": true, "authorizerResultTtlInSeconds": 30, + "identitySource": ["$request.header.X-Token", "$request.querystring.q"]}}, + "Iam": {"type": "apiKey", "name": "Authorization", "in": "header", + "x-amazon-apigateway-authtype": "awsSigv4"} + }}, + "paths": { + "/jwt": {"get": {"security": [{"Jwt": ["read", "write"]}], "x-amazon-apigateway-integration": { + "type": "http_proxy", "httpMethod": "GET", "uri": "https://example.com/jwt", + "payloadFormatVersion": "1.0", "timeoutInMillis": 5000}}}, + "/req": {"get": {"security": [{"Req": []}], "x-amazon-apigateway-integration": { + "type": "http_proxy", "httpMethod": "GET", "uri": "https://example.com/req"}}}, + "/iam": {"post": {"security": [{"Iam": []}], "x-amazon-apigateway-integration": { + "type": "http_proxy", "httpMethod": "POST", "uri": "https://example.com/iam"}}}, + "/open": {"get": {"security": [{"NONE": []}], "x-amazon-apigateway-integration": { + "type": "http_proxy", "httpMethod": "GET", "uri": "https://example.com/open"}}}, + "$default": {"x-amazon-apigateway-any-method": {"isDefaultRoute": true, "x-amazon-apigateway-integration": { + "type": "http_proxy", "httpMethod": "ANY", "uri": "https://example.com/"}}}, + "/any": {"x-amazon-apigateway-any-method": {"x-amazon-apigateway-integration": { + "type": "http_proxy", "httpMethod": "ANY", "uri": "https://example.com/any"}}, "parameters": []} + } +}` + +func TestApplyOpenAPI_AuthCorsTagsAndRoutes(t *testing.T) { + t.Parallel() + + b := apigatewayv2.NewInMemoryBackend() + api, err := b.CreateAPI(t.Context(), apigatewayv2.CreateAPIInput{Name: "x", ProtocolType: "HTTP"}) + require.NoError(t, err) + + require.NoError(t, apigatewayv2.ApplyOpenAPI(b, api.APIID, openAPIAuthSpec, "")) + + got, err := b.GetAPI(api.APIID) + require.NoError(t, err) + assert.Equal(t, "Pets", got.Name) + assert.Equal(t, "pet store", got.Description) + assert.Equal(t, "7", got.Version) + assert.True(t, got.DisableExecuteAPIEndpoint) + assert.Equal(t, map[string]string{"team": "core"}, got.Tags) + require.NotNil(t, got.CorsConfiguration) + assert.Equal(t, []string{"https://a.example"}, got.CorsConfiguration.AllowOrigins) + assert.EqualValues(t, 60, got.CorsConfiguration.MaxAge) + + auths, err := b.GetAuthorizers(api.APIID) + require.NoError(t, err) + byName := map[string]apigatewayv2.Authorizer{} + for _, a := range auths { + byName[a.Name] = a + } + require.Len(t, byName, 2, "the awsSigv4 scheme is not an authorizer") + assert.Equal(t, "JWT", byName["Jwt"].AuthorizerType) + assert.Equal(t, []string{"$request.header.Authorization"}, byName["Jwt"].IdentitySource) + assert.Equal(t, "REQUEST", byName["Req"].AuthorizerType) + assert.Equal(t, "2.0", byName["Req"].AuthorizerPayloadFormatVersion) + assert.True(t, byName["Req"].EnableSimpleResponses) + assert.EqualValues(t, 30, byName["Req"].AuthorizerResultTTLInSeconds) + assert.Equal(t, []string{"$request.header.X-Token", "$request.querystring.q"}, byName["Req"].IdentitySource) + + routes, err := b.GetRoutes(api.APIID) + require.NoError(t, err) + byKey := map[string]apigatewayv2.Route{} + for _, r := range routes { + byKey[r.RouteKey] = r + } + require.Len(t, byKey, 6) + + jwt := byKey["GET /jwt"] + assert.Equal(t, "JWT", jwt.AuthorizationType) + assert.Equal(t, byName["Jwt"].AuthorizerID, jwt.AuthorizerID) + assert.Equal(t, []string{"read", "write"}, jwt.AuthorizationScopes) + assert.Equal(t, "CUSTOM", byKey["GET /req"].AuthorizationType) + assert.Equal(t, "AWS_IAM", byKey["POST /iam"].AuthorizationType) + assert.Equal(t, "NONE", byKey["GET /open"].AuthorizationType) + assert.Contains(t, byKey, "$default") + assert.Contains(t, byKey, "ANY /any") + + integ, err := b.GetIntegration(api.APIID, jwt.Target[len("integrations/"):]) + require.NoError(t, err) + assert.Equal(t, "HTTP_PROXY", integ.IntegrationType) + assert.Equal(t, "1.0", integ.PayloadFormatVersion) + assert.EqualValues(t, 5000, integ.TimeoutInMillis) +} + +func TestApplyOpenAPI_YAMLBody(t *testing.T) { + t.Parallel() + + b := apigatewayv2.NewInMemoryBackend() + api, err := b.CreateAPI(t.Context(), apigatewayv2.CreateAPIInput{Name: "x", ProtocolType: "HTTP"}) + require.NoError(t, err) + + spec := "openapi: 3.0.1\ninfo:\n title: FromYAML\npaths:\n /a:\n get:\n x-amazon-apigateway-integration:\n" + + " type: http_proxy\n httpMethod: GET\n uri: https://example.com/a\n" + require.NoError(t, apigatewayv2.ApplyOpenAPI(b, api.APIID, spec, "")) + + got, err := b.GetAPI(api.APIID) + require.NoError(t, err) + assert.Equal(t, "FromYAML", got.Name) + routes, err := b.GetRoutes(api.APIID) + require.NoError(t, err) + require.Len(t, routes, 1) + assert.Equal(t, "GET /a", routes[0].RouteKey) +} + +func TestApplyOpenAPI_InvalidDocument(t *testing.T) { + t.Parallel() + + b := apigatewayv2.NewInMemoryBackend() + api, err := b.CreateAPI(t.Context(), apigatewayv2.CreateAPIInput{Name: "x", ProtocolType: "HTTP"}) + require.NoError(t, err) + + err = apigatewayv2.ApplyOpenAPI(b, api.APIID, "{not json", "") + require.ErrorIs(t, err, apigatewayv2.ErrOpenAPIInvalid) +} diff --git a/services/appconfig/deployments.go b/services/appconfig/deployments.go index 436ccf5a9e..07d4c00340 100644 --- a/services/appconfig/deployments.go +++ b/services/appconfig/deployments.go @@ -394,12 +394,12 @@ func (b *InMemoryBackend) scheduleDeploymentReconcilerLocked() { defer ticker.Stop() for { - <-ticker.C + now := <-ticker.C - b.runDeploymentTicks() + b.runDeploymentTicks(now) b.mu.Lock("deploymentReconcile") - b.reconcileDeploymentsLocked() + b.reconcileDeploymentsLocked(now) done := len(b.deploymentTimers) == 0 b.mu.Unlock() @@ -413,9 +413,7 @@ func (b *InMemoryBackend) scheduleDeploymentReconcilerLocked() { // reconcileDeploymentsLocked advances every in-flight deployment whose // timer has elapsed by one progression step. Must be called under lock. -func (b *InMemoryBackend) reconcileDeploymentsLocked() { - now := time.Now() - +func (b *InMemoryBackend) reconcileDeploymentsLocked(now time.Time) { for key, timer := range b.deploymentTimers { if now.Before(timer.nextAt) { continue diff --git a/services/appconfig/extension_events.go b/services/appconfig/extension_events.go index f8e69e7ba6..55762969fe 100644 --- a/services/appconfig/extension_events.go +++ b/services/appconfig/extension_events.go @@ -349,9 +349,7 @@ type tickJob struct { } // collectTicksLocked gathers AT_DEPLOYMENT_TICK invocations for deployments whose timer is due. -func (b *InMemoryBackend) collectTicksLocked() []tickJob { - now := time.Now() - +func (b *InMemoryBackend) collectTicksLocked(now time.Time) []tickJob { var jobs []tickJob for key, timer := range b.deploymentTimers { @@ -384,10 +382,11 @@ func (b *InMemoryBackend) collectTicksLocked() []tickJob { } // runDeploymentTicks invokes AT_DEPLOYMENT_TICK actions synchronously and rolls a deployment back when an -// action errors or answers Directive ROLL_BACK. -func (b *InMemoryBackend) runDeploymentTicks() { +// action errors or answers Directive ROLL_BACK. now is shared with the reconcile pass so no deployment +// advances before its tick. +func (b *InMemoryBackend) runDeploymentTicks(now time.Time) { b.mu.RLock("collectTicks") - jobs := b.collectTicksLocked() + jobs := b.collectTicksLocked(now) b.mu.RUnlock() for _, job := range jobs { diff --git a/services/appconfig/whitebox_test.go b/services/appconfig/whitebox_test.go index c29886a6d9..2e812bd586 100644 --- a/services/appconfig/whitebox_test.go +++ b/services/appconfig/whitebox_test.go @@ -188,3 +188,36 @@ func TestDeploymentTimers_DrainToZero(t *testing.T) { assert.Equal(t, 0, deploymentTimerCount(), "every deployment timer must drain once its deployment completes") }) } + +func TestReconcileDeployments_AdvancesOnlyDueAtSharedNow(t *testing.T) { + t.Parallel() + + b := NewInMemoryBackend("000000000000", "us-east-1") + appID, envID, profileID, _ := seedDeployableConfig(t, b, []byte("v")) + + strategy, err := b.CreateDeploymentStrategy("slow", "", 5, 0, 50, "LINEAR", "NONE", nil) + require.NoError(t, err) + + _, err = b.StartDeployment(appID, envID, profileID, strategy.ID, "1", "", nil, nil, nil) + require.NoError(t, err) + + b.mu.Lock("test.reconcile") + defer b.mu.Unlock() + + require.Len(t, b.deploymentTimers, 1) + + var due time.Time + + for _, timer := range b.deploymentTimers { + due = timer.nextAt + } + + b.reconcileDeploymentsLocked(due.Add(-time.Nanosecond)) + + d, ok := b.deployments.Get(deploymentKey(appID, envID, 1)) + require.True(t, ok) + assert.Zero(t, d.PercentageComplete) + + b.reconcileDeploymentsLocked(due) + assert.InDelta(t, 50, d.PercentageComplete, 0) +} diff --git a/services/appsync/api_keys.go b/services/appsync/api_keys.go index 811b6e157a..226dda2e5a 100644 --- a/services/appsync/api_keys.go +++ b/services/appsync/api_keys.go @@ -56,7 +56,7 @@ func randomAPIKeyID() string { // documented 1-365-day bounds, returning ErrAPIKeyValidityOutOfBounds if it // falls outside them. func resolveAPIKeyExpiry(expires int64) (int64, error) { - now := time.Now() + now := time.Now().UTC() if expires <= 0 { return now.AddDate(0, 0, defaultAPIKeyExpiryDays).Unix(), nil diff --git a/services/athena/PARITY.md b/services/athena/PARITY.md index e0de172ac6..e6d3de9f50 100644 --- a/services/athena/PARITY.md +++ b/services/athena/PARITY.md @@ -55,18 +55,18 @@ families: janitor/leaks: {status: clean, note: "worker.Group-based ticker with ctx cancellation; sweeps queryExecutions+queryResults, sessions, calculations under RLock-collect/Lock-delete with re-verification to avoid racing a concurrent revival. No goroutine leak risk found."} gaps: [] items_still_open: - - "Iceberg: DELETE, UPDATE, MERGE, OPTIMIZE, VACUUM, CTAS with table_type='ICEBERG', ALTER TABLE schema/partition evolution, equality deletes, ORC/Avro data files, writing nested (array/map/struct) columns, the $partitions metadata table, partition pruning from manifest bounds (all data files are read), timestamp precision above milliseconds." + - "Iceberg: equality deletes, ORC data files, writing nested (array/map/struct) columns, the $partitions metadata table, partition pruning from manifest bounds (all data files are read), timestamp precision above milliseconds, OPTIMIZE output is one file per partition (no write_target_data_file_size_bytes splitting), VACUUM/OPTIMIZE/MERGE error wording (MERGE_TARGET_ROW_MULTIPLE_MATCHES, the non-transactional Hive DELETE message) is UNVERIFIED against live Athena, DML conflicts fail instead of retrying, and the written manifests/delete files are checked only by this emulator's reader and spec-driven unit tests (no third-party Iceberg reader is available offline)." - "S3 Tables catalog: CREATE/DROP TABLE and CTAS through s3tablescatalog/ are rejected (use the S3 Tables API); ListDatabases/GetTableMetadata/ListTableMetadata do not enumerate s3tablescatalog tables; ROW-column dot access is not resolved on S3 Tables tables at plan time." - "Delta Lake: deletion vectors and reader versions above 3 fail NOT_SUPPORTED; id column-mapping and timestampNtz reads are untested against real writers." - - "ORC/Avro input and output formats: no pure-Go ORC or Avro library in go.mod (only parquet-go), so these stay NOT_SUPPORTED until a dependency is approved." - - "Complex-type results: Boolean expression results render 1/0 rather than true/false (SQLite has no boolean type); ROW(...) constructors, CAST(... AS ROW(...)) and anonymous rows are unsupported; map and row expression results both report ColumnInfo type map; integral doubles inside arrays render without the .0 suffix; col.field dereference works on ROW columns of a referenced table only (not CTE/subquery outputs or lambda parameters)." + - "ORC input and output formats: no maintained pure-Go ORC library exists (scritchley/orc was last updated 2021 and is read-only), so ORC SELECT/CTAS/UNLOAD stay NOT_SUPPORTED. Avro tables with avro.schema.literal/url and no Glue columns are not derived." + - "Complex-type results: ROW(...) constructors, CAST(... AS ROW(...)) and anonymous rows are unsupported; map and row expression results both report ColumnInfo type map; integral doubles inside arrays render without the .0 suffix; col.field dereference works on ROW columns of a referenced table only (not CTE/subquery outputs or lambda parameters); boolean result typing is inferred syntactically from the outermost SELECT list (comparisons, logical operators, IS/IN/BETWEEN/LIKE, boolean literals and functions, CASE/COALESCE/IF over those, CAST AS boolean, aliases through CTEs and subqueries) so an unlisted boolean-returning function renders 1/0; SELECT * after such an expression disables the inference." - "Lambda functions cover transform, filter, reduce, any_match/all_match/none_match, map_filter, transform_values, transform_keys and zip_with; array_sort(arr, comparator), map_zip_with, reduce_agg and zip_with over more than two arrays are not translated." - "INTERVAL literals are supported only as the right operand of + or - (INTERVAL n UNIT, YEAR TO MONTH, 'D HH:MM:SS' DAY TO SECOND) via date_add semantics with end-of-month clamping; timestamp - timestamp interval results, standalone INTERVAL values/comparisons and fractional-second intervals are unsupported." - "GROUP BY ROLLUP/CUBE/GROUPING SETS are expanded to UNION ALL of plain GROUP BY branches: not combinable with UNION/INTERSECT/EXCEPT in the same query block, and the outer ORDER BY may reference only output columns or ordinals." - "Partition projection (enum/integer/date/injected) enumerates partitions from table properties and prunes by simple WHERE conjuncts (=, IN, BETWEEN, comparisons; OR or non-literal predicates disable pruning); the INVALID_TABLE_PROPERTY/EXCEEDED_LIMIT failure wording for a missing injected filter or oversized enumeration is modeled, UNVERIFIED against live Athena; SHOW PARTITIONS does not list projected partitions." - "QueryExecution.SubstatementType is not modeled: the pinned SDK documents it only as \"The kind of query statement that was run\" with no enum or value list, so the vocabulary cannot be verified." structural_gaps: - - "ORC and Avro tables cannot be read or written (go.mod carries only parquet-go; no pure-Go ORC/Avro reader): SELECT fails NOT_SUPPORTED; CTAS/UNLOAD with format ORC/AVRO fail NOT_SUPPORTED. SQL is SQLite semantics behind a Presto/Trino translation layer, not Trino; TABLESAMPLE and multi-argument UNNEST fail as SYNTAX_ERROR." + - "ORC tables cannot be read or written (no maintained pure-Go ORC library): SELECT fails NOT_SUPPORTED; CTAS/UNLOAD with format ORC fail NOT_SUPPORTED. SQL is SQLite semantics behind a Presto/Trino translation layer, not Trino; TABLESAMPLE and multi-argument UNNEST fail as SYNTAX_ERROR." - "DeleteDataCatalogInput.DeleteCatalogOnly has no effect: a FEDERATED catalog's CFN Stack/Lambda/Glue Connection resources are not simulated." - "WorkGroup.IdentityCenterApplicationArn is never set: the IAM Identity Center application AWS provisions for an Identity Center workgroup does not exist here." - "WorkGroup on GetDatabase/ListDatabases/GetTableMetadata/ListTableMetadata/GetDataCatalog/ListDataCatalogs is accepted and ignored: it only matters for IAM Identity Center-enabled Glue catalogs, which do not exist here." @@ -79,6 +79,32 @@ leaks: {status: clean, note: "janitor uses pkgs/worker.Group with proper ctx.Don ## Notes +### 2026-11-06 booleans, Avro, Iceberg row-level DML + +Docs authority (fetched 2026-11-06): Athena user guide pages `optimize-statement`, `vacuum-statement`, +`querying-iceberg-data-optimization`, `querying-iceberg-creating-tables`, `create-table-as`, +`querying-iceberg-evolving-table-schema`, `querying-iceberg-alter-table-change-column`. + +- Boolean expression results now report ColumnInfo type `boolean` and VarCharValue `true`/`false` + (`engine_bool.go`); previously 1/0 with type bigint. Boolean table columns already rendered correctly. +- Avro: `AvroSerDe`/`AvroContainerInputFormat` tables are read (OCF, null/deflate/snappy codecs, logical + date/timestamp-millis/decimal, nested array/map/record; `engine_avro.go`), and `format='AVRO'` works for + CTAS, INSERT INTO and UNLOAD with `write_compression` NONE/DEFLATE/SNAPPY. Pure-Go `iskorotkov/avro/v2` + was already a direct dependency (Iceberg manifests); no new module. CTAS of ARRAY/MAP/ROW expression values + still produces string (JSON) columns because result columns carry no inner type. +- Iceberg: `DELETE FROM`, `UPDATE ... SET`, `MERGE INTO` write merge-on-read position delete files + (field ids 2147483546/2147483545, one file per partition) plus new data files in one `delete`/`overwrite` + snapshot; `OPTIMIZE ... REWRITE DATA USING BIN_PACK [WHERE partition-predicate]` replaces data files and drops + delete files they covered (`replace` snapshot; thresholds `optimize_rewrite_data_file_threshold` default 5, + `optimize_rewrite_delete_file_threshold` default 2, size window 0.75x-1.8x of 512 MiB); `VACUUM` expires snapshots + by `vacuum_max_snapshot_age_seconds` (432000) / `vacuum_min_snapshots_to_keep` (1), trims + `vacuum_max_metadata_files_to_keep` (100) and deletes unreachable and old orphan files; + `CREATE TABLE ... WITH (table_type='ICEBERG', location=..., partitioning=ARRAY[...], format PARQUET|AVRO)` CTAS; + `ALTER TABLE ADD COLUMNS / DROP COLUMN / CHANGE COLUMN [FIRST|AFTER] / RENAME COLUMN / SET TBLPROPERTIES` + (type promotion int->bigint, float->double, wider decimal) as metadata-only schema commits. Reads of the current + snapshot now use the table's current schema (older data files resolve by field id). +- Iceberg tables may use `format='AVRO'` data files (read and write). + ### 2026-09-26 parity sweep: GetResourceDashboard fabricated a URL for a nonexistent session The prior fix (see op row) validated ResourceARN was non-empty but never checked diff --git a/services/athena/README.md b/services/athena/README.md index a5e52b5f16..c159914af7 100644 --- a/services/athena/README.md +++ b/services/athena/README.md @@ -16,11 +16,11 @@ ### Known gaps -- Iceberg: DELETE, UPDATE, MERGE, OPTIMIZE, VACUUM, CTAS with table_type='ICEBERG', ALTER TABLE schema/partition evolution, equality deletes, ORC/Avro data files, writing nested (array/map/struct) columns, the $partitions metadata table, partition pruning from manifest bounds (all data files are read), timestamp precision above milliseconds. +- Iceberg: equality deletes, ORC data files, writing nested (array/map/struct) columns, the $partitions metadata table, partition pruning from manifest bounds (all data files are read), timestamp precision above milliseconds, OPTIMIZE output is one file per partition (no write_target_data_file_size_bytes splitting), VACUUM/OPTIMIZE/MERGE error wording (MERGE_TARGET_ROW_MULTIPLE_MATCHES, the non-transactional Hive DELETE message) is UNVERIFIED against live Athena, DML conflicts fail instead of retrying, and the written manifests/delete files are checked only by this emulator's reader and spec-driven unit tests (no third-party Iceberg reader is available offline). - S3 Tables catalog: CREATE/DROP TABLE and CTAS through s3tablescatalog/ are rejected (use the S3 Tables API); ListDatabases/GetTableMetadata/ListTableMetadata do not enumerate s3tablescatalog tables; ROW-column dot access is not resolved on S3 Tables tables at plan time. - Delta Lake: deletion vectors and reader versions above 3 fail NOT_SUPPORTED; id column-mapping and timestampNtz reads are untested against real writers. -- ORC/Avro input and output formats: no pure-Go ORC or Avro library in go.mod (only parquet-go), so these stay NOT_SUPPORTED until a dependency is approved. -- Complex-type results: Boolean expression results render 1/0 rather than true/false (SQLite has no boolean type); ROW(...) constructors, CAST(... AS ROW(...)) and anonymous rows are unsupported; map and row expression results both report ColumnInfo type map; integral doubles inside arrays render without the .0 suffix; col.field dereference works on ROW columns of a referenced table only (not CTE/subquery outputs or lambda parameters). +- ORC input and output formats: no maintained pure-Go ORC library exists (scritchley/orc was last updated 2021 and is read-only), so ORC SELECT/CTAS/UNLOAD stay NOT_SUPPORTED. Avro tables with avro.schema.literal/url and no Glue columns are not derived. +- Complex-type results: ROW(...) constructors, CAST(... AS ROW(...)) and anonymous rows are unsupported; map and row expression results both report ColumnInfo type map; integral doubles inside arrays render without the .0 suffix; col.field dereference works on ROW columns of a referenced table only (not CTE/subquery outputs or lambda parameters); boolean result typing is inferred syntactically from the outermost SELECT list (comparisons, logical operators, IS/IN/BETWEEN/LIKE, boolean literals and functions, CASE/COALESCE/IF over those, CAST AS boolean, aliases through CTEs and subqueries) so an unlisted boolean-returning function renders 1/0; SELECT * after such an expression disables the inference. - Lambda functions cover transform, filter, reduce, any_match/all_match/none_match, map_filter, transform_values, transform_keys and zip_with; array_sort(arr, comparator), map_zip_with, reduce_agg and zip_with over more than two arrays are not translated. - INTERVAL literals are supported only as the right operand of + or - (INTERVAL n UNIT, YEAR TO MONTH, 'D HH:MM:SS' DAY TO SECOND) via date_add semantics with end-of-month clamping; timestamp - timestamp interval results, standalone INTERVAL values/comparisons and fractional-second intervals are unsupported. - GROUP BY ROLLUP/CUBE/GROUPING SETS are expanded to UNION ALL of plain GROUP BY branches: not combinable with UNION/INTERSECT/EXCEPT in the same query block, and the outer ORDER BY may reference only output columns or ordinals. @@ -31,7 +31,7 @@ These do not block an A grade — no implementation could produce real data here because the underlying data source cannot exist in an emulator. -- ORC and Avro tables cannot be read or written (go.mod carries only parquet-go; no pure-Go ORC/Avro reader): SELECT fails NOT_SUPPORTED; CTAS/UNLOAD with format ORC/AVRO fail NOT_SUPPORTED. SQL is SQLite semantics behind a Presto/Trino translation layer, not Trino; TABLESAMPLE and multi-argument UNNEST fail as SYNTAX_ERROR. +- ORC tables cannot be read or written (no maintained pure-Go ORC library): SELECT fails NOT_SUPPORTED; CTAS/UNLOAD with format ORC fail NOT_SUPPORTED. SQL is SQLite semantics behind a Presto/Trino translation layer, not Trino; TABLESAMPLE and multi-argument UNNEST fail as SYNTAX_ERROR. - DeleteDataCatalogInput.DeleteCatalogOnly has no effect: a FEDERATED catalog's CFN Stack/Lambda/Glue Connection resources are not simulated. - WorkGroup.IdentityCenterApplicationArn is never set: the IAM Identity Center application AWS provisions for an Identity Center workgroup does not exist here. - WorkGroup on GetDatabase/ListDatabases/GetTableMetadata/ListTableMetadata/GetDataCatalog/ListDataCatalogs is accepted and ignored: it only matters for IAM Identity Center-enabled Glue catalogs, which do not exist here. diff --git a/services/athena/engine_avro.go b/services/athena/engine_avro.go new file mode 100644 index 0000000000..32e94492bb --- /dev/null +++ b/services/athena/engine_avro.go @@ -0,0 +1,752 @@ +package athena + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "math/big" + "strconv" + "strings" + "time" + + "github.com/iskorotkov/avro/v2" + "github.com/iskorotkov/avro/v2/ocf" +) + +const ( + avroSerdeLib = "org.apache.hadoop.hive.serde2.avro.AvroSerDe" + avroInputFmt = "org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat" + avroOutputFmt = "org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat" + fmtNameAvro = "AVRO" + avroRootName = "hive_schema" + defaultDecimalPrec = 10 + hiveArray = "array" + hiveMap = "map" + hiveStruct = "struct" + hiveDecimal = "decimal" + avroTimeLayout = "2006-01-02 15:04:05.999999999" + avroKeyType = "type" + avroKeyName = "name" + avroKeyDefault = "default" + avroKeyLogical = "logicalType" + codecDeflate = "DEFLATE" + codecSnappy = "SNAPPY" +) + +var ( + errAvroType = errors.New("unsupported Avro column type") + errAvroValue = errors.New("cannot encode Avro value") +) + +type hiveField struct { + typ *hiveType + name string +} + +type hiveType struct { + elem *hiveType + base string + name string + fields []hiveField + prec int + scale int +} + +func splitTop(s string, sep byte) []string { + var ( + parts []string + depth int + start int + ) + + for i := range len(s) { + switch c := s[i]; { + case c == '<' || c == '(': + depth++ + case c == '>' || c == ')': + depth-- + case c == sep && depth == 0: + parts = append(parts, s[start:i]) + start = i + 1 + } + } + + return append(parts, s[start:]) +} + +func splitTypeArgs(t string) (string, string, bool) { + if base, rest, ok := strings.Cut(t, "<"); ok { + return strings.TrimSpace(base), strings.TrimSuffix(rest, ">"), true + } + + if base, rest, ok := strings.Cut(t, "("); ok { + return strings.TrimSpace(base), strings.TrimSuffix(rest, ")"), true + } + + return strings.TrimSpace(t), "", false +} + +func parseHiveType(raw string) (*hiveType, error) { + base, args, hasArgs := splitTypeArgs(strings.ToLower(strings.TrimSpace(raw))) + ht := &hiveType{base: base} + + var err error + + switch base { + case hiveArray: + ht.elem, err = parseHiveType(args) + case hiveMap: + kv := splitTop(args, ',') + if len(kv) != argsTwo { + return nil, fmt.Errorf("%w: %s", errAvroType, raw) + } + + ht.elem, err = parseHiveType(kv[1]) + case hiveStruct: + ht.fields, err = parseStructFields(args, raw) + case hiveDecimal, typeNumeric: + ht.base, ht.prec = hiveDecimal, defaultDecimalPrec + + if hasArgs { + ht.prec, ht.scale = parseDecimalArgs(args) + } + } + + return ht, err +} + +func parseStructFields(args, raw string) ([]hiveField, error) { + var out []hiveField + + for _, f := range splitTop(args, ',') { + name, typ, ok := strings.Cut(strings.TrimSpace(f), ":") + if !ok { + return nil, fmt.Errorf("%w: %s", errAvroType, raw) + } + + ft, err := parseHiveType(typ) + if err != nil { + return nil, err + } + + out = append(out, hiveField{name: strings.TrimSpace(name), typ: ft}) + } + + return out, nil +} + +func parseDecimalArgs(args string) (int, int) { + ps := strings.Split(args, ",") + prec, _ := strconv.Atoi(strings.TrimSpace(ps[0])) + scale := 0 + + if len(ps) > 1 { + scale, _ = strconv.Atoi(strings.TrimSpace(ps[1])) + } + + return prec, scale +} + +type avroNamer struct{ n int } + +func (a *avroNamer) next(prefix string) string { + a.n++ + + return prefix + strconv.Itoa(a.n) +} + +func nullable(v any) []any { return []any{kwNull, v} } + +func avroPrimitiveSchema(base string) (any, bool) { + switch base { + case typeBoolean: + return "boolean", true + case typeTinyint, typeSmallint, typeInt, typeInteger: + return "int", true + case typeBigint: + return "long", true + case typeFloat, typeReal: + return "float", true + case typeDouble: + return "double", true + case columnTypeString, typeVarchar, "char": + return "string", true + case "binary", "varbinary": + return "bytes", true + case typeDate: + return map[string]any{avroKeyType: "int", avroKeyLogical: typeDate}, true + case typeTimestamp: + return map[string]any{avroKeyType: "long", avroKeyLogical: "timestamp-millis"}, true + } + + return nil, false +} + +func (t *hiveType) avroSchema(nm *avroNamer) (any, error) { + if sch, ok := avroPrimitiveSchema(t.base); ok { + return sch, nil + } + + switch t.base { + case hiveDecimal: + return map[string]any{ + avroKeyType: "bytes", avroKeyLogical: hiveDecimal, "precision": t.prec, "scale": t.scale, + }, nil + case hiveArray: + el, err := t.elem.avroSchema(nm) + + return map[string]any{avroKeyType: hiveArray, "items": nullable(el)}, err + case hiveMap: + el, err := t.elem.avroSchema(nm) + + return map[string]any{avroKeyType: hiveMap, "values": nullable(el)}, err + case hiveStruct: + return t.structSchema(nm) + default: + return nil, fmt.Errorf("%w: %s", errAvroType, t.base) + } +} + +func (t *hiveType) structSchema(nm *avroNamer) (any, error) { + fields := make([]any, 0, len(t.fields)) + + for _, f := range t.fields { + ft, err := f.typ.avroSchema(nm) + if err != nil { + return nil, err + } + + fields = append(fields, map[string]any{avroKeyName: f.name, avroKeyType: nullable(ft), avroKeyDefault: nil}) + } + + t.name = nm.next("struct_") + + return map[string]any{avroKeyType: "record", avroKeyName: t.name, "fields": fields}, nil +} + +func avroSchemaFor(cols []outCol) (string, []*hiveType, error) { + nm := &avroNamer{} + fields := make([]any, 0, len(cols)) + types := make([]*hiveType, len(cols)) + + for i, c := range cols { + ht, err := parseHiveType(c.typ) + if err != nil { + return "", nil, err + } + + sch, err := ht.avroSchema(nm) + if err != nil { + return "", nil, fmt.Errorf("column %s: %w", c.name, err) + } + + types[i] = ht + fields = append(fields, map[string]any{avroKeyName: c.name, avroKeyType: nullable(sch), avroKeyDefault: nil}) + } + + b, err := json.Marshal(map[string]any{avroKeyType: "record", avroKeyName: avroRootName, "fields": fields}) + + return string(b), types, err +} + +func avroTime(s string) (time.Time, error) { + if t, err := time.Parse(dateLayout, s); err == nil { + return t, nil + } + + return time.Parse(avroTimeLayout, s) +} + +func (t *hiveType) fromText(s string) (any, error) { + switch t.base { + case typeBoolean: + return s == strTrue, nil + case typeTinyint, typeSmallint, typeInt, typeInteger: + n, err := strconv.ParseInt(s, 10, 32) + + return int32(n), err + case typeBigint: + return strconv.ParseInt(s, 10, 64) + case typeFloat, typeReal: + f, err := strconv.ParseFloat(s, 32) + + return float32(f), err + case typeDouble: + return strconv.ParseFloat(s, 64) + case "binary", "varbinary": + return []byte(s), nil + case typeDate, typeTimestamp: + return avroTime(s) + case hiveDecimal: + r, ok := new(big.Rat).SetString(s) + if !ok { + return nil, fmt.Errorf("%w: decimal %q", errAvroValue, s) + } + + return r, nil + case hiveArray, hiveMap, hiveStruct: + dec := json.NewDecoder(strings.NewReader(s)) + dec.UseNumber() + + var v any + if err := dec.Decode(&v); err != nil { + return nil, fmt.Errorf("%w: %w", errAvroValue, err) + } + + return t.fromJSON(v) + default: + return s, nil + } +} + +func jsonScalarText(v any) string { + switch n := v.(type) { + case string: + return n + case json.Number: + return n.String() + case bool: + return strconv.FormatBool(n) + default: + return fmt.Sprint(n) + } +} + +func (t *hiveType) fromJSON(v any) (any, error) { + if v == nil { + return nil, nil //nolint:nilnil // JSON null is an Avro null + } + + switch t.base { + case hiveArray: + return t.arrayFromJSON(v) + case hiveMap: + return t.mapFromJSON(v) + case hiveStruct: + return t.structFromJSON(v) + default: + return t.fromText(jsonScalarText(v)) + } +} + +func (t *hiveType) arrayFromJSON(v any) (any, error) { + arr, ok := v.([]any) + if !ok { + return nil, fmt.Errorf("%w: expected array", errAvroValue) + } + + out := make([]any, len(arr)) + + for i, e := range arr { + ev, err := t.elem.fromJSON(e) + if err != nil { + return nil, err + } + + out[i] = t.elem.union(ev) + } + + return out, nil +} + +func (t *hiveType) mapFromJSON(v any) (any, error) { + m, ok := v.(map[string]any) + if !ok { + return nil, fmt.Errorf("%w: expected map", errAvroValue) + } + + out := make(map[string]any, len(m)) + + for k, e := range m { + ev, err := t.elem.fromJSON(e) + if err != nil { + return nil, err + } + + out[k] = t.elem.union(ev) + } + + return out, nil +} + +func (t *hiveType) structFromJSON(v any) (any, error) { + m, ok := v.(map[string]any) + if !ok { + return nil, fmt.Errorf("%w: expected struct", errAvroValue) + } + + lower := make(map[string]any, len(m)) + for k, e := range m { + lower[strings.ToLower(k)] = e + } + + out := make(map[string]any, len(t.fields)) + + for _, f := range t.fields { + ev, err := f.typ.fromJSON(lower[strings.ToLower(f.name)]) + if err != nil { + return nil, err + } + + out[f.name] = f.typ.union(ev) + } + + return out, nil +} + +// union wraps a map or record value for its ["null", T] slot: the encoder +// reads a bare map[string]any there as a union branch selector. +func (t *hiveType) union(v any) any { + switch { + case v == nil: + return nil + case t.base == hiveMap: + return map[string]any{hiveMap: v} + case t.base == hiveStruct: + return map[string]any{t.name: v} + default: + return v + } +} + +func avroCodec(name string) (ocf.CodecName, error) { + switch strings.ToUpper(name) { + case "", compressionNone: + return ocf.Null, nil + case codecDeflate: + return ocf.Deflate, nil + case codecSnappy: + return ocf.Snappy, nil + default: + return "", fmt.Errorf("%w: compression %s", errAvroType, name) + } +} + +// encodeAvroRows writes one Avro object container file holding the given rows. +func encodeAvroRows(res *sqlResult, rows []int, cols []outCol, codec string) ([]byte, error) { + schema, types, err := avroSchemaFor(cols) + if err != nil { + return nil, err + } + + cn, err := avroCodec(codec) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + + enc, err := ocf.NewEncoder(schema, &buf, ocf.WithCodec(cn)) + if err != nil { + return nil, err + } + + for _, r := range rows { + rec := make(map[string]any, len(cols)) + + for i, c := range cols { + v, null := res.cell(r, c.src) + if null { + rec[c.name] = nil + + continue + } + + av, perr := types[i].fromText(v) + if perr != nil { + return nil, fmt.Errorf("column %s: %w", c.name, perr) + } + + rec[c.name] = types[i].union(av) + } + + if err = enc.Encode(rec); err != nil { + return nil, err + } + } + + if err = enc.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// avroNorm turns a decoded Avro value into JSON-friendly Go values typed by t (nil: untyped). +func avroNorm(v any, t *hiveType) any { + switch n := v.(type) { + case time.Time: + if t != nil && t.base == typeDate { + return n.UTC().Format(dateLayout) + } + + return n.UTC().Format(tsMillisLayout) + case *big.Rat: + scale := 0 + if t != nil { + scale = t.scale + } + + return n.FloatString(scale) + case []byte: + return string(n) + case [16]byte: + return hex.EncodeToString(n[:]) + case float32: + return float64(n) + case []any: + out := make([]any, len(n)) + + var el *hiveType + if t != nil { + el = t.elem + } + + for i, e := range n { + out[i] = avroNorm(e, el) + } + + return out + case map[string]any: + out := make(map[string]any, len(n)) + + for k, e := range n { + out[k] = avroNorm(e, subType(t, k)) + } + + return out + default: + return v + } +} + +func subType(t *hiveType, field string) *hiveType { + if t == nil { + return nil + } + + if t.base == hiveStruct { + for _, f := range t.fields { + if strings.EqualFold(f.name, field) { + return f.typ + } + } + + return nil + } + + return t.elem +} + +func derefSchema(s avro.Schema) avro.Schema { + if ref, ok := s.(*avro.RefSchema); ok { + return ref.Schema() + } + + return s +} + +func unionKey(s avro.Schema) string { + switch sc := derefSchema(s).(type) { + case *avro.ArraySchema: + return string(avro.Array) + case *avro.MapSchema: + return string(avro.Map) + case *avro.RecordSchema: + return sc.FullName() + default: + return "" + } +} + +// avroUnwrap strips the {branch: value} wrapper the decoder puts around +// array, map and record members of a union, guided by the file's schema. +func avroUnwrap(v any, s avro.Schema) any { + switch sc := derefSchema(s).(type) { + case *avro.UnionSchema: + return unwrapUnion(v, sc) + case *avro.RecordSchema: + m, ok := v.(map[string]any) + if !ok { + return v + } + + for _, f := range sc.Fields() { + if fv, has := m[f.Name()]; has { + m[f.Name()] = avroUnwrap(fv, f.Type()) + } + } + + return m + case *avro.ArraySchema: + if arr, ok := v.([]any); ok { + for i, e := range arr { + arr[i] = avroUnwrap(e, sc.Items()) + } + } + + return v + case *avro.MapSchema: + if m, ok := v.(map[string]any); ok { + for k, e := range m { + m[k] = avroUnwrap(e, sc.Values()) + } + } + + return v + default: + return v + } +} + +func unwrapUnion(v any, sc *avro.UnionSchema) any { + if v == nil { + return nil + } + + var only avro.Schema + + nonNull := 0 + + for _, br := range sc.Types() { + if br.Type() == avro.Null { + continue + } + + nonNull++ + only = br + + if m, ok := v.(map[string]any); ok && len(m) == 1 { + if inner, has := m[unionKey(br)]; has && unionKey(br) != "" { + return avroUnwrap(inner, br) + } + } + } + + if nonNull == 1 { + return avroUnwrap(v, only) + } + + return v +} + +func avroCell(v any, t *hiveType) string { + nv := avroNorm(v, t) + + switch n := nv.(type) { + case nil: + return "" + case string: + return n + case bool: + return strconv.FormatBool(n) + case int32: + return strconv.FormatInt(int64(n), 10) + case int64: + return avroIntCell(n, t) + case int: + return strconv.Itoa(n) + case float64: + if math.IsNaN(n) || math.IsInf(n, 0) { + return formatFloat(n) + } + + return strconv.FormatFloat(n, 'g', -1, 64) + default: + b, err := json.Marshal(n) + if err != nil { + return fmt.Sprint(n) + } + + return string(b) + } +} + +func avroIntCell(n int64, t *hiveType) string { + if t != nil && t.base == typeTimestamp { + return time.UnixMilli(n).UTC().Format(tsMillisLayout) + } + + return strconv.FormatInt(n, 10) +} + +type avroOpts struct { + skip map[int64]bool + path string + rowIDs bool +} + +func (l *tableLoader) decodeAvro( + ctx context.Context, rd io.Reader, cols []Column, sink *rowSink, opts avroOpts, +) error { + dec, err := ocf.NewDecoder(rd) + if err != nil { + return fmt.Errorf("HIVE_CURSOR_ERROR: invalid Avro file: %w", err) + } + + types := make([]*hiveType, len(cols)) + + for i, c := range cols { + if ht, perr := parseHiveType(c.Type); perr == nil { + types[i] = ht + } + } + + for ord := int64(0); dec.HasNext(); ord++ { + var obj map[string]any + if err = dec.Decode(&obj); err != nil { + return fmt.Errorf("HIVE_CURSOR_ERROR: Failed to read Avro record: %w", err) + } + + if opts.skip[ord] { + continue + } + + obj, _ = avroUnwrap(obj, dec.Schema()).(map[string]any) + rec, nulls := avroRow(obj, cols, types, opts, ord) + + if err = l.insertJSON(ctx, sink, rec, nulls); err != nil { + return err + } + } + + if err = dec.Error(); err != nil { + return fmt.Errorf("HIVE_CURSOR_ERROR: %w", err) + } + + return nil +} + +// avroRow maps a decoded record onto the table columns (case-insensitive names). +func avroRow(obj map[string]any, cols []Column, types []*hiveType, opts avroOpts, ord int64) ([]string, []bool) { + lower := make(map[string]any, len(obj)) + for k, v := range obj { + lower[strings.ToLower(k)] = v + } + + extra := 0 + if opts.rowIDs { + extra = argsTwo + } + + rec := make([]string, len(cols)+extra) + nulls := make([]bool, len(cols)+extra) + + for i, c := range cols { + v, ok := lower[strings.ToLower(c.Name)] + if nulls[i] = !ok || v == nil; !nulls[i] { + rec[i] = avroCell(v, types[i]) + } + } + + if opts.rowIDs { + rec[len(cols)], rec[len(cols)+1] = opts.path, strconv.FormatInt(ord, 10) + } + + return rec, nulls +} diff --git a/services/athena/engine_avro_sdk_test.go b/services/athena/engine_avro_sdk_test.go new file mode 100644 index 0000000000..0efb5e1304 --- /dev/null +++ b/services/athena/engine_avro_sdk_test.go @@ -0,0 +1,250 @@ +package athena_test + +import ( + "bytes" + "math/big" + "strings" + "testing" + "time" + + "github.com/iskorotkov/avro/v2/ocf" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/athena" +) + +const avroSerde = "org.apache.hadoop.hive.serde2.avro.AvroSerDe" + +const eventsAvroSchema = `{"type":"record","name":"events","fields":[ + {"name":"id","type":"long"}, + {"name":"Name","type":["null","string"],"default":null}, + {"name":"score","type":["null","double"],"default":null}, + {"name":"ok","type":["null","boolean"],"default":null}, + {"name":"day","type":{"type":"int","logicalType":"date"}}, + {"name":"at","type":{"type":"long","logicalType":"timestamp-millis"}}, + {"name":"price","type":{"type":"bytes","logicalType":"decimal","precision":10,"scale":2}}, + {"name":"tags","type":{"type":"array","items":"string"}}, + {"name":"attrs","type":{"type":"map","values":"long"}}, + {"name":"owner","type":{"type":"record","name":"owner_t","fields":[ + {"name":"first","type":"string"},{"name":"age","type":"int"}]}} +]}` + +func eventsAvroFile(t *testing.T, codec ocf.CodecName) []byte { + t.Helper() + + var buf bytes.Buffer + + enc, err := ocf.NewEncoder(eventsAvroSchema, &buf, ocf.WithCodec(codec)) + require.NoError(t, err) + + day := time.Date(2024, 3, 5, 0, 0, 0, 0, time.UTC) + at := time.Date(2024, 3, 5, 12, 30, 45, 123000000, time.UTC) + + rows := []map[string]any{ + { + "id": int64(1), "Name": "ann", "score": 1.5, "ok": true, "day": day, "at": at, + "price": big.NewRat(1234, 100), "tags": []string{"a", "b"}, "attrs": map[string]int64{"x": 7}, + "owner": map[string]any{"first": "Ann", "age": int32(30)}, + }, + { + "id": int64(2), "Name": nil, "score": nil, "ok": false, "day": day.AddDate(0, 0, 1), "at": at, + "price": big.NewRat(5, 1), "tags": []string{}, "attrs": map[string]int64{}, + "owner": map[string]any{"first": "Bob", "age": int32(41)}, + }, + } + + for _, r := range rows { + require.NoError(t, enc.Encode(r)) + } + + require.NoError(t, enc.Close()) + + return buf.Bytes() +} + +func eventsAvroDef() *athena.GlueTableDef { + return &athena.GlueTableDef{ + Name: "events", Location: "s3://lake/events/", SerdeLibrary: avroSerde, + InputFormat: "org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat", + Columns: []athena.Column{ + {Name: "id", Type: "bigint"}, {Name: "name", Type: "string"}, {Name: "score", Type: "double"}, + {Name: "ok", Type: "boolean"}, {Name: "day", Type: "date"}, {Name: "at", Type: "timestamp"}, + {Name: "price", Type: "decimal(10,2)"}, {Name: "tags", Type: "array"}, + {Name: "attrs", Type: "map"}, {Name: "owner", Type: "struct"}, + }, + } +} + +func TestSDK_AvroRead(t *testing.T) { + t.Parallel() + + for _, codec := range []ocf.CodecName{ocf.Null, ocf.Deflate, ocf.Snappy} { + t.Run(string(codec), func(t *testing.T) { + t.Parallel() + + client := sdkClientFor(t, map[string][]byte{"lake/events/a.avro": eventsAvroFile(t, codec)}, + map[string]*athena.GlueTableDef{"shop/events": eventsAvroDef()}) + + cases := []struct { + name string + sql string + want [][]string + }{ + { + "scalars", + "SELECT id, name, score, ok, day, at, price FROM events ORDER BY id", + [][]string{ + {"1", "ann", "1.5", "true", "2024-03-05", "2024-03-05 12:30:45.123", "12.34"}, + {"2", "", "", "false", "2024-03-06", "2024-03-05 12:30:45.123", "5.00"}, + }, + }, + { + "complex", + "SELECT tags[1], attrs['x'], owner.first, owner.age FROM events WHERE id = 1", + [][]string{{"a", "7", "Ann", "30"}}, + }, + {"filter", "SELECT count(*) FROM events WHERE name IS NULL AND ok = false", [][]string{{"1"}}}, + } + + for _, tc := range cases { + got := sdkRun(t, client, tc.sql) + assert.Equal(t, tc.want, got.rows, tc.name) + } + }) + } +} + +func TestSDK_AvroWriteRoundTrip(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + compress string + }{ + {"none", ""}, + {"snappy", ", write_compression = 'SNAPPY'"}, + {"deflate", ", write_compression = 'DEFLATE'"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b, store := newEngineBackend(t, + map[string][]byte{"lake/events/a.avro": eventsAvroFile(t, ocf.Null)}, + map[string]*athena.GlueTableDef{"shop/events": eventsAvroDef()}) + client := newTestAthenaClient(t, athena.NewHandler(b)) + + sdkRun(t, client, "CREATE DATABASE shop") + sdkRun(t, client, "CREATE TABLE shop.copy WITH (format = 'AVRO', external_location = 's3://lake/copy/'"+ + tc.compress+") AS SELECT id, name, score, ok, day, at, price FROM events") + + store.mu.Lock() + + var file []byte + + for k, v := range store.objects { + if strings.HasPrefix(k, "lake/copy/") { + file = v + } + } + store.mu.Unlock() + + require.NotEmpty(t, file) + + dec, err := ocf.NewDecoder(bytes.NewReader(file)) + require.NoError(t, err) + + var rec map[string]any + + require.True(t, dec.HasNext()) + require.NoError(t, dec.Decode(&rec)) + assert.EqualValues(t, 1, rec["id"]) + assert.Equal(t, "ann", rec["name"]) + assert.Equal(t, true, rec["ok"]) + assert.Contains(t, string(dec.Metadata()["avro.schema"]), `"logicalType":"timestamp-millis"`) + + got := sdkRun(t, client, "SELECT id, name, ok, day, at, price FROM shop.copy ORDER BY id") + assert.Equal(t, [][]string{ + {"1", "ann", "true", "2024-03-05", "2024-03-05 12:30:45.123", "12.34"}, + {"2", "", "false", "2024-03-06", "2024-03-05 12:30:45.123", "5.00"}, + }, got.rows) + + sdkRun(t, client, "INSERT INTO shop.copy SELECT id + 10, name, score, ok, day, at, price "+ + "FROM events WHERE id = 1") + + got = sdkRun(t, client, "SELECT count(*), max(id) FROM shop.copy") + assert.Equal(t, [][]string{{"3", "11"}}, got.rows) + }) + } +} + +func TestSDK_AvroInsertNestedColumns(t *testing.T) { + t.Parallel() + + b, store := newEngineBackend(t, + map[string][]byte{"lake/events/a.avro": eventsAvroFile(t, ocf.Null)}, + map[string]*athena.GlueTableDef{"shop/events": eventsAvroDef()}) + client := newTestAthenaClient(t, athena.NewHandler(b)) + + sdkRun(t, client, "INSERT INTO events SELECT id + 10, name, score, ok, day, at, price, tags, attrs, owner "+ + "FROM events WHERE id = 1") + + got := sdkRun(t, client, "SELECT id, tags[2], attrs['x'], owner.first, owner.age FROM events WHERE id > 5") + assert.Equal(t, [][]string{{"11", "b", "7", "Ann", "30"}}, got.rows) + + store.mu.Lock() + defer store.mu.Unlock() + + var written []byte + + for k, v := range store.objects { + if strings.HasPrefix(k, "lake/events/") && k != "lake/events/a.avro" { + written = v + } + } + + dec, err := ocf.NewDecoder(bytes.NewReader(written)) + require.NoError(t, err) + + var rec map[string]any + + require.True(t, dec.HasNext()) + require.NoError(t, dec.Decode(&rec)) + assert.Equal(t, map[string]any{"array": []any{"a", "b"}}, rec["tags"]) + assert.Equal(t, map[string]any{"map": map[string]any{"x": int64(7)}}, rec["attrs"]) + assert.Equal(t, map[string]any{"struct_1": map[string]any{"first": "Ann", "age": 30}}, rec["owner"]) +} + +func TestSDK_AvroWriteErrors(t *testing.T) { + t.Parallel() + + cases := []struct { + name, sql, reason string + }{ + { + "gzip", + "CREATE TABLE shop.o WITH (format = 'AVRO', write_compression = 'GZIP', external_location = 's3://lake/o/') " + + "AS SELECT id FROM events", + "compression GZIP is not supported for AVRO", + }, + { + "orc", + "CREATE TABLE shop.o WITH (format = 'ORC', external_location = 's3://lake/o/') AS SELECT id FROM events", + "format ORC is not supported", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + client := sdkClientFor(t, map[string][]byte{"lake/events/a.avro": eventsAvroFile(t, ocf.Null)}, + map[string]*athena.GlueTableDef{"shop/events": eventsAvroDef()}) + + sdkRun(t, client, "CREATE DATABASE shop") + assert.Contains(t, sdkFailure(t, client, tc.sql), tc.reason) + }) + } +} diff --git a/services/athena/engine_bool.go b/services/athena/engine_bool.go new file mode 100644 index 0000000000..6ae10b8499 --- /dev/null +++ b/services/athena/engine_bool.go @@ -0,0 +1,444 @@ +package athena + +import ( + "slices" + "strings" +) + +// SQLite has no boolean type, so a comparison or logical expression comes +// back as 0/1 with no declared type. boolOutputs classifies the outermost +// SELECT list statically so those columns render true/false. + +const ( + kwNull = "null" + kwCase = "case" + kwEnd = "end" + kwAnd = "and" + kwOr = "or" + kwOrder = "order" + kwUnion = "union" + kwWindow = "window" + kwBetween = "between" + kwIs = "is" + kwIn = "in" + kwLike = "like" + kwNot = "not" + kwThen = "then" + kwElse = "else" + kwWhen = "when" + kwSelect = "select" + kwAs = "as" +) + +func isBoolFunc(n string) bool { + switch n { + case "regexp_like", "starts_with", "contains", "is_nan", "is_finite", "is_infinite", "any_match", "all_match", + "none_match", "json_array_contains", "bool_and", "bool_or", "every", "arrays_overlap": + return true + } + + return false +} + +func isBoolPassFunc(n string) bool { + switch n { + case "coalesce", "try", "nullif", "greatest", "least": + return true + } + + return false +} + +func isSelectEnder(lw string) bool { + switch lw { + case kwFrom, kwWhere, kwGroup, kwHaving, kwOrder, kwLimit, kwOffset, kwUnion, kwIntersect, kwExcept, kwFetch, + kwWindow: + return true + } + + return false +} + +func isBoolWord(lw string) bool { + switch lw { + case kwAnd, kwOr, kwNot, kwIs, kwIn, kwBetween, kwLike, "exists", "ilike": + return true + } + + return false +} + +func isCmpPunct(t token) bool { + return t.isPunct("=") || t.isPunct("<") || t.isPunct(">") || t.isPunct("!") +} + +type boolScope struct { + yes map[string]bool + no map[string]bool +} + +// boolOutputs reports, per top-level output column, whether it is boolean; +// nil means the shape is not analysable (SELECT *, no SELECT). +func boolOutputs(query string) []bool { + toks, err := tokenize(query) + if err != nil { + return nil + } + + sig := significant(toks) + sc := &boolScope{yes: map[string]bool{}, no: map[string]bool{}} + + const passes = 3 + + for range passes { + for i, t := range sig { + if t.isWord(kwSelect) && !prevIsDotTok(sig, i) { + sc.scanSelect(sig, i, true) + } + } + } + + return sc.scanSelect(sig, outermostSelect(sig), false) +} + +func prevIsDotTok(sig []token, i int) bool { return i > 0 && sig[i-1].isPunct(".") } + +// outermostSelect returns the first SELECT outside any parenthesis (CTE bodies +// and subqueries sit deeper), or the first SELECT at all; -1 if none. +func outermostSelect(sig []token) int { + depth, first := 0, -1 + + for i, t := range sig { + switch { + case t.isPunct("("): + depth++ + case t.isPunct(")"): + depth-- + case t.isWord(kwSelect): + if depth == 0 { + return i + } + + if first < 0 { + first = i + } + } + } + + return first +} + +// selectItems splits the select list starting after the SELECT at sel. +func selectItems(sig []token, sel int) [][]token { + i := sel + 1 + if i < len(sig) && (sig[i].isWord("distinct") || sig[i].isWord("all")) { + i++ + } + + items := make([][]token, 0, 1) + cur := make([]token, 0, len(sig)-i) + depth, cases := 0, 0 + + for ; i < len(sig); i++ { + t := sig[i] + + if t.isPunct(")") && depth == 0 { + break + } + + if depth == 0 && cases == 0 && t.kind == tkWord && isSelectEnder(strings.ToLower(t.text)) { + break + } + + if depth == 0 && cases == 0 && t.isPunct(",") { + items, cur = append(items, cur), nil + + continue + } + + depth, cases = trackNesting(t, depth, cases) + cur = append(cur, t) + } + + return append(items, cur) +} + +func trackNesting(t token, depth, cases int) (int, int) { + switch { + case t.isPunct("("): + depth++ + case t.isPunct(")"): + depth-- + case t.isWord(kwCase): + cases++ + case t.isWord(kwEnd) && cases > 0: + cases-- + } + + return depth, cases +} + +// scanSelect classifies the select items starting at sel. With learn set it +// records alias classifications instead of returning the output vector. +func (sc *boolScope) scanSelect(sig []token, sel int, learn bool) []bool { + if sel < 0 { + return nil + } + + items := selectItems(sig, sel) + out := make([]bool, len(items)) + + for k, item := range items { + if len(item) > 0 && item[len(item)-1].isPunct("*") { + return nil + } + + expr, alias := splitAliasSig(item) + out[k] = sc.isBool(expr) + + if learn { + sc.record(expr, alias, out[k]) + } + } + + if learn { + return nil + } + + return out +} + +func (sc *boolScope) record(expr []token, alias string, isB bool) { + name := alias + if name == "" && len(expr) > 0 { + last := expr[len(expr)-1] + if last.kind == tkWord || last.kind == tkQuoted { + name = last.ident() + } + } + + switch { + case name == "": + case isB: + sc.yes[name] = true + case alias != "" || len(expr) != 1: + sc.no[name] = true + } +} + +func isNameTok(t token) bool { return t.kind == tkWord || t.kind == tkQuoted } + +// splitAliasSig splits `expr [AS] alias` on already-significant tokens. +func splitAliasSig(item []token) ([]token, string) { + n := len(item) + if n >= argsThree && item[n-2].isWord(kwAs) { + return item[:n-2], item[n-1].ident() + } + + if n >= argsTwo && isNameTok(item[n-1]) && !isOperandKeyword(strings.ToLower(item[n-1].text)) && + !isClauseWord(item[n-1]) && !item[n-2].isPunct(".") && + (isNameTok(item[n-2]) || item[n-2].isPunct(")") || item[n-2].kind == tkNumber || item[n-2].kind == tkString) { + return item[:n-1], item[n-1].ident() + } + + return item, "" +} + +func isClauseWord(t token) bool { + switch strings.ToLower(t.text) { + case kwEnd, kwNull, strTrue, strFalse, kwAnd, kwOr, kwNot, kwIs, kwIn, kwLike, kwBetween: + return true + } + + return false +} + +func stripParens(expr []token) []token { + for len(expr) >= argsTwo && expr[0].isPunct("(") && matchParen(expr, 0) == len(expr)-1 { + if len(expr) > argsTwo && expr[1].isWord(kwSelect) { + return nil + } + + expr = expr[1 : len(expr)-1] + } + + return expr +} + +func (sc *boolScope) isBool(expr []token) bool { + expr = stripParens(expr) + + switch { + case len(expr) == 0: + return false + case len(expr) == 1: + return sc.isBoolAtom(expr[0]) + case hasBoolOperator(expr): + return true + case expr[0].isWord(kwCase) && expr[len(expr)-1].isWord(kwEnd) && matchCaseEnd(expr): + return sc.caseIsBool(expr) + } + + return sc.isBoolCall(expr) || sc.isBoolRef(expr) +} + +func (sc *boolScope) isBoolAtom(t token) bool { + if t.isWord(strTrue) || t.isWord(strFalse) { + return true + } + + if isNameTok(t) { + n := t.ident() + + return sc.yes[n] && !sc.no[n] + } + + return false +} + +func (sc *boolScope) isBoolRef(expr []token) bool { + return len(expr) == argsThree && expr[1].isPunct(".") && sc.isBoolAtom(expr[2]) +} + +func hasBoolOperator(expr []token) bool { + depth, cases := 0, 0 + + for i, t := range expr { + if depth == 0 && cases == 0 && isBoolOperatorAt(expr, i) { + return true + } + + depth, cases = trackNesting(t, depth, cases) + } + + return false +} + +func isBoolOperatorAt(expr []token, i int) bool { + t := expr[i] + + switch { + case isCmpPunct(t): + isArrow := t.isPunct(">") && i > 0 && expr[i-1].isPunct("-") + + return !isArrow + case t.kind == tkWord: + return isBoolWord(strings.ToLower(t.text)) && !prevIsDotTok(expr, i) + } + + return false +} + +func matchCaseEnd(expr []token) bool { + cases := 0 + + for i, t := range expr { + switch { + case t.isWord(kwCase): + cases++ + case t.isWord(kwEnd): + cases-- + if cases == 0 { + return i == len(expr)-1 + } + } + } + + return false +} + +// caseBranches returns the result expressions (THEN and ELSE) of a CASE block. +func caseBranches(expr []token) [][]token { + var ( + out [][]token + depth, cases int + start = -1 + ) + + flush := func(end int) { + if start >= 0 && (end-start != 1 || !expr[start].isWord(kwNull)) { + out = append(out, expr[start:end]) + } + + start = -1 + } + + for i, t := range expr { + top := depth == 0 && cases == 1 + + switch { + case top && (t.isWord(kwWhen) || t.isWord(kwElse)): + flush(i) + + if t.isWord(kwElse) { + start = i + 1 + } + case top && t.isWord(kwThen): + start = i + 1 + case t.isWord(kwEnd) && cases == 1 && depth == 0: + flush(i) + } + + depth, cases = trackNesting(t, depth, cases) + } + + return out +} + +func (sc *boolScope) caseIsBool(expr []token) bool { + return slices.ContainsFunc(caseBranches(expr), sc.isBool) +} + +func (sc *boolScope) isBoolCall(expr []token) bool { + if len(expr) < argsThree || expr[0].kind != tkWord || !expr[1].isPunct("(") || + matchParen(expr, 1) != len(expr)-1 { + return false + } + + name := strings.ToLower(expr[0].text) + args := splitCommasSig(expr[2 : len(expr)-1]) + + switch { + case isBoolFunc(name): + return true + case name == "cast" || name == "try_cast": + return len(args) == 1 && endsAsBoolean(args[0]) + case name == "if": + return len(args) >= argsTwo && (sc.isBool(args[1]) || len(args) > argsTwo && sc.isBool(args[2])) + case isBoolPassFunc(name): + return slices.ContainsFunc(args, sc.isBool) + } + + return false +} + +func endsAsBoolean(toks []token) bool { + n := len(toks) + + return n >= argsTwo && toks[n-1].isWord(typeBoolean) && toks[n-2].isWord(kwAs) +} + +func splitCommasSig(toks []token) [][]token { + var ( + parts [][]token + cur = make([]token, 0, len(toks)) + depth int + ) + + for _, t := range toks { + switch { + case t.isPunct("("): + depth++ + case t.isPunct(")"): + depth-- + case depth == 0 && t.isPunct(","): + parts, cur = append(parts, cur), nil + + continue + } + + cur = append(cur, t) + } + + return append(parts, cur) +} diff --git a/services/athena/engine_bool_sdk_test.go b/services/athena/engine_bool_sdk_test.go new file mode 100644 index 0000000000..44a065be4c --- /dev/null +++ b/services/athena/engine_bool_sdk_test.go @@ -0,0 +1,81 @@ +package athena_test + +import ( + "testing" + + athenasdk "github.com/aws/aws-sdk-go-v2/service/athena" + + "github.com/stretchr/testify/assert" + + "github.com/blackbirdworks/gopherstack/services/athena" +) + +const flagsData = `{"id":1,"ok":true,"n":5,"name":"ann"} +{"id":2,"ok":false,"n":null,"name":"bob"} +` + +func flagsClient(t *testing.T) *athenasdk.Client { + t.Helper() + + return sdkClientFor(t, map[string][]byte{"lake/flags/a.json": []byte(flagsData)}, + map[string]*athena.GlueTableDef{"shop/flags": { + Name: "flags", Location: "s3://lake/flags/", SerdeLibrary: serdeJSON, + Columns: []athena.Column{ + {Name: "id", Type: "int"}, {Name: "ok", Type: "boolean"}, + {Name: "n", Type: "int"}, {Name: "name", Type: "string"}, + }, + }}) +} + +func TestSDK_BooleanResults(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + sql string + types []string + want [][]string + }{ + {"comparison", "SELECT 1 < 2, 2 < 1", []string{"boolean", "boolean"}, [][]string{{"true", "false"}}}, + {"literals", "SELECT true, false", []string{"boolean", "boolean"}, [][]string{{"true", "false"}}}, + {"logical", "SELECT true AND false AS a, true OR false AS o, NOT true AS n", + []string{"boolean", "boolean", "boolean"}, [][]string{{"false", "true", "false"}}}, + {"boolean column", "SELECT ok FROM flags ORDER BY id", []string{"boolean"}, [][]string{{"true"}, {"false"}}}, + { + "column comparison", + "SELECT n > 3 AS big FROM flags ORDER BY id", + []string{"boolean"}, + [][]string{{"true"}, {""}}, + }, + {"is null", "SELECT n IS NULL AS missing, id IN (1, 3) AS hit FROM flags ORDER BY id", + []string{"boolean", "boolean"}, [][]string{{"false", "true"}, {"true", "false"}}}, + {"between like", "SELECT id BETWEEN 1 AND 1 AS b, name LIKE 'a%' AS l FROM flags ORDER BY id", + []string{"boolean", "boolean"}, [][]string{{"true", "true"}, {"false", "false"}}}, + {"case boolean", "SELECT CASE WHEN id = 1 THEN true ELSE false END AS c FROM flags ORDER BY id", + []string{"boolean"}, [][]string{{"true"}, {"false"}}}, + {"case not boolean", "SELECT CASE WHEN id = 1 THEN 10 ELSE 20 END AS c FROM flags ORDER BY id", + []string{"bigint"}, [][]string{{"10"}, {"20"}}}, + {"function", "SELECT starts_with(name, 'a') AS s, regexp_like(name, 'b') AS r FROM flags ORDER BY id", + []string{"boolean", "boolean"}, [][]string{{"true", "false"}, {"false", "true"}}}, + {"cast", "SELECT CAST(1 AS boolean) AS c", []string{"boolean"}, [][]string{{"true"}}}, + {"cte alias", "WITH t AS (SELECT id, n > 3 AS big FROM flags) SELECT big FROM t ORDER BY id", + []string{"boolean"}, [][]string{{"true"}, {""}}}, + {"subquery", "SELECT x FROM (SELECT ok AND true AS x FROM flags) ORDER BY x", + []string{"boolean"}, [][]string{{"false"}, {"true"}}}, + {"aggregate", "SELECT bool_and(ok) AS a, bool_or(ok) AS o FROM flags", + []string{"boolean", "boolean"}, [][]string{{"false", "true"}}}, + {"not boolean", "SELECT id + 1 AS i, name FROM flags ORDER BY id", + []string{"bigint", "varchar"}, [][]string{{"2", "ann"}, {"3", "bob"}}}, + {"filtered", "SELECT id FROM flags WHERE ok = true", []string{"integer"}, [][]string{{"1"}}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got := sdkRun(t, flagsClient(t), tc.sql) + assert.Equal(t, tc.want, got.rows) + assert.Equal(t, tc.types, got.types) + }) + } +} diff --git a/services/athena/engine_columnar.go b/services/athena/engine_columnar.go index 2ed6507df7..0acfd70512 100644 --- a/services/athena/engine_columnar.go +++ b/services/athena/engine_columnar.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "strconv" "strings" "github.com/aws/aws-sdk-go-v2/aws" @@ -39,6 +40,7 @@ type fileCol struct { hive string fileName string id int + rowPos bool } // fillTable creates the SQLite table for cols and lets fill insert rows. @@ -205,7 +207,7 @@ func (l *tableLoader) loadColumnarFile( return nil } - rec, nulls := l.fileRow(cols, wanted, vals) + rec, nulls := l.fileRow(cols, wanted, vals, cur) return l.insertJSON(ctx, sink, rec, nulls) }) @@ -217,14 +219,17 @@ func (l *tableLoader) loadColumnarFile( } // fileRow assembles one table row: constants first, then the values read from the file. -func (l *tableLoader) fileRow(cols []fileCol, wanted []boundCol, vals []any) ([]string, []bool) { +func (l *tableLoader) fileRow(cols []fileCol, wanted []boundCol, vals []any, rowNum int64) ([]string, []bool) { rec := make([]string, len(cols)) nulls := make([]bool, len(cols)) for i, c := range cols { - if c.constant != nil && l.want.has(c.name) { + switch { + case c.rowPos: + rec[i] = strconv.FormatInt(rowNum, 10) + case c.constant != nil && l.want.has(c.name): rec[i] = *c.constant - } else { + default: nulls[i] = true } } diff --git a/services/athena/engine_consts.go b/services/athena/engine_consts.go index c0ff7342ab..f00daac6b4 100644 --- a/services/athena/engine_consts.go +++ b/services/athena/engine_consts.go @@ -1,21 +1,27 @@ package athena const ( - typeBigint = "bigint" - typeVarchar = "varchar" - typeDecimal = "decimal" - typeDate = "date" - typeTimestamp = "timestamp" - typeInteger = "integer" - typeDouble = "double" - typeBoolean = "boolean" - typeInt = "int" - typeSmallint = "smallint" - typeTinyint = "tinyint" - typeReal = "real" - typeFloat = "float" - typeNumeric = "numeric" - sqlInteger = "INTEGER" + colPosName = "pos" + propParquetComp = "parquet_compression" + colFilePath = "file_path" + propCompression = "compression" + propIsExternal = "is_external" + fnBucket = "bucket" + typeBigint = "bigint" + typeVarchar = "varchar" + typeDecimal = "decimal" + typeDate = "date" + typeTimestamp = "timestamp" + typeInteger = "integer" + typeDouble = "double" + typeBoolean = "boolean" + typeInt = "int" + typeSmallint = "smallint" + typeTinyint = "tinyint" + typeReal = "real" + typeFloat = "float" + typeNumeric = "numeric" + sqlInteger = "INTEGER" declDateText = "DATE_TEXT" declTimestampText = "TIMESTAMP_TEXT" diff --git a/services/athena/engine_ddl.go b/services/athena/engine_ddl.go index 9bcad86c54..193939951d 100644 --- a/services/athena/engine_ddl.go +++ b/services/athena/engine_ddl.go @@ -44,7 +44,13 @@ func (b *InMemoryBackend) execCatalogStatement( return b.execGlueDropTable(query, qctx) case hasKeyword(upper, "MSCK"): return b.execMsckRepair(query, qctx) + case hasKeyword(upper, "VACUUM"): + return b.execVacuum(query, qctx) case hasKeyword(upper, "ALTER"): + if res, out, handled := b.execIcebergAlter(query, qctx); handled { + return res, out, true + } + return b.execAlterAddPartition(query, qctx) case hasKeyword(upper, "SHOW"), hasKeyword(upper, "DESCRIBE"), hasKeyword(upper, "DESC"): return b.execShowDescribe(query, qctx) diff --git a/services/athena/engine_dml.go b/services/athena/engine_dml.go new file mode 100644 index 0000000000..77249a8a9c --- /dev/null +++ b/services/athena/engine_dml.go @@ -0,0 +1,616 @@ +package athena + +import ( + "cmp" + "context" + "fmt" + "slices" + "sort" + "strconv" + "strings" + + "github.com/blackbirdworks/gopherstack/pkgs/iceberg" +) + +const ( + posDeleteFilePathID = 2147483546 + posDeletePosID = 2147483545 + dmlHidden = 2 + errMultiMatch = "MERGE_TARGET_ROW_MULTIPLE_MATCHES: One MERGE target table row matched more than one source row" + nonIcebergDML = "NOT_SUPPORTED: Modifying Hive table rows is only supported for transactional tables" +) + +func isDMLKind(k writeKind) bool { + return k == writeDelete || k == writeUpdate || k == writeMerge || k == writeOptimize +} + +func andWhere(where string) string { + if where == "" { + return "" + } + + return " WHERE (" + where + ")" +} + +func (p *writePlan) lookupDML(ctx context.Context, get S3Getter) (*icebergTable, statementOutcome) { + def, err := p.cat.GetTableDef(p.db, p.table) + if err != nil { + return nil, stmtFail(athenaErrTypeEntityMiss, + "TABLE_NOT_FOUND: line 1:1: Table '%s' does not exist", p.display()) + } + + if tableTypeOf(def) != tableTypeIceberg { + if p.stmt.kind == writeOptimize { + return nil, stmtFail( + athenaErrTypeSyntax, + "NOT_SUPPORTED: OPTIMIZE is supported only for Iceberg tables", + ) + } + + return nil, stmtFail(athenaErrTypeSyntax, "%s", nonIcebergDML) + } + + l := &tableLoader{get: get, limits: defaultEngineLimits()} + + tbl, err := l.openIceberg(ctx, def, p.db+"."+p.table) + if err != nil { + return nil, engineFailure(err) + } + + return tbl, stmtOK() +} + +// prepareDML resolves the Iceberg target and builds the SELECT that feeds the write. +func (p *writePlan) prepareDML(get S3Getter) statementOutcome { + tbl, out := p.lookupDML(context.Background(), get) + if out.failed { + return out + } + + if _, ok := p.cat.(tableCommitter); !ok { + return stmtFail(athenaErrTypeSyntax, "NOT_SUPPORTED: the catalog cannot commit Iceberg metadata") + } + + cols, err := icebergColumns(tbl.meta.CurrentSchema(), p.display()) + if err != nil { + return engineFailure(err) + } + + names := make([]string, len(cols)) + for i, c := range cols { + names[i] = c.Name + } + + d := p.stmt.dml + + switch p.stmt.kind { + case writeDelete: + p.stmt.body = "SELECT " + hiddenCols("") + " FROM " + d.targetText + andWhere(d.where) + case writeUpdate: + items, uout := updateItems(names, d.sets, "") + if uout.failed { + return uout + } + + p.stmt.body = "SELECT " + hiddenCols("") + ", " + strings.Join(items, ", ") + " FROM " + d.targetText + + andWhere(d.where) + case writeOptimize: + if out = checkPartitionOnly(d.where, tbl.meta, names); out.failed { + return out + } + + pred := "1" + if d.where != "" { + pred = "(" + d.where + ")" + } + + p.stmt.body = "SELECT " + hiddenCols("") + ", " + pred + ", " + quotedList(names, "") + " FROM " + d.targetText + default: + body, mout := p.mergeBody(names) + if mout.failed { + return mout + } + + p.stmt.body = body + } + + return stmtOK() +} + +func hiddenCols(alias string) string { + q := "" + if alias != "" { + q = quoteIdent(alias) + "." + } + + return q + quoteIdent(colRowPath) + ", " + q + quoteIdent(colRowPos) +} + +func quotedList(names []string, alias string) string { + q := "" + if alias != "" { + q = quoteIdent(alias) + "." + } + + out := make([]string, len(names)) + for i, n := range names { + out[i] = q + quoteIdent(n) + } + + return strings.Join(out, ", ") +} + +func updateItems(names []string, sets []dmlAssign, alias string) ([]string, statementOutcome) { + byCol := map[string]string{} + + for _, s := range sets { + known := slices.ContainsFunc(names, func(n string) bool { return strings.EqualFold(n, s.col) }) + if !known { + return nil, stmtFail(athenaErrTypeEntityMiss, + "COLUMN_NOT_FOUND: line 1:1: Column '%s' cannot be resolved", s.col) + } + + byCol[strings.ToLower(s.col)] = s.expr + } + + q := "" + if alias != "" { + q = quoteIdent(alias) + "." + } + + items := make([]string, len(names)) + + for i, n := range names { + if e, ok := byCol[strings.ToLower(n)]; ok { + items[i] = "(" + e + ")" + } else { + items[i] = q + quoteIdent(n) + } + } + + return items, stmtOK() +} + +func checkPartitionOnly(where string, meta *iceberg.TableMetadata, names []string) statementOutcome { + if where == "" { + return stmtOK() + } + + toks, err := tokenize(where) + if err != nil { + return stmtOK() + } + + part := map[string]bool{} + spec := meta.DefaultSpec() + schema := meta.CurrentSchema() + + for _, pf := range spec.Fields { + for _, f := range schema.Fields { + if f.ID == pf.SourceID { + part[strings.ToLower(f.Name)] = true + } + } + } + + for _, t := range significant(toks) { + if t.kind != tkWord && t.kind != tkQuoted { + continue + } + + n := t.ident() + if slices.ContainsFunc(names, func(c string) bool { return strings.EqualFold(c, n) }) && !part[n] { + return stmtFail(athenaErrTypeSyntax, + "NOT_SUPPORTED: line 1:1: Only partition columns are allowed in the OPTIMIZE WHERE clause: %s", n) + } + } + + return stmtOK() +} + +func (p *writePlan) mergeBody(names []string) (string, statementOutcome) { + d := p.stmt.dml + + alias := d.targetAlias + if alias == "" { + alias = p.stmt.target[len(p.stmt.target)-1] + } + + hid := quoteIdent(alias) + "." + quoteIdent(colRowPath) + cases := make([]string, 0, len(d.clauses)) + blocks := make([]string, 0, len(d.clauses)*len(names)) + + for i, mc := range d.clauses { + cond := hid + " IS NULL" + if mc.matched { + cond = hid + " IS NOT NULL" + } + + if mc.cond != "" { + cond += " AND (" + mc.cond + ")" + } + + cases = append(cases, fmt.Sprintf("WHEN %s THEN %d", cond, i+1)) + + items, out := mergeBlock(mc, names, alias) + if out.failed { + return "", out + } + + blocks = append(blocks, items...) + } + + sel := hiddenCols(alias) + ", CASE " + strings.Join(cases, " ") + " ELSE 0 END" + if len(blocks) > 0 { + sel += ", " + strings.Join(blocks, ", ") + } + + return "SELECT " + sel + " FROM " + d.source + " LEFT JOIN " + d.targetText + " AS " + quoteIdent(alias) + + " ON (" + d.on + ")", stmtOK() +} + +func mergeBlock(mc mergeClause, names []string, alias string) ([]string, statementOutcome) { + switch mc.action { + case mergeUpdate: + return updateItems(names, mc.sets, alias) + case mergeInsert: + cols := mc.insCols + if len(cols) == 0 { + cols = names + } + + if len(mc.insVals) != len(cols) { + return nil, stmtFail(athenaErrTypeSyntax, + "line 1:1: Insert query has mismatched column types: Table: [%d], Query: %d column(s)", + len(cols), len(mc.insVals)) + } + + vals := map[string]string{} + + for i, c := range cols { + if !slices.ContainsFunc(names, func(n string) bool { return strings.EqualFold(n, c) }) { + return nil, stmtFail(athenaErrTypeEntityMiss, + "COLUMN_NOT_FOUND: line 1:1: Column '%s' cannot be resolved", c) + } + + vals[strings.ToLower(c)] = mc.insVals[i] + } + + items := make([]string, len(names)) + + for i, n := range names { + if v, ok := vals[strings.ToLower(n)]; ok { + items[i] = "(" + v + ")" + } else { + items[i] = "NULL" + } + } + + return items, stmtOK() + default: + return nil, stmtOK() + } +} + +// openForApply re-reads the target and fails when it moved since the query read it. +func (p *writePlan) openForApply(ctx context.Context) (*tableLoader, statementOutcome) { + tbl, out := p.lookupDML(ctx, p.get) + if out.failed { + return nil, out + } + + if p.target != nil && tbl.location != p.target.loadedAt { + return nil, stmtFail(athenaErrTypeSyntax, + "ICEBERG_COMMIT_ERROR: Failed to commit Iceberg update to table: %s (concurrent modification)", p.display()) + } + + p.ice = tbl + + return &tableLoader{get: p.get, list: p.list, limits: defaultEngineLimits()}, stmtOK() +} + +func parsePos(s string) int64 { + n, _ := strconv.ParseInt(s, 10, 64) + + return n +} + +type rowRef struct { + path string + pos int64 +} + +// applyIcebergDML dispatches DELETE, UPDATE, MERGE and OPTIMIZE once their feeding SELECT has run. +func (p *writePlan) applyIcebergDML(ctx context.Context, res *sqlResult, limits EngineLimits) statementOutcome { + l, out := p.openForApply(ctx) + if out.failed { + return out + } + + switch p.stmt.kind { + case writeOptimize: + return p.applyOptimize(ctx, l, res, limits) + case writeDelete: + return p.commitRowChanges(ctx, l, res, nil, limits, iceberg.OpDelete) + case writeUpdate: + keep := make([]int, len(res.rows)) + for i := range keep { + keep[i] = i + } + + return p.commitRowChanges(ctx, l, res, p.sliceRows(res, keep, dmlHidden), limits, iceberg.OpOverwrite) + default: + return p.applyMerge(ctx, l, res, limits) + } +} + +// sliceRows copies the columns from off onward of the chosen rows into a new result. +func (p *writePlan) sliceRows(res *sqlResult, rows []int, off int) *sqlResult { + out := &sqlResult{columns: res.columns[off:]} + + for _, r := range rows { + out.rows = append(out.rows, res.rows[r][off:]) + + var nulls []bool + if r < len(res.nulls) && res.nulls[r] != nil { + nulls = res.nulls[r][off:] + } + + out.nulls = append(out.nulls, nulls) + } + + return out +} + +func (p *writePlan) applyMerge( + ctx context.Context, + l *tableLoader, + res *sqlResult, + limits EngineLimits, +) statementOutcome { + d := p.stmt.dml + ncols := len(p.ice.meta.CurrentSchema().Fields) + + offsets := make([]int, len(d.clauses)) + next := dmlHidden + 1 + + for i, mc := range d.clauses { + offsets[i] = next + if mc.action != mergeDelete { + next += ncols + } + } + + seen := map[rowRef]bool{} + removed := map[rowRef]bool{} + + var fresh []int + + var freshOff []int + + for r, row := range res.rows { + act := int(parsePos(row[dmlHidden])) + if act == 0 { + continue + } + + mc := d.clauses[act-1] + + if mc.matched { + ref := rowRef{path: row[0], pos: parsePos(row[1])} + if seen[ref] { + return stmtFail(athenaErrTypeSyntax, "%s", errMultiMatch) + } + + seen[ref], removed[ref] = true, true + } + + if mc.action != mergeDelete { + fresh, freshOff = append(fresh, r), append(freshOff, offsets[act-1]) + } + } + + var added *sqlResult + + if len(fresh) > 0 { + added = &sqlResult{columns: res.columns[offsets[0]:min(len(res.columns), offsets[0]+ncols)]} + + for i, r := range fresh { + off := freshOff[i] + added.rows = append(added.rows, res.rows[r][off:off+ncols]) + + var nulls []bool + if r < len(res.nulls) && res.nulls[r] != nil { + nulls = res.nulls[r][off : off+ncols] + } + + added.nulls = append(added.nulls, nulls) + } + } + + return p.commitRefs(ctx, l, removed, added, limits, iceberg.OpOverwrite) +} + +// commitRowChanges removes the rows named by res's leading path/pos columns and appends added. +func (p *writePlan) commitRowChanges( + ctx context.Context, l *tableLoader, res *sqlResult, added *sqlResult, limits EngineLimits, op string, +) statementOutcome { + removed := map[rowRef]bool{} + for _, row := range res.rows { + removed[rowRef{path: row[0], pos: parsePos(row[1])}] = true + } + + return p.commitRefs(ctx, l, removed, added, limits, op) +} + +func (p *writePlan) commitRefs( + ctx context.Context, l *tableLoader, removed map[rowRef]bool, added *sqlResult, limits EngineLimits, op string, +) statementOutcome { + if len(removed) == 0 && (added == nil || len(added.rows) == 0) { + return stmtOK() + } + + ch := iceberg.Change{Operation: op} + + if added != nil && len(added.rows) > 0 { + src := make([]int, len(added.columns)) + for i := range src { + src[i] = i + } + + files, out := p.writeIcebergFiles(ctx, added, src, limits) + if out.failed { + return out + } + + ch.AddData = files + } + + if len(removed) > 0 { + dels, out := p.positionDeleteFiles(ctx, l, removed) + if out.failed { + return out + } + + ch.AddDeletes = dels + } + + if out := p.commitIcebergChange(ctx, ch, false); out.failed { + return out + } + + return stmtWrote(int64(len(removed))) +} + +func (p *writePlan) dataFileIndex(ctx context.Context, l *tableLoader) (map[string]fileEntry, error) { + snap, ok := p.ice.meta.CurrentSnapshot() + if !ok { + return map[string]fileEntry{}, nil + } + + files, err := l.snapshotFiles(ctx, snap) + if err != nil { + return nil, err + } + + idx := make(map[string]fileEntry, len(files.data)) + for _, f := range files.data { + idx[f.entry.DataFile.Path] = f + } + + return idx, nil +} + +type deleteGroup struct { + partition map[string]any + rows []rowRef + specID int +} + +// positionDeleteFiles writes one position delete file per partition of the affected data files. +func (p *writePlan) positionDeleteFiles( + ctx context.Context, l *tableLoader, removed map[rowRef]bool, +) ([]iceberg.DataFile, statementOutcome) { + idx, err := p.dataFileIndex(ctx, l) + if err != nil { + return nil, engineFailure(err) + } + + groups := map[string]*deleteGroup{} + + var keys []string + + for ref := range removed { + fe, ok := idx[ref.path] + if !ok { + return nil, stmtFail(athenaErrTypeSyntax, + "ICEBERG_COMMIT_ERROR: Failed to commit Iceberg update to table: %s (data file %s no longer exists)", + p.display(), ref.path) + } + + key := fmt.Sprint(fe.spec, "|", partitionKey(fe.entry.DataFile.Partition)) + + g, ok := groups[key] + if !ok { + g = &deleteGroup{partition: fe.entry.DataFile.Partition, specID: fe.spec} + groups[key] = g + keys = append(keys, key) + } + + g.rows = append(g.rows, ref) + } + + sort.Strings(keys) + + meta := p.ice.meta + out := make([]iceberg.DataFile, 0, len(keys)) + + for seq, k := range keys { + g := groups[k] + + slices.SortFunc(g.rows, func(a, b rowRef) int { + return cmp.Or(strings.Compare(a.path, b.path), cmp.Compare(a.pos, b.pos)) + }) + + res := &sqlResult{columns: []sqlColumn{{name: colFilePath}, {name: colPosName}}} + idxs := make([]int, len(g.rows)) + + for i, ref := range g.rows { + res.rows = append(res.rows, []string{ref.path, strconv.FormatInt(ref.pos, 10)}) + idxs[i] = i + } + + cols := []outCol{ + {name: colFilePath, typ: "string", src: 0, id: posDeleteFilePathID}, + {name: colPosName, typ: "bigint", src: 1, id: posDeletePosID}, + } + + body, eerr := encodeParquetRows(res, idxs, cols, meta.Properties[propParquetCodec]) + if eerr != nil { + return nil, engineFailure(eerr) + } + + spec, _ := meta.SpecByID(g.specID) + path := fmt.Sprintf("%s/data/%s%s_delete_%0*d.parquet", strings.TrimSuffix(meta.Location, "/"), + icebergPartDir(spec, g.partition), p.queryID, fileSeqWidth, seq) + + if o := p.putObjectAt(ctx, path, body); o.failed { + return nil, o + } + + out = append(out, iceberg.DataFile{ + Content: iceberg.ContentPositionDeletes, Path: path, Format: "PARQUET", Partition: g.partition, + RecordCount: int64(len(g.rows)), FileSize: int64(len(body)), + }) + } + + return out, stmtOK() +} + +func partitionKey(part map[string]any) string { + names := make([]string, 0, len(part)) + for n := range part { + names = append(names, n) + } + + sort.Strings(names) + + var sb strings.Builder + + for _, n := range names { + fmt.Fprintf(&sb, "%s=%v;", n, part[n]) + } + + return sb.String() +} + +// bindTarget marks the plan's table for the target so its rows carry file path and position. +func (p *writePlan) bindTarget(plan *enginePlan) { + want := p.db + "." + p.table + + for _, t := range plan.tables { + if t.display == want && t.travel.kind == "" { + t.rowIDs, p.target = true, t + } + } +} diff --git a/services/athena/engine_dml_parse.go b/services/athena/engine_dml_parse.go new file mode 100644 index 0000000000..b46cf0bfa2 --- /dev/null +++ b/services/athena/engine_dml_parse.go @@ -0,0 +1,328 @@ +package athena + +import ( + "slices" + "strings" +) + +const ( + writeDelete writeKind = iota + 100 + writeUpdate + writeMerge + writeOptimize +) + +type mergeAction int + +const ( + mergeDelete mergeAction = iota + 1 + mergeUpdate + mergeInsert +) + +type dmlAssign struct { + col string + expr string +} + +type mergeClause struct { + cond string + insCols []string + insVals []string + sets []dmlAssign + action mergeAction + matched bool +} + +// dmlSpec is a parsed DELETE, UPDATE, MERGE or OPTIMIZE. The SELECT that feeds it +// is built once the table's columns are known. +type dmlSpec struct { + targetText string + targetAlias string + where string + source string + on string + sets []dmlAssign + clauses []mergeClause +} + +// topIndex returns the index of the first token at or after from that is one +// of words outside parentheses and CASE blocks, or len(toks). +func topIndex(toks []token, from int, words ...string) int { + depth, cases := 0, 0 + + for i := from; i < len(toks); i++ { + t := toks[i] + + switch { + case t.isPunct("("): + depth++ + case t.isPunct(")"): + depth-- + case t.isWord("case"): + cases++ + case t.isWord("end") && cases > 0: + cases-- + case depth == 0 && cases == 0 && t.kind == tkWord: + if slices.ContainsFunc(words, t.isWord) { + return i + } + } + } + + return len(toks) +} + +func topCommas(toks []token, from, to int) []int { + var out []int + + depth, cases := 0, 0 + + for i := from; i < to; i++ { + switch t := toks[i]; { + case t.isPunct("("): + depth++ + case t.isPunct(")"): + depth-- + case t.isWord("case"): + cases++ + case t.isWord("end") && cases > 0: + cases-- + case depth == 0 && cases == 0 && t.isPunct(","): + out = append(out, i) + } + } + + return out +} + +// splitAssigns reads `col = expr, ...` between significant indexes from and to. +func (c *stmtCursor) splitAssigns(from, to int) ([]dmlAssign, bool) { + var out []dmlAssign + + start := from + + for _, comma := range append(topCommas(c.toks, from, to), to) { + if comma-start < argsThree || !c.toks[start+1].isPunct("=") || + (c.toks[start].kind != tkWord && c.toks[start].kind != tkQuoted) { + return nil, false + } + + out = append(out, dmlAssign{col: c.toks[start].ident(), expr: c.text(start+argsTwo, comma)}) + start = comma + 1 + } + + return out, len(out) > 0 +} + +func (c *stmtCursor) optionalAlias(stop ...string) string { + c.acceptWords("as") + + if c.done() || (c.toks[c.pos].kind != tkWord && c.toks[c.pos].kind != tkQuoted) { + return "" + } + + if slices.ContainsFunc(stop, c.toks[c.pos].isWord) { + return "" + } + + a := c.toks[c.pos].ident() + c.pos++ + + return a +} + +func (c *stmtCursor) targetName() ([]string, string) { + start := c.pos + + parts := c.qualifiedName() + if parts == nil { + return nil, "" + } + + return parts, c.text(start, c.pos) +} + +func (c *stmtCursor) parseDelete() (*writeStmt, bool) { + parts, text := c.targetName() + if parts == nil { + return nil, false + } + + w := &writeStmt{kind: writeDelete, target: parts, dml: &dmlSpec{targetText: text}} + + if c.acceptWords("where") { + w.dml.where = c.text(c.pos, len(c.toks)) + c.pos = len(c.toks) + } + + return w, c.done() +} + +func (c *stmtCursor) parseUpdate() (*writeStmt, bool) { + parts, text := c.targetName() + if parts == nil || !c.acceptWords("set") { + return nil, false + } + + end := topIndex(c.toks, c.pos, "where") + + sets, ok := c.splitAssigns(c.pos, end) + if !ok { + return nil, false + } + + w := &writeStmt{kind: writeUpdate, target: parts, dml: &dmlSpec{targetText: text, sets: sets}} + c.pos = end + + if c.acceptWords("where") { + w.dml.where = c.text(c.pos, len(c.toks)) + c.pos = len(c.toks) + } + + return w, c.done() +} + +func (c *stmtCursor) parseMerge() (*writeStmt, bool) { + parts, text := c.targetName() + if parts == nil { + return nil, false + } + + d := &dmlSpec{targetText: text} + d.targetAlias = c.optionalAlias("using") + + if !c.acceptWords("using") { + return nil, false + } + + srcStart := c.pos + + if c.done() { + return nil, false + } + + if _, ok := c.group(); !ok && c.qualifiedName() == nil { + return nil, false + } + + srcEnd := c.pos + alias := c.optionalAlias("on") + d.source = c.text(srcStart, srcEnd) + + if alias != "" { + d.source += " AS " + quoteIdent(alias) + } + + if !c.acceptWords("on") { + return nil, false + } + + onEnd := topIndex(c.toks, c.pos, "when") + d.on = c.text(c.pos, onEnd) + c.pos = onEnd + + for c.acceptWords("when") { + mc, ok := c.parseMergeClause() + if !ok { + return nil, false + } + + d.clauses = append(d.clauses, mc) + } + + return &writeStmt{kind: writeMerge, target: parts, dml: d}, len(d.clauses) > 0 && c.done() +} + +func (c *stmtCursor) parseMergeClause() (mergeClause, bool) { + var mc mergeClause + + switch { + case c.acceptWords("not", "matched"): + case c.acceptWords("matched"): + mc.matched = true + default: + return mc, false + } + + if c.acceptWords("and") { + end := topIndex(c.toks, c.pos, "then") + mc.cond = c.text(c.pos, end) + c.pos = end + } + + if !c.acceptWords("then") { + return mc, false + } + + end := topIndex(c.toks, c.pos, "when") + + switch { + case mc.matched && c.acceptWords("delete"): + mc.action = mergeDelete + case mc.matched && c.acceptWords("update", "set"): + sets, ok := c.splitAssigns(c.pos, end) + if !ok { + return mc, false + } + + mc.action, mc.sets, c.pos = mergeUpdate, sets, end + case !mc.matched && c.acceptWords("insert"): + return c.parseMergeInsert(mc, end) + default: + return mc, false + } + + return mc, c.pos == end +} + +func (c *stmtCursor) parseMergeInsert(mc mergeClause, end int) (mergeClause, bool) { + mc.action = mergeInsert + + if !c.done() && c.toks[c.pos].isPunct("(") { + inner, ok := c.group() + if !ok { + return mc, false + } + + for _, part := range splitCommas(inner) { + sig := significant(part) + if len(sig) != 1 { + return mc, false + } + + mc.insCols = append(mc.insCols, sig[0].ident()) + } + } + + if !c.acceptWords("values") || c.done() { + return mc, false + } + + inner, ok := c.group() + if !ok { + return mc, false + } + + for _, part := range splitCommas(inner) { + mc.insVals = append(mc.insVals, strings.TrimSpace(joinTokens(part))) + } + + return mc, c.pos == end +} + +func (c *stmtCursor) parseOptimize() (*writeStmt, bool) { + parts, text := c.targetName() + if parts == nil || !c.acceptWords("rewrite", "data", "using", "bin_pack") { + return nil, false + } + + w := &writeStmt{kind: writeOptimize, target: parts, dml: &dmlSpec{targetText: text}} + + if c.acceptWords("where") { + w.dml.where = c.text(c.pos, len(c.toks)) + c.pos = len(c.toks) + } + + return w, c.done() +} diff --git a/services/athena/engine_iceberg.go b/services/athena/engine_iceberg.go index 63d14c7c63..9130faecb7 100644 --- a/services/athena/engine_iceberg.go +++ b/services/athena/engine_iceberg.go @@ -1,6 +1,7 @@ package athena import ( + "bytes" "context" "database/sql" "errors" @@ -15,6 +16,8 @@ import ( const ( secondsPerDayInt = 86400 + colRowPath = "$path" + colRowPos = "$pos" travelTimestamp = "ts" travelVersion = "ver" tsMillisLayout = "2006-01-02 15:04:05.000" @@ -248,7 +251,7 @@ func (l *tableLoader) positionDeletes(ctx context.Context, files []fileEntry) (m perFile := map[string]map[int64]bool{} - err = pf.ForEach([]string{"file_path", "pos"}, func(vals []any) error { + err = pf.ForEach([]string{colFilePath, colPosName}, func(vals []any) error { p, _ := vals[0].(string) n, _ := vals[1].(int64) @@ -317,13 +320,23 @@ func (t *planTable) loadIceberg(ctx context.Context, conn *sql.Conn, l *tableLoa return t.loadIcebergMetaTable(ctx, conn, l, tbl, snap, hasSnap) } - schema := tbl.schemaFor(snap, hasSnap) + schema := tbl.meta.CurrentSchema() + if t.travel.kind != "" { + schema = tbl.schemaFor(snap, hasSnap) + } + + t.loadedAt = tbl.location cols, err := icebergColumns(schema, t.display) if err != nil { return err } + nreal := len(cols) + if t.rowIDs { + cols = append(cols, Column{Name: colRowPath, Type: columnTypeString}, Column{Name: colRowPos, Type: typeBigint}) + } + return fillTable(ctx, conn, t.schema, t.name, cols, func(stmt *sql.Stmt, kinds []colKind) error { if !hasSnap { return nil @@ -340,9 +353,10 @@ func (t *planTable) loadIceberg(ctx context.Context, conn *sql.Conn, l *tableLoa } sink := &rowSink{stmt: stmt, kinds: kinds, nargs: len(cols)} + cols = cols[:nreal] for _, f := range files.data { - if lerr := l.loadIcebergDataFile(ctx, tbl, schema, cols, f, dels, sink); lerr != nil { + if lerr := l.loadIcebergDataFile(ctx, tbl, schema, cols, f, dels, sink, t.rowIDs); lerr != nil { return lerr } } @@ -353,20 +367,33 @@ func (t *planTable) loadIceberg(ctx context.Context, conn *sql.Conn, l *tableLoa func (l *tableLoader) loadIcebergDataFile( ctx context.Context, tbl *icebergTable, schema iceberg.Schema, cols []Column, f fileEntry, - dels map[string][]deleteSet, sink *rowSink, + dels map[string][]deleteSet, sink *rowSink, rowIDs bool, ) error { df := f.entry.DataFile - if !strings.EqualFold(df.Format, "PARQUET") { + isAvro := strings.EqualFold(df.Format, "AVRO") + + if !strings.EqualFold(df.Format, "PARQUET") && !isAvro { return fmt.Errorf("%w: Iceberg data file format %s (%s)", errUnsupportedFormat, df.Format, df.Path) } + if isAvro { + data, rerr := l.readURI(ctx, df.Path) + if rerr != nil { + return icebergFailf("ICEBERG_CANNOT_OPEN_SPLIT", "Cannot open data file %s", df.Path) + } + + opts := avroOpts{skip: skipRows(dels[df.Path], f.seq), rowIDs: rowIDs, path: df.Path} + + return l.decodeAvro(ctx, bytes.NewReader(data), cols, sink, opts) + } + bucket, key, ok := splitS3URI(df.Path) if !ok { return icebergFailf("ICEBERG_CANNOT_OPEN_SPLIT", "Invalid data file location %q", df.Path) } spec, _ := tbl.meta.SpecByID(f.spec) - fcs := make([]fileCol, len(cols)) + fcs := make([]fileCol, len(cols), len(cols)+dmlHidden) for i, c := range cols { fcs[i] = fileCol{name: c.Name, hive: c.Type, fileName: c.Name, id: schema.Fields[i].ID} @@ -383,6 +410,11 @@ func (l *tableLoader) loadIcebergDataFile( } } + if rowIDs { + p := df.Path + fcs = append(fcs, fileCol{name: colRowPath, constant: &p}, fileCol{name: colRowPos, rowPos: true}) + } + return l.loadColumnarFile(ctx, bucket, key, fcs, skipRows(dels[df.Path], f.seq), sink) } diff --git a/services/athena/engine_iceberg_alter.go b/services/athena/engine_iceberg_alter.go new file mode 100644 index 0000000000..85c6588b09 --- /dev/null +++ b/services/athena/engine_iceberg_alter.go @@ -0,0 +1,271 @@ +package athena + +import ( + "context" + "errors" + "maps" + "strings" + + "github.com/blackbirdworks/gopherstack/pkgs/iceberg" +) + +type alterOp func(m *iceberg.TableMetadata, next *iceberg.TableMetadata) error + +func alterFail(err error) statementOutcome { + switch { + case errors.Is(err, iceberg.ErrColumnNotFound): + return stmtFail(athenaErrTypeEntityMiss, "COLUMN_NOT_FOUND: line 1:1: %s", err.Error()) + case errors.Is(err, iceberg.ErrColumnExists): + return stmtFail(athenaErrTypeSyntax, "line 1:1: %s", err.Error()) + default: + return stmtFail(athenaErrTypeSyntax, "NOT_SUPPORTED: line 1:1: %s", err.Error()) + } +} + +// parseAlterOp reads the action of ALTER TABLE t for an Iceberg table. +func parseAlterOp(c *stmtCursor) (alterOp, bool) { + switch { + case c.acceptWords("add", "columns"): + return parseAddColumns(c) + case c.acceptWords("drop", "column"), c.acceptWords("drop"): + return parseDropColumn(c) + case c.acceptWords("rename", "column"): + return parseRenameColumn(c) + case c.acceptWords("change"): + return parseChangeColumn(c) + case c.acceptWords("set", "tblproperties"): + return parseSetProps(c) + default: + return nil, false + } +} + +func parseAddColumns(c *stmtCursor) (alterOp, bool) { + inner, ok := c.group() + if !ok || !c.done() { + return nil, false + } + + cols := parseColumnTokens(inner) + + return func(m, next *iceberg.TableMetadata) error { + s := m.CurrentSchema() + + for _, col := range cols { + var err error + if s, _, err = next.WithColumn(s, col.Name, col.Type, col.Comment); err != nil { + return err + } + + next.LastColumnID++ + } + + next.SetSchema(s) + + return nil + }, len(cols) > 0 +} + +func parseDropColumn(c *stmtCursor) (alterOp, bool) { + name := c.qualifiedName() + if len(name) != 1 || !c.done() { + return nil, false + } + + return func(m, next *iceberg.TableMetadata) error { + s, err := next.WithoutColumn(m.CurrentSchema(), name[0]) + if err != nil { + return err + } + + next.SetSchema(s) + + return nil + }, true +} + +func parseRenameColumn(c *stmtCursor) (alterOp, bool) { + from := c.qualifiedName() + if len(from) != 1 || !c.acceptWords("to") { + return nil, false + } + + to := c.qualifiedName() + if len(to) != 1 || !c.done() { + return nil, false + } + + return func(m, next *iceberg.TableMetadata) error { + s, err := next.WithRenamedColumn(m.CurrentSchema(), from[0], to[0], "") + if err != nil { + return err + } + + next.SetSchema(s) + + return nil + }, true +} + +type changeSpec struct { + oldName, newName, typ, comment, after string + first bool +} + +func parseChangeSpec(c *stmtCursor) (changeSpec, bool) { + c.acceptWords("column") + + oldN, newN := c.qualifiedName(), c.qualifiedName() + if len(oldN) != 1 || len(newN) != 1 { + return changeSpec{}, false + } + + end := topIndex(c.toks, c.pos, "comment", "first", "after") + cs := changeSpec{oldName: oldN[0], newName: newN[0], typ: typeText(c.toks[c.pos:end])} + c.pos = end + + if c.acceptWords("comment") { + cs.comment = c.stringLit() + } + + if c.acceptWords("after") { + a := c.qualifiedName() + if len(a) != 1 { + return cs, false + } + + cs.after = a[0] + } else { + cs.first = c.acceptWords("first") + } + + return cs, cs.typ != "" && c.done() +} + +func parseChangeColumn(c *stmtCursor) (alterOp, bool) { + cs, ok := parseChangeSpec(c) + if !ok { + return nil, false + } + + return func(m, next *iceberg.TableMetadata) error { + s, err := next.WithRenamedColumn(m.CurrentSchema(), cs.oldName, cs.newName, cs.typ) + if err != nil { + return err + } + + for i := range s.Fields { + if cs.comment != "" && strings.EqualFold(s.Fields[i].Name, cs.newName) { + s.Fields[i].Doc = cs.comment + } + } + + if cs.first || cs.after != "" { + if s, err = next.WithMovedColumn(s, cs.newName, cs.first, cs.after); err != nil { + return err + } + } + + next.SetSchema(s) + + return nil + }, true +} + +func parseSetProps(c *stmtCursor) (alterOp, bool) { + inner, ok := c.group() + if !ok || !c.done() { + return nil, false + } + + raw := map[string]string{} + + for _, part := range splitCommas(inner) { + sig := significant(part) + if len(sig) != argsThree || !sig[1].isPunct("=") || sig[0].kind != tkString || sig[2].kind != tkString { + return nil, false + } + + raw[stringLiteral(sig[0])] = stringLiteral(sig[2]) + } + + props, err := icebergProps(raw) + + return func(_, next *iceberg.TableMetadata) error { + if err != nil { + return err + } + + maps.Copy(next.Properties, props) + + return nil + }, ok +} + +// execIcebergAlter handles ALTER TABLE on Iceberg tables; handled=false defers other ALTER forms. +func (b *InMemoryBackend) execIcebergAlter( + query string, qctx QueryExecutionContext, +) (*sqlResult, statementOutcome, bool) { + c, err := newStmtCursor(query) + if err != nil || !c.acceptWords("alter", "table") { + return nil, stmtOK(), false + } + + parts := c.qualifiedName() + if parts == nil { + return nil, stmtOK(), false + } + + save := c.pos + + op, ok := parseAlterOp(c) + if !ok { + c.pos = save + + return nil, stmtOK(), false + } + + tg, out, handled := b.icebergTargetFor(parts, qctx) + if !handled || out.failed { + return nil, out, handled + } + + return &sqlResult{}, tg.alter(context.Background(), op), true +} + +func (tg *icebergTarget) alter(ctx context.Context, op alterOp) statementOutcome { + p := tg.wp + + plan, err := tg.tbl.meta.PlanMetadataUpdate(tg.tbl.location, func(next *iceberg.TableMetadata) error { + return op(tg.tbl.meta, next) + }, p.now().UTC()) + if err != nil { + if _, ok := errors.AsType[*icebergError](err); ok { + return engineFailure(err) + } + + if se, ok := errors.AsType[*syntaxError](err); ok { + return stmtFail(athenaErrTypeSyntax, "%s", se.msg) + } + + return alterFail(err) + } + + conflict, out := p.publishPlan(ctx, tg.tbl, plan) + if out.failed { + return out + } + + if conflict { + return stmtFail(athenaErrTypeSyntax, + "ICEBERG_COMMIT_ERROR: Failed to commit Iceberg update to table: %s (concurrent modification)", p.display()) + } + + if setter, ok := p.cat.(tableColumnSetter); ok { + if cols, cerr := icebergColumns(plan.Metadata.CurrentSchema(), p.display()); cerr == nil { + _ = setter.SetTableColumns(p.db, p.table, cols) + } + } + + return stmtOK() +} diff --git a/services/athena/engine_iceberg_ctas.go b/services/athena/engine_iceberg_ctas.go new file mode 100644 index 0000000000..2d9433fe41 --- /dev/null +++ b/services/athena/engine_iceberg_ctas.go @@ -0,0 +1,243 @@ +package athena + +import ( + "context" + "errors" + "strings" + + "github.com/blackbirdworks/gopherstack/pkgs/iceberg" +) + +const ( + propTableType = "table_type" + propLocation = "location" + propPartitioning = "partitioning" +) + +func isIcebergCTASProp(k string) bool { + if _, ok := icebergTableProps()[k]; ok { + return true + } + + switch k { + case propTableType, propLocation, propIsExternal, propPartitioning, + "optimize_rewrite_min_data_file_size_bytes", "optimize_rewrite_max_data_file_size_bytes", + propParquetComp, propCompression: + return true + default: + return false + } +} + +func (p *writePlan) isIcebergCTAS() bool { + return p.stmt.kind == writeCTAS && strings.EqualFold(firstProp(p.stmt.props, propTableType), tableTypeIceberg) +} + +func (p *writePlan) validateIcebergCTAS() statementOutcome { + props := p.stmt.props + + for k := range props { + if k == "external_location" { + return stmtFail( + athenaErrTypeSyntax, + "INVALID_TABLE_PROPERTY: line 1:1: external_location is not supported for table type ICEBERG; use location", + ) + } + + if !isIcebergCTASProp(k) { + return stmtFail(athenaErrTypeSyntax, + "INVALID_TABLE_PROPERTY: line 1:1: table property '%s' is not supported", k) + } + } + + if strings.EqualFold(firstProp(props, propIsExternal), strTrue) { + return stmtFail( + athenaErrTypeSyntax, + "INVALID_TABLE_PROPERTY: line 1:1: is_external must be false for ICEBERG tables", + ) + } + + if f := strings.ToUpper(firstProp(props, propFormat)); f != "" && f != fmtNameParquet && f != fmtNameAvro { + return stmtFail( + athenaErrTypeSyntax, + "NOT_SUPPORTED: format %s is not supported for Iceberg tables by the embedded SQL engine; use PARQUET or AVRO", + f, + ) + } + + p.iceCTAS = true + + return stmtOK() +} + +// partitioningSpec turns partitioning = ARRAY['month(ts)', 'bucket(id, 8)', 'country'] into a spec. +func partitioningSpec(schema iceberg.Schema, entries []string) (iceberg.PartitionSpec, error) { + spec := iceberg.PartitionSpec{Fields: []iceberg.PartitionField{}} + + for _, e := range entries { + toks, err := tokenize(e) + if err != nil { + return spec, err + } + + sig := significant(toks) + if len(sig) > 1 { + sig = normalizeTransform(sig) + } + + pf, err := partitionField(schema, sig) + if err != nil { + return spec, err + } + + pf.FieldID = partitionFieldIDBase + len(spec.Fields) + spec.Fields = append(spec.Fields, pf) + } + + return spec, nil +} + +// normalizeTransform reorders bucket(col, N) / truncate(col, N) to the DDL form (N, col). +func normalizeTransform(sig []token) []token { + if len(sig) != 6 || !sig[1].isPunct("(") || !sig[3].isPunct(",") { + return sig + } + + fn := strings.ToLower(sig[0].text) + if (fn == fnBucket || fn == fnTruncate) && sig[2].kind != tkNumber && sig[4].kind == tkNumber { + return []token{sig[0], sig[1], sig[4], sig[3], sig[2], sig[5]} + } + + return sig +} + +func (p *writePlan) icebergCTASProps() (map[string]string, error) { + raw := map[string]string{} + + for k, v := range p.stmt.props { + switch k { + case propTableType, propLocation, propIsExternal, propPartitioning, + "optimize_rewrite_min_data_file_size_bytes", "optimize_rewrite_max_data_file_size_bytes": + case propParquetComp, propCompression: + raw[propWriteCompression] = v[0] + default: + raw[k] = v[0] + } + } + + return icebergProps(raw) +} + +func ctasColumnSpecs(res *sqlResult) ([]iceberg.ColumnSpec, statementOutcome) { + specs := make([]iceberg.ColumnSpec, len(res.columns)) + seen := map[string]bool{} + + for i, c := range res.columns { + name := strings.ToLower(c.name) + if seen[name] { + return nil, stmtFail(athenaErrTypeSyntax, "line 1:1: Duplicate column name '%s'", name) + } + + seen[name] = true + specs[i] = iceberg.ColumnSpec{Name: name, Type: c.glue} + } + + return specs, stmtOK() +} + +func (p *writePlan) applyIcebergCTAS(ctx context.Context, res *sqlResult, limits EngineLimits) statementOutcome { + icProps, err := p.icebergCTASProps() + if err != nil { + return engineFailure(err) + } + + specs, out := ctasColumnSpecs(res) + if out.failed { + return out + } + + schema, err := iceberg.SchemaFromColumns(specs) + if err != nil { + return stmtFail(athenaErrTypeSyntax, "NOT_SUPPORTED: %s", err.Error()) + } + + pspec, err := partitioningSpec(schema, p.stmt.props[propPartitioning]) + if err != nil { + return engineFailure(err) + } + + meta := iceberg.NewTableMetadata(iceberg.NewParams{ + Location: strings.TrimSuffix(p.location, "/"), Schema: schema, Spec: pspec, + Properties: icProps, Now: p.now(), + }) + + current, out := p.writeCTASMetadata(ctx, meta, res, limits) + if out.failed { + return out + } + + cols := make([]Column, len(specs)) + for i, s := range specs { + cols[i] = Column{Name: s.Name, Type: res.columns[i].glue} + } + + def := GlueTableDef{ + Name: p.table, TableType: tableTypeExternal, Location: meta.Location, Columns: cols, + Parameters: map[string]string{paramTableType: tableTypeIceberg, paramMetadataLoc: current}, + } + + if err = p.cat.CreateTableDef(p.db, def); err != nil { + if errors.Is(err, ErrCatalogEntityExists) { + return stmtFail(athenaErrTypeEntityFound, + "TABLE_ALREADY_EXISTS: line 1:1: Table '%s' already exists", p.display()) + } + + return stmtFail(athenaErrTypeSyntax, "line 1:1: Unable to create table: %s", err.Error()) + } + + return stmtOK() +} + +// writeCTASMetadata stores the empty table's metadata, then the first snapshot when the query returned rows. +func (p *writePlan) writeCTASMetadata( + ctx context.Context, meta *iceberg.TableMetadata, res *sqlResult, limits EngineLimits, +) (string, statementOutcome) { + body, err := iceberg.EncodeMetadata(meta) + if err != nil { + return "", engineFailure(err) + } + + loc := iceberg.InitialMetadataLocation(meta.Location) + if out := p.putObjectAt(ctx, loc, body); out.failed { + return "", out + } + + p.ice = &icebergTable{meta: meta, location: loc, display: p.db + "." + p.table} + + if p.stmt.noData || len(res.rows) == 0 { + return loc, stmtOK() + } + + src := make([]int, len(res.columns)) + for i := range src { + src[i] = i + } + + files, out := p.writeIcebergFiles(ctx, res, src, limits) + if out.failed { + return "", out + } + + plan, err := meta.PlanAppend(loc, nil, files, p.now().UTC()) + if err != nil { + return "", engineFailure(err) + } + + for _, o := range plan.Objects { + if out = p.putObjectAt(ctx, o.Location, o.Body); out.failed { + return "", out + } + } + + return plan.MetadataLocation, stmtOK() +} diff --git a/services/athena/engine_iceberg_ddl.go b/services/athena/engine_iceberg_ddl.go index 9d357d1097..9304175adf 100644 --- a/services/athena/engine_iceberg_ddl.go +++ b/services/athena/engine_iceberg_ddl.go @@ -61,9 +61,9 @@ func icebergProps(raw map[string]string) (map[string]string, error) { switch lk { case propFormat: - if !strings.EqualFold(v, "parquet") { + if !strings.EqualFold(v, "parquet") && !strings.EqualFold(v, "avro") { return nil, fmt.Errorf( - "%w: Iceberg format %s; the embedded engine writes PARQUET", + "%w: Iceberg format %s; the embedded engine writes PARQUET and AVRO", errUnsupportedFormat, v, ) @@ -223,7 +223,7 @@ func parseTransformCall(sig []token) (transformCall, error) { args := splitCommas(sig[2 : len(sig)-1]) colArg := 0 - if c.fn == "bucket" || c.fn == fnTruncate { + if c.fn == fnBucket || c.fn == fnTruncate { if len(args) != argsTwo { return c, ddlSyntax(c.fn + " takes a width and a column") } @@ -263,7 +263,7 @@ func partitionField(schema iceberg.Schema, sig []token) (iceberg.PartitionField, transform, suffix := c.fn, "_"+c.fn switch c.fn { - case "bucket", fnTruncate: + case fnBucket, fnTruncate: transform = c.fn + "[" + c.width + "]" if c.fn == fnTruncate { suffix = "_trunc" diff --git a/services/athena/engine_iceberg_dml_test.go b/services/athena/engine_iceberg_dml_test.go new file mode 100644 index 0000000000..4aff5d4fc8 --- /dev/null +++ b/services/athena/engine_iceberg_dml_test.go @@ -0,0 +1,353 @@ +package athena_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/iceberg" + "github.com/blackbirdworks/gopherstack/pkgs/parquetio" +) + +func newOrders(t *testing.T, partition string) *iceEnv { + t.Helper() + + e := newIceEnv(t) + + e.mustRun(t, "CREATE DATABASE shop") + e.mustRun(t, "CREATE TABLE orders (id bigint, category string, qty int, price double) "+partition+ + " LOCATION 's3://lake/orders/' TBLPROPERTIES ('table_type'='ICEBERG')") + e.mustRun(t, "INSERT INTO orders VALUES (1, 'a', 1, 10.5), (2, 'b', 2, 20.0), (3, 'a', 3, 30.0)") + e.mustRun(t, "INSERT INTO orders VALUES (4, 'c', 4, 40.0), (5, 'b', 5, 50.0)") + + return e +} + +func TestIceberg_DeleteWritesPositionDeletes(t *testing.T) { + t.Parallel() + + for _, part := range []string{"", "PARTITIONED BY (category)"} { + t.Run("part="+part, func(t *testing.T) { + t.Parallel() + + e := newOrders(t, part) + e.mustRun(t, "DELETE FROM orders WHERE category = 'a' OR id = 5") + + assert.Equal(t, [][]string{{"2"}, {"4"}}, + e.mustRun(t, "SELECT id FROM orders ORDER BY id")) + + meta, _ := e.metadata(t, "orders") + cur, _ := meta.CurrentSnapshot() + assert.Equal(t, "delete", cur.Summary["operation"]) + assert.Equal(t, "3", cur.Summary["added-position-deletes"]) + assert.Equal(t, "5", cur.Summary["total-records"]) + + list, err := iceberg.ReadManifestList(e.objectAt(t, cur.ManifestList)) + require.NoError(t, err) + + var deleteFiles int + + for _, mf := range list { + if mf.Content != iceberg.ContentPositionDeletes { + continue + } + + entries, merr := iceberg.ReadManifest(e.objectAt(t, mf.Path)) + require.NoError(t, merr) + + for _, en := range entries { + deleteFiles++ + + pf, perr := parquetio.Open(e.objectAt(t, en.DataFile.Path)) + require.NoError(t, perr) + assert.Equal(t, map[int]string{2147483546: "file_path", 2147483545: "pos"}, pf.ColumnIDs()) + assert.Equal(t, iceberg.ContentPositionDeletes, en.DataFile.Content) + } + } + + assert.GreaterOrEqual(t, deleteFiles, 1) + + e.mustRun(t, "DELETE FROM orders") + assert.Equal(t, [][]string{{"0"}}, e.mustRun(t, "SELECT count(*) FROM orders")) + }) + } +} + +func TestIceberg_UpdateAndMerge(t *testing.T) { + t.Parallel() + + e := newOrders(t, "PARTITIONED BY (category)") + + e.mustRun(t, "UPDATE orders SET qty = qty + 10, price = price * 2 WHERE category = 'a'") + assert.Equal( + t, + [][]string{ + {"1", "11", "21.0"}, + {"2", "2", "20.0"}, + {"3", "13", "60.0"}, + {"4", "4", "40.0"}, + {"5", "5", "50.0"}, + }, + e.mustRun(t, "SELECT id, qty, price FROM orders ORDER BY id"), + ) + + meta, _ := e.metadata(t, "orders") + cur, _ := meta.CurrentSnapshot() + assert.Equal(t, "overwrite", cur.Summary["operation"]) + + e.mustRun(t, "CREATE TABLE changes (id bigint, category string, qty int, price double) "+ + "LOCATION 's3://lake/changes/' TBLPROPERTIES ('table_type'='ICEBERG')") + e.mustRun(t, "INSERT INTO changes VALUES (2, 'b', 0, 0.0), (4, 'c', 99, 1.0), (6, 'd', 6, 60.0), (7, 'e', -1, 0.0)") + + e.mustRun(t, `MERGE INTO orders t USING changes s ON t.id = s.id + WHEN MATCHED AND s.qty = 0 THEN DELETE + WHEN MATCHED THEN UPDATE SET qty = s.qty, price = t.price + s.price + WHEN NOT MATCHED AND s.qty > 0 THEN INSERT (id, category, qty, price) VALUES (s.id, s.category, s.qty, s.price)`) + + assert.Equal(t, [][]string{ + {"1", "a", "11", "21.0"}, {"3", "a", "13", "60.0"}, {"4", "c", "99", "41.0"}, + {"5", "b", "5", "50.0"}, {"6", "d", "6", "60.0"}, + }, e.mustRun(t, "SELECT id, category, qty, price FROM orders ORDER BY id")) + + e.mustRun(t, "INSERT INTO changes VALUES (4, 'c', 1, 1.0)") + assert.Contains(t, e.failure(t, `MERGE INTO orders t USING changes s ON t.id = s.id + WHEN MATCHED THEN DELETE`), "MERGE_TARGET_ROW_MULTIPLE_MATCHES") +} + +func TestIceberg_OptimizeCompactsAndDropsDeletes(t *testing.T) { + t.Parallel() + + e := newIceEnv(t) + e.mustRun(t, "CREATE TABLE t (id bigint, cat string) PARTITIONED BY (cat) LOCATION 's3://lake/t/' "+ + "TBLPROPERTIES ('table_type'='ICEBERG', 'optimize_rewrite_data_file_threshold'='3')") + + for i := range 4 { + e.mustRun(t, "INSERT INTO t VALUES ("+strings.Repeat("1", i+1)+", 'x'), (100, 'y')") + } + + e.mustRun(t, "DELETE FROM t WHERE id = 11") + before, _ := e.metadata(t, "t") + cur, _ := before.CurrentSnapshot() + assert.Equal(t, "8", cur.Summary["total-data-files"]) + + assert.Contains(t, e.failure(t, "OPTIMIZE t REWRITE DATA USING BIN_PACK WHERE id > 1"), "partition columns") + + e.mustRun(t, "OPTIMIZE t REWRITE DATA USING BIN_PACK WHERE cat = 'x'") + + assert.Equal(t, [][]string{{"1"}, {"100"}, {"100"}, {"100"}, {"100"}, {"111"}, {"1111"}}, + e.mustRun(t, "SELECT id FROM t ORDER BY id")) + + after, _ := e.metadata(t, "t") + cur, _ = after.CurrentSnapshot() + assert.Equal(t, "replace", cur.Summary["operation"]) + assert.Equal(t, "5", cur.Summary["total-data-files"]) + assert.Equal(t, "0", cur.Summary["total-delete-files"]) +} + +func TestIceberg_CTAS(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + with string + files string + parts []string + }{ + {name: "plain", with: "table_type='ICEBERG', location='s3://lake/c1/', is_external=false", files: ".parquet"}, + { + name: "partitioned", + with: "table_type='ICEBERG', location='s3://lake/c2/', is_external=false, " + + "partitioning=ARRAY['category', 'bucket(id, 4)']", + parts: []string{"identity", "bucket[4]"}, files: ".parquet", + }, + { + name: "avro", + with: "table_type='ICEBERG', location='s3://lake/c3/', format='AVRO', write_compression='snappy'", + files: ".avro", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + e := newOrders(t, "") + e.mustRun( + t, + "CREATE TABLE copy WITH ("+tc.with+") AS SELECT id, category, qty > 2 AS big, price FROM orders", + ) + + assert.Equal( + t, + [][]string{ + {"1", "a", "false"}, + {"2", "b", "false"}, + {"3", "a", "true"}, + {"4", "c", "true"}, + {"5", "b", "true"}, + }, + e.mustRun(t, "SELECT id, category, big FROM copy ORDER BY id"), + ) + + meta, _ := e.metadata(t, "copy") + assert.Equal(t, "boolean", string(meta.CurrentSchema().Fields[2].Type[1:8])) + require.Len(t, meta.DefaultSpec().Fields, len(tc.parts)) + + for i, tr := range tc.parts { + assert.Equal(t, tr, meta.DefaultSpec().Fields[i].Transform) + } + + cur, _ := meta.CurrentSnapshot() + assert.Equal(t, "5", cur.Summary["total-records"]) + + var sawExt bool + + for k := range e.store.objects { + if strings.Contains(k, "/data/") && strings.HasSuffix(k, tc.files) { + sawExt = true + } + } + + assert.True(t, sawExt) + + e.mustRun(t, "DELETE FROM copy WHERE big") + assert.Equal(t, [][]string{{"2"}}, e.mustRun(t, "SELECT count(*) FROM copy")) + }) + } + + e := newOrders(t, "") + e.mustRun( + t, + "CREATE TABLE empty WITH (table_type='ICEBERG', location='s3://lake/e/') AS SELECT id FROM orders WITH NO DATA", + ) + assert.Equal(t, [][]string{{"0"}}, e.mustRun(t, "SELECT count(*) FROM empty")) + assert.Contains( + t, + e.failure( + t, + "CREATE TABLE bad WITH (table_type='ICEBERG', format='ORC', location='s3://lake/b/') AS SELECT id FROM orders", + ), + "ORC", + ) +} + +func TestIceberg_AlterTable(t *testing.T) { + t.Parallel() + + e := newOrders(t, "PARTITIONED BY (category)") + + e.mustRun(t, "ALTER TABLE orders ADD COLUMNS (note string, weight double)") + e.mustRun(t, "INSERT INTO orders VALUES (6, 'a', 6, 60.0, 'new', 1.5)") + assert.Equal(t, [][]string{{"5", ""}, {"6", "new"}}, + e.mustRun(t, "SELECT id, note FROM orders WHERE id >= 5 ORDER BY id")) + + e.mustRun(t, "ALTER TABLE orders CHANGE COLUMN qty quantity bigint AFTER id") + e.mustRun(t, "ALTER TABLE orders RENAME COLUMN note TO memo") + e.mustRun(t, "ALTER TABLE orders DROP COLUMN weight") + + assert.Equal(t, [][]string{{"6", "6", "new"}}, + e.mustRun(t, "SELECT id, quantity, memo FROM orders WHERE id = 6")) + assert.Equal(t, [][]string{{"1", "1", "a"}}, + e.mustRun(t, "SELECT id, quantity, category FROM orders WHERE id = 1")) + + meta, _ := e.metadata(t, "orders") + assert.Len(t, meta.Schemas, 5) + assert.Equal(t, 6, meta.LastColumnID) + + names := make([]string, 0, len(meta.CurrentSchema().Fields)) + for _, f := range meta.CurrentSchema().Fields { + names = append(names, f.Name) + } + + assert.Equal(t, []string{"id", "quantity", "category", "price", "memo"}, names) + + defCols := e.cat.defs["shop/orders"].Columns + assert.Len(t, defCols, 5) + + assert.Contains(t, e.failure(t, "ALTER TABLE orders DROP COLUMN category"), "partition") + assert.Contains(t, e.failure(t, "ALTER TABLE orders DROP COLUMN nope"), "COLUMN_NOT_FOUND") + assert.Contains(t, e.failure(t, "ALTER TABLE orders ADD COLUMNS (id int)"), "already exists") + assert.Contains(t, e.failure(t, "ALTER TABLE orders CHANGE COLUMN price price int"), "double to int") + + e.mustRun(t, "ALTER TABLE orders SET TBLPROPERTIES ('vacuum_max_snapshot_age_seconds'='60')") + + meta, _ = e.metadata(t, "orders") + assert.Equal(t, "60000", meta.Properties["history.expire.max-snapshot-age-ms"]) + assert.Contains(t, e.failure(t, "ALTER TABLE orders SET TBLPROPERTIES ('bogus'='1')"), "bogus") +} + +func TestIceberg_Vacuum(t *testing.T) { + t.Parallel() + + e := newOrders(t, "") + e.mustRun(t, "DELETE FROM orders WHERE id = 1") + e.mustRun( + t, + "ALTER TABLE orders SET TBLPROPERTIES ('vacuum_max_snapshot_age_seconds'='1', 'vacuum_min_snapshots_to_keep'='1')", + ) + + before, _ := e.metadata(t, "orders") + require.Len(t, before.Snapshots, 3) + + oldFiles := make([]string, 0, 2) + + for _, s := range before.Snapshots[:2] { + oldFiles = append(oldFiles, s.ManifestList) + } + + e.store.mu.Lock() + e.store.objects["lake/orders/data/orphan.parquet"] = []byte("x") + e.store.mu.Unlock() + + e.mustRun(t, "VACUUM orders") + + after, _ := e.metadata(t, "orders") + require.Len(t, after.Snapshots, 1) + assert.Equal(t, before.CurrentSnapshotID, after.CurrentSnapshotID) + assert.Len(t, after.SnapshotLog, 1) + + e.store.mu.Lock() + defer e.store.mu.Unlock() + + for _, f := range oldFiles { + _, ok := e.store.objects[strings.TrimPrefix(f, "s3://")] + assert.False(t, ok, f) + } + + _, orphan := e.store.objects["lake/orders/data/orphan.parquet"] + assert.False(t, orphan) +} + +func TestIceberg_DMLErrors(t *testing.T) { + t.Parallel() + + cases := []struct { + name, sql, want string + }{ + {"update unknown column", "UPDATE orders SET nope = 1", "COLUMN_NOT_FOUND"}, + {"missing table", "DELETE FROM ghost WHERE id = 1", "TABLE_NOT_FOUND"}, + { + "merge arity", + "MERGE INTO orders t USING orders s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id) VALUES (1, 2)", + "mismatched", + }, + {"vacuum missing", "VACUUM ghost", "does not exist"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + e := newOrders(t, "") + assert.Contains(t, e.failure(t, tc.sql), tc.want) + }) + } + + e := newOrders(t, "") + e.mustRun(t, "CREATE TABLE hive (id int) LOCATION 's3://lake/hive/'") + assert.Contains(t, e.failure(t, "DELETE FROM hive"), "transactional") + assert.Contains(t, e.failure(t, "OPTIMIZE hive REWRITE DATA USING BIN_PACK"), "Iceberg") + assert.Contains(t, e.failure(t, "VACUUM hive"), "Iceberg") +} diff --git a/services/athena/engine_iceberg_maint.go b/services/athena/engine_iceberg_maint.go new file mode 100644 index 0000000000..592b6a2d0e --- /dev/null +++ b/services/athena/engine_iceberg_maint.go @@ -0,0 +1,317 @@ +package athena + +import ( + "context" + "errors" + "slices" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + sdk_s3 "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/blackbirdworks/gopherstack/pkgs/iceberg" +) + +const ( + propMaxSnapshotAgeMS = "history.expire.max-snapshot-age-ms" + propMinSnapshots = "history.expire.min-snapshots-to-keep" + propMaxMetadataFiles = "write.metadata.previous-versions-max" + defaultSnapshotAgeMS = 432000 * millisPerSecondInt + defaultMinSnapshots = 1 + defaultMetadataFiles = 100 +) + +// S3Deleter is the optional delete side of the wired S3 backend, used by VACUUM. +type S3Deleter interface { + DeleteObject(ctx context.Context, input *sdk_s3.DeleteObjectInput) (*sdk_s3.DeleteObjectOutput, error) +} + +// tableColumnSetter is implemented by catalogs that keep a column list beside the Iceberg metadata. +type tableColumnSetter interface { + SetTableColumns(db, name string, cols []Column) error +} + +type icebergTarget struct { + wp *writePlan + l *tableLoader + tbl *icebergTable + del S3Deleter +} + +// icebergTargetFor resolves an Iceberg table for a maintenance statement. +// handled=false means the table is not an Iceberg table of a Glue catalog. +func (b *InMemoryBackend) icebergTargetFor( + parts []string, qctx QueryExecutionContext, +) (*icebergTarget, statementOutcome, bool) { + catalog, db, table := refTarget(parts, qctx) + + cat, ok := b.glueCatalogFor(catalog) + if !ok { + return nil, stmtOK(), false + } + + get, list, put, ok := b.ddlDeps() + if !ok { + return nil, stmtOK(), false + } + + def, err := cat.GetTableDef(db, table) + if err != nil { + return nil, stmtFail(athenaErrTypeEntityMiss, "Table does not exist: %s", table), true + } + + if tableTypeOf(def) != tableTypeIceberg { + return nil, stmtOK(), false + } + + l := &tableLoader{get: get, list: list, limits: b.limitsSnapshot()} + + tbl, err := l.openIceberg(context.Background(), def, db+"."+table) + if err != nil { + return nil, engineFailure(err), true + } + + wp := &writePlan{ + cat: cat, + put: put, + get: get, + list: list, + now: b.now, + catalog: catalog, + db: db, + table: table, + ice: tbl, + } + del, _ := b.s3.(S3Deleter) + + return &icebergTarget{wp: wp, l: l, tbl: tbl, del: del}, stmtOK(), true +} + +func (b *InMemoryBackend) execVacuum(query string, qctx QueryExecutionContext) (*sqlResult, statementOutcome, bool) { + toks, err := tokenize(query) + if err != nil { + return nil, stmtOK(), false + } + + c := &sigCursor{toks: significant(toks)} + if !c.acceptWords("vacuum") { + return nil, stmtOK(), false + } + + parts := c.qualifiedName() + if parts == nil || !c.done() { + return nil, stmtFail(athenaErrTypeSyntax, "line 1:1: mismatched input in VACUUM statement"), true + } + + catalog, db, table := refTarget(parts, qctx) + + cat, ok := b.glueCatalogFor(catalog) + if !ok { + return nil, stmtOK(), false + } + + if def, gerr := cat.GetTableDef(db, table); gerr == nil && tableTypeOf(def) != tableTypeIceberg { + return nil, stmtFail(athenaErrTypeSyntax, "NOT_SUPPORTED: VACUUM is supported only for Iceberg tables"), true + } + + tg, out, handled := b.icebergTargetFor(parts, qctx) + if !handled || out.failed { + return nil, out, handled + } + + return &sqlResult{}, tg.vacuum(context.Background()), true +} + +func propInt64(props map[string]string, key string, def int64) int64 { + return int64(intProp(props, key, int(min(def, int64(1<<31-1))))) +} + +// protectedSnapshots picks the snapshots VACUUM keeps: the newest vacuum_min_snapshots_to_keep +// and anything newer than the cutoff. +func protectedSnapshots(meta *iceberg.TableMetadata, cutoffMS int64) map[int64]bool { + minKeep := intProp(meta.Properties, propMinSnapshots, defaultMinSnapshots) + newest := slices.Clone(meta.Snapshots) + slices.SortFunc(newest, func(a, b iceberg.Snapshot) int { return int(b.TimestampMS - a.TimestampMS) }) + + protected := map[int64]bool{} + + for i, s := range newest { + if i < minKeep || s.TimestampMS >= cutoffMS { + protected[s.SnapshotID] = true + } + } + + return protected +} + +func (tg *icebergTarget) vacuum(ctx context.Context) statementOutcome { + p := tg.wp + meta := tg.tbl.meta + now := p.now().UTC() + ageMS := propInt64(meta.Properties, propMaxSnapshotAgeMS, defaultSnapshotAgeMS) + maxMeta := intProp(meta.Properties, propMaxMetadataFiles, defaultMetadataFiles) + cutoff := now.UnixMilli() - ageMS + protected := protectedSnapshots(meta, cutoff) + + var gone []iceberg.Snapshot + + var dropped []iceberg.MetadataLogEntry + + plan, err := meta.PlanMetadataUpdate(tg.tbl.location, func(next *iceberg.TableMetadata) error { + gone = next.ExpireSnapshots(func(s iceberg.Snapshot) bool { return protected[s.SnapshotID] }) + dropped = next.TrimMetadataLog(maxMeta - 1) + + return nil + }, now) + if err != nil { + return engineFailure(err) + } + + committed := len(gone) > 0 || len(dropped) > 0 + retained := meta + + if committed { + retained = plan.Metadata + + if conflict, out := p.publishPlan(ctx, tg.tbl, plan); out.failed || conflict { + if out.failed { + return out + } + + return stmtFail( + athenaErrTypeSyntax, + "ICEBERG_COMMIT_ERROR: Failed to commit Iceberg update to table: %s (concurrent modification)", + p.display(), + ) + } + } + + keep, force, err := tg.cleanupSets(ctx, retained, gone, dropped) + if err != nil { + return engineFailure(err) + } + + keep[tg.tbl.location] = true + keep[plan.MetadataLocation] = committed + + return tg.removeUnreferenced(ctx, keep, force, cutoff) +} + +// cleanupSets returns the files to keep and the files that must go even when young. +func (tg *icebergTarget) cleanupSets( + ctx context.Context, retained *iceberg.TableMetadata, gone []iceberg.Snapshot, dropped []iceberg.MetadataLogEntry, +) (map[string]bool, map[string]bool, error) { + keep, err := tg.reachable(ctx, retained.Snapshots) + if err != nil { + return nil, nil, err + } + + force, err := tg.reachable(ctx, gone) + if err != nil { + return nil, nil, err + } + + for k := range keep { + delete(force, k) + } + + for _, e := range retained.MetadataLog { + keep[e.MetadataFile] = true + } + + for _, e := range dropped { + force[e.MetadataFile] = true + } + + return keep, force, nil +} + +// reachable lists the manifest lists, manifests and data/delete files of the snapshots. +func (tg *icebergTarget) reachable(ctx context.Context, snaps []iceberg.Snapshot) (map[string]bool, error) { + keep := map[string]bool{} + + for _, s := range snaps { + if s.ManifestList == "" { + continue + } + + keep[s.ManifestList] = true + + raw, err := tg.l.readURI(ctx, s.ManifestList) + if err != nil { + continue + } + + list, err := iceberg.ReadManifestList(raw) + if err != nil { + return nil, icebergFailf("ICEBERG_INVALID_METADATA", "Invalid manifest list %s: %v", s.ManifestList, err) + } + + for _, mf := range list { + keep[mf.Path] = true + } + + files, err := tg.l.snapshotFiles(ctx, s) + if err != nil { + return nil, err + } + + for _, f := range append(files.data, files.deletes...) { + keep[f.entry.DataFile.Path] = true + } + } + + return keep, nil +} + +// removeUnreferenced deletes table files outside keep that are older than cutoffMS. +func (tg *icebergTarget) removeUnreferenced( + ctx context.Context, keep, force map[string]bool, cutoffMS int64, +) statementOutcome { + if tg.del == nil { + return stmtOK() + } + + bucket, prefix, ok := splitS3URI(strings.TrimSuffix(tg.tbl.meta.Location, "/") + "/") + if !ok { + return stmtOK() + } + + var token *string + + for { + out, err := tg.wp.list.ListObjectsV2(ctx, &sdk_s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), Prefix: aws.String(prefix), ContinuationToken: token, + }) + if err != nil { + return stmtFail( + athenaErrTypeSyntax, + "ICEBERG_FILESYSTEM_ERROR: cannot list s3://%s/%s: %s", + bucket, + prefix, + err, + ) + } + + for _, o := range out.Contents { + uri := "s3://" + bucket + "/" + aws.ToString(o.Key) + young := o.LastModified != nil && o.LastModified.After(time.UnixMilli(cutoffMS)) + if keep[uri] || (young && !force[uri]) { + continue + } + + if _, derr := tg.del.DeleteObject(ctx, &sdk_s3.DeleteObjectInput{ + Bucket: aws.String(bucket), Key: o.Key, + }); derr != nil && !errors.Is(derr, context.Canceled) { + return stmtFail(athenaErrTypeSyntax, "ICEBERG_FILESYSTEM_ERROR: cannot delete %s: %s", uri, derr) + } + } + + if !aws.ToBool(out.IsTruncated) || out.NextContinuationToken == nil { + return stmtOK() + } + + token = out.NextContinuationToken + } +} diff --git a/services/athena/engine_iceberg_meta.go b/services/athena/engine_iceberg_meta.go index 8b03e9da91..063b6d0855 100644 --- a/services/athena/engine_iceberg_meta.go +++ b/services/athena/engine_iceberg_meta.go @@ -202,7 +202,7 @@ func (l *tableLoader) manifestsTable( func (l *tableLoader) filesTable(ctx context.Context, snap iceberg.Snapshot, hasSnap bool) ([]Column, [][]any, error) { cols := []Column{ - {Name: "content", Type: typeInteger}, {Name: "file_path", Type: typeVarchar}, + {Name: "content", Type: typeInteger}, {Name: colFilePath, Type: typeVarchar}, {Name: "file_format", Type: typeVarchar}, {Name: "spec_id", Type: typeInteger}, {Name: "partition", Type: typeVarchar}, {Name: "record_count", Type: typeBigint}, {Name: "file_size_in_bytes", Type: typeBigint}, diff --git a/services/athena/engine_iceberg_test.go b/services/athena/engine_iceberg_test.go index f5e40cacef..1d70788374 100644 --- a/services/athena/engine_iceberg_test.go +++ b/services/athena/engine_iceberg_test.go @@ -43,6 +43,20 @@ func (c *iceCatalog) CommitTableMetadata(db, name, expected, next string) error return nil } +func (c *iceCatalog) SetTableColumns(db, name string, cols []athena.Column) error { + c.mu.Lock() + defer c.mu.Unlock() + + d, ok := c.defs[db+"/"+name] + if !ok { + return athena.ErrCatalogEntityNotFound + } + + d.Columns = cols + + return nil +} + type stepClock struct { t time.Time mu sync.Mutex diff --git a/services/athena/engine_iceberg_write.go b/services/athena/engine_iceberg_write.go index 995d218b46..ed0de2a5cc 100644 --- a/services/athena/engine_iceberg_write.go +++ b/services/athena/engine_iceberg_write.go @@ -17,6 +17,7 @@ import ( const ( maxCommitAttempts = 4 propParquetCodec = "write.parquet.compression-codec" + propDefaultFormat = "write.format.default" nullPartition = "null" ) @@ -46,10 +47,7 @@ func writeHiveType(t string) string { } func (p *writePlan) applyIcebergInsert(ctx context.Context, res *sqlResult, limits EngineLimits) statementOutcome { - meta := p.ice.meta - schema := meta.CurrentSchema() - - dest, err := icebergColumns(schema, p.display()) + dest, err := icebergColumns(p.ice.meta.CurrentSchema(), p.display()) if err != nil { return engineFailure(err) } @@ -59,11 +57,32 @@ func (p *writePlan) applyIcebergInsert(ctx context.Context, res *sqlResult, limi return out } + files, out := p.writeIcebergFiles(ctx, res, src, limits) + if out.failed || len(files) == 0 { + return out + } + + return p.commitIcebergChange(ctx, iceberg.Change{Operation: iceberg.OpAppend, AddData: files}, true) +} + +// writeIcebergFiles writes res as partitioned data files of the table's default +// spec; src maps each schema column to a result column. +func (p *writePlan) writeIcebergFiles( + ctx context.Context, res *sqlResult, src []int, limits EngineLimits, +) ([]iceberg.DataFile, statementOutcome) { + meta := p.ice.meta + schema := meta.CurrentSchema() + + dest, err := icebergColumns(schema, p.display()) + if err != nil { + return nil, engineFailure(err) + } + cols := make([]outCol, len(dest)) for i, c := range dest { if iceberg.PrimitiveName(schema.Fields[i].Type) == "" { - return stmtFail(athenaErrTypeSyntax, + return nil, stmtFail(athenaErrTypeSyntax, "NOT_SUPPORTED: writing nested column '%s' to an Iceberg table is not supported", c.Name) } @@ -72,7 +91,7 @@ func (p *writePlan) applyIcebergInsert(ctx context.Context, res *sqlResult, limi groups, err := p.icebergGroups(res, schema, cols) if err != nil { - return stmtFail(athenaErrTypeSyntax, "NOT_SUPPORTED: %s", err.Error()) + return nil, stmtFail(athenaErrTypeSyntax, "NOT_SUPPORTED: %s", err.Error()) } var ( @@ -80,39 +99,46 @@ func (p *writePlan) applyIcebergInsert(ctx context.Context, res *sqlResult, limi total int64 ) + ext, avroFmt := ".parquet", strings.EqualFold(meta.Properties[propDefaultFormat], "avro") + if avroFmt { + ext = ".avro" + } + for seq, g := range groups { if err = ctx.Err(); err != nil { - return engineFailure(err) + return nil, engineFailure(err) + } + + var body []byte + if avroFmt { + body, err = encodeAvroRows(res, g.rows, cols, avroCodecFor(meta.Properties[propParquetCodec])) + } else { + body, err = encodeParquetRows(res, g.rows, cols, meta.Properties[propParquetCodec]) } - body, eerr := encodeParquetRows(res, g.rows, cols, meta.Properties[propParquetCodec]) - if eerr != nil { - return engineFailure(eerr) + if err != nil { + return nil, engineFailure(err) } if total += int64(len(body)); total > limits.MaxResultBytes { - return engineFailure(fmt.Errorf("%w: more than %d bytes", errResultBytes, limits.MaxResultBytes)) + return nil, engineFailure(fmt.Errorf("%w: more than %d bytes", errResultBytes, limits.MaxResultBytes)) } - path := fmt.Sprintf("%s/data/%s%s_%0*d.parquet", strings.TrimSuffix(meta.Location, "/"), - icebergPartDir(meta.DefaultSpec(), g.partition), p.queryID, fileSeqWidth, seq) + path := fmt.Sprintf("%s/data/%s%s_%0*d%s", strings.TrimSuffix(meta.Location, "/"), + icebergPartDir(meta.DefaultSpec(), g.partition), p.queryID, fileSeqWidth, seq, ext) - if out = p.putObjectAt(ctx, path, body); out.failed { - return out + if out := p.putObjectAt(ctx, path, body); out.failed { + return nil, out } files = append(files, iceberg.DataFile{ - Path: path, Format: "PARQUET", Partition: g.partition, RecordCount: int64(len(g.rows)), + Path: path, Format: strings.ToUpper(ext[1:]), Partition: g.partition, RecordCount: int64(len(g.rows)), FileSize: int64(len(body)), ValueCounts: countsFor(res, g.rows, cols, false), NullCounts: countsFor(res, g.rows, cols, true), }) } - if len(files) == 0 { - return stmtOK() - } - - return p.commitIcebergAppend(ctx, files) + return files, stmtOK() } type icebergGroup struct { @@ -250,18 +276,23 @@ func (l *tableLoader) currentManifests(ctx context.Context, tbl *icebergTable) ( return list, nil } -// commitIcebergAppend writes the manifest, manifest list and metadata, then -// swaps the catalog pointer; a lost race re-plans the append on the new head. -func (p *writePlan) commitIcebergAppend(ctx context.Context, files []iceberg.DataFile) statementOutcome { +// commitIcebergChange writes the manifests, manifest list and metadata of ch, +// then swaps the catalog pointer. With retry a lost race re-plans on the new +// head (safe for appends); otherwise it fails. +func (p *writePlan) commitIcebergChange(ctx context.Context, ch iceberg.Change, retry bool) statementOutcome { l := &tableLoader{get: p.get, list: p.list, limits: defaultEngineLimits()} tbl := p.ice for range maxCommitAttempts { - conflict, out := p.tryCommit(ctx, l, tbl, files) + conflict, out := p.tryCommit(ctx, l, tbl, ch) if !conflict { return out } + if !retry { + break + } + def, err := p.cat.GetTableDef(p.db, p.table) if err != nil { return stmtFail( @@ -282,27 +313,46 @@ func (p *writePlan) commitIcebergAppend(ctx context.Context, files []iceberg.Dat // tryCommit makes one commit attempt; conflict=true means the catalog pointer moved. func (p *writePlan) tryCommit( - ctx context.Context, l *tableLoader, tbl *icebergTable, files []iceberg.DataFile, + ctx context.Context, l *tableLoader, tbl *icebergTable, ch iceberg.Change, ) (bool, statementOutcome) { prev, err := l.currentManifests(ctx, tbl) if err != nil { return false, engineFailure(err) } - plan, err := tbl.meta.PlanAppend(tbl.location, prev, files, p.now().UTC()) + read := func(path string) ([]iceberg.ManifestEntry, error) { + body, rerr := l.readURI(ctx, path) + if rerr != nil { + return nil, icebergFailf("ICEBERG_MISSING_METADATA", "Manifest not found: %s", path) + } + + return iceberg.ReadManifest(body) + } + + plan, err := tbl.meta.PlanCommit(tbl.location, prev, read, ch, p.now().UTC()) if err != nil { return false, engineFailure(icebergFailf("ICEBERG_COMMIT_ERROR", "Cannot plan Iceberg update: %v", err)) } + return p.publishPlan(ctx, tbl, plan) +} + +// publishPlan stores the plan's objects and swaps the catalog's metadata pointer. +func (p *writePlan) publishPlan( + ctx context.Context, tbl *icebergTable, plan *iceberg.AppendPlan, +) (bool, statementOutcome) { for _, o := range plan.Objects { if out := p.putObjectAt(ctx, o.Location, o.Body); out.failed { return false, out } } - committer, _ := p.cat.(tableCommitter) + committer, ok := p.cat.(tableCommitter) + if !ok { + return false, stmtFail(athenaErrTypeSyntax, "NOT_SUPPORTED: the catalog cannot commit Iceberg metadata") + } - switch err = committer.CommitTableMetadata(p.db, p.table, tbl.location, plan.MetadataLocation); { + switch err := committer.CommitTableMetadata(p.db, p.table, tbl.location, plan.MetadataLocation); { case err == nil: return false, stmtOK() case errors.Is(err, ErrCatalogCommitConflict): @@ -312,3 +362,12 @@ func (p *writePlan) tryCommit( "ICEBERG_COMMIT_ERROR: Failed to commit Iceberg update to table: %s: %s", p.display(), err) } } + +func avroCodecFor(v string) string { + switch c := strings.ToUpper(v); c { + case codecSnappy, codecDeflate: + return c + default: + return "" + } +} diff --git a/services/athena/engine_load.go b/services/athena/engine_load.go index 6cf53bc662..45d4358d41 100644 --- a/services/athena/engine_load.go +++ b/services/athena/engine_load.go @@ -26,6 +26,7 @@ const ( fmtCSVSerde fmtJSON fmtParquet + fmtAvro ) const ( @@ -123,8 +124,9 @@ func detectFormat(def *GlueTableDef) (tableFormat, error) { return f, fmt.Errorf("%w: ORC (SerDe %q); the embedded SQL engine reads CSV, TSV, JSON lines and Parquet only", errUnsupportedFormat, def.SerdeLibrary) case strings.Contains(lib, "avro") || strings.Contains(in, "avro"): - return f, fmt.Errorf("%w: Avro (SerDe %q); the embedded SQL engine reads CSV, TSV, JSON lines and Parquet only", - errUnsupportedFormat, def.SerdeLibrary) + f.kind = fmtAvro + + return f, nil case strings.Contains(lib, "jsonserde"): f.kind = fmtJSON @@ -357,6 +359,10 @@ func (l *tableLoader) loadObject( return l.loadParquet(ctx, out.Body, bucket, key, cols, sink) } + if f.kind == fmtAvro { + return l.loadAvro(ctx, out.Body, bucket, key, cols, sink) + } + counted := &countingReader{r: out.Body, n: &l.scanned} br := bufio.NewReaderSize(counted, readBufBytes) @@ -384,6 +390,21 @@ func (l *tableLoader) loadObject( return nil } +func (l *tableLoader) loadAvro( + ctx context.Context, body io.Reader, bucket, key string, cols []Column, sink *rowSink, +) error { + data, err := io.ReadAll(io.LimitReader(body, l.limits.MaxScanBytes+1)) + if err != nil { + return fmt.Errorf("HIVE_CURSOR_ERROR: s3://%s/%s: %w", bucket, key, err) + } + + if l.scanned += int64(len(data)); l.scanned > l.limits.MaxScanBytes { + return fmt.Errorf("%w: more than %d bytes", errScanLimit, l.limits.MaxScanBytes) + } + + return l.decodeAvro(ctx, bytes.NewReader(data), cols, sink, avroOpts{}) +} + func (l *tableLoader) decode(ctx context.Context, rd io.Reader, f tableFormat, cols []Column, sink *rowSink) error { switch f.kind { case fmtJSON: diff --git a/services/athena/engine_optimize.go b/services/athena/engine_optimize.go new file mode 100644 index 0000000000..5c73fa1672 --- /dev/null +++ b/services/athena/engine_optimize.go @@ -0,0 +1,251 @@ +package athena + +import ( + "context" + "sort" + "strconv" + + "github.com/blackbirdworks/gopherstack/pkgs/iceberg" + "github.com/blackbirdworks/gopherstack/pkgs/parquetio" +) + +const ( + propRewriteDataThreshold = "optimize_rewrite_data_file_threshold" + propRewriteDeleteThreshold = "optimize_rewrite_delete_file_threshold" + defaultDataThreshold = 5 + defaultDeleteThreshold = 2 + defaultTargetFileBytes = 512 << 20 + minFileRatioNum = 3 + minFileRatioDen = 4 + maxFileRatioNum = 9 + maxFileRatioDen = 5 +) + +func intProp(props map[string]string, key string, def int) int { + if n, err := strconv.Atoi(props[key]); err == nil && n > 0 { + return n + } + + return def +} + +// deleteRefs maps each position delete file to the data files it names. +func (l *tableLoader) deleteRefs(ctx context.Context, dels []fileEntry) (map[string]map[string]bool, error) { + out := make(map[string]map[string]bool, len(dels)) + + for _, d := range dels { + data, err := l.readURI(ctx, d.entry.DataFile.Path) + if err != nil { + return nil, icebergFailf("ICEBERG_CANNOT_OPEN_SPLIT", "Cannot open delete file %s", d.entry.DataFile.Path) + } + + pf, err := parquetio.Open(data) + if err != nil { + return nil, icebergFailf( + "ICEBERG_CANNOT_OPEN_SPLIT", + "Cannot open delete file %s: %v", + d.entry.DataFile.Path, + err, + ) + } + + refs := map[string]bool{} + + err = pf.ForEach([]string{colFilePath}, func(vals []any) error { + if s, ok := vals[0].(string); ok { + refs[s] = true + } + + return nil + }) + if err != nil { + return nil, icebergFailf( + "ICEBERG_CURSOR_ERROR", + "Cannot read delete file %s: %v", + d.entry.DataFile.Path, + err, + ) + } + + out[d.entry.DataFile.Path] = refs + } + + return out, nil +} + +type optGroup struct { + files []string + rows []int + deletes bool +} + +type optInputs struct { + refs map[string]map[string]bool + byFile map[string][]int + matched map[string]bool + files snapshotFiles +} + +func specKey(spec int, part map[string]any) string { + return strconv.Itoa(spec) + "|" + partitionKey(part) +} + +func (p *writePlan) optimizeInputs( + ctx context.Context, l *tableLoader, res *sqlResult, snap iceberg.Snapshot, +) (*optInputs, error) { + files, err := l.snapshotFiles(ctx, snap) + if err != nil { + return nil, err + } + + refs, err := l.deleteRefs(ctx, files.deletes) + if err != nil { + return nil, err + } + + in := &optInputs{files: files, refs: refs, byFile: map[string][]int{}, matched: map[string]bool{}} + + for r, row := range res.rows { + in.byFile[row[0]] = append(in.byFile[row[0]], r) + + if v := row[dmlHidden]; v == "1" || v == strTrue { + in.matched[row[0]] = true + } + } + + return in, nil +} + +// groups buckets the data files worth rewriting by partition. +func (in *optInputs) groups(props map[string]string) ([]string, map[string]*optGroup) { + delThr := intProp(props, propRewriteDeleteThreshold, defaultDeleteThreshold) + minSize := int64(defaultTargetFileBytes * minFileRatioNum / minFileRatioDen) + maxSize := int64(defaultTargetFileBytes * maxFileRatioNum / maxFileRatioDen) + matchedParts := map[string]bool{} + + for _, f := range in.files.data { + if in.matched[f.entry.DataFile.Path] { + matchedParts[specKey(f.spec, f.entry.DataFile.Partition)] = true + } + } + + groups := map[string]*optGroup{} + + var keys []string + + for _, f := range in.files.data { + df := f.entry.DataFile + key := specKey(f.spec, df.Partition) + inScope := in.matched[df.Path] || (len(in.byFile[df.Path]) == 0 && matchedParts[key]) + heavy := in.deleteCount(df.Path) >= delThr + + if !inScope || (df.FileSize >= minSize && df.FileSize <= maxSize && !heavy) { + continue + } + + g, found := groups[key] + if !found { + g = &optGroup{} + groups[key], keys = g, append(keys, key) + } + + g.files = append(g.files, df.Path) + g.rows = append(g.rows, in.byFile[df.Path]...) + g.deletes = g.deletes || heavy + } + + sort.Strings(keys) + + return keys, groups +} + +func (in *optInputs) deleteCount(dataPath string) int { + n := 0 + + for _, r := range in.refs { + if r[dataPath] { + n++ + } + } + + return n +} + +func (in *optInputs) change(props map[string]string) (iceberg.Change, []int) { + dataThr := intProp(props, propRewriteDataThreshold, defaultDataThreshold) + ch := iceberg.Change{Operation: iceberg.OpReplace, RemoveData: map[string]bool{}, RemoveDeletes: map[string]bool{}} + + var rewrite []int + + keys, groups := in.groups(props) + + for _, k := range keys { + g := groups[k] + if len(g.files) < dataThr && !g.deletes { + continue + } + + for _, f := range g.files { + ch.RemoveData[f] = true + } + + rewrite = append(rewrite, g.rows...) + } + + for dp, r := range in.refs { + if len(r) > 0 && allRemoved(r, ch.RemoveData) { + ch.RemoveDeletes[dp] = true + } + } + + sort.Ints(rewrite) + + return ch, rewrite +} + +func allRemoved(refs, removed map[string]bool) bool { + for ref := range refs { + if !removed[ref] { + return false + } + } + + return true +} + +func (p *writePlan) applyOptimize( + ctx context.Context, l *tableLoader, res *sqlResult, limits EngineLimits, +) statementOutcome { + snap, ok := p.ice.meta.CurrentSnapshot() + if !ok { + return stmtOK() + } + + in, err := p.optimizeInputs(ctx, l, res, snap) + if err != nil { + return engineFailure(err) + } + + ch, rewrite := in.change(p.ice.meta.Properties) + if len(ch.RemoveData) == 0 { + return stmtOK() + } + + if len(rewrite) > 0 { + added := p.sliceRows(res, rewrite, dmlHidden+1) + + src := make([]int, len(added.columns)) + for i := range src { + src[i] = i + } + + newFiles, out := p.writeIcebergFiles(ctx, added, src, limits) + if out.failed { + return out + } + + ch.AddData = newFiles + } + + return p.commitIcebergChange(ctx, ch, false) +} diff --git a/services/athena/engine_run.go b/services/athena/engine_run.go index a87f671643..34a8599ee0 100644 --- a/services/athena/engine_run.go +++ b/services/athena/engine_run.go @@ -20,12 +20,14 @@ var ( type planTable struct { def *GlueTableDef + travel travelSpec schema string name string display string metaTable string - travel travelSpec + loadedAt string parts []GluePartitionDef + rowIDs bool } type enginePlan struct { @@ -33,6 +35,7 @@ type enginePlan struct { list S3Lister write *writePlan sql string + bools []bool tables []*planTable } @@ -77,6 +80,16 @@ func normalizeCatalog(c string) string { return c } +func isWriteLeader(upper string) bool { + for _, k := range []string{"CREATE", "INSERT", "UNLOAD", "DELETE", "UPDATE", "MERGE", "OPTIMIZE"} { + if hasKeyword(upper, k) { + return true + } + } + + return false +} + // planEngineQuery translates a SELECT over Glue tables with S3 locations, or // a CTAS/INSERT/UNLOAD whose source is such a SELECT. It returns planned=false // when the statement should take the legacy in-memory path. @@ -86,7 +99,7 @@ func (b *InMemoryBackend) planEngineQuery( upper := strings.ToUpper(strings.TrimSpace(query)) isRead := hasKeyword(upper, "SELECT") || hasKeyword(upper, "WITH") - if !isRead && !hasKeyword(upper, "CREATE") && !hasKeyword(upper, "INSERT") && !hasKeyword(upper, "UNLOAD") { + if !isRead && !isWriteLeader(upper) { return nil, stmtOK(), false } @@ -191,6 +204,7 @@ func (b *InMemoryBackend) buildPlan( } plan.sql = sqlText + plan.bools = boolOutputs(query) if err = applyProjection(plan, query, b.now().UTC()); err != nil { return nil, nil, err @@ -254,7 +268,7 @@ func (p *enginePlan) run(ctx context.Context, limits EngineLimits) (*sqlResult, return nil, loader.scanned, engineFailure(err) } - res, err := readResult(ctx, conn, p.sql, limits, p.write == nil) + res, err := readResult(ctx, conn, p.sql, limits, p.write == nil, p.bools) if err != nil { return nil, loader.scanned, engineFailure(err) } @@ -554,6 +568,10 @@ func formatCell(v any, decl string) string { return strconv.FormatBool(t != 0) } + if strings.HasPrefix(d, "DECIMAL") { + return strconv.FormatFloat(float64(t), 'f', decimalScale(d), 64) + } + return strconv.FormatInt(t, 10) case float64: if strings.HasPrefix(d, "DECIMAL") { @@ -613,7 +631,7 @@ func (rb *resultBuilder) cellText(i int, v any) string { return strings.TrimPrefix(s, containerMark) } -func newResultBuilder(rows *sql.Rows, limits EngineLimits, display bool) (*resultBuilder, error) { +func newResultBuilder(rows *sql.Rows, limits EngineLimits, display bool, bools []bool) (*resultBuilder, error) { names, err := rows.Columns() if err != nil { return nil, err @@ -638,9 +656,14 @@ func newResultBuilder(rows *sql.Rows, limits EngineLimits, display bool) (*resul rb.decls[i] = declVarchar } + name := resultColName(names[i], rb.decls[i], i) + + if rb.decls[i] == "" && i < len(bools) && bools[i] { + rb.decls[i] = strings.ToUpper(typeBoolean) + } + rb.res.columns[i] = sqlColumn{ - name: resultColName(names[i], rb.decls[i], i), typ: athenaTypeOfDecl(rb.decls[i]), - glue: glueTypeOfDecl(rb.decls[i]), + name: name, typ: athenaTypeOfDecl(rb.decls[i]), glue: glueTypeOfDecl(rb.decls[i]), } rb.inferred[i] = rb.decls[i] == "" } @@ -698,7 +721,7 @@ func (rb *resultBuilder) finish() *sqlResult { } func readResult( - ctx context.Context, conn *sql.Conn, query string, limits EngineLimits, display bool, + ctx context.Context, conn *sql.Conn, query string, limits EngineLimits, display bool, bools []bool, ) (*sqlResult, error) { rows, err := conn.QueryContext(ctx, query) if err != nil { @@ -706,7 +729,7 @@ func readResult( } defer func() { _ = rows.Close() }() - rb, err := newResultBuilder(rows, limits, display) + rb, err := newResultBuilder(rows, limits, display, bools) if err != nil { return nil, err } diff --git a/services/athena/engine_sql_funcs_test.go b/services/athena/engine_sql_funcs_test.go index 5d480f78d6..e5cc180d5e 100644 --- a/services/athena/engine_sql_funcs_test.go +++ b/services/athena/engine_sql_funcs_test.go @@ -101,7 +101,7 @@ func TestEngine_PrestoFunctions(t *testing.T) { { "array literal contains", "SELECT contains(ARRAY['x', 'y'], 'y') AS c FROM events WHERE id = 1", - [][]string{{"1"}}, + [][]string{{"true"}}, }, {"array_join", "SELECT array_join(ARRAY[1, 2, 3], '-') AS j FROM events WHERE id = 1", [][]string{{"1-2-3"}}}, {"array_agg", "SELECT cardinality(array_agg(id)) AS n FROM events", [][]string{{"2"}}}, @@ -137,7 +137,7 @@ func TestEngine_PrestoFunctions(t *testing.T) { {"approx_percentile", "SELECT approx_percentile(id, 0.5) AS p FROM events", [][]string{{"1.0"}}}, {"arbitrary", "SELECT arbitrary(id) AS a FROM events WHERE id = 2", [][]string{{"2"}}}, {"stddev_pop", "SELECT stddev_pop(id) AS s FROM events", [][]string{{"0.5"}}}, - {"bool_or", "SELECT bool_or(id > 1) AS b FROM events", [][]string{{"1"}}}, + {"bool_or", "SELECT bool_or(id > 1) AS b FROM events", [][]string{{"true"}}}, {"lpad", "SELECT lpad('7', 3, '0') AS p FROM events WHERE id = 1", [][]string{{"007"}}}, {"reverse", "SELECT reverse('abc') AS r FROM events WHERE id = 1", [][]string{{"cba"}}}, {"split_part", "SELECT split_part('a-b-c', '-', 2) AS s FROM events WHERE id = 1", [][]string{{"b"}}}, diff --git a/services/athena/engine_test.go b/services/athena/engine_test.go index 6636193dee..c33f6702e4 100644 --- a/services/athena/engine_test.go +++ b/services/athena/engine_test.go @@ -39,6 +39,15 @@ func (f *engineS3) PutObject(_ context.Context, in *sdk_s3.PutObjectInput) (*sdk return &sdk_s3.PutObjectOutput{}, nil } +func (f *engineS3) DeleteObject(_ context.Context, in *sdk_s3.DeleteObjectInput) (*sdk_s3.DeleteObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + + delete(f.objects, aws.ToString(in.Bucket)+"/"+aws.ToString(in.Key)) + + return &sdk_s3.DeleteObjectOutput{}, nil +} + func (f *engineS3) GetObject(ctx context.Context, in *sdk_s3.GetObjectInput) (*sdk_s3.GetObjectOutput, error) { if f.blockGet { <-ctx.Done() @@ -321,7 +330,7 @@ func TestEngine_Failures(t *testing.T) { name: "avro", sql: "SELECT * FROM sales", serde: "org.apache.hadoop.hive.serde2.avro.AvroSerDe", - wantReason: "NOT_SUPPORTED: unsupported data format: Avro", + wantReason: "HIVE_CURSOR_ERROR: invalid Avro file", }, { name: "orc", diff --git a/services/athena/engine_trino_sdk_test.go b/services/athena/engine_trino_sdk_test.go index 69a11016bb..eb60658c14 100644 --- a/services/athena/engine_trino_sdk_test.go +++ b/services/athena/engine_trino_sdk_test.go @@ -211,10 +211,14 @@ func TestSDK_TrinoLambdas(t *testing.T) { [][]string{{"60"}}, }, {"reduce empty", "SELECT reduce(tags, 7, (s, x) -> s + 1, s -> s) FROM docs WHERE id = 2", [][]string{{"7"}}}, - {"any_match", "SELECT any_match(nums, x -> x > 4) FROM docs ORDER BY id", [][]string{{"0"}, {"1"}}}, - {"all_match", "SELECT all_match(nums, x -> x > 1) FROM docs ORDER BY id", [][]string{{"0"}, {"1"}}}, - {"none_match", "SELECT none_match(nums, x -> x > 4) FROM docs ORDER BY id", [][]string{{"1"}, {"0"}}}, - {"any_match empty", "SELECT any_match(tags, x -> x = 'x') FROM docs ORDER BY id", [][]string{{"1"}, {"0"}}}, + {"any_match", "SELECT any_match(nums, x -> x > 4) FROM docs ORDER BY id", [][]string{{"false"}, {"true"}}}, + {"all_match", "SELECT all_match(nums, x -> x > 1) FROM docs ORDER BY id", [][]string{{"false"}, {"true"}}}, + {"none_match", "SELECT none_match(nums, x -> x > 4) FROM docs ORDER BY id", [][]string{{"true"}, {"false"}}}, + { + "any_match empty", + "SELECT any_match(tags, x -> x = 'x') FROM docs ORDER BY id", + [][]string{{"true"}, {"false"}}, + }, { "map_filter", "SELECT map_filter(m, (k, v) -> v > 1) FROM docs ORDER BY id", diff --git a/services/athena/engine_write.go b/services/athena/engine_write.go index e7187c893e..6ed5ba4ba0 100644 --- a/services/athena/engine_write.go +++ b/services/athena/engine_write.go @@ -35,7 +35,7 @@ var ( func isWriteProp(k string) bool { switch k { case propFormat, "external_location", "partitioned_by", "field_delimiter", - propWriteCompression, "compression", "parquet_compression", "orc_compression", "is_external": + propWriteCompression, propCompression, propParquetComp, "orc_compression", propIsExternal: return true default: return false @@ -60,22 +60,24 @@ type outCol struct { // writePlan is the write side of a CTAS, INSERT or UNLOAD. type writePlan struct { - cat GlueTableCatalog put S3Storer list S3Lister get S3Getter + cat GlueTableCatalog + ice *icebergTable now func() time.Time def *GlueTableDef stmt *writeStmt - catalog string + target *planTable db string table string - ice *icebergTable + catalog string queryID string outputLocation string location string partitionBy []string format outFormat + iceCTAS bool s3tables bool } @@ -118,11 +120,21 @@ func (b *InMemoryBackend) planWrite( wp.catalog, wp.db, wp.table = catalog, db, table } + if isDMLKind(w.kind) { + if out := wp.prepareDML(get); out.failed { + return nil, out, true + } + } + plan, unresolved, err := b.buildPlan(w.body, qctx, get, list) if err != nil { return nil, planFailure(err), true } + if isDMLKind(w.kind) { + wp.bindTarget(plan) + } + if len(plan.tables) == 0 && len(unresolved) > 0 { return nil, stmtOK(), false } @@ -143,7 +155,7 @@ func (b *InMemoryBackend) planWrite( func (p *writePlan) validate(b *InMemoryBackend) statementOutcome { switch p.stmt.kind { - case writeInsert: + case writeInsert, writeDelete, writeUpdate, writeMerge, writeOptimize: return stmtOK() case writeCTAS: if p.s3tables { @@ -162,6 +174,10 @@ func (p *writePlan) validate(b *InMemoryBackend) statementOutcome { func (p *writePlan) validateProps() statementOutcome { props := p.stmt.props + if p.isIcebergCTAS() { + return p.validateIcebergCTAS() + } + for k := range props { if !isWriteProp(k) { return stmtFail(athenaErrTypeSyntax, @@ -175,11 +191,11 @@ func (p *writePlan) validateProps() statementOutcome { } switch name { - case fmtNameText, fmtNameJSON, fmtNameCSV, fmtNameParquet: - case "ORC", "AVRO": + case fmtNameText, fmtNameJSON, fmtNameCSV, fmtNameParquet, fmtNameAvro: + case "ORC": return stmtFail(athenaErrTypeSyntax, - "NOT_SUPPORTED: format %s is not supported by the embedded SQL engine; use PARQUET, TEXTFILE, CSV or JSON", - name) + "NOT_SUPPORTED: format ORC is not supported by the embedded SQL engine; "+ + "use PARQUET, AVRO, TEXTFILE, CSV or JSON") default: return stmtFail(athenaErrTypeSyntax, "INVALID_TABLE_PROPERTY: line 1:1: invalid format '%s'", name) } @@ -189,6 +205,8 @@ func (p *writePlan) validateProps() statementOutcome { p.format.kind = fmtJSON case fmtNameParquet: p.format.kind = fmtParquet + case fmtNameAvro: + p.format.kind = fmtAvro case fmtNameCSV: p.format.kind, p.format.delim = fmtCSVSerde, csvDefaultDelim default: @@ -211,7 +229,7 @@ func (p *writePlan) validateProps() statementOutcome { func (p *writePlan) validateCompression() statementOutcome { props := p.stmt.props c := strings.ToUpper(orDefault(firstProp(props, propWriteCompression), - orDefault(firstProp(props, "compression"), firstProp(props, "parquet_compression")))) + orDefault(firstProp(props, propCompression), firstProp(props, propParquetComp)))) if out := p.checkCompression(c); out.failed { return out @@ -226,9 +244,21 @@ func (p *writePlan) validateCompression() statementOutcome { } func (p *writePlan) checkCompression(c string) statementOutcome { + if p.format.kind == fmtAvro { + switch c { + case "", compressionNone, codecDeflate, codecSnappy: + p.format.codec = c + + return stmtOK() + default: + return stmtFail(athenaErrTypeSyntax, + "NOT_SUPPORTED: compression %s is not supported for AVRO; use SNAPPY, DEFLATE or NONE", c) + } + } + if p.format.kind == fmtParquet { switch c { - case "", "SNAPPY", compressionGzip, "ZSTD", "LZ4", compressionNone: + case "", codecSnappy, compressionGzip, "ZSTD", "LZ4", compressionNone: p.format.codec = c return stmtOK() @@ -272,6 +302,8 @@ func (p *writePlan) precheck(ctx context.Context) (statementOutcome, bool) { switch p.stmt.kind { case writeInsert: return p.precheckInsert(ctx), false + case writeDelete, writeUpdate, writeMerge, writeOptimize: + return stmtOK(), false case writeUnload: p.location = withSlash(p.stmt.location) default: @@ -297,6 +329,9 @@ func (p *writePlan) precheck(ctx context.Context) (statementOutcome, bool) { func (p *writePlan) ctasLocation() (string, error) { loc := firstProp(p.stmt.props, "external_location") + if p.iceCTAS { + loc = firstProp(p.stmt.props, propLocation) + } switch { case loc != "": @@ -369,6 +404,8 @@ func (p *writePlan) apply(ctx context.Context, res *sqlResult, limits EngineLimi return p.applyInsert(ctx, res, limits) case writeUnload: return p.applyUnload(ctx, res, limits) + case writeDelete, writeUpdate, writeMerge, writeOptimize: + return p.applyIcebergDML(ctx, res, limits) default: return p.applyCTAS(ctx, res, limits) } @@ -395,6 +432,10 @@ func (p *writePlan) applyUnload(ctx context.Context, res *sqlResult, limits Engi } func (p *writePlan) applyCTAS(ctx context.Context, res *sqlResult, limits EngineLimits) statementOutcome { + if p.iceCTAS { + return p.applyIcebergCTAS(ctx, res, limits) + } + all := resultOutCols(res) seen := map[string]bool{} @@ -450,6 +491,10 @@ func (p *writePlan) ctasDef(data, parts []outCol) GlueTableDef { def.InputFormat = parquetInputFmt def.OutputFormat = parquetOutFmt def.SerdeParameters = map[string]string{serdeParamFmtKey: "1"} + case fmtAvro: + def.SerdeLibrary = avroSerdeLib + def.InputFormat = avroInputFmt + def.OutputFormat = avroOutputFmt case fmtJSON: def.SerdeLibrary = jsonSerdeLib case fmtCSVSerde: @@ -785,6 +830,8 @@ func (p *writePlan) encodeGroup(res *sqlResult, g *rowGroup, data []outCol) ([]b switch p.format.kind { case fmtParquet: return encodeParquetRows(res, g.rows, data, p.format.codec) + case fmtAvro: + return encodeAvroRows(res, g.rows, data, p.format.codec) case fmtJSON: body = encodeJSONLines(res, g.rows, data) case fmtCSVSerde: @@ -818,6 +865,8 @@ func (p *writePlan) putObject(ctx context.Context, loc string, seq int, body []b switch { case p.format.kind == fmtParquet: name += ".parquet" + case p.format.kind == fmtAvro: + name += ".avro" case p.format.gzip: name += ".gz" } diff --git a/services/athena/engine_write_parse.go b/services/athena/engine_write_parse.go index 61eaa210bf..897254a6bb 100644 --- a/services/athena/engine_write_parse.go +++ b/services/athena/engine_write_parse.go @@ -15,6 +15,7 @@ const ( // writeStmt is a parsed CTAS, INSERT INTO or UNLOAD statement. type writeStmt struct { props map[string][]string + dml *dmlSpec location string body string target []string @@ -77,6 +78,14 @@ func parseWriteStmt(sql string) (*writeStmt, bool) { return c.parseInsert() case c.acceptWords("unload"): return c.parseUnload() + case c.acceptWords("delete", "from"): + return c.parseDelete() + case c.acceptWords("update"): + return c.parseUpdate() + case c.acceptWords("merge", "into"): + return c.parseMerge() + case c.acceptWords("optimize"): + return c.parseOptimize() case c.acceptWords("create"): c.acceptWords("external") diff --git a/services/azureappconfig/PARITY.md b/services/azureappconfig/PARITY.md index d1b0595c1d..80c5831a1d 100644 --- a/services/azureappconfig/PARITY.md +++ b/services/azureappconfig/PARITY.md @@ -39,7 +39,6 @@ items_still_open: - "Replicas are metadata only and do not replicate or serve a separate data set." - "Soft-deleted stores lose their key-values; recovery restores an empty store." - "Customer-managed key encryption, private endpoint connections, SKU request quotas and ARM store updates through PATCH are not modelled." - - "No dashboard page." leaks: {status: clean, note: "Shutdown closes the HTTPS server and waits for the serve goroutine; there are no background workers."} --- diff --git a/services/azureappconfig/README.md b/services/azureappconfig/README.md index 054edfb8d2..40fa37f467 100644 --- a/services/azureappconfig/README.md +++ b/services/azureappconfig/README.md @@ -9,7 +9,7 @@ | --- | --- | | PARITY entries audited | 14 (14 ok) | | Feature families | 4 (4 ok) | -| Known gaps | 13 | +| Known gaps | 12 | | Deferred items | 0 | | Resource leaks | clean | @@ -27,7 +27,6 @@ - Replicas are metadata only and do not replicate or serve a separate data set. - Soft-deleted stores lose their key-values; recovery restores an empty store. - Customer-managed key encryption, private endpoint connections, SKU request quotas and ARM store updates through PATCH are not modelled. -- No dashboard page. ## More diff --git a/services/azurearm/PARITY.md b/services/azurearm/PARITY.md index 4f9cfc1302..e43ae134c3 100644 --- a/services/azurearm/PARITY.md +++ b/services/azurearm/PARITY.md @@ -24,12 +24,17 @@ ops: ServiceBusAuthorizationRuleListKeys: {wire: ok, errors: ok, state: ok, persist: n/a, note: "PUT (metadata-only, any rule name/rights accepted) + POST .../authorizationRules/{name}/listKeys on Microsoft.ServiceBus/namespaces/{ns}. Response shape {primaryConnectionString,secondaryConnectionString,primaryKey,secondaryKey,keyName} verified against hashicorp/go-azure-sdk's namespacesauthorizationrule.AccessKeys model. Both keys/connection-strings are the same fixed dev SAS identity (sbDefaultKeyName/sbDefaultKeyValue, mirroring services/azureservicebus/sas.go's DefaultKeyName/DefaultKeyValue), matching StorageAccountListKeys's identical key1/key2 convention. Critically, ListKeys does NOT require the caller to have PUT an authorizationRules resource first -- only that the parent namespace exists -- since resourceServiceBusNamespaceFlatten (services/servicebus/servicebus_namespace_resource.go) unconditionally calls ListKeys for the fixed \"RootManageSharedAccessKey\" rule on every namespace Read, with no corresponding azurerm_servicebus_namespace_authorization_rule resource in a typical Terraform config."} EventHubCRUD: {wire: ok, errors: ok, state: ok, persist: ok, note: "Microsoft.EventHub namespaces, eventhubs (partitionCount, messageRetentionInDays, retentionDescription), consumergroups, namespace- and hub-level authorizationRules with listKeys (real generated keys, connection strings with the AMQP endpoint), networkRuleSets/default (rp_eventhub*.go). Hubs, groups and rules are created in services/azureeventhubs through the EventHubs seam; ARM-side metadata persists in the eventHub snapshot field. Not implemented: namespace identity, regenerateKeys, disasterRecoveryConfigs, capture. Verified by test/terraform/azure/eventhubs_test.go."} EventGridCRUD: {wire: ok, errors: ok, state: ok, persist: ok, note: "Microsoft.EventGrid topics (inputSchema, endpoint) with listKeys and regenerateKey, topic-scoped eventSubscriptions (topics/{t}/providers/Microsoft.EventGrid/eventSubscriptions/{s}, getFullUrl), namespaces (listKeys, regenerateKey), namespaces/topics and namespaces/topics/eventSubscriptions (rp_eventgrid*.go). Topics, keys, subscriptions and queued events live in services/azureeventgrid through the EventGrid seam; ARM-side metadata persists in the eventGrid snapshot field. Not implemented: domains, system and partner topics, non-topic subscription scopes. Verified by test/terraform/azure/eventgrid_test.go."} + SearchServiceCRUD: {wire: ok, errors: partial, state: ok, persist: ok, note: "Microsoft.Search searchServices (rp_search*.go; armsearch v1.4.0, api 2025-05-01): PUT/PATCH/GET/list/DELETE, sku immutable on update, replica/partition validation per SKU (free 1x1, basic up to 3 replicas and 1 partition, others 1-12 replicas with partitions 1,2,3,4,6,12 and at most 36 search units, highDensity only on standard3 with up to 3 partitions), hostingMode, publicNetworkAccess, authOptions apiKeyOnly/aadOrApiKey versus disableLocalAuth, identity, tags, unmodelled properties; names 2-60 lowercase letters/digits/dashes, unique subscription-wide; listAdminKeys, regenerateAdminKey, createQueryKey/listQueryKeys/deleteQueryKey, checkNameAvailability, usages. Services and keys live in services/azuresearch through the SearchPlane seam; ARM-side metadata persists in the search snapshot field. Capacity limits come from learn.microsoft.com/azure/search/search-capacity-planning and are partly unverified; error codes for validation failures are generic BadRequest."} AppConfigurationCRUD: {wire: ok, errors: ok, state: ok, persist: ok, note: "Microsoft.AppConfiguration configurationStores (sku, identity with a generated principalId, tags, unmodelled properties; empty nested objects are dropped so empty encryption blocks do not drift) with listKeys, regenerateKey and listKeyValue, keyValues ({key}${label} names with ~XX escapes) and replicas, soft-deleted stores (deletedConfigurationStores get, list and purge; createMode Recover) and checkNameAvailability (rp_appconfig*.go). Stores, access keys and key-values live in services/azureappconfig through the AppConfig seam; ARM-side metadata persists in the appConfig snapshot field. Not implemented: customer-managed keys, private endpoints, PATCH updates. azurerm_app_configuration_key/_feature need an App Configuration domain suffix that custom metadata hosts cannot supply, so test/terraform/azure/appconfig_test.go applies the store and drives key-values through the SDK."} ServiceBusNamespaceNetworkRuleSet: {wire: ok, errors: ok, state: ok, persist: gap, note: "GET/PUT Microsoft.ServiceBus/namespaces/{ns}/networkRuleSets/default (rp_servicebus.go's sbKindNetworkRuleSet, sibling of queues/topics/authorizationRules in classifyServiceBusPath's 2-pair shape -- no new handler_ops.go route needed since the existing generic path walker already parses this shape correctly). Fixed the real-CI failure found in PR #2466's terraform-tests (6) run (404 ResourceNotFound retrieving the network rule set) -- root-caused by reading terraform-provider-azurerm@v4.81.0's pinned source directly: resourceServiceBusNamespaceFlatten (internal/services/servicebus/servicebus_namespace_resource.go:575) calls client.GetNetworkRuleSet on EVERY namespace Read, unconditionally -- not gated on SKU (this milestone's fixture uses Standard) or on whether the caller's config sets a `network_rule_set` block -- and classifyServiceBusPath had no case for the `networkRuleSets` type segment, so the request fell through to the generic ARM dispatcher and 404'd since nothing had ever PUT a generic resource at that path. hashicorp/go-azure-sdk@resource-manager/servicebus/2024-01-01/namespaces's GetNetworkRuleSet (method_getnetworkruleset.go) confirms the exact path shape (`fmt.Sprintf(\"%s/networkRuleSets/default\", id.ID())`, GET only, always \"default\") and response struct (NetworkRuleSet{properties: NetworkRuleSetProperties{defaultAction, ipRules, publicNetworkAccess, trustedServiceAccessEnabled, virtualNetworkRules}}, all fields pointer/omitempty). Fixed the same way as Storage's GetServiceProperties/{Blob,File}ServiceProperties bugs (AZURE.md section 10.10 M7/M8, bugs (7)/(8)): GET returns 200 with a minimal, schema-valid body (defaultAction:\"Allow\", publicNetworkAccess:\"Enabled\", trustedServiceAccessEnabled:false, empty ipRules/virtualNetworkRules) when no Put has ever been made for that namespace -- the common case, matching what real Azure reports for a namespace with no custom network rules configured -- and echoes back whatever properties a caller does PUT (exercised when terraform-provider-azurerm's createNetworkRuleSetForNamespace runs because a config sets `network_rule_set` explicitly). Both GET and PUT require the parent namespace to already exist (ErrServiceBusNamespaceNotFound otherwise), matching every other ServiceBus child resource's requireNamespace check. Confirmed via TestServiceBusProvider_NetworkRuleSet_{GetWithoutPut,PutThenGet,ParentNamespaceMissing} (rp_servicebus_test.go) -- a live tofu apply confirmation was not possible on this milestone's macOS sandbox because tofu apply fails earlier, at ARM's own metadata-discovery call, due to the pre-existing host-OS TLS-trust gap (AZURE.md section 10.10); expected to be confirmed by the next green CI run on Linux. persist=gap: same known gap as ServiceBusNamespaceCRUD."} DocumentDBAccountsSQL: {wire: ok, errors: ok, state: ok, persist: ok, note: "Microsoft.DocumentDB databaseAccounts (PUT/GET/PATCH/DELETE/LIST, listKeys, readonlykeys, listConnectionStrings, regenerateKey) and sqlDatabases/containers/throughputSettings (rp_cosmos*.go). Databases and containers live in services/cosmosdb through the CosmosResources seam, so ARM- and data-plane-created ones see each other; account keys are the data plane's master key plus generated keys it accepts (cosmosdb Handler.AcceptKey). One data plane serves every account: ARM-created databases belong to their account, data-plane-created ones show under all. PUT answers 200 synchronously; DELETE answers 202 with an Azure-AsyncOperation that is already Succeeded. Throughput and analyticalStorageTtl/partitionKey.version are ARM-side only. Verified by test/terraform/azure TestTerraform_Azure_CosmosDB (apply, no-drift plan, azcosmos item round trip with the listKeys key, destroy). Not implemented: migrateToAutoscale/Manual, failoverPriorityChange, other APIs (Mongo, Table, Cassandra, Gremlin), regenerating the primary key does not revoke the configured master key."} families: RedisCacheCRUD: {wire: ok, errors: ok, state: ok, persist: ok, note: "Microsoft.Cache redis caches (PUT/PATCH/DELETE as async operations, listKeys, regenerateKey, forceReboot, import, export) with firewallRules, patchSchedules and linkedServers children (rp_redis*.go). Real Redis servers, keys and data live in services/azureredis through the Redis seam; ARM-side metadata persists in the redis snapshot field. Not implemented: Microsoft.Cache/redisEnterprise, access policies, private endpoints. Applied without drift by test/terraform/azure/redis_test.go."} - ContainerAppsCRUD: {wire: ok, errors: partial, state: ok, persist: ok, note: "Microsoft.App at api-version 2025-07-01 (rp_containerapps*.go), checked against armappcontainers/v4@v4.0.0 and driven by the SDK pollers in aca_sdk_test.go, aca_engine_test.go and aca_jobs_test.go. managedEnvironments: PUT/GET/PATCH/DELETE/list (appLogsConfiguration with the Log Analytics shared key write-only, vnetConfiguration checked against Microsoft.Network subnets when the VNet exists and sized /27 for workload-profile or /23 for consumption-only environments, workloadProfiles validated, immutable vnet and zoneRedundant, defaultDomain -<6 hex>..azurecontainerapps.io, staticIp 127.0.0.1, delete refused while apps or jobs use the environment) and storages (AzureFile / NfsAzureFile, account key write-only). containerApps: PUT/GET/PATCH/DELETE/list, start, stop, listSecrets. configuration (secrets inline or keyVaultUrl with identity checked, registries, ingress external/internal with targetPort, transport, ipSecurityRestrictions and traffic, activeRevisionsMode Single/Multiple/Labels, dapr stored, maxInactiveRevisions pruning), template (containers with env/secretRef, vCPU and memory totals checked against the documented Consumption combinations, probes, volumes EmptyDir/Secret/AzureFile, scale min/max and rules), identity through rp_identity.go. A template change creates revision -- (7 hex or revisionSuffix; a name conflict is 400); Single mode retires older revisions once the new one is ready and keeps the old one serving when the new one fails; Multiple mode keeps them. Revisions list/get/activate/deactivate/restart and replicas list/get. Traffic weights must total 100 and name existing revisions; labels are validated. PUT/PATCH/DELETE answer with Azure-AsyncOperation (+Location on delete) already Succeeded; ETag and If-Match/If-None-Match are honoured on environments, apps and jobs. jobs: PUT/GET/PATCH/DELETE/list, start (with container overrides), stop, stop execution, executions list, listSecrets for Manual, Schedule and Event triggers. Replicas run in the Container Instances engine (same --azure-arm-aci-engine flag, same fake-able ACIRuntime): ingress hosts ., .internal. (internal), . and ---