diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 633dcb406..3fb7fe7da 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -7,12 +7,14 @@ # # Nothing complained before because actionlint has never run in this repository; # lint-release.yml is what introduces it. +# +# RunsOn labels need no entry here. Each one starts with +# `runs-on=${{ github.run_id }}`, and actionlint does not check a label that +# contains an expression. self-hosted-runner: labels: - blacksmith-4vcpu-ubuntu-2404 - # Every job in the imported EQL suite. Sixteen of them, and the size is - # deliberate upstream: the sharded SQLx matrix compiles the full Rust - # dependency tree, so the runner is chosen for the compile rather than for - # the tests. Kept as its own label rather than renamed to the 4vcpu one — - # rehoming those jobs is a cost decision, not a lint fix. + # Only light jobs still use this label: they run bash, compute a matrix or + # summarise results. The heavy jobs that used it, such as the Rust compiles + # and the Postgres suites, run on RunsOn instead. - blacksmith-16vcpu-ubuntu-2204 diff --git a/.github/workflows/_build-eql-docs.yml b/.github/workflows/_build-eql-docs.yml index 63e6fb4ca..13f296262 100644 --- a/.github/workflows/_build-eql-docs.yml +++ b/.github/workflows/_build-eql-docs.yml @@ -37,7 +37,7 @@ permissions: jobs: publish-docs: name: Build and publish documentation - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 20 steps: @@ -53,10 +53,12 @@ jobs: cache: false # publish path; see _build-eql-sql.yml working_directory: packages/eql + # A fresh RunsOn machine can still be running apt from boot, and apt-get + # fails at once on a held lock unless told to wait for it. - name: Install Doxygen run: | - sudo apt-get update - sudo apt-get install -y doxygen + sudo apt-get -o DPkg::Lock::Timeout=300 update + sudo apt-get -o DPkg::Lock::Timeout=300 install -y doxygen - name: Generate documentation env: diff --git a/.github/workflows/_build-eql-sql.yml b/.github/workflows/_build-eql-sql.yml index f27115dd1..8915a53fd 100644 --- a/.github/workflows/_build-eql-sql.yml +++ b/.github/workflows/_build-eql-sql.yml @@ -57,7 +57,7 @@ permissions: jobs: build: name: Build EQL - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 20 steps: diff --git a/.github/workflows/_build-ffi-artifacts.yml b/.github/workflows/_build-ffi-artifacts.yml index 14b454141..069c62986 100644 --- a/.github/workflows/_build-ffi-artifacts.yml +++ b/.github/workflows/_build-ffi-artifacts.yml @@ -114,12 +114,15 @@ jobs: # a native x86_64 build — so the other two legs were paying an # `apt-get update` (full package indexes) plus a ~200MB toolchain they do # not link against. + # + # A fresh RunsOn machine can still be running apt from boot, and apt-get + # fails at once on a held lock unless told to wait for it. - name: Install cross-compile toolchain (linux-arm64-gnu) if: ${{ matrix.cfg.platform == 'linux-arm64-gnu' }} run: | set -euo pipefail - sudo apt-get update - sudo apt-get install -y gcc-13-aarch64-linux-gnu + sudo apt-get -o DPkg::Lock::Timeout=300 update + sudo apt-get -o DPkg::Lock::Timeout=300 install -y gcc-13-aarch64-linux-gnu sudo ln -sf /usr/bin/aarch64-linux-gnu-gcc-13 /usr/bin/aarch64-linux-gnu-gcc # Rust comes from the runner image; only the TARGET has to be added. This diff --git a/.github/workflows/_test-auth-refresh-soak.yml b/.github/workflows/_test-auth-refresh-soak.yml index b129d8d78..56cb24e4d 100644 --- a/.github/workflows/_test-auth-refresh-soak.yml +++ b/.github/workflows/_test-auth-refresh-soak.yml @@ -12,7 +12,7 @@ permissions: jobs: soak: - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 30 env: MISE_ENV: test diff --git a/.github/workflows/bench-eql.yml b/.github/workflows/bench-eql.yml index 8d46945ae..7e0c023b0 100644 --- a/.github/workflows/bench-eql.yml +++ b/.github/workflows/bench-eql.yml @@ -93,7 +93,7 @@ defaults: jobs: bench: name: "Bench EQL (Postgres 17)" - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 60 env: @@ -146,6 +146,24 @@ jobs: # slow. Asserted by scripts/__tests__/eql-suite-ci.test.mjs. save-if: false + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + - name: Setup database run: | mise run postgres:up "postgres-${POSTGRES_VERSION}" --extra-args "--detach --wait" diff --git a/.github/workflows/crap-crates.yml b/.github/workflows/crap-crates.yml index 8a325ab1a..68f56520b 100644 --- a/.github/workflows/crap-crates.yml +++ b/.github/workflows/crap-crates.yml @@ -59,7 +59,7 @@ env: jobs: crap-stack-auth: name: CRAP (stack-auth) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 45 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -83,7 +83,7 @@ jobs: crap-stack-encrypt: name: CRAP (stack-encrypt) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 45 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/integration-drizzle.yml b/.github/workflows/integration-drizzle.yml index ddbf73c2c..e175ea0dd 100644 --- a/.github/workflows/integration-drizzle.yml +++ b/.github/workflows/integration-drizzle.yml @@ -204,7 +204,7 @@ on: jobs: integration: name: Drizzle v3 integration (db=${{ matrix.db }}) - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu24-full-x64/volume=80gb/extras=s3-cache # No concurrency group: `integration-db` gives each job its own compose # project and ephemeral host ports, so live-DB jobs no longer contend and do # not need serialising. See that action for why the old @@ -299,6 +299,24 @@ jobs: with: wasm: 'true' + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + # No pre-`up` cleanup step any more: the project name is unique per job, so # a container leaked by a hard-killed prior run cannot hold this job's # name or its (ephemeral) port. Blanket-pruning would now be actively diff --git a/.github/workflows/integration-prisma-next.yml b/.github/workflows/integration-prisma-next.yml index ed74631f2..206af0e12 100644 --- a/.github/workflows/integration-prisma-next.yml +++ b/.github/workflows/integration-prisma-next.yml @@ -114,7 +114,7 @@ on: jobs: integration: name: prisma-next v3 integration (db=${{ matrix.db }}) - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu24-full-x64/volume=80gb/extras=s3-cache # No concurrency group: `integration-db` gives each job its own compose # project and ephemeral host ports, so live-DB jobs no longer contend and do # not need serialising. See that action for why the old @@ -171,6 +171,24 @@ jobs: - name: Build the @cipherstash/auth binding uses: ./.github/actions/build-auth-binding + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + # No pre-`up` cleanup step any more: the project name is unique per job, so # a container leaked by a hard-killed prior run cannot hold this job's # name or its (ephemeral) port. Blanket-pruning would now be actively diff --git a/.github/workflows/integration-protect-ffi.yml b/.github/workflows/integration-protect-ffi.yml index 8b8f4ff03..dc481976b 100644 --- a/.github/workflows/integration-protect-ffi.yml +++ b/.github/workflows/integration-protect-ffi.yml @@ -160,7 +160,7 @@ concurrency: jobs: integration: name: protect-ffi integration suite - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu24-full-x64/volume=80gb/extras=s3-cache # Generous, because a cold `build-ffi-binding` compiles the cipherstash # client graph from scratch. It is also the backstop for `mise run # start-db`, whose readiness loop retries `pg_isready` forever rather than @@ -341,6 +341,24 @@ jobs: # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs cache_key_prefix: mise-v1-ubuntu-2404 + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + # One task, four things the suite cannot run without (see # integration-tests/tasks.toml): `pnpm install --frozen-lockfile` (a # no-op here — the step above already ran it), `docker compose up` for diff --git a/.github/workflows/integration-supabase.yml b/.github/workflows/integration-supabase.yml index 16d4da21e..ce98a0df7 100644 --- a/.github/workflows/integration-supabase.yml +++ b/.github/workflows/integration-supabase.yml @@ -173,7 +173,7 @@ on: jobs: integration: name: Supabase v3 integration (db=${{ matrix.db }}) - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu24-full-x64/volume=80gb/extras=s3-cache # No concurrency group: `integration-db` gives each job its own compose # project and ephemeral host ports, so live-DB jobs no longer contend and do # not need serialising. See that action for why the old @@ -250,6 +250,24 @@ jobs: - name: Build the @cipherstash/auth binding uses: ./.github/actions/build-auth-binding + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + # No pre-`up` cleanup step any more: the project name is unique per job, so # a container leaked by a hard-killed prior run cannot hold this job's # name or its (ephemeral) port. Blanket-pruning would now be actively diff --git a/.github/workflows/macro-expand-eql.yml b/.github/workflows/macro-expand-eql.yml index f0949d16b..868e40449 100644 --- a/.github/workflows/macro-expand-eql.yml +++ b/.github/workflows/macro-expand-eql.yml @@ -51,7 +51,7 @@ permissions: jobs: macro-expand: name: "Macro expand drift (nightly)" - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 30 steps: diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index 65cabc4c0..26ef909fb 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -64,7 +64,7 @@ env: jobs: mutants: name: Mutants gate - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=16/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 120 permissions: contents: read diff --git a/.github/workflows/release-golang.yml b/.github/workflows/release-golang.yml index 0db79e34a..8c08bdcfa 100644 --- a/.github/workflows/release-golang.yml +++ b/.github/workflows/release-golang.yml @@ -71,7 +71,7 @@ concurrency: jobs: build: name: Build and test the release - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 90 env: VERSION: ${{ inputs.version }} diff --git a/.github/workflows/release-postgres-eql-image.yml b/.github/workflows/release-postgres-eql-image.yml index c4006fd27..eec9dbfda 100644 --- a/.github/workflows/release-postgres-eql-image.yml +++ b/.github/workflows/release-postgres-eql-image.yml @@ -58,7 +58,7 @@ jobs: name: Build EQL SQL needs: [eql-armed] if: needs.eql-armed.outputs.armed == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 20 defaults: run: @@ -110,7 +110,7 @@ jobs: name: Build postgres-eql image (PG ${{ matrix.pg_version }}) needs: [eql-armed, build-sql] if: needs.eql-armed.outputs.armed == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 30 permissions: contents: read @@ -135,6 +135,24 @@ jobs: # The Dockerfile COPYs it from its own build context. path: packages/eql/docker/ + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + - name: Set up QEMU uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 with: diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index b3d0c2d60..92d832d4b 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -313,7 +313,7 @@ jobs: || needs.changes.outputs.relevant == 'true') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache env: # test:sqlx:archive depends on test:sqlx:prep, which copies the built EQL # into migrations/, applies it to a live Postgres, and regenerates the @@ -360,6 +360,24 @@ jobs: # `save-if: false` so a fast-finishing light job can't win the save race # and overwrite the key with a deps-less target/. + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + - name: Setup database (Postgres 17) run: | mise run postgres:up "postgres-${POSTGRES_VERSION}" --extra-args "--detach --wait" @@ -397,7 +415,7 @@ jobs: if: >- github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache strategy: fail-fast: false matrix: @@ -441,6 +459,24 @@ jobs: name: nextest-archive path: packages/eql + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + - name: Setup database (Postgres ${{ matrix.postgres-version }}) run: | mise run postgres:up "postgres-${POSTGRES_VERSION}" --extra-args "--detach --wait" @@ -459,7 +495,7 @@ jobs: if: >- github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache strategy: fail-fast: false matrix: @@ -486,6 +522,24 @@ jobs: shared-key: sqlx-tests save-if: false + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + - name: Setup database (Postgres ${{ matrix.postgres-version }}) run: | mise run postgres:up "postgres-${POSTGRES_VERSION}" --extra-args "--detach --wait" @@ -518,7 +572,7 @@ jobs: if: >- github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -547,10 +601,21 @@ jobs: if: >- github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache services: encryption-postgres: image: postgres:17 + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. One expression, not + # `username:` and `password:` keys: GitHub rejects an empty value for + # either key ("Unexpected value ''"), which fails the job on a fork's + # pull request. With either secret empty this gives `{}`, and the + # runner then pulls without a login. + credentials: >- + ${{ secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_PASSWORD != '' + && fromJSON(format('{{"username":{0},"password":{1}}}', + toJSON(secrets.DOCKER_HUB_USERNAME), toJSON(secrets.DOCKER_HUB_PASSWORD))) + || fromJSON('{}') }} env: POSTGRES_PASSWORD: postgres # Container port only: Docker picks the host port, and the test step @@ -675,7 +740,7 @@ jobs: if: >- github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -707,7 +772,7 @@ jobs: if: >- github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -743,7 +808,7 @@ jobs: if: >- github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -785,7 +850,7 @@ jobs: if: >- github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache env: POSTGRES_VERSION: "17" steps: @@ -817,6 +882,24 @@ jobs: workspaces: packages/eql shared-key: sqlx-tests save-if: false + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + - name: Setup database run: | mise run postgres:up "postgres-${POSTGRES_VERSION}" --extra-args "--detach --wait" @@ -845,7 +928,7 @@ jobs: # per-version `validate` job). docs-static: name: "SQL doc validation" - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -955,7 +1038,7 @@ jobs: || needs.changes.outputs.relevant == 'true') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache env: POSTGRES_VERSION: "17" CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_CLIENT_ACCESS_KEY }} @@ -987,6 +1070,24 @@ jobs: workspaces: packages/eql shared-key: sqlx-tests save-if: false + # Docker Hub refuses pulls without a login after too many from one + # address, and RunsOn machines reach that limit. With either secret empty, + # as on a fork's pull request, this step does nothing and pulls stay + # anonymous. scripts/__tests__/workflow-docker-hub-login.test.mjs + # requires it. + - name: Log in to Docker Hub + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + run: | + set -euo pipefail + if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then + echo "No Docker Hub credentials, so image pulls stay anonymous." + exit 0 + fi + printf '%s' "$DOCKER_HUB_PASSWORD" | + docker login --username "$DOCKER_HUB_USERNAME" --password-stdin + - name: Setup database (Postgres 17) run: | mise run postgres:up "postgres-${POSTGRES_VERSION}" --extra-args "--detach --wait" diff --git a/.github/workflows/tests-crates.yml b/.github/workflows/tests-crates.yml index 40cc572cc..9f421b4e4 100644 --- a/.github/workflows/tests-crates.yml +++ b/.github/workflows/tests-crates.yml @@ -79,7 +79,7 @@ env: jobs: rust: name: fmt, clippy, nextest, doctests, rustdoc - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 60 env: # Job-level, not workflow-level: RUSTFLAGS replaces the @@ -158,7 +158,7 @@ jobs: node-bindings: name: node bindings, napi typings, stack-auth-wasm - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 45 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 diff --git a/.github/workflows/tests-golang.yml b/.github/workflows/tests-golang.yml index 9bfff334a..6c4b82987 100644 --- a/.github/workflows/tests-golang.yml +++ b/.github/workflows/tests-golang.yml @@ -95,7 +95,7 @@ env: jobs: wasi-check: name: WASI check, guests and Go (Linux) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache timeout-minutes: 60 env: # wasm:guest:test and wasm:auth-guest:test run nextest, which is pinned diff --git a/scripts/__tests__/ffi-release-matrix.test.mjs b/scripts/__tests__/ffi-release-matrix.test.mjs index 3a71919ac..014ee1f71 100644 --- a/scripts/__tests__/ffi-release-matrix.test.mjs +++ b/scripts/__tests__/ffi-release-matrix.test.mjs @@ -56,7 +56,9 @@ function logFileOf(scriptName) { return redirect ? redirect[1] : null } -const MATRIX = releaseMatrix(TRIPLES) +const RUN_ID = '1234' +const MATRIX = releaseMatrix(TRIPLES, RUN_ID) +const RUNS_ON = `runs-on=${RUN_ID}/family=c8a+m8a/cpu=8/image=ubuntu24-full-x64/volume=80gb/extras=s3-cache` describe('the platform fixture matches the package', () => { it('covers exactly the platforms this package publishes', () => { @@ -157,14 +159,14 @@ describe('release matrix', () => { { platform: 'linux-x64-gnu', target: 'x86_64-unknown-linux-gnu', - os: 'blacksmith-4vcpu-ubuntu-2404', + os: RUNS_ON, script: 'zigbuild', log: 'zig.log', }, { platform: 'linux-arm64-gnu', target: 'aarch64-unknown-linux-gnu', - os: 'blacksmith-4vcpu-ubuntu-2404', + os: RUNS_ON, script: 'zigbuild', log: 'zig.log', }, @@ -181,7 +183,15 @@ describe('release matrix', () => { it('sends a platform the table does not name to a Linux runner', () => { // The fallback arm, which the table above cannot reach. A seventh platform // added tomorrow gets Blacksmith unless `runnerFor` is taught otherwise. - expect(runnerFor('linux-arm64-musl')).toBe('blacksmith-4vcpu-ubuntu-2404') + expect(runnerFor('linux-arm64-musl', RUN_ID)).toBe( + 'blacksmith-4vcpu-ubuntu-2404', + ) + }) + + it('refuses a RunsOn label with no run ID in it', () => { + // `runs-on=undefined/…` would be one label shared by every run, so one + // run's machine could take another run's job. + expect(() => runnerFor('linux-x64-gnu', '')).toThrow(/GITHUB_RUN_ID/) }) it('is empty for an empty mapping, rather than inventing platforms', () => { diff --git a/scripts/__tests__/workflow-docker-hub-login.test.mjs b/scripts/__tests__/workflow-docker-hub-login.test.mjs new file mode 100644 index 000000000..710da4ca1 --- /dev/null +++ b/scripts/__tests__/workflow-docker-hub-login.test.mjs @@ -0,0 +1,437 @@ +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs' +import { dirname, join, relative } from 'node:path' +import { describe, expect, it } from 'vitest' +import { expr } from './lib/expressions.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { readWorkflow, workflowFiles } from './lib/workflows.mjs' + +/** + * Every RunsOn job logs in to Docker Hub before it pulls an image. + * + * Docker Hub refuses pulls without a login after too many from one address. + * RunsOn machines reach that limit: on #1419, Postgres shards failed with + * `toomanyrequests` before a single test ran, and failed again on a re-run. + * So a RunsOn job that may pull an image logs in first, with the + * `DOCKER_HUB_USERNAME` and `DOCKER_HUB_PASSWORD` secrets. + * + * The login must do nothing when either secret is empty. That is the case on a + * fork's pull request, and on Dependabot's, and the job must still run there. + * + * - A service or job container from Docker Hub sets `credentials` to one + * expression that reads both secrets and falls back to `fromJSON('{}')`. + * `username:` and `password:` keys do not work: GitHub rejects an empty + * value for either ("Unexpected value ''"), so the job fails at "Set up + * job". With `{}`, the runner pulls without a login. + * - Any other pull needs a `run:` step before it that pipes the password to + * `docker login --password-stdin`, and exits early when either secret is + * empty. + * + * "May pull" is decided without knowing the registry, so it fails closed. A + * step that starts or builds a container counts, whichever registry its image + * comes from. That covers a `docker` command in the step, a `docker/` action + * that pulls, a local composite action that does either, and a `mise run` of a + * task that reaches either through its body, its `depends`, or a nested + * `mise run`. A login a job did not need costs one step and nothing else. + * + * Only a literal `runs-on=` label counts as RunsOn. GitHub-hosted runners are + * not limited this way, and Blacksmith jobs are leaving. + */ + +// --------------------------------------------------------------------------- +// What pulls an image +// --------------------------------------------------------------------------- + +/** A command that starts or builds a container, so may pull an image. */ +const PULLS = + /\bdocker(?:-compose\s+up\b|\s+(?:compose\b[^\n]*\bup\b|run\b|pull\b|create\b|build\b|buildx\s+build\b))/ + +/** + * `docker/` actions that pull: QEMU's binfmt image, the BuildKit image, and + * the base images of a build. Logging in and computing tags pull nothing. + */ +const PULLING_ACTION = /^docker\/(?!login-action@|metadata-action@)[\w-]+@/ + +const DOCKER_HUB_HOSTS = new Set([ + 'docker.io', + 'index.docker.io', + 'registry-1.docker.io', +]) + +/** Whether an image reference names Docker Hub, where a bare name resolves. */ +export function onDockerHub(image) { + const first = String(image).split('/')[0] + const isHost = + String(image).includes('/') && + (first.includes('.') || first.includes(':') || first === 'localhost') + return !isHost || DOCKER_HUB_HOSTS.has(first) +} + +// --------------------------------------------------------------------------- +// mise tasks +// --------------------------------------------------------------------------- + +const SKIP_DIRS = new Set(['node_modules', '.git', 'target', 'dist']) + +/** Every tracked-looking `mise.toml` / `mise..toml`, repo-relative. */ +function miseConfigs(dir = REPO_ROOT, found = []) { + for (const name of readdirSync(dir)) { + if (SKIP_DIRS.has(name)) continue + const path = join(dir, name) + if (statSync(path).isDirectory()) miseConfigs(path, found) + else if (/^\.?mise(\.[\w-]+)?\.toml$/.test(name)) + found.push(relative(REPO_ROOT, path)) + } + return found +} + +const parseStringArray = (inner) => + [...inner.matchAll(/"([^"]+)"|'([^']+)'/g)].map((m) => m[1] ?? m[2]) + +function parseIncludes(text) { + const section = /^\[task_config\][^\n]*\n([\s\S]*?)(?=^\[|$(?![\s\S]))/m.exec( + text, + ) + const array = /^includes\s*=\s*\[([^\]]*)\]/m.exec(section?.[1] ?? '') + return array ? parseStringArray(array[1]) : [] +} + +/** + * Task blocks. A mise config writes `[tasks."name"]`; an included file makes + * every top-level table a task, `["name"]`. + */ +function parseTomlTasks(text, { bareTables }) { + const header = bareTables + ? /^\[(?:"([^"]+)"|([\w:.-]+))\]/ + : /^\[tasks\.(?:"([^"]+)"|([\w:.-]+))\]/ + return text + .split(/^(?=\[)/m) + .map((block) => ({ match: header.exec(block), block })) + .filter(({ match }) => match) + .map(({ match, block }) => ({ name: match[1] ?? match[2], body: block })) +} + +/** File tasks: `tasks/test/splinter.sh` is the task `test:splinter`. */ +function fileTasks(dir, base = dir, found = []) { + for (const name of readdirSync(dir)) { + const path = join(dir, name) + if (statSync(path).isDirectory()) fileTasks(path, base, found) + else if (!name.endsWith('.toml')) + found.push({ + name: relative(base, path) + .replace(/\.[^./]+$/, '') + .split('/') + .join(':'), + body: readFileSync(path, 'utf8'), + }) + } + return found +} + +/** + * Task name -> bodies. Names repeat across configs (two packages can each + * have a `setup`), so a name keeps every body, and pulls if any of them does. + * That is the fail-closed direction. + */ +function taskIndex() { + const index = new Map() + const add = ({ name, body }) => { + if (!index.has(name)) index.set(name, []) + index.get(name).push(body) + } + for (const config of miseConfigs()) { + const text = readFileSync(join(REPO_ROOT, config), 'utf8') + for (const task of parseTomlTasks(text, { bareTables: false })) add(task) + for (const include of parseIncludes(text)) { + const path = join(REPO_ROOT, dirname(config), include) + if (!existsSync(path)) continue + if (statSync(path).isDirectory()) fileTasks(path).forEach(add) + else + parseTomlTasks(readFileSync(path, 'utf8'), { + bareTables: true, + }).forEach(add) + } + } + return index +} + +const stripCommentLines = (text) => text.replace(/^[ \t]*#(?!MISE).*$/gm, '') + +// A quote starts a command only at the start of a line, as in a TOML +// `run = [ "mise run …" ]` entry. Mid-line it is a message to the reader. +const COMMAND_START = String.raw`(?:^[ \t]*["']?|(?:&&|\|\||[;|(){}])[ \t]*)` + +function invokes(text, taskName) { + const escaped = taskName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + return new RegExp( + `${COMMAND_START}mise\\s+run\\s+[^\\n'"]*?(? + [...body.matchAll(/depends\s*=\s*\[([^\]]*)\]/g)].flatMap((m) => + parseStringArray(m[1]), + ) + +/** The names of every task that may pull an image, directly or through others. */ +function pullingTasks(index) { + const pulling = new Set( + [...index] + .filter(([, bodies]) => + bodies.some((body) => PULLS.test(stripCommentLines(body))), + ) + .map(([name]) => name), + ) + // Grow the set until nothing new reaches a member: a fixpoint, so a cycle + // between tasks cannot loop. + let grew = true + while (grew) { + grew = false + for (const [name, bodies] of index) { + if (pulling.has(name)) continue + const reaches = bodies.some( + (body) => + dependsOf(body).some((dep) => pulling.has(dep)) || + [...pulling].some((target) => invokes(body, target)), + ) + if (reaches) { + pulling.add(name) + grew = true + } + } + } + return pulling +} + +const PULLING_TASKS = pullingTasks(taskIndex()) + +// --------------------------------------------------------------------------- +// Steps and jobs +// --------------------------------------------------------------------------- + +/** Whether one step may pull an image. `seen` guards composite-action cycles. */ +export function stepPulls(step, seen = new Set()) { + const uses = typeof step?.uses === 'string' ? step.uses.trim() : '' + if (PULLING_ACTION.test(uses)) return true + if (uses.startsWith('./')) { + const dir = uses.replace(/^\.\//, '').replace(/\/$/, '') + if (seen.has(dir)) return false + seen.add(dir) + for (const file of ['action.yml', 'action.yaml']) { + const rel = `${dir}/${file}` + if (!existsSync(join(REPO_ROOT, rel))) continue + const steps = readWorkflow(rel)?.runs?.steps ?? [] + return steps.some((inner) => stepPulls(inner, seen)) + } + return false + } + const run = typeof step?.run === 'string' ? stripCommentLines(step.run) : '' + return ( + PULLS.test(run) || [...PULLING_TASKS].some((task) => invokes(run, task)) + ) +} + +const USERNAME = expr('secrets.DOCKER_HUB_USERNAME') +const PASSWORD = expr('secrets.DOCKER_HUB_PASSWORD') + +/** Whether a step logs in to Docker Hub, and does nothing without the secrets. */ +export function loginFinding(step) { + const run = typeof step?.run === 'string' ? step.run : '' + if (!/\bdocker\s+login\b/.test(run)) return 'not a login' + if ( + step.env?.DOCKER_HUB_USERNAME !== USERNAME || + step.env?.DOCKER_HUB_PASSWORD !== PASSWORD + ) + return `does not read ${USERNAME} and ${PASSWORD} into DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD` + if (!/--password-stdin/.test(run)) + return 'does not pass the password on stdin' + if ( + !/-z\s+"\$DOCKER_HUB_USERNAME"/.test(run) || + !/-z\s+"\$DOCKER_HUB_PASSWORD"/.test(run) + ) + return 'does not skip the login when a secret is empty, so a fork pull request would fail' + return null +} + +/** Whether a container's `credentials` log in, and still work without the secrets. */ +export function credentialsFinding(credentials) { + if (credentials === undefined) return 'sets no credentials' + if (typeof credentials !== 'string') + return "sets username and password keys, which fail at 'Set up job' when a secret is empty; use one expression with a fromJSON('{}') fallback" + const flat = credentials.replace(/\s+/g, ' ') + if ( + !flat.includes('secrets.DOCKER_HUB_USERNAME') || + !flat.includes('secrets.DOCKER_HUB_PASSWORD') + ) + return 'does not read secrets.DOCKER_HUB_USERNAME and secrets.DOCKER_HUB_PASSWORD' + if (!flat.includes("|| fromJSON('{}')")) + return "has no || fromJSON('{}') fallback for when a secret is empty" + return null +} + +const isRunsOn = (job) => String(job?.['runs-on'] ?? '').startsWith('runs-on=') + +/** Every finding for one job, or [] when it is correct. */ +export function jobFindings(where, job) { + if (!isRunsOn(job)) return [] + const findings = [] + const containers = [ + ...Object.entries(job.services ?? {}).map(([name, svc]) => [ + `service ${name}`, + svc, + ]), + ...(job.container ? [['container', job.container]] : []), + ] + for (const [what, def] of containers) { + const image = typeof def === 'string' ? def : def?.image + if (!image || !onDockerHub(image)) continue + const finding = credentialsFinding(def?.credentials) + if (finding) + findings.push( + `${where}: ${what} pulls ${image} from Docker Hub, and ${finding}`, + ) + } + const steps = job.steps ?? [] + const firstPull = steps.findIndex((step) => stepPulls(step)) + if (firstPull === -1) return findings + const logins = steps + .map((step, i) => ({ i, finding: loginFinding(step) })) + .filter(({ finding }) => finding !== 'not a login') + for (const { i, finding } of logins) + if (finding) findings.push(`${where}: step ${i + 1} ${finding}`) + const pullName = steps[firstPull].name ?? steps[firstPull].uses ?? 'run' + if (!logins.some(({ i }) => i < firstPull)) + findings.push( + `${where}: step ${firstPull + 1} (${pullName}) may pull an image, and no Docker Hub login comes before it`, + ) + return findings +} + +function runsOnJobs() { + return workflowFiles().flatMap((file) => + Object.entries(readWorkflow(file)?.jobs ?? {}) + .filter(([, job]) => isRunsOn(job)) + .map(([name, job]) => ({ where: `${file} ${name}`, job })), + ) +} + +// --------------------------------------------------------------------------- + +describe('Docker Hub login on RunsOn', () => { + const jobs = runsOnJobs() + + it('finds the jobs it checks', () => { + // A discovery test that finds nothing passes having checked nothing. + expect(jobs.length).toBeGreaterThan(20) + expect( + jobs.filter(({ job }) => (job.steps ?? []).some((s) => stepPulls(s))) + .length, + ).toBeGreaterThan(8) + expect(PULLING_TASKS).toContain('postgres:up') + expect(PULLING_TASKS).toContain('setup') + }) + + it('logs in before every pull', () => { + expect(jobs.flatMap(({ where, job }) => jobFindings(where, job))).toEqual( + [], + ) + }) + + // Every job in the tree is correct, so the test above runs none of the + // branches that report a wrong one. These do. + const runsOn = 'runs-on=1/family=c8a+m8a/cpu=8' + const login = { + name: 'Log in to Docker Hub', + env: { DOCKER_HUB_USERNAME: USERNAME, DOCKER_HUB_PASSWORD: PASSWORD }, + run: [ + 'if [ -z "$DOCKER_HUB_USERNAME" ] || [ -z "$DOCKER_HUB_PASSWORD" ]; then', + ' exit 0', + 'fi', + 'printf \'%s\' "$DOCKER_HUB_PASSWORD" |', + ' docker login --username "$DOCKER_HUB_USERNAME" --password-stdin', + ].join('\n'), + } + const up = { name: 'Start Postgres', run: 'mise run postgres:up' } + + it('reports a pull with no login before it', () => { + expect(jobFindings('w j', { 'runs-on': runsOn, steps: [up] })).toEqual([ + expect.stringMatching(/no Docker Hub login comes before it/), + ]) + expect( + jobFindings('w j', { 'runs-on': runsOn, steps: [up, login] }), + ).toEqual([expect.stringMatching(/no Docker Hub login comes before it/)]) + expect( + jobFindings('w j', { 'runs-on': runsOn, steps: [login, up] }), + ).toEqual([]) + // Not RunsOn: out of scope. + expect( + jobFindings('w j', { 'runs-on': 'ubuntu-latest', steps: [up] }), + ).toEqual([]) + }) + + it('reports a login that would fail on a fork pull request', () => { + const noSkip = { + ...login, + run: 'printf \'%s\' "$DOCKER_HUB_PASSWORD" | docker login --username "$DOCKER_HUB_USERNAME" --password-stdin', + } + expect(loginFinding(noSkip)).toMatch(/fork pull request/) + expect(loginFinding({ ...login, env: {} })).toMatch(/does not read/) + expect(loginFinding(login)).toBeNull() + }) + + it('reports a Docker Hub service with no credentials', () => { + const job = (credentials) => ({ + 'runs-on': runsOn, + services: { pg: { image: 'postgres:17', credentials } }, + steps: [], + }) + expect(jobFindings('w j', job(undefined))).toEqual([ + expect.stringMatching( + /service pg pulls postgres:17.*sets no credentials/, + ), + ]) + // The form GitHub rejects when a secret is empty. + expect( + jobFindings('w j', job({ username: USERNAME, password: PASSWORD })), + ).toEqual([expect.stringMatching(/fail at 'Set up job'/)]) + const flat = `secrets.DOCKER_HUB_USERNAME != '' && secrets.DOCKER_HUB_PASSWORD != '' && fromJSON(format('{{"username":{0},"password":{1}}}', toJSON(secrets.DOCKER_HUB_USERNAME), toJSON(secrets.DOCKER_HUB_PASSWORD)))` + expect(jobFindings('w j', job(expr(flat)))).toEqual([ + expect.stringMatching(/no \|\| fromJSON\('\{\}'\) fallback/), + ]) + expect(jobFindings('w j', job(expr(`${flat} || fromJSON('{}')`)))).toEqual( + [], + ) + expect( + jobFindings('w j', { + 'runs-on': runsOn, + services: { pg: { image: 'ghcr.io/cipherstash/postgres-eql:17' } }, + steps: [], + }), + ).toEqual([]) + }) + + it('recognises what may pull an image', () => { + expect( + stepPulls({ run: 'docker compose -p x -f y.yml up -d --wait' }), + ).toBe(true) + expect(stepPulls({ run: 'docker run --rm alpine true' })).toBe(true) + expect( + stepPulls({ run: 'mise run --output prefix postgres:up pg-17' }), + ).toBe(true) + expect(stepPulls({ uses: './.github/actions/integration-db' })).toBe(true) + expect(stepPulls({ uses: 'docker/setup-qemu-action@abc' })).toBe(true) + expect(stepPulls({ uses: 'docker/login-action@abc' })).toBe(false) + expect(stepPulls({ run: 'docker exec -i pg psql -f-' })).toBe(false) + expect(stepPulls({ run: 'docker compose down' })).toBe(false) + expect(stepPulls({ run: '# docker run in a comment' })).toBe(false) + }) + + it('reads which registry an image names', () => { + expect(onDockerHub('postgres:17')).toBe(true) + expect(onDockerHub('supabase/postgres:17.4.1.048')).toBe(true) + expect(onDockerHub('docker.io/library/postgres:17')).toBe(true) + expect(onDockerHub('ghcr.io/cipherstash/postgres-eql:17')).toBe(false) + expect(onDockerHub('public.ecr.aws/docker/library/postgres:17')).toBe(false) + expect(onDockerHub('localhost:5000/pg')).toBe(false) + }) +}) diff --git a/scripts/__tests__/workflow-mise-cache-image.test.mjs b/scripts/__tests__/workflow-mise-cache-image.test.mjs index 4007ba9d3..1250b9cd6 100644 --- a/scripts/__tests__/workflow-mise-cache-image.test.mjs +++ b/scripts/__tests__/workflow-mise-cache-image.test.mjs @@ -24,8 +24,9 @@ import { readWorkflow, workflowFiles } from './lib/workflows.mjs' * * So each cached step sets `cache_key_prefix: mise-v1-`, where `` * is the `ubuntu-NNNN` its job's `runs-on` names (`ubuntu-24.04` counts as - * `ubuntu-2404`). A job whose `runs-on` names no image (a matrix, - * `ubuntu-latest`) must turn the cache off. No step sets `cache_key`, which + * `ubuntu-2404`, and so does a RunsOn label's `image=ubuntu24-full-x64`). A job + * whose `runs-on` names no image (a matrix, `ubuntu-latest`, a RunsOn label with + * no `image=`) must turn the cache off. No step sets `cache_key`, which * mise-action prefers over the prefix and so would bring the shared key back. * A step with the cache off sets no prefix: it would read as a cache in use, * on paths `scripts/lint-no-workflow-caching.mjs` keeps cache-free. @@ -63,7 +64,14 @@ const IMAGE_INDEPENDENT_TOOLS = new Set(['aqua:wasm-bindgen/wasm-pack']) /** The `ubuntu-NNNN` a runs-on label names, or undefined. */ function imageOf(runsOn) { - const m = /ubuntu-(\d{2})\.?(\d{2})/.exec(String(runsOn)) + const label = String(runsOn) + // RunsOn takes its image from an `image=` segment, and ships only LTS ones. + // With no segment it uses the stack's default, which the label cannot show. + if (label.startsWith('runs-on=')) { + const m = /\/image=ubuntu(\d{2})-/.exec(label) + return m ? `ubuntu-${m[1]}04` : undefined + } + const m = /ubuntu-(\d{2})\.?(\d{2})/.exec(label) return m ? `ubuntu-${m[1]}${m[2]}` : undefined } @@ -149,6 +157,22 @@ describe('mise-action cache keys', () => { expect(imageOf(undefined)).toBeUndefined() }) + it('reads the image of a RunsOn label', () => { + expect( + imageOf( + // biome-ignore lint/suspicious/noTemplateCurlyInString: a GitHub Actions expression, the literal `runs-on` value under test. + 'runs-on=${{ github.run_id }}/family=c8a+m8a/cpu=8/image=ubuntu22-full-x64/volume=80gb/extras=s3-cache', + ), + ).toBe('ubuntu-2204') + expect(imageOf('runs-on=1/family=c8a/cpu=4/image=ubuntu24-full-x64')).toBe( + 'ubuntu-2404', + ) + // No `image=`: RunsOn uses the stack's default image, which the label + // does not name, so the cache must be off. + expect(imageOf('runs-on=1/family=c8a+m8a/cpu=8')).toBeUndefined() + expect(imageOf('runs-on=1/runner=4cpu-linux-x64')).toBeUndefined() + }) + it('lets a composite action cache only image-independent tools', () => { const checked = [] const wrong = [] diff --git a/scripts/ffi-release-matrix.mjs b/scripts/ffi-release-matrix.mjs index c4475fbf7..f39688e7c 100644 --- a/scripts/ffi-release-matrix.mjs +++ b/scripts/ffi-release-matrix.mjs @@ -44,14 +44,24 @@ const PLATFORMS_DIR = join( /** * Where each platform builds. * - * Linux goes to Blacksmith (the repo's standard Linux runner); the Darwin and - * Windows builds need GitHub-hosted images for their SDKs. Both Darwin - * platforms share `macos-latest` and cross-compile — which is exactly why - * `target` is explicit. + * Darwin and Windows need GitHub-hosted images for their SDKs; both Darwin + * platforms share `macos-latest` and cross-compile, which is why `target` is + * explicit. The gnu builds took just over twice as long on GitHub's standard + * runner as on Blacksmith 4 vCPU, so they run on RunsOn. A matrix value cannot + * read `${{ github.run_id }}`, so the run ID that makes the RunsOn label unique + * is passed in. musl stays on Blacksmith: it could not be timed, because its + * image pins a `musl-dev` that Alpine no longer serves. */ -export function runnerFor(platform) { +export function runnerFor(platform, runId = process.env.GITHUB_RUN_ID) { if (platform.startsWith('win32')) return 'windows-latest' if (platform.startsWith('darwin')) return 'macos-latest' + if (platform.startsWith('linux-') && platform.endsWith('-gnu')) { + if (!runId) + throw new Error( + `${platform} builds on RunsOn, whose label needs the run ID; set GITHUB_RUN_ID`, + ) + return `runs-on=${runId}/family=c8a+m8a/cpu=8/image=ubuntu24-full-x64/volume=80gb/extras=s3-cache` + } return 'blacksmith-4vcpu-ubuntu-2404' } @@ -97,11 +107,11 @@ export function triplesFromPlatformManifests(dir = PLATFORMS_DIR) { * `triples` is a platform name -> Rust target triple mapping, as * `triplesFromPlatformManifests()` (or `neon list-platforms`) produces. */ -export function releaseMatrix(triples) { +export function releaseMatrix(triples, runId = process.env.GITHUB_RUN_ID) { return Object.entries(triples).map(([platform, target]) => ({ platform, target, - os: runnerFor(platform), + os: runnerFor(platform, runId), ...buildFor(platform), })) }