diff --git a/.github/workflows/fta-v3.yml b/.github/workflows/fta-v3.yml index 258e3e941..12b94e253 100644 --- a/.github/workflows/fta-v3.yml +++ b/.github/workflows/fta-v3.yml @@ -6,11 +6,16 @@ name: "FTA Complexity" # the check when any file exceeds the score cap (`--score-cap` in the # `analyze:complexity` script). FTA is pure static source analysis, so this job # needs no build step, database, or credentials. +# +# The `selector` job decides whether a pull request runs the gate, and +# `summary` reports the result (ADR 0003). on: push: branches: - 'main' + # The same files as the `selector` job's filter. + # scripts/__tests__/workflow-paths-filter-parity.test.mjs compares them. paths: - 'languages/typescript/packages/stack/src/eql/v3/**' - 'languages/typescript/packages/stack-drizzle/**' @@ -21,23 +26,92 @@ on: - 'languages/typescript/packages/stack/src/schema/match-defaults.ts' - 'languages/typescript/packages/stack/package.json' - '.github/workflows/fta-v3.yml' + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. pull_request: branches: - "**" - paths: - - 'languages/typescript/packages/stack/src/eql/v3/**' - - 'languages/typescript/packages/stack-drizzle/**' - - 'languages/typescript/packages/stack-supabase/**' - - 'languages/typescript/packages/stack/src/schema/match-defaults.ts' - - 'languages/typescript/packages/stack/package.json' - - '.github/workflows/fta-v3.yml' permissions: contents: read jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + fta: ${{ steps.decide.outputs.fta }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + filters: | + # The sources each `analyze:complexity` step scores. Shared + # match-index defaults live outside src/eql/v3 but shape every + # emitted v3 match block, so edits there trigger the gate too. + fta: + - "languages/typescript/packages/stack/src/eql/v3/**" + - "languages/typescript/packages/stack-drizzle/**" + - "languages/typescript/packages/stack-supabase/**" + - "languages/typescript/packages/stack/src/schema/match-defaults.ts" + - "languages/typescript/packages/stack/package.json" + - ".github/workflows/fta-v3.yml" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + FTA: ${{ steps.filter.outputs.fta }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + FTA=true + fi + answers=( + "fta=$FTA" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + fta: name: Analyze v3 complexity + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.fta == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 steps: @@ -89,3 +163,31 @@ jobs: - name: Analyze stack-supabase complexity run: pnpm exec turbo run analyze:complexity --filter @cipherstash/stack-supabase + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: FTA Complexity summary + needs: [selector, fta] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/require-auth-npm-changeset.yml b/.github/workflows/require-auth-npm-changeset.yml index d71644af0..0ee44bc4a 100644 --- a/.github/workflows/require-auth-npm-changeset.yml +++ b/.github/workflows/require-auth-npm-changeset.yml @@ -9,19 +9,16 @@ name: "Require @cipherstash/auth changeset" # Changes that never ship (tests, examples, devDependencies) are filtered out # by `check-auth-npm-changeset.mjs --shipped`. # -# The paths below and the job's `git diff` pathspec are held to the same set -# by scripts/__tests__/check-auth-npm-changeset.test.mjs. There is no `push:` -# trigger: the check compares a pull request with its base. +# The `selector` job's filter and the job's `git diff` pathspec are held to +# the same set by scripts/__tests__/check-auth-npm-changeset.test.mjs. There is +# no `push:` trigger: the check compares a pull request with its base. The +# `summary` job reports the result (ADR 0003). on: - pull_request: - paths: - - packages/stack-auth/Cargo.toml - - packages/stack-auth/src/** - - languages/typescript/packages/auth/** - - languages/typescript/packages/stack-auth-wasm/** - - scripts/check-auth-npm-changeset.mjs - - .github/workflows/require-auth-npm-changeset.yml + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. + pull_request: {} permissions: contents: read @@ -35,8 +32,80 @@ concurrency: cancel-in-progress: true jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + auth_changeset: ${{ steps.decide.outputs.auth_changeset }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + filters: | + # What `@cipherstash/auth` ships, which the job diffs, and the + # job's own inputs. + auth_changeset: + - "packages/stack-auth/Cargo.toml" + - "packages/stack-auth/src/**" + - "languages/typescript/packages/auth/**" + - "languages/typescript/packages/stack-auth-wasm/**" + - "scripts/check-auth-npm-changeset.mjs" + - ".github/workflows/require-auth-npm-changeset.yml" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + AUTH_CHANGESET: ${{ steps.filter.outputs.auth_changeset }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + AUTH_CHANGESET=true + fi + answers=( + "auth_changeset=$AUTH_CHANGESET" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + require-npm-changeset: name: Require @cipherstash/auth changeset + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.auth_changeset == 'true' runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -115,3 +184,31 @@ jobs: mapfile -t CHANGESET_FILES <<< "$CHANGESETS" node scripts/check-auth-npm-changeset.mjs "${CHANGESET_FILES[@]}" + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Require @cipherstash/auth changeset summary + needs: [selector, require-npm-changeset] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/tests-bench.yml b/.github/workflows/tests-bench.yml index 1e97e64cc..d458d7b46 100644 --- a/.github/workflows/tests-bench.yml +++ b/.github/workflows/tests-bench.yml @@ -6,10 +6,15 @@ name: "Test Benchmark JS" # them can red this job, so an edit to any of them has to be able to trigger it. # Scoped the same way the `integration-*` workflows scope theirs rather than # taking whole packages, to keep the trigger surface honest. +# +# The `selector` job decides whether a pull request runs the job, and +# `summary` reports the result (ADR 0003). on: push: branches: - 'main' + # The same files as the `selector` job's filter. + # scripts/__tests__/workflow-paths-filter-parity.test.mjs compares them. paths: - 'languages/typescript/packages/bench/**' - 'languages/typescript/packages/stack/src/eql/v3/**' @@ -24,28 +29,93 @@ on: - '.github/workflows/tests-bench.yml' - '.github/actions/integration-setup/**' - '.github/actions/build-auth-binding/**' + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. pull_request: branches: - "**" - # Repeated verbatim: GitHub Actions does not support YAML anchors/aliases. - paths: - - 'languages/typescript/packages/bench/**' - - 'languages/typescript/packages/stack/src/eql/v3/**' - # The bench drives the typed v3 CLIENT, which lives in `encryption/`, not - # `eql/v3/` (that is the authoring DSL). Without this the job stayed green - # by never running for the changes most likely to break it. - - 'languages/typescript/packages/stack/src/encryption/**' - - 'languages/typescript/packages/stack-drizzle/**' - - 'languages/typescript/packages/test-kit/**' - - 'languages/typescript/packages/cli/src/installer/**' - - 'local/**' - - '.github/workflows/tests-bench.yml' - - '.github/actions/integration-setup/**' - - '.github/actions/build-auth-binding/**' jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + bench: ${{ steps.decide.outputs.bench }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + filters: | + # The package graph the bench reads; see the comment at the top + # of this file. The bench drives the typed v3 client, which lives + # in `encryption/`, not `eql/v3/`. + bench: + - "languages/typescript/packages/bench/**" + - "languages/typescript/packages/stack/src/eql/v3/**" + - "languages/typescript/packages/stack/src/encryption/**" + - "languages/typescript/packages/stack-drizzle/**" + - "languages/typescript/packages/test-kit/**" + - "languages/typescript/packages/cli/src/installer/**" + - "local/**" + - ".github/workflows/tests-bench.yml" + - ".github/actions/integration-setup/**" + - ".github/actions/build-auth-binding/**" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + BENCH: ${{ steps.filter.outputs.bench }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + BENCH=true + fi + answers=( + "bench=$BENCH" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + tests-bench: name: Run Bench Smoke Tests + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.bench == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 steps: @@ -108,3 +178,31 @@ jobs: - name: Run bench smoke tests working-directory: languages/typescript/packages/bench run: pnpm test:local db-only + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Test Benchmark JS summary + needs: [selector, tests-bench] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/tests-supply-chain.yml b/.github/workflows/tests-supply-chain.yml index 490c48370..d54ba9e18 100644 --- a/.github/workflows/tests-supply-chain.yml +++ b/.github/workflows/tests-supply-chain.yml @@ -17,11 +17,16 @@ name: Test supply chain security # - contents:read only # - no secrets # - no caching +# +# The `selector` job decides whether a pull request runs the gate, and +# `summary` reports the result (ADR 0003). on: push: branches: - main + # The same files as the `selector` job's filter. + # scripts/__tests__/workflow-paths-filter-parity.test.mjs compares them. paths: - '.github/workflows/release.yml' # The reusable workflows release.yml calls for the two native-binary @@ -34,28 +39,94 @@ on: - 'scripts/lint-no-workflow-caching.mjs' - 'scripts/__tests__/lint-no-workflow-caching.test.mjs' - 'scripts/__tests__/fixtures/lint-no-workflow-caching/**' + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. pull_request: branches: - '**' - paths: - - '.github/workflows/release.yml' - # The reusable workflows release.yml calls for the two native-binary - # lines: everything they build is packed and published, and they are on - # the lint's target list for that reason, so an edit to either has to run - # this gate too. - - '.github/workflows/_build-ffi-artifacts.yml' - - '.github/workflows/_build-auth-artifacts.yml' - - '.github/workflows/tests-supply-chain.yml' - - 'scripts/lint-no-workflow-caching.mjs' - - 'scripts/__tests__/lint-no-workflow-caching.test.mjs' - - 'scripts/__tests__/fixtures/lint-no-workflow-caching/**' permissions: contents: read jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + supply_chain: ${{ steps.decide.outputs.supply_chain }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + filters: | + # The workflows the lint targets, and the lint itself. The + # reusable workflows release.yml calls for the two native-binary + # lines are on the target list: everything they build is packed + # and published. + supply_chain: + - ".github/workflows/release.yml" + - ".github/workflows/_build-ffi-artifacts.yml" + - ".github/workflows/_build-auth-artifacts.yml" + - ".github/workflows/tests-supply-chain.yml" + - "scripts/lint-no-workflow-caching.mjs" + - "scripts/__tests__/lint-no-workflow-caching.test.mjs" + - "scripts/__tests__/fixtures/lint-no-workflow-caching/**" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + SUPPLY_CHAIN: ${{ steps.filter.outputs.supply_chain }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + SUPPLY_CHAIN=true + fi + answers=( + "supply_chain=$SUPPLY_CHAIN" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + verify-no-caching-in-release-workflows: name: Verify no caching in release workflows + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.supply_chain == 'true' runs-on: ubuntu-latest steps: @@ -90,3 +161,31 @@ jobs: - name: Verify no caching in the release and supply-chain workflows run: pnpm run lint:workflow-cache + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Test supply chain security summary + needs: [selector, verify-no-caching-in-release-workflows] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ba7affc68..19ea87bc7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,19 +1,89 @@ name: Test JS +# The `selector` job decides whether a pull request runs these jobs, and +# `summary` reports the result (ADR 0003). + on: push: branches: - 'main' + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. pull_request: branches: - "**" - # Plan documents under docs/plans/ need none of these jobs. Later patterns - # win, so the negation has to come after the catch-all. - paths: - - '**' - - '!docs/plans/**' jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + js: ${{ steps.decide.outputs.js }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + # + # `every` makes a file count only when it matches every rule. That is + # what makes the `!` rule exclude plans: under dorny's default, `some`, + # a `!` rule matches every other file in the repository. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + predicate-quantifier: every + filters: | + # Every job reads most of the repository: Biome checks every file, + # and run-tests runs the script guards, the typechecks and every + # package's tests. Plan documents under docs/plans/ need none of them, + # and no test reads one (plan-docs-unread-by-tests.test.mjs). + js: + - "**" + - "!docs/plans/**" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + JS: ${{ steps.filter.outputs.js }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + JS=true + fi + answers=( + "js=$JS" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + # Biome format + lint. Its own job (no DB, no credentials, node-agnostic) so it # surfaces as a distinct check and runs once rather than per Node matrix leg. # Gates on ERRORS only — warnings are allowed, so the `no-type-erasing-assertions` @@ -21,6 +91,12 @@ jobs: # Tightening to fail on warnings is tracked in #625. lint: name: Lint (Biome) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.js == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - name: Checkout Repo @@ -53,6 +129,12 @@ jobs: run-tests: name: Run Tests (Node ${{ matrix.node-version }}) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.js == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 strategy: matrix: @@ -397,6 +479,12 @@ jobs: e2e-tests: name: Run E2E Tests + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.js == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 # Auth-dependent suites in `e2e/` skip themselves unless these env vars @@ -478,6 +566,12 @@ jobs: # a silent fallback to the NAPI module is impossible. wasm-e2e-tests: name: Run WASM E2E Tests (Deno) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.js == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 permissions: @@ -621,6 +715,12 @@ jobs: run-tests-bun: name: Run Tests (Bun) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.js == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 continue-on-error: true @@ -713,3 +813,31 @@ jobs: (cd "$dir" && bunx --bun vitest run) || true fi done + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Test JS summary + needs: [selector, lint, run-tests, e2e-tests, wasm-e2e-tests, run-tests-bun] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/scripts/__tests__/auth-binding-step-order.test.mjs b/scripts/__tests__/auth-binding-step-order.test.mjs index aa1b604ab..82dd4317e 100644 --- a/scripts/__tests__/auth-binding-step-order.test.mjs +++ b/scripts/__tests__/auth-binding-step-order.test.mjs @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { filterCovers } from './lib/paths-filter.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -84,13 +85,20 @@ describe('the @cipherstash/auth binding is built wherever the SDK runs', () => { const same = (entry, input) => entry === input -/** The `paths:` lists of a workflow's filtered events. */ -const filters = (relPath) => { - const wf = readWorkflow(relPath) - const on = wf?.on ?? wf?.[true] - return ['push', 'pull_request'] - .map((event) => on?.[event]?.paths) - .filter(Array.isArray) +/** + * The path lists that decide whether the given jobs run, once each: the + * workflow's `push` and `pull_request` `paths:` lists, and the selector filter + * each job reads. Every job of the workflow when none are given. + */ +const filters = (relPath, jobNames) => { + const lists = new Map() + for (const jobName of jobNames ?? + Object.keys(readWorkflow(relPath)?.jobs ?? {})) { + for (const { source, paths } of jobTriggerLists(relPath, jobName)) { + lists.set(source, paths) + } + } + return [...lists.values()] } /** @@ -105,14 +113,19 @@ const AUTH_SOURCE_WORKFLOWS = [ ] describe('a change to the auth binding starts the jobs that load it', () => { - const users = [ - ...new Set( - JOBS.filter((job) => job.auth).map((job) => job.id.split(' / ')[0]), - ), - ] + /** Each workflow with a job that builds the binding, and those jobs. */ + const usersByWorkflow = new Map() + for (const job of JOBS.filter((entry) => entry.auth)) { + const [relPath, jobName] = job.id.split(' / ') + usersByWorkflow.set(relPath, [ + ...(usersByWorkflow.get(relPath) ?? []), + jobName, + ]) + } + const users = [...usersByWorkflow.keys()] it.each(users)('%s filters on the action', (relPath) => { - for (const paths of filters(relPath)) { + for (const paths of filters(relPath, usersByWorkflow.get(relPath))) { expect( filterCovers(paths, '.github/actions/build-auth-binding/**', same), ).toBe(true) @@ -133,7 +146,7 @@ describe('a change to the auth binding starts the jobs that load it', () => { '%s filters on the auth sources', (relPath) => { expect(shipped.length).toBeGreaterThanOrEqual(4) - const lists = filters(relPath) + const lists = filters(relPath, usersByWorkflow.get(relPath)) expect(lists.length).toBe(2) for (const paths of lists) { expect( diff --git a/scripts/__tests__/check-auth-npm-changeset.test.mjs b/scripts/__tests__/check-auth-npm-changeset.test.mjs index 4554edd51..da1cbbb65 100644 --- a/scripts/__tests__/check-auth-npm-changeset.test.mjs +++ b/scripts/__tests__/check-auth-npm-changeset.test.mjs @@ -5,6 +5,7 @@ import { join } from 'node:path' import { afterAll, describe, expect, it } from 'vitest' import { isShipped } from '../check-auth-npm-changeset.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow } from './lib/workflows.mjs' /** @@ -227,8 +228,9 @@ describe('check-auth-npm-changeset.mjs --shipped', () => { }) describe('require-auth-npm-changeset.yml', () => { + const JOB = 'require-npm-changeset' const workflow = readWorkflow(WORKFLOW) - const [job] = Object.values(workflow?.jobs ?? {}) + const job = workflow?.jobs?.[JOB] const check = job?.steps?.find((step) => step.run?.includes(`node ${SCRIPT}`)) /** The pathspec of the job's first `git diff`: what counts as shipped. */ @@ -241,7 +243,12 @@ describe('require-auth-npm-changeset.yml', () => { .map((line) => line.trim().replace(/\s*\\$/, '')) .filter(Boolean) - const filters = workflow?.on?.pull_request?.paths ?? [] + // The selector job's filter: the paths that start the job on a pull request. + const filters = jobTriggerLists(WORKFLOW, JOB).flatMap(({ paths }) => paths) + + it('reads a selector filter', () => { + expect(filters).not.toEqual([]) + }) it('diffs the crate and both binding folders', () => { expect(shipped).toEqual([ diff --git a/scripts/__tests__/integration-workflow-paths.test.mjs b/scripts/__tests__/integration-workflow-paths.test.mjs index c3b1e812a..a44f65539 100644 --- a/scripts/__tests__/integration-workflow-paths.test.mjs +++ b/scripts/__tests__/integration-workflow-paths.test.mjs @@ -2,6 +2,7 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -40,9 +41,6 @@ const STACK_SRC = 'languages/typescript/packages/stack/src' const STACK_MANIFEST = 'languages/typescript/packages/stack/package.json' const CATALOG_MANIFEST = 'pnpm-workspace.yaml' -/** Both trigger events, in the order GitHub evaluates them. */ -const TRIGGER_EVENTS = ['push', 'pull_request'] - /** * The integration workflows, discovered rather than listed: any workflow whose * `CS_IT_SUITE` globs select suites out of `languages/typescript/packages/stack/` is in scope. A @@ -56,14 +54,21 @@ function discoverWorkflows() { } /** - * `on:` parses as the boolean `true` under YAML 1.1 (the "Norway problem"), - * which is why this reads both keys rather than `wf.on`. + * The path lists that decide whether the jobs running a suite run, once each: + * the workflow's `push` and `pull_request` `paths:` lists, and the selector + * filter each such job reads. */ -function triggerBlocks(wf) { - const on = wf.on ?? wf[true] - return TRIGGER_EVENTS.map((event) => on?.[event]).filter( - (block) => block && Array.isArray(block.paths), - ) +function triggerBlocks(relPath) { + const lists = new Map() + for (const [jobId, job] of Object.entries( + readWorkflow(relPath)?.jobs ?? {}, + )) { + if (!job?.env?.CS_IT_SUITE) continue + for (const { source, paths } of jobTriggerLists(relPath, jobId)) { + lists.set(source, { source, paths }) + } + } + return [...lists.values()] } /** Every `CS_IT_SUITE` glob set declared anywhere in the workflow. */ @@ -190,8 +195,9 @@ describe('integration workflow paths filters', () => { for (const relPath of WORKFLOWS) { it(`${relPath} triggers on the manifests pinning its suites' dependencies`, () => { const wf = readWorkflow(relPath) - const blocks = triggerBlocks(wf) - expect(blocks.length).toBe(TRIGGER_EVENTS.length) + const blocks = triggerBlocks(relPath) + // A `push` list, and a `pull_request` list or a selector filter. + expect(blocks.length).toBe(2) const deps = stackDependencies() const files = suiteFiles(suiteGlobs(wf)) @@ -215,7 +221,7 @@ describe('integration workflow paths filters', () => { it(`${relPath} triggers on every source path its suites import`, () => { const wf = readWorkflow(relPath) - const blocks = triggerBlocks(wf) + const blocks = triggerBlocks(relPath) expect(blocks.length).toBeGreaterThan(0) const files = suiteFiles(suiteGlobs(wf)) diff --git a/scripts/__tests__/js-selectors.test.mjs b/scripts/__tests__/js-selectors.test.mjs new file mode 100644 index 000000000..5815f08e6 --- /dev/null +++ b/scripts/__tests__/js-selectors.test.mjs @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest' +import { readSelectorJob, selectedJobs } from './lib/selector.mjs' + +/** + * What the selector jobs of the JavaScript and TypeScript workflows decide for + * a pull request. Each one keeps the paths of the workflow-level filter it + * replaced, so these pin that nothing was lost in the move. + */ + +const WORKFLOWS = { + tests: '.github/workflows/tests.yml', + fta: '.github/workflows/fta-v3.yml', + bench: '.github/workflows/tests-bench.yml', + supplyChain: '.github/workflows/tests-supply-chain.yml', + authChangeset: '.github/workflows/require-auth-npm-changeset.yml', +} + +/** The jobs a pull request runs, without the two that always run. */ +const checks = (relPath, files) => + selectedJobs(relPath, files).filter( + (id) => id !== 'selector' && id !== 'summary', + ) + +describe('the JavaScript and TypeScript workflows', () => { + it.each(Object.values(WORKFLOWS))('%s has a selector job', (relPath) => { + expect(readSelectorJob(relPath)).not.toBeNull() + }) +}) + +describe('the Test JS selector', () => { + const ALL = [ + 'e2e-tests', + 'lint', + 'run-tests', + 'run-tests-bun', + 'wasm-e2e-tests', + ] + + it('runs every job for a change anywhere but the plans', () => { + expect(checks(WORKFLOWS.tests, ['scripts/example.mjs'])).toEqual(ALL) + expect(checks(WORKFLOWS.tests, ['docs/example.md'])).toEqual(ALL) + expect(checks(WORKFLOWS.tests, ['packages/eql/src/v3/a.sql'])).toEqual(ALL) + }) + + it('runs no job for a plan-only change', () => { + expect(checks(WORKFLOWS.tests, ['docs/plans/nested/example.md'])).toEqual( + [], + ) + }) +}) + +describe('the FTA selector', () => { + it('runs the gate for the analysed sources only', () => { + const run = (file) => checks(WORKFLOWS.fta, [file]) + expect( + run('languages/typescript/packages/stack/src/schema/match-defaults.ts'), + ).toEqual(['fta']) + expect( + run('languages/typescript/packages/stack-supabase/src/a.ts'), + ).toEqual(['fta']) + expect(run('languages/typescript/packages/stack/src/index.ts')).toEqual([]) + }) +}) + +describe('the bench selector', () => { + it('runs for the package graph the bench reads', () => { + const run = (file) => checks(WORKFLOWS.bench, [file]) + expect( + run('languages/typescript/packages/stack/src/encryption/index.ts'), + ).toEqual(['tests-bench']) + expect(run('local/docker-compose.postgres.yml')).toEqual(['tests-bench']) + expect(run('languages/typescript/packages/stack-prisma/src/a.ts')).toEqual( + [], + ) + }) +}) + +describe('the supply chain selector', () => { + it('runs for the release workflows and the lint, not for other workflows', () => { + const run = (file) => checks(WORKFLOWS.supplyChain, [file]) + expect(run('.github/workflows/_build-auth-artifacts.yml')).toEqual([ + 'verify-no-caching-in-release-workflows', + ]) + expect(run('scripts/lint-no-workflow-caching.mjs')).toEqual([ + 'verify-no-caching-in-release-workflows', + ]) + expect(run('.github/workflows/tests.yml')).toEqual([]) + }) +}) + +describe('the auth changeset selector', () => { + it('runs for what @cipherstash/auth ships', () => { + const run = (file) => checks(WORKFLOWS.authChangeset, [file]) + expect(run('packages/stack-auth/src/lib.rs')).toEqual([ + 'require-npm-changeset', + ]) + expect(run('languages/typescript/packages/auth/index.js')).toEqual([ + 'require-npm-changeset', + ]) + expect(run('packages/stack-auth/tasks.toml')).toEqual([]) + }) +}) diff --git a/scripts/__tests__/lib/selector.mjs b/scripts/__tests__/lib/selector.mjs new file mode 100644 index 000000000..ec2f910bb --- /dev/null +++ b/scripts/__tests__/lib/selector.mjs @@ -0,0 +1,361 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import yaml from 'js-yaml' +import { runsWhen } from './expressions.mjs' +import { pathsGlobToRegExp } from './paths-filter.mjs' +import { readWorkflow } from './workflows.mjs' + +/** + * Reading and running a workflow's selector job, for the guards that pin what + * it decides. + * + * The selector job is the first job of a workflow. It compares the files a + * pull request changes with each check's trigger paths, using + * dorny/paths-filter, and writes one output per filter. Each gated job's `if:` + * reads the output named after its filter. ADR 0003 and `.github/CONTEXT.md` + * describe the design. + * + * Everything here runs the workflow's own text: the filter rules as written, + * the decision script as written, and each job's real `if:`. A guard that + * reimplemented the decision would agree with a rewritten one by luck. + */ + +export const SELECTOR_JOB = 'selector' + +/** The events the selector decides nothing for: each one selects every job. */ +export const NON_PULL_REQUEST_EVENTS = [ + 'push', + 'schedule', + 'merge_group', + 'workflow_dispatch', +] + +const DORNY = 'dorny/paths-filter@' + +const stepOutputRef = (expression) => + /steps\.([A-Za-z0-9_-]+)\.outputs\.([A-Za-z0-9_-]+)/.exec( + String(expression ?? ''), + ) + +const selectorOutputRefs = (expression) => + [ + ...String(expression ?? '').matchAll( + new RegExp(`needs\\.${SELECTOR_JOB}\\.outputs\\.([A-Za-z0-9_-]+)`, 'g'), + ), + ].map((match) => match[1]) + +/** + * `{a,b}` expanded into its alternatives, as picomatch reads a brace pattern. + * One level only: a nested brace throws, rather than being read wrongly. + */ +export function expandBraces(pattern) { + const open = pattern.indexOf('{') + if (open === -1) return [pattern] + const close = pattern.indexOf('}', open) + const inner = pattern.slice(open + 1, close) + if (close === -1 || inner.includes('{')) { + throw new Error(`Cannot expand the brace pattern ${pattern}`) + } + const head = pattern.slice(0, open) + return inner + .split(',') + .flatMap((alternative) => + expandBraces(`${head}${alternative}${pattern.slice(close + 1)}`), + ) +} + +/** One dorny/paths-filter rule against one file. A `!` rule negates. */ +export function dornyRuleMatches(rule, file) { + if (rule.startsWith('!')) return !dornyRuleMatches(rule.slice(1), file) + return expandBraces(rule).some((glob) => pathsGlobToRegExp(glob).test(file)) +} + +/** + * Whether a file counts for a filter. Under dorny's default quantifier, `some`, + * one matching rule is enough; under `every`, all of them must match. + */ +export function dornyFilterMatches(rules, quantifier, file) { + return quantifier === 'every' + ? rules.every((rule) => dornyRuleMatches(rule, file)) + : rules.some((rule) => dornyRuleMatches(rule, file)) +} + +/** + * A workflow's selector job, parsed, or `null` when it has none. + * + * `filters` maps each filter name to its rules. A filter that includes an + * anchored list (`- *name`) parses to a nested list, and dorny flattens it; + * so does this. `outputs` maps each selector output to the decision step + * output it reads. + */ +export function readSelectorJob(relPath) { + const workflow = readWorkflow(relPath) + const job = workflow?.jobs?.[SELECTOR_JOB] + if (!job) return null + const steps = Array.isArray(job.steps) ? job.steps : [] + const filterSteps = steps.filter((step) => + String(step?.uses ?? '').startsWith(DORNY), + ) + const filterStep = filterSteps.length === 1 ? filterSteps[0] : null + const filters = Object.fromEntries( + Object.entries(yaml.load(filterStep?.with?.filters ?? '') ?? {}).map( + ([name, rules]) => [name, [rules].flat(Infinity).map(String)], + ), + ) + const outputs = Object.fromEntries( + Object.entries(job.outputs ?? {}).map(([name, expression]) => [ + name, + stepOutputRef(expression), + ]), + ) + const decideIds = [ + ...new Set(Object.values(outputs).map((ref) => ref?.[1])), + ].filter(Boolean) + const decideStep = + decideIds.length === 1 + ? steps.find((step) => step?.id === decideIds[0]) + : null + return { + workflow, + job, + filterSteps, + filterStep, + quantifier: filterStep?.with?.['predicate-quantifier'] ?? 'some', + filters, + outputs: Object.fromEntries( + Object.entries(outputs).map(([name, ref]) => [name, ref?.[2]]), + ), + decideStep, + } +} + +/** The selector outputs a job's `if:` reads. */ +export function selectorOutputsRead(job) { + return [...new Set(selectorOutputRefs(job?.if))] +} + +/** + * Runs the selector's decision script the way GitHub would: `answers` stands + * in for the filter step's outputs, keyed by filter name. An env entry that + * names no answer gets the empty string, as GitHub substitutes for an + * expression that resolves to nothing. + * + * Returns the exit status, the log, and the selector job's outputs. + */ +export function runSelectorDecision(relPath, eventName, answers) { + const selector = readSelectorJob(relPath) + const scratch = mkdtempSync(join(tmpdir(), 'selector-')) + try { + const outputFile = join(scratch, 'outputs.txt') + writeFileSync(outputFile, '') + const env = { PATH: process.env.PATH, GITHUB_OUTPUT: outputFile } + for (const [key, value] of Object.entries( + selector?.decideStep?.env ?? {}, + )) { + const text = String(value) + if (/github\.event_name/.test(text)) env[key] = eventName + else env[key] = answers[stepOutputRef(text)?.[2]] ?? '' + } + const run = spawnSync( + 'bash', + ['-c', String(selector?.decideStep?.run ?? 'exit 99')], + { cwd: scratch, env, encoding: 'utf8' }, + ) + const written = Object.fromEntries( + readFileSync(outputFile, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => [ + line.slice(0, line.indexOf('=')), + line.slice(line.indexOf('=') + 1), + ]), + ) + return { + status: run.status, + log: `${run.stdout}${run.stderr}`, + outputs: Object.fromEntries( + Object.entries(selector?.outputs ?? {}).map(([name, stepOutput]) => [ + name, + written[stepOutput], + ]), + ), + } + } finally { + rmSync(scratch, { recursive: true, force: true }) + } +} + +/** What each filter answers for a pull request that changes `files`. */ +export function filterAnswers(relPath, files) { + const selector = readSelectorJob(relPath) + return Object.fromEntries( + Object.entries(selector?.filters ?? {}).map(([name, rules]) => [ + name, + String( + files.some((file) => + dornyFilterMatches(rules, selector.quantifier, file), + ), + ), + ]), + ) +} + +const needsOf = (job) => [job?.needs ?? []].flat() + +/** + * The jobs that run for a same-repository pull request changing `files`, + * sorted. The selector job's outputs come from its own script; each job's + * `if:` is evaluated as GitHub does, including the implicit `success()` that + * skips a job when one it needs did not succeed. + * + * `github` adds to the context, for a job whose `if:` reads more than the + * event and the selector (such as `head_ref`). + */ +export function selectedJobs(relPath, files, { github = {} } = {}) { + const selector = readSelectorJob(relPath) + const decision = runSelectorDecision( + relPath, + 'pull_request', + filterAnswers(relPath, files), + ) + if (decision.status !== 0) { + throw new Error(`The selector failed:\n${decision.log}`) + } + const repository = 'cipherstash/stack' + const context = { + github: { + event_name: 'pull_request', + repository, + head_ref: 'feature', + event: { + pull_request: { + draft: false, + head: { repo: { full_name: repository } }, + user: { type: 'User' }, + }, + }, + ...github, + }, + needs: {}, + vars: {}, + } + const jobs = selector.workflow.jobs + const results = { + [SELECTOR_JOB]: { result: 'success', outputs: decision.outputs }, + } + const resolve = (id) => { + if (results[id]) return results[id].result + const job = jobs[id] + const needs = needsOf(job) + const needResults = needs.map(resolve) + const condition = String(job?.if ?? '') + const statusFunction = /\b(always|failure|cancelled|success)\(\)/.test( + condition, + ) + let runs + if (!statusFunction && needResults.some((result) => result !== 'success')) { + runs = false + } else if (!condition) { + runs = true + } else { + runs = runsWhen(condition, { + ...context, + needs: Object.fromEntries(needs.map((need) => [need, results[need]])), + }) + } + results[id] = { result: runs ? 'success' : 'skipped', outputs: {} } + return results[id].result + } + return Object.keys(jobs) + .filter((id) => resolve(id) === 'success') + .sort() +} + +// GitHub reads a `paths:` list in order, and the last matching entry decides. +export function pathsListSelects(paths, file) { + let selected = false + for (const entry of paths) { + if (entry.startsWith('!')) { + if (pathsGlobToRegExp(entry.slice(1)).test(file)) selected = false + } else if (pathsGlobToRegExp(entry).test(file)) { + selected = true + } + } + return selected +} + +/** + * A filter as a GitHub `paths:` list: entries in order, a `!` entry excluding + * what came before it. Lets the guards that check a workflow's `paths:` + * coverage read a selector filter with the same code. + * + * Under `every`, the one positive rule's brace alternatives come first and the + * `!` rules after, which selects exactly the files dorny selects. Under + * `some`, a filter has no `!` rule (`workflow-selectors.test.mjs`), so it is + * already that list. + */ +export function filterAsPathsList(rules, quantifier) { + if (quantifier !== 'every') return rules.flatMap(expandBraces) + const positive = rules.filter((rule) => !rule.startsWith('!')) + const negative = rules.filter((rule) => rule.startsWith('!')) + return [...positive.flatMap(expandBraces), ...negative] +} + +/** + * The path lists that decide whether a job runs on a change, as GitHub + * `paths:` lists: the workflow's `on.push.paths` and `on.pull_request.paths` + * where they exist, and each selector filter the job's `if:` reads. + * + * A job in a selector workflow that reads no filter runs on every pull + * request, so only the `push` list, if any, decides it. + */ +export function jobTriggerLists(relPath, jobId) { + const workflow = readWorkflow(relPath) + const on = workflow?.on ?? workflow?.[true] + const lists = ['push', 'pull_request'] + .filter((event) => Array.isArray(on?.[event]?.paths)) + .map((event) => ({ + source: `on.${event}.paths`, + paths: on[event].paths.map(String), + })) + const selector = readSelectorJob(relPath) + for (const name of selectorOutputsRead(workflow?.jobs?.[jobId])) { + const rules = selector?.filters?.[name] + if (!rules) continue + lists.push({ + source: `selector filter "${name}"`, + paths: filterAsPathsList(rules, selector.quantifier), + }) + } + return lists +} + +/** + * Runs the summary job's check with the given results of the jobs it needs, + * and returns its exit status. The results stand in for + * `join(needs.*.result, ' ')`, wherever the step reads it. + */ +export function runSummaryCheck(relPath, summaryId, results) { + const workflow = readWorkflow(relPath) + const steps = workflow?.jobs?.[summaryId]?.steps ?? [] + const joined = results.join(' ') + const resultsExpression = /\$\{\{\s*join\(needs\.\*\.result,\s*' '\)\s*\}\}/g + let status = 0 + for (const step of steps) { + if (typeof step?.run !== 'string') continue + const env = { PATH: process.env.PATH } + for (const [key, value] of Object.entries(step.env ?? {})) { + env[key] = String(value).replace(resultsExpression, joined) + } + const run = spawnSync( + 'bash', + ['-c', step.run.replace(resultsExpression, joined)], + { env, encoding: 'utf8' }, + ) + status = run.status + if (status !== 0) break + } + return status +} diff --git a/scripts/__tests__/paths-filter.test.mjs b/scripts/__tests__/paths-filter.test.mjs index 5b2da3cd4..ab1451a65 100644 --- a/scripts/__tests__/paths-filter.test.mjs +++ b/scripts/__tests__/paths-filter.test.mjs @@ -4,7 +4,7 @@ import { negationRemovesAll, negationSubtracts, } from './lib/paths-filter.mjs' -import { readWorkflow } from './lib/workflows.mjs' +import { jobTriggerLists } from './lib/selector.mjs' const exact = (entry, input) => entry === input const IN = 'languages/typescript/packages/protect-ffi/crates/**' @@ -92,10 +92,12 @@ describe('negationRemovesAll subtracts only a whole-tree negation', () => { }) }) -describe('tests.yml pull_request filter', () => { - const workflow = readWorkflow('.github/workflows/tests.yml') - // `on:` parses as the boolean `true` under YAML 1.1. - const { paths } = (workflow.on ?? workflow[true]).pull_request +describe('tests.yml selector filter', () => { + // As a `paths:` list. Every job in the workflow reads the same filter. + const [{ paths }] = jobTriggerLists( + '.github/workflows/tests.yml', + 'run-tests', + ) it('skips plan-only pull requests and runs on everything else', () => { expect(filterCovers(paths, 'docs/plans/nested/example.md', exact)).toBe( diff --git a/scripts/__tests__/plan-docs-unread-by-tests.test.mjs b/scripts/__tests__/plan-docs-unread-by-tests.test.mjs index 529099eba..6616426a3 100644 --- a/scripts/__tests__/plan-docs-unread-by-tests.test.mjs +++ b/scripts/__tests__/plan-docs-unread-by-tests.test.mjs @@ -40,7 +40,7 @@ describe('tests.yml skips pull requests that change only docs/plans/', () => { it('no test names a file under docs/plans/', () => { expect( filesNamingAPlan(TESTS), - 'A test that reads a plan is skipped on a plan-only pull request, because `tests.yml` ignores `docs/plans/**`. Move the fact the test needs out of the plan, or drop the negation from the `pull_request` filter.', + 'A test that reads a plan is skipped on a plan-only pull request, because `tests.yml` ignores `docs/plans/**`. Move the fact the test needs out of the plan, or drop the negation from the selector filter.', ).toEqual([]) }) it('the scan finds a plan name when there is one', () => { diff --git a/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs b/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs index 28d96ad94..8f28725ad 100644 --- a/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs +++ b/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs @@ -4,6 +4,7 @@ import yaml from 'js-yaml' import { describe, expect, it } from 'vitest' import { filterCovers } from './lib/paths-filter.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -57,9 +58,6 @@ const ACTION = '.github/actions/build-ffi-binding/action.yml' const ACTION_USES = './.github/actions/build-ffi-binding' const CACHE_ACTION = /^actions\/cache(\/(restore|save))?@/ -/** Both trigger events that carry a `paths:` filter, in GitHub's own order. */ -const FILTERED_EVENTS = ['push', 'pull_request'] - /** The package whose Rust both cache keys are about. */ const FFI_PACKAGE = 'languages/typescript/packages/protect-ffi' const FFI_PACKAGE_NAME = '@cipherstash/protect-ffi' @@ -276,21 +274,34 @@ function stepDir(wf, job, step) { /** Every job step of a workflow, paired with the job that owns it. */ function steps(wf) { - return Object.values(wf?.jobs ?? {}).flatMap((job) => - (Array.isArray(job?.steps) ? job.steps : []).map((step) => ({ job, step })), + return Object.entries(wf?.jobs ?? {}).flatMap(([jobId, job]) => + (Array.isArray(job?.steps) ? job.steps : []).map((step) => ({ + jobId, + job, + step, + })), ) } -/** Runs `./.github/actions/build-ffi-binding`, optionally with `wasm: 'true'`. */ -function usesAction(relPath, { wasmOnly = false } = {}) { - return steps(readWorkflow(relPath)).some( - ({ step }) => - typeof step?.uses === 'string' && - step.uses.trim() === ACTION_USES && - (!wasmOnly || String(step?.with?.wasm) === 'true'), +const jobIdsOf = (entries) => [...new Set(entries.map(({ jobId }) => jobId))] + +/** + * The jobs that run `./.github/actions/build-ffi-binding`, optionally with + * `wasm: 'true'`. + */ +function actionJobs(relPath, { wasmOnly = false } = {}) { + return jobIdsOf( + steps(readWorkflow(relPath)).filter( + ({ step }) => + typeof step?.uses === 'string' && + step.uses.trim() === ACTION_USES && + (!wasmOnly || String(step?.with?.wasm) === 'true'), + ), ) } +const usesAction = (relPath, options) => actionJobs(relPath, options).length > 0 + /** * Compiles this package's Rust — through the action, or directly. * @@ -300,10 +311,9 @@ function usesAction(relPath, { wasmOnly = false } = {}) { * EQL workspace has its own, with its own workflows — is not mistaken for this * one. */ -function compilesFfiRust(relPath) { - if (usesAction(relPath)) return true +function ffiRustJobs(relPath) { const wf = readWorkflow(relPath) - return steps(wf).some(({ job, step }) => { + const direct = steps(wf).filter(({ job, step }) => { const run = String(step?.run ?? '') if (!run) return false const targetsPackage = @@ -316,24 +326,28 @@ function compilesFfiRust(relPath) { (inPackage && mentionsAny(run, CARGO_TASKS)) ) }) + return [...new Set([...actionJobs(relPath), ...jobIdsOf(direct)])] } +const compilesFfiRust = (relPath) => ffiRustJobs(relPath).length > 0 + /** - * The `paths:` filters a workflow declares, one entry per filtered event. + * The path lists that decide whether the given jobs run: the workflow's `push` + * and `pull_request` `paths:` lists, and the selector filter each job reads. + * One entry per list, labelled with where it comes from. * - * `on:` parses as the boolean `true` under YAML 1.1 (the "Norway problem"), - * hence `wf.on ?? wf[true]`. An event with no `paths:` is UNFILTERED and is - * skipped rather than failed — it already runs on every diff, so it cannot miss - * one. `tests.yml` and `tests-rust.yml` are both half-cases: `push` is - * `branches: [main]` with no filter, so only their `pull_request` lists are - * here. `tests.yml`'s is `'**'` with one negation, which `filterCovers` reads. + * A job that no list decides is skipped rather than failed: it runs on every + * diff, so it cannot miss one. A selector filter is read as a `paths:` list, + * so a `'**'` with negations is read the same way in either. */ -function filteredEvents(relPath) { - const wf = readWorkflow(relPath) - const on = wf?.on ?? wf?.[true] - return FILTERED_EVENTS.filter((event) => - Array.isArray(on?.[event]?.paths), - ).map((event) => ({ relPath, event, paths: on[event].paths })) +function filteredEvents(relPath, jobIds) { + const lists = new Map() + for (const jobId of jobIds) { + for (const { source, paths } of jobTriggerLists(relPath, jobId)) { + lists.set(source, { relPath, event: source, paths }) + } + } + return [...lists.values()] } const ALL_WORKFLOWS = workflowFiles() @@ -381,6 +395,7 @@ const KEYS = [ what: '`index.node`, the binding every credentialed suite encrypts through', reaches: 'compiles the crate graph behind `index.node`', workflows: ALL_WORKFLOWS.filter(compilesFfiRust), + jobs: ffiRustJobs, expected: [ '.github/workflows/_build-ffi-artifacts.yml', '.github/workflows/integration-drizzle.yml', @@ -401,6 +416,7 @@ const KEYS = [ workflows: ALL_WORKFLOWS.filter((relPath) => usesAction(relPath, { wasmOnly: true }), ), + jobs: (relPath) => actionJobs(relPath, { wasmOnly: true }), expected: [ '.github/workflows/integration-drizzle.yml', '.github/workflows/integration-protect-ffi.yml', @@ -447,7 +463,9 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { for (const key of KEYS) { const inputs = buildInputs(key.step) - const filtered = key.workflows.flatMap(filteredEvents) + const filtered = key.workflows.flatMap((relPath) => + filteredEvents(relPath, key.jobs(relPath)), + ) describe(`the ${key.id} key`, () => { it('hashes build inputs that name real files', () => { @@ -506,7 +524,7 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { ) expect( uncovered, - `These files are hashed into the ${key.id} cache key of \`${ACTION}\`, so an edit to one MISSES the cache and rebuilds ${key.what}. ${relPath} ${key.reaches}, but its \`${event}\` \`paths:\` filter does not list them, so a PR touching only one of them rebuilds the artifact and starts no suite that exercises it. Nothing goes red; nothing ran.\nAdd each to BOTH copies of the filter — GitHub Actions has no YAML anchors, and \`workflow-paths-filter-parity.test.mjs\` fails a one-sided edit.`, + `These files are hashed into the ${key.id} cache key of \`${ACTION}\`, so an edit to one MISSES the cache and rebuilds ${key.what}. ${relPath} ${key.reaches}, but its \`${event}\` \`paths:\` filter does not list them, so a PR touching only one of them rebuilds the artifact and starts no suite that exercises it. Nothing goes red; nothing ran.\nAdd each to every copy of the list — GitHub Actions has no YAML anchors, and \`workflow-paths-filter-parity.test.mjs\` fails a one-sided edit.`, ).toEqual([]) }) } @@ -521,7 +539,7 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { KEYS.flatMap((key) => { const inputs = buildInputs(key.step) return key.workflows - .flatMap(filteredEvents) + .flatMap((relPath) => filteredEvents(relPath, key.jobs(relPath))) .flatMap(({ relPath, paths }) => inputs .filter((pattern) => !filterCovers(paths, pattern, entryCovers)) diff --git a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs index e727e71fe..d7a6ea95e 100644 --- a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs +++ b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs @@ -180,7 +180,11 @@ function extractForkClause(condition) { * run. */ const PERMISSIVE_NEEDS = { - selector: { outputs: { eql: 'true', rust_crates: 'true' } }, + // Every selector output, whatever its name: each workflow's selector names + // its outputs after its own filters. + selector: { + outputs: new Proxy({}, { has: () => true, get: () => 'true' }), + }, // The release workflows' gates, none of which is about how the run was // triggered. Held open, or "skips because there is nothing to publish" would // be indistinguishable from "skips on a dispatch". diff --git a/scripts/__tests__/workflow-paths-filter-parity.test.mjs b/scripts/__tests__/workflow-paths-filter-parity.test.mjs index 74ab3af62..4e8799765 100644 --- a/scripts/__tests__/workflow-paths-filter-parity.test.mjs +++ b/scripts/__tests__/workflow-paths-filter-parity.test.mjs @@ -1,4 +1,11 @@ +import { execFileSync } from 'node:child_process' import { describe, expect, it } from 'vitest' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { + dornyFilterMatches, + pathsListSelects, + readSelectorJob, +} from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -57,6 +64,14 @@ function pathsFilters(relPath) { ) } +/** Every tracked file: what a `paths:` list or a filter can select. */ +const TRACKED = execFileSync('git', ['ls-files', '-z'], { + cwd: REPO_ROOT, + encoding: 'utf8', +}) + .split('\0') + .filter(Boolean) + const CLASSIFIED = workflowFiles().map((relPath) => { const filters = pathsFilters(relPath) const declared = FILTERED_EVENTS.filter((event) => filters[event] !== null) @@ -66,8 +81,23 @@ const CLASSIFIED = workflowFiles().map((relPath) => { /** Workflows writing the list twice — the ones the parity check applies to. */ const PARITY = CLASSIFIED.filter((entry) => entry.declared.length === 2) +/** + * Workflows whose selector job decides what a pull request runs, and whose + * `push` keeps a `paths:` list. Their `pull_request` has no filter, by design + * (ADR 0003), so the second copy of the list is the selector's filters. The + * check below compares the two by what they select. + */ +const SELECTOR_PUSH = CLASSIFIED.filter( + (entry) => + entry.filters.push !== null && + entry.filters.pull_request === null && + readSelectorJob(entry.relPath) !== null, +) + /** Workflows filtering one event and not the other. */ -const ASYMMETRIC = CLASSIFIED.filter((entry) => entry.declared.length === 1) +const ASYMMETRIC = CLASSIFIED.filter( + (entry) => entry.declared.length === 1 && !SELECTOR_PUSH.includes(entry), +) /** * The guard on the scan, in the shape the rest of this directory uses. Every @@ -80,15 +110,12 @@ const ASYMMETRIC = CLASSIFIED.filter((entry) => entry.declared.length === 1) * also fire) or the discovery broke. */ const EXPECTED_PARITY_WORKFLOWS = [ - '.github/workflows/fta-v3.yml', '.github/workflows/integration-drizzle.yml', '.github/workflows/integration-prisma-next.yml', '.github/workflows/integration-protect-ffi.yml', '.github/workflows/integration-supabase.yml', '.github/workflows/prisma-example-readme-e2e.yml', '.github/workflows/prisma-next-e2e.yml', - '.github/workflows/tests-bench.yml', - '.github/workflows/tests-supply-chain.yml', ] /** @@ -122,13 +149,6 @@ const EXPECTED_ASYMMETRIES = new Map([ // unchecked — is unreachable when `push` is unfiltered. 'push is deliberately unfiltered (branches: [main]), so main runs a superset of what PRs run', ], - [ - '.github/workflows/tests.yml', - // `push: branches: [main]` has no `paths:`, so main runs everything and only - // pull requests skip `docs/plans/**`. A `push` filter narrower than the - // `pull_request` one is unreachable when `push` is unfiltered. - 'push is deliberately unfiltered (branches: [main]), so main runs a superset of what PRs run', - ], [ '.github/workflows/lint-release.yml', // There is no `push:` trigger at all — this gate exists to catch a broken @@ -139,30 +159,6 @@ const EXPECTED_ASYMMETRIES = new Map([ // release machinery never runs the check) is what the single list IS. 'pull_request is the only trigger; a post-merge copy would report a release-blocking finding too late to act on', ], - [ - '.github/workflows/require-auth-npm-changeset.yml', - // Its list is held to the job's own `git diff` pathspec by - // check-auth-npm-changeset.test.mjs, which is the second copy here. - 'pull_request is the only trigger; the check diffs a pull request against its base, which a push does not have', - ], - [ - '.github/workflows/test-eql.yml', - // `pull_request` is deliberately UNFILTERED and `push: branches: [main]` - // carries the list, which is the same shape as tests-rust.yml with the two - // events swapped. Unfiltered on the PR side because this workflow's - // `ci-required` is the check branch protection is meant to reference, and a - // workflow skipped by a path filter leaves a required check stuck Pending; - // relevance is applied per-job via `if:` instead. Filtered on the push side - // because nothing waits on a push run, so skipping one costs nothing. - // - // The direction that would hurt — a `pull_request` filter narrower than the - // `push` one, so the PR introducing a change never runs the suite — is - // unreachable when `pull_request` has no filter at all. The push list is - // not unchecked: `eql-workflow-filters.test.mjs` compares it against the - // union of the selector job's filters, which is the second copy of the - // same list and the one this file cannot see. - 'pull_request is deliberately unfiltered (per-job relevance instead), so PRs run a superset of what push runs; the push list is compared against the selector job filters by eql-workflow-filters.test.mjs', - ], [ '.github/workflows/bench-eql.yml', // The mirror image of lint-release.yml: `push: branches: [main]` with a @@ -235,6 +231,27 @@ describe('paths filters are written twice, identically', () => { ).toEqual([]) }) + for (const { relPath, filters } of SELECTOR_PUSH) { + it(`${relPath} selects on push what its selector selects on a pull request`, () => { + // A selector workflow writes its list twice as well: as the `push` + // `paths:` list, and split across the selector job's filters. The two + // are written in different syntaxes, so they are compared by what they + // select from the tracked files, not by their text. + const selector = readSelectorJob(relPath) + const differ = TRACKED.filter( + (file) => + pathsListSelects(filters.push, file) !== + Object.values(selector.filters).some((rules) => + dornyFilterMatches(rules, selector.quantifier, file), + ), + ) + expect( + differ, + `In ${relPath}, the \`push\` \`paths:\` list and the union of the selector job's filters disagree about these files. Change both in the same edit.`, + ).toEqual([]) + }) + } + for (const { relPath } of PARITY) { it(`${relPath} filters push and pull_request identically`, () => { const filters = pathsFilters(relPath) diff --git a/scripts/__tests__/workflow-selectors.test.mjs b/scripts/__tests__/workflow-selectors.test.mjs new file mode 100644 index 000000000..341b95f8d --- /dev/null +++ b/scripts/__tests__/workflow-selectors.test.mjs @@ -0,0 +1,220 @@ +import { describe, expect, it } from 'vitest' +import { runsWhen } from './lib/expressions.mjs' +import { + NON_PULL_REQUEST_EVENTS, + readSelectorJob, + runSelectorDecision, + runSummaryCheck, + SELECTOR_JOB, + selectorOutputsRead, +} from './lib/selector.mjs' +import { workflowFiles } from './lib/workflows.mjs' + +/** + * Every workflow with a selector job follows the same design (ADR 0003): the + * selector job decides which checks a pull request runs, each gated job reads + * the selector's answer in its `if:`, and a summary job ends the workflow. + * `.github/CONTEXT.md` defines the terms. + * + * The workflows are discovered, not listed, so a workflow that gains a + * selector job is held to this the day it lands. What each selector decides + * for particular changes is pinned per workflow, in the `*-selector*.test.mjs` + * files. + * + * Whether every job is decided by the selector, or has a written reason to run + * on every change, is the failsafe's question, not this file's. + */ + +const SELECTOR_WORKFLOWS = workflowFiles().filter( + (relPath) => readSelectorJob(relPath) !== null, +) + +/** + * The guard on the scan. A minimum: a workflow that gains a selector job must + * not fail this. Each workflow's own `*-selector*.test.mjs` file checks that + * its selector job exists. + */ +const EXPECTED_SELECTOR_WORKFLOWS = ['.github/workflows/test-eql.yml'] + +const triggersOf = (workflow) => workflow?.on ?? workflow?.[true] + +const needsOf = (job) => [job?.needs ?? []].flat() + +/** The jobs whose `if:` reads a selector output. */ +function gatedJobs(workflow) { + return Object.entries(workflow?.jobs ?? {}) + .map(([id, job]) => ({ id, job, reads: selectorOutputsRead(job) })) + .filter((entry) => entry.reads.length > 0) +} + +describe('the workflows with a selector job', () => { + it('include the converted workflows', () => { + expect(SELECTOR_WORKFLOWS).toEqual( + expect.arrayContaining(EXPECTED_SELECTOR_WORKFLOWS), + ) + }) +}) + +for (const relPath of SELECTOR_WORKFLOWS) { + const selector = readSelectorJob(relPath) + const { workflow } = selector + const jobIds = Object.keys(workflow.jobs) + const summaryId = jobIds.at(-1) + const filterNames = Object.keys(selector.filters) + const outputNames = Object.keys(selector.outputs) + + describe(relPath, () => { + it('has no path filter on pull_request', () => { + // A workflow that a path filter skips leaves its required checks + // pending, and GitHub's documentation on required status checks says + // that blocks a merge. The selector job decides instead. + const pullRequest = triggersOf(workflow)?.pull_request ?? {} + expect(Object.keys(pullRequest)).not.toContain('paths') + expect(Object.keys(pullRequest)).not.toContain('paths-ignore') + }) + + it('starts with the selector job, on a GitHub-hosted runner', () => { + // ADR 0003: every gated job waits for it, and a self-provisioned + // machine takes longer to start. + expect(jobIds[0]).toBe(SELECTOR_JOB) + expect(String(selector.job['runs-on'])).toMatch(/^ubuntu-/) + expect(selector.job.needs).toBeUndefined() + expect(selector.job.if).toBeUndefined() + }) + + it('has one path filter step, run only on a pull request', () => { + expect(selector.filterSteps).toHaveLength(1) + expect(String(selector.filterStep?.if)).toBe( + "github.event_name == 'pull_request'", + ) + expect(filterNames).not.toEqual([]) + }) + + it('writes no `!` rule that dorny would read as matching everything', () => { + // Under dorny's default quantifier, `some`, a file counts when ANY rule + // matches, so `!**/*.md` matches every file that is not Markdown. A `!` + // rule only excludes under `every`, where a file must match every rule, + // and there a second positive rule would select nothing at all. + expect(['some', 'every']).toContain(selector.quantifier) + for (const [name, rules] of Object.entries(selector.filters)) { + const positive = rules.filter((rule) => !rule.startsWith('!')) + const negative = rules.filter((rule) => rule.startsWith('!')) + if (selector.quantifier === 'every') { + expect(positive, `filter "${name}"`).toHaveLength(1) + } else { + expect(negative, `filter "${name}"`).toEqual([]) + } + } + }) + + it('names each output after the filter that decides it', () => { + // The guards follow a job's `if:` to its filter by this name. + expect(outputNames.sort()).toEqual([...filterNames].sort()) + for (const [name, stepOutput] of Object.entries(selector.outputs)) { + expect(stepOutput, `output "${name}"`).toBe(name) + } + expect( + selector.decideStep, + 'No single step writes the selector outputs.', + ).toBeTruthy() + }) + + it('selects every job on events other than pull_request', () => { + for (const event of NON_PULL_REQUEST_EVENTS) { + const { status, log, outputs } = runSelectorDecision(relPath, event, {}) + expect(status, log).toBe(0) + for (const name of outputNames) { + expect(outputs[name], `${name} on ${event}`).toBe('true') + } + } + }) + + it('passes the filter answers through on a pull request', () => { + for (const answer of ['true', 'false']) { + const answers = Object.fromEntries( + filterNames.map((name) => [name, answer]), + ) + const { status, log, outputs } = runSelectorDecision( + relPath, + 'pull_request', + answers, + ) + expect(status, log).toBe(0) + expect(outputs).toEqual(answers) + } + }) + + it('fails when a pull request filter gives no answer', () => { + // A missing answer would skip every gated job, and the summary job + // counts skipped as passed: a green run that ran nothing. + for (const name of filterNames) { + for (const bad of ['', 'maybe']) { + const answers = Object.fromEntries( + filterNames.map((other) => [other, other === name ? bad : 'true']), + ) + const { status } = runSelectorDecision( + relPath, + 'pull_request', + answers, + ) + expect(status, `filter "${name}" answering '${bad}'`).not.toBe(0) + } + } + }) + + it('gates jobs only on outputs the selector writes, and needs it', () => { + const gated = gatedJobs(workflow) + expect(gated).not.toEqual([]) + for (const { id, job, reads } of gated) { + expect(needsOf(job), `job "${id}"`).toContain(SELECTOR_JOB) + for (const name of reads) { + expect(outputNames, `job "${id}" reads "${name}"`).toContain(name) + } + } + }) + + it("writes each gated job's `if:` as an exclusion of pull_request", () => { + // Outside a pull request the selector's answers must not matter: a job + // written as a list of allowed events skips on the event nobody listed. + for (const { id, job } of gatedJobs(workflow)) { + for (const event of NON_PULL_REQUEST_EVENTS) { + const verdict = (answer) => + runsWhen(job.if, { + github: { + event_name: event, + repository: 'cipherstash/stack', + event: {}, + }, + needs: { + [SELECTOR_JOB]: { + result: 'success', + outputs: Object.fromEntries( + outputNames.map((name) => [name, answer]), + ), + }, + }, + vars: {}, + }) + expect(verdict('false'), `job "${id}" on ${event}`).toBe( + verdict('true'), + ) + } + } + }) + + it('ends with a summary job that needs every other job', () => { + const summary = workflow.jobs[summaryId] + expect(String(summary.if)).toBe('always()') + expect([...needsOf(summary)].sort()).toEqual( + jobIds.filter((id) => id !== summaryId).sort(), + ) + }) + + it('passes the summary only when every needed job passed or skipped', () => { + const status = (results) => runSummaryCheck(relPath, summaryId, results) + expect(status(['success', 'skipped', 'success'])).toBe(0) + expect(status(['success', 'failure'])).not.toBe(0) + expect(status(['skipped', 'cancelled'])).not.toBe(0) + }) + }) +} diff --git a/scripts/lint-no-workflow-caching.mjs b/scripts/lint-no-workflow-caching.mjs index 2fbb7d88b..7f36ff5b8 100644 --- a/scripts/lint-no-workflow-caching.mjs +++ b/scripts/lint-no-workflow-caching.mjs @@ -169,6 +169,12 @@ const AUDITED_ACTIONS = new Map([ // per-run store with no cross-run key. ['actions/upload-artifact', { cacheInput: null }], ['actions/download-artifact', { cacheInput: null }], + // The selector job's path filter (tests-supply-chain.yml). Read at the + // pinned v3.0.4: on a pull request it lists the changed files through the + // GitHub REST API and matches them with picomatch. Its dependencies are + // @actions/core, @actions/exec, @actions/github and picomatch. No cache, no + // cache input. + ['dorny/paths-filter', { cacheInput: null }], // Supplies zig + cargo-zigbuild (the glibc-pinned gnu builds) and wasm-pack, // all pinned in languages/typescript/packages/protect-ffi/mise.toml. `cache` DEFAULTS TO TRUE, so // an omitted key is a cache restore — and the generic `with.cache` rule below