From a9f15285ec9b48a72c4b61e47a81e12be7560e92 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Wed, 28 Oct 2026 23:25:35 +1100 Subject: [PATCH 1/3] test(ci): check every selector job the same way Test EQL has a selector job, and the other pull request workflows are getting one each. Each workflow's selector must follow the same rules: no path filter on pull_request, one dorny/paths-filter step, an output named after each filter, every job selected on other events, a failure when a filter gives no answer, `if:` conditions that exclude pull_request, and a summary job that needs every other job. workflow-selectors.test.mjs checks those rules for every workflow that has a selector job, found by scanning. It runs the selector's own decision script and the summary job's own check. lib/selector.mjs holds the shared code. It reads a selector job, runs its script, and works out which jobs a pull request runs from the files it changes. It reads dorny's `every` quantifier and brace patterns, which a filter needs to exclude documentation. The parity test now compares a selector workflow's push paths with the union of its selector filters, by what each selects from the tracked files. That replaces Test EQL's written exemption. The dispatch test holds every selector output open, whatever its name. --- scripts/__tests__/lib/selector.mjs | 361 ++++++++++++++++++ .../workflow-dispatch-job-conditions.test.mjs | 6 +- .../workflow-paths-filter-parity.test.mjs | 71 +++- scripts/__tests__/workflow-selectors.test.mjs | 220 +++++++++++ 4 files changed, 638 insertions(+), 20 deletions(-) create mode 100644 scripts/__tests__/lib/selector.mjs create mode 100644 scripts/__tests__/workflow-selectors.test.mjs diff --git a/scripts/__tests__/lib/selector.mjs b/scripts/__tests__/lib/selector.mjs new file mode 100644 index 000000000..ec2f910bb --- /dev/null +++ b/scripts/__tests__/lib/selector.mjs @@ -0,0 +1,361 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import yaml from 'js-yaml' +import { runsWhen } from './expressions.mjs' +import { pathsGlobToRegExp } from './paths-filter.mjs' +import { readWorkflow } from './workflows.mjs' + +/** + * Reading and running a workflow's selector job, for the guards that pin what + * it decides. + * + * The selector job is the first job of a workflow. It compares the files a + * pull request changes with each check's trigger paths, using + * dorny/paths-filter, and writes one output per filter. Each gated job's `if:` + * reads the output named after its filter. ADR 0003 and `.github/CONTEXT.md` + * describe the design. + * + * Everything here runs the workflow's own text: the filter rules as written, + * the decision script as written, and each job's real `if:`. A guard that + * reimplemented the decision would agree with a rewritten one by luck. + */ + +export const SELECTOR_JOB = 'selector' + +/** The events the selector decides nothing for: each one selects every job. */ +export const NON_PULL_REQUEST_EVENTS = [ + 'push', + 'schedule', + 'merge_group', + 'workflow_dispatch', +] + +const DORNY = 'dorny/paths-filter@' + +const stepOutputRef = (expression) => + /steps\.([A-Za-z0-9_-]+)\.outputs\.([A-Za-z0-9_-]+)/.exec( + String(expression ?? ''), + ) + +const selectorOutputRefs = (expression) => + [ + ...String(expression ?? '').matchAll( + new RegExp(`needs\\.${SELECTOR_JOB}\\.outputs\\.([A-Za-z0-9_-]+)`, 'g'), + ), + ].map((match) => match[1]) + +/** + * `{a,b}` expanded into its alternatives, as picomatch reads a brace pattern. + * One level only: a nested brace throws, rather than being read wrongly. + */ +export function expandBraces(pattern) { + const open = pattern.indexOf('{') + if (open === -1) return [pattern] + const close = pattern.indexOf('}', open) + const inner = pattern.slice(open + 1, close) + if (close === -1 || inner.includes('{')) { + throw new Error(`Cannot expand the brace pattern ${pattern}`) + } + const head = pattern.slice(0, open) + return inner + .split(',') + .flatMap((alternative) => + expandBraces(`${head}${alternative}${pattern.slice(close + 1)}`), + ) +} + +/** One dorny/paths-filter rule against one file. A `!` rule negates. */ +export function dornyRuleMatches(rule, file) { + if (rule.startsWith('!')) return !dornyRuleMatches(rule.slice(1), file) + return expandBraces(rule).some((glob) => pathsGlobToRegExp(glob).test(file)) +} + +/** + * Whether a file counts for a filter. Under dorny's default quantifier, `some`, + * one matching rule is enough; under `every`, all of them must match. + */ +export function dornyFilterMatches(rules, quantifier, file) { + return quantifier === 'every' + ? rules.every((rule) => dornyRuleMatches(rule, file)) + : rules.some((rule) => dornyRuleMatches(rule, file)) +} + +/** + * A workflow's selector job, parsed, or `null` when it has none. + * + * `filters` maps each filter name to its rules. A filter that includes an + * anchored list (`- *name`) parses to a nested list, and dorny flattens it; + * so does this. `outputs` maps each selector output to the decision step + * output it reads. + */ +export function readSelectorJob(relPath) { + const workflow = readWorkflow(relPath) + const job = workflow?.jobs?.[SELECTOR_JOB] + if (!job) return null + const steps = Array.isArray(job.steps) ? job.steps : [] + const filterSteps = steps.filter((step) => + String(step?.uses ?? '').startsWith(DORNY), + ) + const filterStep = filterSteps.length === 1 ? filterSteps[0] : null + const filters = Object.fromEntries( + Object.entries(yaml.load(filterStep?.with?.filters ?? '') ?? {}).map( + ([name, rules]) => [name, [rules].flat(Infinity).map(String)], + ), + ) + const outputs = Object.fromEntries( + Object.entries(job.outputs ?? {}).map(([name, expression]) => [ + name, + stepOutputRef(expression), + ]), + ) + const decideIds = [ + ...new Set(Object.values(outputs).map((ref) => ref?.[1])), + ].filter(Boolean) + const decideStep = + decideIds.length === 1 + ? steps.find((step) => step?.id === decideIds[0]) + : null + return { + workflow, + job, + filterSteps, + filterStep, + quantifier: filterStep?.with?.['predicate-quantifier'] ?? 'some', + filters, + outputs: Object.fromEntries( + Object.entries(outputs).map(([name, ref]) => [name, ref?.[2]]), + ), + decideStep, + } +} + +/** The selector outputs a job's `if:` reads. */ +export function selectorOutputsRead(job) { + return [...new Set(selectorOutputRefs(job?.if))] +} + +/** + * Runs the selector's decision script the way GitHub would: `answers` stands + * in for the filter step's outputs, keyed by filter name. An env entry that + * names no answer gets the empty string, as GitHub substitutes for an + * expression that resolves to nothing. + * + * Returns the exit status, the log, and the selector job's outputs. + */ +export function runSelectorDecision(relPath, eventName, answers) { + const selector = readSelectorJob(relPath) + const scratch = mkdtempSync(join(tmpdir(), 'selector-')) + try { + const outputFile = join(scratch, 'outputs.txt') + writeFileSync(outputFile, '') + const env = { PATH: process.env.PATH, GITHUB_OUTPUT: outputFile } + for (const [key, value] of Object.entries( + selector?.decideStep?.env ?? {}, + )) { + const text = String(value) + if (/github\.event_name/.test(text)) env[key] = eventName + else env[key] = answers[stepOutputRef(text)?.[2]] ?? '' + } + const run = spawnSync( + 'bash', + ['-c', String(selector?.decideStep?.run ?? 'exit 99')], + { cwd: scratch, env, encoding: 'utf8' }, + ) + const written = Object.fromEntries( + readFileSync(outputFile, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => [ + line.slice(0, line.indexOf('=')), + line.slice(line.indexOf('=') + 1), + ]), + ) + return { + status: run.status, + log: `${run.stdout}${run.stderr}`, + outputs: Object.fromEntries( + Object.entries(selector?.outputs ?? {}).map(([name, stepOutput]) => [ + name, + written[stepOutput], + ]), + ), + } + } finally { + rmSync(scratch, { recursive: true, force: true }) + } +} + +/** What each filter answers for a pull request that changes `files`. */ +export function filterAnswers(relPath, files) { + const selector = readSelectorJob(relPath) + return Object.fromEntries( + Object.entries(selector?.filters ?? {}).map(([name, rules]) => [ + name, + String( + files.some((file) => + dornyFilterMatches(rules, selector.quantifier, file), + ), + ), + ]), + ) +} + +const needsOf = (job) => [job?.needs ?? []].flat() + +/** + * The jobs that run for a same-repository pull request changing `files`, + * sorted. The selector job's outputs come from its own script; each job's + * `if:` is evaluated as GitHub does, including the implicit `success()` that + * skips a job when one it needs did not succeed. + * + * `github` adds to the context, for a job whose `if:` reads more than the + * event and the selector (such as `head_ref`). + */ +export function selectedJobs(relPath, files, { github = {} } = {}) { + const selector = readSelectorJob(relPath) + const decision = runSelectorDecision( + relPath, + 'pull_request', + filterAnswers(relPath, files), + ) + if (decision.status !== 0) { + throw new Error(`The selector failed:\n${decision.log}`) + } + const repository = 'cipherstash/stack' + const context = { + github: { + event_name: 'pull_request', + repository, + head_ref: 'feature', + event: { + pull_request: { + draft: false, + head: { repo: { full_name: repository } }, + user: { type: 'User' }, + }, + }, + ...github, + }, + needs: {}, + vars: {}, + } + const jobs = selector.workflow.jobs + const results = { + [SELECTOR_JOB]: { result: 'success', outputs: decision.outputs }, + } + const resolve = (id) => { + if (results[id]) return results[id].result + const job = jobs[id] + const needs = needsOf(job) + const needResults = needs.map(resolve) + const condition = String(job?.if ?? '') + const statusFunction = /\b(always|failure|cancelled|success)\(\)/.test( + condition, + ) + let runs + if (!statusFunction && needResults.some((result) => result !== 'success')) { + runs = false + } else if (!condition) { + runs = true + } else { + runs = runsWhen(condition, { + ...context, + needs: Object.fromEntries(needs.map((need) => [need, results[need]])), + }) + } + results[id] = { result: runs ? 'success' : 'skipped', outputs: {} } + return results[id].result + } + return Object.keys(jobs) + .filter((id) => resolve(id) === 'success') + .sort() +} + +// GitHub reads a `paths:` list in order, and the last matching entry decides. +export function pathsListSelects(paths, file) { + let selected = false + for (const entry of paths) { + if (entry.startsWith('!')) { + if (pathsGlobToRegExp(entry.slice(1)).test(file)) selected = false + } else if (pathsGlobToRegExp(entry).test(file)) { + selected = true + } + } + return selected +} + +/** + * A filter as a GitHub `paths:` list: entries in order, a `!` entry excluding + * what came before it. Lets the guards that check a workflow's `paths:` + * coverage read a selector filter with the same code. + * + * Under `every`, the one positive rule's brace alternatives come first and the + * `!` rules after, which selects exactly the files dorny selects. Under + * `some`, a filter has no `!` rule (`workflow-selectors.test.mjs`), so it is + * already that list. + */ +export function filterAsPathsList(rules, quantifier) { + if (quantifier !== 'every') return rules.flatMap(expandBraces) + const positive = rules.filter((rule) => !rule.startsWith('!')) + const negative = rules.filter((rule) => rule.startsWith('!')) + return [...positive.flatMap(expandBraces), ...negative] +} + +/** + * The path lists that decide whether a job runs on a change, as GitHub + * `paths:` lists: the workflow's `on.push.paths` and `on.pull_request.paths` + * where they exist, and each selector filter the job's `if:` reads. + * + * A job in a selector workflow that reads no filter runs on every pull + * request, so only the `push` list, if any, decides it. + */ +export function jobTriggerLists(relPath, jobId) { + const workflow = readWorkflow(relPath) + const on = workflow?.on ?? workflow?.[true] + const lists = ['push', 'pull_request'] + .filter((event) => Array.isArray(on?.[event]?.paths)) + .map((event) => ({ + source: `on.${event}.paths`, + paths: on[event].paths.map(String), + })) + const selector = readSelectorJob(relPath) + for (const name of selectorOutputsRead(workflow?.jobs?.[jobId])) { + const rules = selector?.filters?.[name] + if (!rules) continue + lists.push({ + source: `selector filter "${name}"`, + paths: filterAsPathsList(rules, selector.quantifier), + }) + } + return lists +} + +/** + * Runs the summary job's check with the given results of the jobs it needs, + * and returns its exit status. The results stand in for + * `join(needs.*.result, ' ')`, wherever the step reads it. + */ +export function runSummaryCheck(relPath, summaryId, results) { + const workflow = readWorkflow(relPath) + const steps = workflow?.jobs?.[summaryId]?.steps ?? [] + const joined = results.join(' ') + const resultsExpression = /\$\{\{\s*join\(needs\.\*\.result,\s*' '\)\s*\}\}/g + let status = 0 + for (const step of steps) { + if (typeof step?.run !== 'string') continue + const env = { PATH: process.env.PATH } + for (const [key, value] of Object.entries(step.env ?? {})) { + env[key] = String(value).replace(resultsExpression, joined) + } + const run = spawnSync( + 'bash', + ['-c', step.run.replace(resultsExpression, joined)], + { env, encoding: 'utf8' }, + ) + status = run.status + if (status !== 0) break + } + return status +} diff --git a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs index e727e71fe..d7a6ea95e 100644 --- a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs +++ b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs @@ -180,7 +180,11 @@ function extractForkClause(condition) { * run. */ const PERMISSIVE_NEEDS = { - selector: { outputs: { eql: 'true', rust_crates: 'true' } }, + // Every selector output, whatever its name: each workflow's selector names + // its outputs after its own filters. + selector: { + outputs: new Proxy({}, { has: () => true, get: () => 'true' }), + }, // The release workflows' gates, none of which is about how the run was // triggered. Held open, or "skips because there is nothing to publish" would // be indistinguishable from "skips on a dispatch". diff --git a/scripts/__tests__/workflow-paths-filter-parity.test.mjs b/scripts/__tests__/workflow-paths-filter-parity.test.mjs index 74ab3af62..93d4facab 100644 --- a/scripts/__tests__/workflow-paths-filter-parity.test.mjs +++ b/scripts/__tests__/workflow-paths-filter-parity.test.mjs @@ -1,4 +1,11 @@ +import { execFileSync } from 'node:child_process' import { describe, expect, it } from 'vitest' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { + dornyFilterMatches, + pathsListSelects, + readSelectorJob, +} from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -57,6 +64,14 @@ function pathsFilters(relPath) { ) } +/** Every tracked file: what a `paths:` list or a filter can select. */ +const TRACKED = execFileSync('git', ['ls-files', '-z'], { + cwd: REPO_ROOT, + encoding: 'utf8', +}) + .split('\0') + .filter(Boolean) + const CLASSIFIED = workflowFiles().map((relPath) => { const filters = pathsFilters(relPath) const declared = FILTERED_EVENTS.filter((event) => filters[event] !== null) @@ -66,8 +81,23 @@ const CLASSIFIED = workflowFiles().map((relPath) => { /** Workflows writing the list twice — the ones the parity check applies to. */ const PARITY = CLASSIFIED.filter((entry) => entry.declared.length === 2) +/** + * Workflows whose selector job decides what a pull request runs, and whose + * `push` keeps a `paths:` list. Their `pull_request` has no filter, by design + * (ADR 0003), so the second copy of the list is the selector's filters. The + * check below compares the two by what they select. + */ +const SELECTOR_PUSH = CLASSIFIED.filter( + (entry) => + entry.filters.push !== null && + entry.filters.pull_request === null && + readSelectorJob(entry.relPath) !== null, +) + /** Workflows filtering one event and not the other. */ -const ASYMMETRIC = CLASSIFIED.filter((entry) => entry.declared.length === 1) +const ASYMMETRIC = CLASSIFIED.filter( + (entry) => entry.declared.length === 1 && !SELECTOR_PUSH.includes(entry), +) /** * The guard on the scan, in the shape the rest of this directory uses. Every @@ -145,24 +175,6 @@ const EXPECTED_ASYMMETRIES = new Map([ // check-auth-npm-changeset.test.mjs, which is the second copy here. 'pull_request is the only trigger; the check diffs a pull request against its base, which a push does not have', ], - [ - '.github/workflows/test-eql.yml', - // `pull_request` is deliberately UNFILTERED and `push: branches: [main]` - // carries the list, which is the same shape as tests-rust.yml with the two - // events swapped. Unfiltered on the PR side because this workflow's - // `ci-required` is the check branch protection is meant to reference, and a - // workflow skipped by a path filter leaves a required check stuck Pending; - // relevance is applied per-job via `if:` instead. Filtered on the push side - // because nothing waits on a push run, so skipping one costs nothing. - // - // The direction that would hurt — a `pull_request` filter narrower than the - // `push` one, so the PR introducing a change never runs the suite — is - // unreachable when `pull_request` has no filter at all. The push list is - // not unchecked: `eql-workflow-filters.test.mjs` compares it against the - // union of the selector job's filters, which is the second copy of the - // same list and the one this file cannot see. - 'pull_request is deliberately unfiltered (per-job relevance instead), so PRs run a superset of what push runs; the push list is compared against the selector job filters by eql-workflow-filters.test.mjs', - ], [ '.github/workflows/bench-eql.yml', // The mirror image of lint-release.yml: `push: branches: [main]` with a @@ -235,6 +247,27 @@ describe('paths filters are written twice, identically', () => { ).toEqual([]) }) + for (const { relPath, filters } of SELECTOR_PUSH) { + it(`${relPath} selects on push what its selector selects on a pull request`, () => { + // A selector workflow writes its list twice as well: as the `push` + // `paths:` list, and split across the selector job's filters. The two + // are written in different syntaxes, so they are compared by what they + // select from the tracked files, not by their text. + const selector = readSelectorJob(relPath) + const differ = TRACKED.filter( + (file) => + pathsListSelects(filters.push, file) !== + Object.values(selector.filters).some((rules) => + dornyFilterMatches(rules, selector.quantifier, file), + ), + ) + expect( + differ, + `In ${relPath}, the \`push\` \`paths:\` list and the union of the selector job's filters disagree about these files. Change both in the same edit.`, + ).toEqual([]) + }) + } + for (const { relPath } of PARITY) { it(`${relPath} filters push and pull_request identically`, () => { const filters = pathsFilters(relPath) diff --git a/scripts/__tests__/workflow-selectors.test.mjs b/scripts/__tests__/workflow-selectors.test.mjs new file mode 100644 index 000000000..341b95f8d --- /dev/null +++ b/scripts/__tests__/workflow-selectors.test.mjs @@ -0,0 +1,220 @@ +import { describe, expect, it } from 'vitest' +import { runsWhen } from './lib/expressions.mjs' +import { + NON_PULL_REQUEST_EVENTS, + readSelectorJob, + runSelectorDecision, + runSummaryCheck, + SELECTOR_JOB, + selectorOutputsRead, +} from './lib/selector.mjs' +import { workflowFiles } from './lib/workflows.mjs' + +/** + * Every workflow with a selector job follows the same design (ADR 0003): the + * selector job decides which checks a pull request runs, each gated job reads + * the selector's answer in its `if:`, and a summary job ends the workflow. + * `.github/CONTEXT.md` defines the terms. + * + * The workflows are discovered, not listed, so a workflow that gains a + * selector job is held to this the day it lands. What each selector decides + * for particular changes is pinned per workflow, in the `*-selector*.test.mjs` + * files. + * + * Whether every job is decided by the selector, or has a written reason to run + * on every change, is the failsafe's question, not this file's. + */ + +const SELECTOR_WORKFLOWS = workflowFiles().filter( + (relPath) => readSelectorJob(relPath) !== null, +) + +/** + * The guard on the scan. A minimum: a workflow that gains a selector job must + * not fail this. Each workflow's own `*-selector*.test.mjs` file checks that + * its selector job exists. + */ +const EXPECTED_SELECTOR_WORKFLOWS = ['.github/workflows/test-eql.yml'] + +const triggersOf = (workflow) => workflow?.on ?? workflow?.[true] + +const needsOf = (job) => [job?.needs ?? []].flat() + +/** The jobs whose `if:` reads a selector output. */ +function gatedJobs(workflow) { + return Object.entries(workflow?.jobs ?? {}) + .map(([id, job]) => ({ id, job, reads: selectorOutputsRead(job) })) + .filter((entry) => entry.reads.length > 0) +} + +describe('the workflows with a selector job', () => { + it('include the converted workflows', () => { + expect(SELECTOR_WORKFLOWS).toEqual( + expect.arrayContaining(EXPECTED_SELECTOR_WORKFLOWS), + ) + }) +}) + +for (const relPath of SELECTOR_WORKFLOWS) { + const selector = readSelectorJob(relPath) + const { workflow } = selector + const jobIds = Object.keys(workflow.jobs) + const summaryId = jobIds.at(-1) + const filterNames = Object.keys(selector.filters) + const outputNames = Object.keys(selector.outputs) + + describe(relPath, () => { + it('has no path filter on pull_request', () => { + // A workflow that a path filter skips leaves its required checks + // pending, and GitHub's documentation on required status checks says + // that blocks a merge. The selector job decides instead. + const pullRequest = triggersOf(workflow)?.pull_request ?? {} + expect(Object.keys(pullRequest)).not.toContain('paths') + expect(Object.keys(pullRequest)).not.toContain('paths-ignore') + }) + + it('starts with the selector job, on a GitHub-hosted runner', () => { + // ADR 0003: every gated job waits for it, and a self-provisioned + // machine takes longer to start. + expect(jobIds[0]).toBe(SELECTOR_JOB) + expect(String(selector.job['runs-on'])).toMatch(/^ubuntu-/) + expect(selector.job.needs).toBeUndefined() + expect(selector.job.if).toBeUndefined() + }) + + it('has one path filter step, run only on a pull request', () => { + expect(selector.filterSteps).toHaveLength(1) + expect(String(selector.filterStep?.if)).toBe( + "github.event_name == 'pull_request'", + ) + expect(filterNames).not.toEqual([]) + }) + + it('writes no `!` rule that dorny would read as matching everything', () => { + // Under dorny's default quantifier, `some`, a file counts when ANY rule + // matches, so `!**/*.md` matches every file that is not Markdown. A `!` + // rule only excludes under `every`, where a file must match every rule, + // and there a second positive rule would select nothing at all. + expect(['some', 'every']).toContain(selector.quantifier) + for (const [name, rules] of Object.entries(selector.filters)) { + const positive = rules.filter((rule) => !rule.startsWith('!')) + const negative = rules.filter((rule) => rule.startsWith('!')) + if (selector.quantifier === 'every') { + expect(positive, `filter "${name}"`).toHaveLength(1) + } else { + expect(negative, `filter "${name}"`).toEqual([]) + } + } + }) + + it('names each output after the filter that decides it', () => { + // The guards follow a job's `if:` to its filter by this name. + expect(outputNames.sort()).toEqual([...filterNames].sort()) + for (const [name, stepOutput] of Object.entries(selector.outputs)) { + expect(stepOutput, `output "${name}"`).toBe(name) + } + expect( + selector.decideStep, + 'No single step writes the selector outputs.', + ).toBeTruthy() + }) + + it('selects every job on events other than pull_request', () => { + for (const event of NON_PULL_REQUEST_EVENTS) { + const { status, log, outputs } = runSelectorDecision(relPath, event, {}) + expect(status, log).toBe(0) + for (const name of outputNames) { + expect(outputs[name], `${name} on ${event}`).toBe('true') + } + } + }) + + it('passes the filter answers through on a pull request', () => { + for (const answer of ['true', 'false']) { + const answers = Object.fromEntries( + filterNames.map((name) => [name, answer]), + ) + const { status, log, outputs } = runSelectorDecision( + relPath, + 'pull_request', + answers, + ) + expect(status, log).toBe(0) + expect(outputs).toEqual(answers) + } + }) + + it('fails when a pull request filter gives no answer', () => { + // A missing answer would skip every gated job, and the summary job + // counts skipped as passed: a green run that ran nothing. + for (const name of filterNames) { + for (const bad of ['', 'maybe']) { + const answers = Object.fromEntries( + filterNames.map((other) => [other, other === name ? bad : 'true']), + ) + const { status } = runSelectorDecision( + relPath, + 'pull_request', + answers, + ) + expect(status, `filter "${name}" answering '${bad}'`).not.toBe(0) + } + } + }) + + it('gates jobs only on outputs the selector writes, and needs it', () => { + const gated = gatedJobs(workflow) + expect(gated).not.toEqual([]) + for (const { id, job, reads } of gated) { + expect(needsOf(job), `job "${id}"`).toContain(SELECTOR_JOB) + for (const name of reads) { + expect(outputNames, `job "${id}" reads "${name}"`).toContain(name) + } + } + }) + + it("writes each gated job's `if:` as an exclusion of pull_request", () => { + // Outside a pull request the selector's answers must not matter: a job + // written as a list of allowed events skips on the event nobody listed. + for (const { id, job } of gatedJobs(workflow)) { + for (const event of NON_PULL_REQUEST_EVENTS) { + const verdict = (answer) => + runsWhen(job.if, { + github: { + event_name: event, + repository: 'cipherstash/stack', + event: {}, + }, + needs: { + [SELECTOR_JOB]: { + result: 'success', + outputs: Object.fromEntries( + outputNames.map((name) => [name, answer]), + ), + }, + }, + vars: {}, + }) + expect(verdict('false'), `job "${id}" on ${event}`).toBe( + verdict('true'), + ) + } + } + }) + + it('ends with a summary job that needs every other job', () => { + const summary = workflow.jobs[summaryId] + expect(String(summary.if)).toBe('always()') + expect([...needsOf(summary)].sort()).toEqual( + jobIds.filter((id) => id !== summaryId).sort(), + ) + }) + + it('passes the summary only when every needed job passed or skipped', () => { + const status = (results) => runSummaryCheck(relPath, summaryId, results) + expect(status(['success', 'skipped', 'success'])).toBe(0) + expect(status(['success', 'failure'])).not.toBe(0) + expect(status(['skipped', 'cancelled'])).not.toBe(0) + }) + }) +} From 61c285381e71dbd8e2eef75158ab8e5d33552125 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Wed, 28 Oct 2026 23:31:51 +1100 Subject: [PATCH 2/3] test(ci): read a job's trigger paths from its selector filter Three guards check that a workflow's path filter covers what its jobs read: the binding build inputs, the auth binding sources, and the sources the integration suites import. They read `on.push.paths` and `on.pull_request.paths`. Once a workflow has a selector job, its pull request list moves into the selector filter of each job, and those guards would stop checking it. They now read, per job, the lists that decide whether that job runs: the workflow's push and pull_request lists, and the selector filter the job reads. For a workflow without a selector job this is the same two lists as before. They also check only the jobs that build or load the binding, not every job in the workflow. --- .../auth-binding-step-order.test.mjs | 41 ++++++---- .../integration-workflow-paths.test.mjs | 32 +++++--- .../wasm-build-inputs-paths-filter.test.mjs | 80 ++++++++++++------- 3 files changed, 95 insertions(+), 58 deletions(-) diff --git a/scripts/__tests__/auth-binding-step-order.test.mjs b/scripts/__tests__/auth-binding-step-order.test.mjs index aa1b604ab..82dd4317e 100644 --- a/scripts/__tests__/auth-binding-step-order.test.mjs +++ b/scripts/__tests__/auth-binding-step-order.test.mjs @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { filterCovers } from './lib/paths-filter.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -84,13 +85,20 @@ describe('the @cipherstash/auth binding is built wherever the SDK runs', () => { const same = (entry, input) => entry === input -/** The `paths:` lists of a workflow's filtered events. */ -const filters = (relPath) => { - const wf = readWorkflow(relPath) - const on = wf?.on ?? wf?.[true] - return ['push', 'pull_request'] - .map((event) => on?.[event]?.paths) - .filter(Array.isArray) +/** + * The path lists that decide whether the given jobs run, once each: the + * workflow's `push` and `pull_request` `paths:` lists, and the selector filter + * each job reads. Every job of the workflow when none are given. + */ +const filters = (relPath, jobNames) => { + const lists = new Map() + for (const jobName of jobNames ?? + Object.keys(readWorkflow(relPath)?.jobs ?? {})) { + for (const { source, paths } of jobTriggerLists(relPath, jobName)) { + lists.set(source, paths) + } + } + return [...lists.values()] } /** @@ -105,14 +113,19 @@ const AUTH_SOURCE_WORKFLOWS = [ ] describe('a change to the auth binding starts the jobs that load it', () => { - const users = [ - ...new Set( - JOBS.filter((job) => job.auth).map((job) => job.id.split(' / ')[0]), - ), - ] + /** Each workflow with a job that builds the binding, and those jobs. */ + const usersByWorkflow = new Map() + for (const job of JOBS.filter((entry) => entry.auth)) { + const [relPath, jobName] = job.id.split(' / ') + usersByWorkflow.set(relPath, [ + ...(usersByWorkflow.get(relPath) ?? []), + jobName, + ]) + } + const users = [...usersByWorkflow.keys()] it.each(users)('%s filters on the action', (relPath) => { - for (const paths of filters(relPath)) { + for (const paths of filters(relPath, usersByWorkflow.get(relPath))) { expect( filterCovers(paths, '.github/actions/build-auth-binding/**', same), ).toBe(true) @@ -133,7 +146,7 @@ describe('a change to the auth binding starts the jobs that load it', () => { '%s filters on the auth sources', (relPath) => { expect(shipped.length).toBeGreaterThanOrEqual(4) - const lists = filters(relPath) + const lists = filters(relPath, usersByWorkflow.get(relPath)) expect(lists.length).toBe(2) for (const paths of lists) { expect( diff --git a/scripts/__tests__/integration-workflow-paths.test.mjs b/scripts/__tests__/integration-workflow-paths.test.mjs index c3b1e812a..a44f65539 100644 --- a/scripts/__tests__/integration-workflow-paths.test.mjs +++ b/scripts/__tests__/integration-workflow-paths.test.mjs @@ -2,6 +2,7 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -40,9 +41,6 @@ const STACK_SRC = 'languages/typescript/packages/stack/src' const STACK_MANIFEST = 'languages/typescript/packages/stack/package.json' const CATALOG_MANIFEST = 'pnpm-workspace.yaml' -/** Both trigger events, in the order GitHub evaluates them. */ -const TRIGGER_EVENTS = ['push', 'pull_request'] - /** * The integration workflows, discovered rather than listed: any workflow whose * `CS_IT_SUITE` globs select suites out of `languages/typescript/packages/stack/` is in scope. A @@ -56,14 +54,21 @@ function discoverWorkflows() { } /** - * `on:` parses as the boolean `true` under YAML 1.1 (the "Norway problem"), - * which is why this reads both keys rather than `wf.on`. + * The path lists that decide whether the jobs running a suite run, once each: + * the workflow's `push` and `pull_request` `paths:` lists, and the selector + * filter each such job reads. */ -function triggerBlocks(wf) { - const on = wf.on ?? wf[true] - return TRIGGER_EVENTS.map((event) => on?.[event]).filter( - (block) => block && Array.isArray(block.paths), - ) +function triggerBlocks(relPath) { + const lists = new Map() + for (const [jobId, job] of Object.entries( + readWorkflow(relPath)?.jobs ?? {}, + )) { + if (!job?.env?.CS_IT_SUITE) continue + for (const { source, paths } of jobTriggerLists(relPath, jobId)) { + lists.set(source, { source, paths }) + } + } + return [...lists.values()] } /** Every `CS_IT_SUITE` glob set declared anywhere in the workflow. */ @@ -190,8 +195,9 @@ describe('integration workflow paths filters', () => { for (const relPath of WORKFLOWS) { it(`${relPath} triggers on the manifests pinning its suites' dependencies`, () => { const wf = readWorkflow(relPath) - const blocks = triggerBlocks(wf) - expect(blocks.length).toBe(TRIGGER_EVENTS.length) + const blocks = triggerBlocks(relPath) + // A `push` list, and a `pull_request` list or a selector filter. + expect(blocks.length).toBe(2) const deps = stackDependencies() const files = suiteFiles(suiteGlobs(wf)) @@ -215,7 +221,7 @@ describe('integration workflow paths filters', () => { it(`${relPath} triggers on every source path its suites import`, () => { const wf = readWorkflow(relPath) - const blocks = triggerBlocks(wf) + const blocks = triggerBlocks(relPath) expect(blocks.length).toBeGreaterThan(0) const files = suiteFiles(suiteGlobs(wf)) diff --git a/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs b/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs index 28d96ad94..8f28725ad 100644 --- a/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs +++ b/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs @@ -4,6 +4,7 @@ import yaml from 'js-yaml' import { describe, expect, it } from 'vitest' import { filterCovers } from './lib/paths-filter.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -57,9 +58,6 @@ const ACTION = '.github/actions/build-ffi-binding/action.yml' const ACTION_USES = './.github/actions/build-ffi-binding' const CACHE_ACTION = /^actions\/cache(\/(restore|save))?@/ -/** Both trigger events that carry a `paths:` filter, in GitHub's own order. */ -const FILTERED_EVENTS = ['push', 'pull_request'] - /** The package whose Rust both cache keys are about. */ const FFI_PACKAGE = 'languages/typescript/packages/protect-ffi' const FFI_PACKAGE_NAME = '@cipherstash/protect-ffi' @@ -276,21 +274,34 @@ function stepDir(wf, job, step) { /** Every job step of a workflow, paired with the job that owns it. */ function steps(wf) { - return Object.values(wf?.jobs ?? {}).flatMap((job) => - (Array.isArray(job?.steps) ? job.steps : []).map((step) => ({ job, step })), + return Object.entries(wf?.jobs ?? {}).flatMap(([jobId, job]) => + (Array.isArray(job?.steps) ? job.steps : []).map((step) => ({ + jobId, + job, + step, + })), ) } -/** Runs `./.github/actions/build-ffi-binding`, optionally with `wasm: 'true'`. */ -function usesAction(relPath, { wasmOnly = false } = {}) { - return steps(readWorkflow(relPath)).some( - ({ step }) => - typeof step?.uses === 'string' && - step.uses.trim() === ACTION_USES && - (!wasmOnly || String(step?.with?.wasm) === 'true'), +const jobIdsOf = (entries) => [...new Set(entries.map(({ jobId }) => jobId))] + +/** + * The jobs that run `./.github/actions/build-ffi-binding`, optionally with + * `wasm: 'true'`. + */ +function actionJobs(relPath, { wasmOnly = false } = {}) { + return jobIdsOf( + steps(readWorkflow(relPath)).filter( + ({ step }) => + typeof step?.uses === 'string' && + step.uses.trim() === ACTION_USES && + (!wasmOnly || String(step?.with?.wasm) === 'true'), + ), ) } +const usesAction = (relPath, options) => actionJobs(relPath, options).length > 0 + /** * Compiles this package's Rust — through the action, or directly. * @@ -300,10 +311,9 @@ function usesAction(relPath, { wasmOnly = false } = {}) { * EQL workspace has its own, with its own workflows — is not mistaken for this * one. */ -function compilesFfiRust(relPath) { - if (usesAction(relPath)) return true +function ffiRustJobs(relPath) { const wf = readWorkflow(relPath) - return steps(wf).some(({ job, step }) => { + const direct = steps(wf).filter(({ job, step }) => { const run = String(step?.run ?? '') if (!run) return false const targetsPackage = @@ -316,24 +326,28 @@ function compilesFfiRust(relPath) { (inPackage && mentionsAny(run, CARGO_TASKS)) ) }) + return [...new Set([...actionJobs(relPath), ...jobIdsOf(direct)])] } +const compilesFfiRust = (relPath) => ffiRustJobs(relPath).length > 0 + /** - * The `paths:` filters a workflow declares, one entry per filtered event. + * The path lists that decide whether the given jobs run: the workflow's `push` + * and `pull_request` `paths:` lists, and the selector filter each job reads. + * One entry per list, labelled with where it comes from. * - * `on:` parses as the boolean `true` under YAML 1.1 (the "Norway problem"), - * hence `wf.on ?? wf[true]`. An event with no `paths:` is UNFILTERED and is - * skipped rather than failed — it already runs on every diff, so it cannot miss - * one. `tests.yml` and `tests-rust.yml` are both half-cases: `push` is - * `branches: [main]` with no filter, so only their `pull_request` lists are - * here. `tests.yml`'s is `'**'` with one negation, which `filterCovers` reads. + * A job that no list decides is skipped rather than failed: it runs on every + * diff, so it cannot miss one. A selector filter is read as a `paths:` list, + * so a `'**'` with negations is read the same way in either. */ -function filteredEvents(relPath) { - const wf = readWorkflow(relPath) - const on = wf?.on ?? wf?.[true] - return FILTERED_EVENTS.filter((event) => - Array.isArray(on?.[event]?.paths), - ).map((event) => ({ relPath, event, paths: on[event].paths })) +function filteredEvents(relPath, jobIds) { + const lists = new Map() + for (const jobId of jobIds) { + for (const { source, paths } of jobTriggerLists(relPath, jobId)) { + lists.set(source, { relPath, event: source, paths }) + } + } + return [...lists.values()] } const ALL_WORKFLOWS = workflowFiles() @@ -381,6 +395,7 @@ const KEYS = [ what: '`index.node`, the binding every credentialed suite encrypts through', reaches: 'compiles the crate graph behind `index.node`', workflows: ALL_WORKFLOWS.filter(compilesFfiRust), + jobs: ffiRustJobs, expected: [ '.github/workflows/_build-ffi-artifacts.yml', '.github/workflows/integration-drizzle.yml', @@ -401,6 +416,7 @@ const KEYS = [ workflows: ALL_WORKFLOWS.filter((relPath) => usesAction(relPath, { wasmOnly: true }), ), + jobs: (relPath) => actionJobs(relPath, { wasmOnly: true }), expected: [ '.github/workflows/integration-drizzle.yml', '.github/workflows/integration-protect-ffi.yml', @@ -447,7 +463,9 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { for (const key of KEYS) { const inputs = buildInputs(key.step) - const filtered = key.workflows.flatMap(filteredEvents) + const filtered = key.workflows.flatMap((relPath) => + filteredEvents(relPath, key.jobs(relPath)), + ) describe(`the ${key.id} key`, () => { it('hashes build inputs that name real files', () => { @@ -506,7 +524,7 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { ) expect( uncovered, - `These files are hashed into the ${key.id} cache key of \`${ACTION}\`, so an edit to one MISSES the cache and rebuilds ${key.what}. ${relPath} ${key.reaches}, but its \`${event}\` \`paths:\` filter does not list them, so a PR touching only one of them rebuilds the artifact and starts no suite that exercises it. Nothing goes red; nothing ran.\nAdd each to BOTH copies of the filter — GitHub Actions has no YAML anchors, and \`workflow-paths-filter-parity.test.mjs\` fails a one-sided edit.`, + `These files are hashed into the ${key.id} cache key of \`${ACTION}\`, so an edit to one MISSES the cache and rebuilds ${key.what}. ${relPath} ${key.reaches}, but its \`${event}\` \`paths:\` filter does not list them, so a PR touching only one of them rebuilds the artifact and starts no suite that exercises it. Nothing goes red; nothing ran.\nAdd each to every copy of the list — GitHub Actions has no YAML anchors, and \`workflow-paths-filter-parity.test.mjs\` fails a one-sided edit.`, ).toEqual([]) }) } @@ -521,7 +539,7 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { KEYS.flatMap((key) => { const inputs = buildInputs(key.step) return key.workflows - .flatMap(filteredEvents) + .flatMap((relPath) => filteredEvents(relPath, key.jobs(relPath))) .flatMap(({ relPath, paths }) => inputs .filter((pattern) => !filterCovers(paths, pattern, entryCovers)) From fce6df0f04edfeeaf1819b2eaec10c934335f564 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Wed, 28 Oct 2026 23:46:30 +1100 Subject: [PATCH 3/3] ci(js): decide in selector jobs which JavaScript checks run Five JavaScript and TypeScript workflows used a workflow-level path filter on pull_request. A workflow that a path filter skips leaves its required checks pending, and GitHub's documentation on required status checks says that blocks a merge. Each workflow now runs on every pull request. A `selector` job on a GitHub-hosted runner decides which jobs run, and fails when a filter gives no answer. A summary job ends each workflow, and passes when every needed job passed or was skipped. Each filter keeps the paths of the filter it replaced, so the checks run for the same changes as before: - Test JS: every file except docs/plans/. Its jobs read most of the repository, so one filter gates all five. dorny's `every` quantifier makes the `!docs/plans/**` rule exclude. - FTA Complexity, Test Benchmark JS, Test supply chain security and Require @cipherstash/auth changeset: one filter each, the old list. Push lists that remain match the selector filters, which the parity test checks. Test supply chain security is a target of the no-caching lint, which allows only audited actions. dorny/paths-filter is added to AUDITED_ACTIONS: at the pinned v3.0.4 it lists changed files through the GitHub REST API, and none of its dependencies caches. The auth changeset test reads the job by id, because the selector is now the first job, and reads the selector filter. js-selectors.test.mjs pins what each selector decides. --- .github/workflows/fta-v3.yml | 116 ++++++++++++++- .../workflows/require-auth-npm-changeset.yml | 119 +++++++++++++-- .github/workflows/tests-bench.yml | 128 ++++++++++++++-- .github/workflows/tests-supply-chain.yml | 123 ++++++++++++++-- .github/workflows/tests.yml | 138 +++++++++++++++++- .../check-auth-npm-changeset.test.mjs | 11 +- scripts/__tests__/js-selectors.test.mjs | 102 +++++++++++++ scripts/__tests__/paths-filter.test.mjs | 12 +- .../plan-docs-unread-by-tests.test.mjs | 2 +- .../workflow-paths-filter-parity.test.mjs | 16 -- scripts/lint-no-workflow-caching.mjs | 6 + 11 files changed, 699 insertions(+), 74 deletions(-) create mode 100644 scripts/__tests__/js-selectors.test.mjs diff --git a/.github/workflows/fta-v3.yml b/.github/workflows/fta-v3.yml index 258e3e941..12b94e253 100644 --- a/.github/workflows/fta-v3.yml +++ b/.github/workflows/fta-v3.yml @@ -6,11 +6,16 @@ name: "FTA Complexity" # the check when any file exceeds the score cap (`--score-cap` in the # `analyze:complexity` script). FTA is pure static source analysis, so this job # needs no build step, database, or credentials. +# +# The `selector` job decides whether a pull request runs the gate, and +# `summary` reports the result (ADR 0003). on: push: branches: - 'main' + # The same files as the `selector` job's filter. + # scripts/__tests__/workflow-paths-filter-parity.test.mjs compares them. paths: - 'languages/typescript/packages/stack/src/eql/v3/**' - 'languages/typescript/packages/stack-drizzle/**' @@ -21,23 +26,92 @@ on: - 'languages/typescript/packages/stack/src/schema/match-defaults.ts' - 'languages/typescript/packages/stack/package.json' - '.github/workflows/fta-v3.yml' + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. pull_request: branches: - "**" - paths: - - 'languages/typescript/packages/stack/src/eql/v3/**' - - 'languages/typescript/packages/stack-drizzle/**' - - 'languages/typescript/packages/stack-supabase/**' - - 'languages/typescript/packages/stack/src/schema/match-defaults.ts' - - 'languages/typescript/packages/stack/package.json' - - '.github/workflows/fta-v3.yml' permissions: contents: read jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + fta: ${{ steps.decide.outputs.fta }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + filters: | + # The sources each `analyze:complexity` step scores. Shared + # match-index defaults live outside src/eql/v3 but shape every + # emitted v3 match block, so edits there trigger the gate too. + fta: + - "languages/typescript/packages/stack/src/eql/v3/**" + - "languages/typescript/packages/stack-drizzle/**" + - "languages/typescript/packages/stack-supabase/**" + - "languages/typescript/packages/stack/src/schema/match-defaults.ts" + - "languages/typescript/packages/stack/package.json" + - ".github/workflows/fta-v3.yml" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + FTA: ${{ steps.filter.outputs.fta }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + FTA=true + fi + answers=( + "fta=$FTA" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + fta: name: Analyze v3 complexity + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.fta == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 steps: @@ -89,3 +163,31 @@ jobs: - name: Analyze stack-supabase complexity run: pnpm exec turbo run analyze:complexity --filter @cipherstash/stack-supabase + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: FTA Complexity summary + needs: [selector, fta] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/require-auth-npm-changeset.yml b/.github/workflows/require-auth-npm-changeset.yml index d71644af0..0ee44bc4a 100644 --- a/.github/workflows/require-auth-npm-changeset.yml +++ b/.github/workflows/require-auth-npm-changeset.yml @@ -9,19 +9,16 @@ name: "Require @cipherstash/auth changeset" # Changes that never ship (tests, examples, devDependencies) are filtered out # by `check-auth-npm-changeset.mjs --shipped`. # -# The paths below and the job's `git diff` pathspec are held to the same set -# by scripts/__tests__/check-auth-npm-changeset.test.mjs. There is no `push:` -# trigger: the check compares a pull request with its base. +# The `selector` job's filter and the job's `git diff` pathspec are held to +# the same set by scripts/__tests__/check-auth-npm-changeset.test.mjs. There is +# no `push:` trigger: the check compares a pull request with its base. The +# `summary` job reports the result (ADR 0003). on: - pull_request: - paths: - - packages/stack-auth/Cargo.toml - - packages/stack-auth/src/** - - languages/typescript/packages/auth/** - - languages/typescript/packages/stack-auth-wasm/** - - scripts/check-auth-npm-changeset.mjs - - .github/workflows/require-auth-npm-changeset.yml + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. + pull_request: {} permissions: contents: read @@ -35,8 +32,80 @@ concurrency: cancel-in-progress: true jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + auth_changeset: ${{ steps.decide.outputs.auth_changeset }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + filters: | + # What `@cipherstash/auth` ships, which the job diffs, and the + # job's own inputs. + auth_changeset: + - "packages/stack-auth/Cargo.toml" + - "packages/stack-auth/src/**" + - "languages/typescript/packages/auth/**" + - "languages/typescript/packages/stack-auth-wasm/**" + - "scripts/check-auth-npm-changeset.mjs" + - ".github/workflows/require-auth-npm-changeset.yml" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + AUTH_CHANGESET: ${{ steps.filter.outputs.auth_changeset }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + AUTH_CHANGESET=true + fi + answers=( + "auth_changeset=$AUTH_CHANGESET" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + require-npm-changeset: name: Require @cipherstash/auth changeset + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.auth_changeset == 'true' runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -115,3 +184,31 @@ jobs: mapfile -t CHANGESET_FILES <<< "$CHANGESETS" node scripts/check-auth-npm-changeset.mjs "${CHANGESET_FILES[@]}" + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Require @cipherstash/auth changeset summary + needs: [selector, require-npm-changeset] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/tests-bench.yml b/.github/workflows/tests-bench.yml index 1e97e64cc..d458d7b46 100644 --- a/.github/workflows/tests-bench.yml +++ b/.github/workflows/tests-bench.yml @@ -6,10 +6,15 @@ name: "Test Benchmark JS" # them can red this job, so an edit to any of them has to be able to trigger it. # Scoped the same way the `integration-*` workflows scope theirs rather than # taking whole packages, to keep the trigger surface honest. +# +# The `selector` job decides whether a pull request runs the job, and +# `summary` reports the result (ADR 0003). on: push: branches: - 'main' + # The same files as the `selector` job's filter. + # scripts/__tests__/workflow-paths-filter-parity.test.mjs compares them. paths: - 'languages/typescript/packages/bench/**' - 'languages/typescript/packages/stack/src/eql/v3/**' @@ -24,28 +29,93 @@ on: - '.github/workflows/tests-bench.yml' - '.github/actions/integration-setup/**' - '.github/actions/build-auth-binding/**' + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. pull_request: branches: - "**" - # Repeated verbatim: GitHub Actions does not support YAML anchors/aliases. - paths: - - 'languages/typescript/packages/bench/**' - - 'languages/typescript/packages/stack/src/eql/v3/**' - # The bench drives the typed v3 CLIENT, which lives in `encryption/`, not - # `eql/v3/` (that is the authoring DSL). Without this the job stayed green - # by never running for the changes most likely to break it. - - 'languages/typescript/packages/stack/src/encryption/**' - - 'languages/typescript/packages/stack-drizzle/**' - - 'languages/typescript/packages/test-kit/**' - - 'languages/typescript/packages/cli/src/installer/**' - - 'local/**' - - '.github/workflows/tests-bench.yml' - - '.github/actions/integration-setup/**' - - '.github/actions/build-auth-binding/**' jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + bench: ${{ steps.decide.outputs.bench }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + filters: | + # The package graph the bench reads; see the comment at the top + # of this file. The bench drives the typed v3 client, which lives + # in `encryption/`, not `eql/v3/`. + bench: + - "languages/typescript/packages/bench/**" + - "languages/typescript/packages/stack/src/eql/v3/**" + - "languages/typescript/packages/stack/src/encryption/**" + - "languages/typescript/packages/stack-drizzle/**" + - "languages/typescript/packages/test-kit/**" + - "languages/typescript/packages/cli/src/installer/**" + - "local/**" + - ".github/workflows/tests-bench.yml" + - ".github/actions/integration-setup/**" + - ".github/actions/build-auth-binding/**" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + BENCH: ${{ steps.filter.outputs.bench }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + BENCH=true + fi + answers=( + "bench=$BENCH" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + tests-bench: name: Run Bench Smoke Tests + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.bench == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 steps: @@ -108,3 +178,31 @@ jobs: - name: Run bench smoke tests working-directory: languages/typescript/packages/bench run: pnpm test:local db-only + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Test Benchmark JS summary + needs: [selector, tests-bench] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/tests-supply-chain.yml b/.github/workflows/tests-supply-chain.yml index 490c48370..d54ba9e18 100644 --- a/.github/workflows/tests-supply-chain.yml +++ b/.github/workflows/tests-supply-chain.yml @@ -17,11 +17,16 @@ name: Test supply chain security # - contents:read only # - no secrets # - no caching +# +# The `selector` job decides whether a pull request runs the gate, and +# `summary` reports the result (ADR 0003). on: push: branches: - main + # The same files as the `selector` job's filter. + # scripts/__tests__/workflow-paths-filter-parity.test.mjs compares them. paths: - '.github/workflows/release.yml' # The reusable workflows release.yml calls for the two native-binary @@ -34,28 +39,94 @@ on: - 'scripts/lint-no-workflow-caching.mjs' - 'scripts/__tests__/lint-no-workflow-caching.test.mjs' - 'scripts/__tests__/fixtures/lint-no-workflow-caching/**' + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. pull_request: branches: - '**' - paths: - - '.github/workflows/release.yml' - # The reusable workflows release.yml calls for the two native-binary - # lines: everything they build is packed and published, and they are on - # the lint's target list for that reason, so an edit to either has to run - # this gate too. - - '.github/workflows/_build-ffi-artifacts.yml' - - '.github/workflows/_build-auth-artifacts.yml' - - '.github/workflows/tests-supply-chain.yml' - - 'scripts/lint-no-workflow-caching.mjs' - - 'scripts/__tests__/lint-no-workflow-caching.test.mjs' - - 'scripts/__tests__/fixtures/lint-no-workflow-caching/**' permissions: contents: read jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + supply_chain: ${{ steps.decide.outputs.supply_chain }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + filters: | + # The workflows the lint targets, and the lint itself. The + # reusable workflows release.yml calls for the two native-binary + # lines are on the target list: everything they build is packed + # and published. + supply_chain: + - ".github/workflows/release.yml" + - ".github/workflows/_build-ffi-artifacts.yml" + - ".github/workflows/_build-auth-artifacts.yml" + - ".github/workflows/tests-supply-chain.yml" + - "scripts/lint-no-workflow-caching.mjs" + - "scripts/__tests__/lint-no-workflow-caching.test.mjs" + - "scripts/__tests__/fixtures/lint-no-workflow-caching/**" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + SUPPLY_CHAIN: ${{ steps.filter.outputs.supply_chain }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + SUPPLY_CHAIN=true + fi + answers=( + "supply_chain=$SUPPLY_CHAIN" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + verify-no-caching-in-release-workflows: name: Verify no caching in release workflows + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.supply_chain == 'true' runs-on: ubuntu-latest steps: @@ -90,3 +161,31 @@ jobs: - name: Verify no caching in the release and supply-chain workflows run: pnpm run lint:workflow-cache + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Test supply chain security summary + needs: [selector, verify-no-caching-in-release-workflows] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ba7affc68..19ea87bc7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,19 +1,89 @@ name: Test JS +# The `selector` job decides whether a pull request runs these jobs, and +# `summary` reports the result (ADR 0003). + on: push: branches: - 'main' + # No path filter: a workflow that a path filter skips leaves its required + # checks pending, and GitHub's documentation on required status checks says + # that blocks a merge. The trigger paths are in the `selector` job instead. pull_request: branches: - "**" - # Plan documents under docs/plans/ need none of these jobs. Later patterns - # win, so the negation has to come after the catch-all. - paths: - - '**' - - '!docs/plans/**' jobs: + # The selector job: it decides which jobs a pull request runs. It runs on + # every event and is never skipped, because the gated jobs need it. When it + # cannot decide it fails, and that fails the summary job: a selector that + # answered nothing would skip every gated job, and the summary job counts a + # skipped job as passed (ADR 0003). + # + # On a GitHub-hosted runner because the gated jobs wait for it. No checkout: + # on a pull request, dorny/paths-filter lists the changed files through the + # GitHub API. + selector: + name: Selector + runs-on: ubuntu-latest + timeout-minutes: 10 + # Each output is named after the filter that decides it. + # `scripts/__tests__/workflow-selectors.test.mjs` follows a job's `if:` to + # its filter by that name. + outputs: + js: ${{ steps.decide.outputs.js }} + steps: + # Only on a pull request, which has a base to compare with. Other events + # select every job in the step below. + # + # `every` makes a file count only when it matches every rule. That is + # what makes the `!` rule exclude plans: under dorny's default, `some`, + # a `!` rule matches every other file in the repository. + - id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@0e4a8c6effa4802afeda77dc8d303f8176d7dfad # v3 + with: + predicate-quantifier: every + filters: | + # Every job reads most of the repository: Biome checks every file, + # and run-tests runs the script guards, the typechecks and every + # package's tests. Plan documents under docs/plans/ need none of them, + # and no test reads one (plan-docs-unread-by-tests.test.mjs). + js: + - "**" + - "!docs/plans/**" + + # Explicit default, not `|| 'true'`, which trips GitHub's inconsistent + # treatment of the string 'false'. On a pull request, an answer other than + # `true` or `false` means the filter did not decide, so the step fails + # rather than write it. + # + # The event and the answers arrive through `env:`, not through a `${{ }}` + # in the script. That lets `scripts/__tests__/workflow-selectors.test.mjs` + # run this script and read back what it wrote. + - id: decide + env: + EVENT_NAME: ${{ github.event_name }} + JS: ${{ steps.filter.outputs.js }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + JS=true + fi + answers=( + "js=$JS" + ) + for answer in "${answers[@]}"; do + case "${answer#*=}" in + true|false) echo "$answer" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::The path filter answered '${answer#*=}' for ${answer%%=*}. The selector cannot decide which jobs to run." + exit 1 + ;; + esac + done + # Biome format + lint. Its own job (no DB, no credentials, node-agnostic) so it # surfaces as a distinct check and runs once rather than per Node matrix leg. # Gates on ERRORS only — warnings are allowed, so the `no-type-erasing-assertions` @@ -21,6 +91,12 @@ jobs: # Tightening to fail on warnings is tracked in #625. lint: name: Lint (Biome) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.js == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - name: Checkout Repo @@ -53,6 +129,12 @@ jobs: run-tests: name: Run Tests (Node ${{ matrix.node-version }}) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.js == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 strategy: matrix: @@ -397,6 +479,12 @@ jobs: e2e-tests: name: Run E2E Tests + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.js == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 # Auth-dependent suites in `e2e/` skip themselves unless these env vars @@ -478,6 +566,12 @@ jobs: # a silent fallback to the NAPI module is impossible. wasm-e2e-tests: name: Run WASM E2E Tests (Deno) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.js == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 permissions: @@ -621,6 +715,12 @@ jobs: run-tests-bun: name: Run Tests (Bun) + needs: [selector] + # Written as an exclusion, not a list of events: every event other + # than a pull request runs this job. + if: >- + github.event_name != 'pull_request' + || needs.selector.outputs.js == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 continue-on-error: true @@ -713,3 +813,31 @@ jobs: (cd "$dir" && bunx --bun vitest run) || true fi done + + # The summary job: the one job of this workflow that branch rules require. + # It has the same name on every event. It passes when every needed job + # passed or was skipped. Skipped counts as passed because the selector skips + # a job whose trigger paths a change does not touch. A failed job still + # fails this one, because it is in `needs` and reports its failure. + summary: + name: Test JS summary + needs: [selector, lint, run-tests, e2e-tests, wasm-e2e-tests, run-tests-bun] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check that every needed job passed or was skipped + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + set -euo pipefail + echo "Needed results: $RESULTS" + for result in $RESULTS; do + case "$result" in + success|skipped) ;; + *) + echo "::error::A needed job reported '$result'." + exit 1 + ;; + esac + done diff --git a/scripts/__tests__/check-auth-npm-changeset.test.mjs b/scripts/__tests__/check-auth-npm-changeset.test.mjs index 4554edd51..da1cbbb65 100644 --- a/scripts/__tests__/check-auth-npm-changeset.test.mjs +++ b/scripts/__tests__/check-auth-npm-changeset.test.mjs @@ -5,6 +5,7 @@ import { join } from 'node:path' import { afterAll, describe, expect, it } from 'vitest' import { isShipped } from '../check-auth-npm-changeset.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' +import { jobTriggerLists } from './lib/selector.mjs' import { readWorkflow } from './lib/workflows.mjs' /** @@ -227,8 +228,9 @@ describe('check-auth-npm-changeset.mjs --shipped', () => { }) describe('require-auth-npm-changeset.yml', () => { + const JOB = 'require-npm-changeset' const workflow = readWorkflow(WORKFLOW) - const [job] = Object.values(workflow?.jobs ?? {}) + const job = workflow?.jobs?.[JOB] const check = job?.steps?.find((step) => step.run?.includes(`node ${SCRIPT}`)) /** The pathspec of the job's first `git diff`: what counts as shipped. */ @@ -241,7 +243,12 @@ describe('require-auth-npm-changeset.yml', () => { .map((line) => line.trim().replace(/\s*\\$/, '')) .filter(Boolean) - const filters = workflow?.on?.pull_request?.paths ?? [] + // The selector job's filter: the paths that start the job on a pull request. + const filters = jobTriggerLists(WORKFLOW, JOB).flatMap(({ paths }) => paths) + + it('reads a selector filter', () => { + expect(filters).not.toEqual([]) + }) it('diffs the crate and both binding folders', () => { expect(shipped).toEqual([ diff --git a/scripts/__tests__/js-selectors.test.mjs b/scripts/__tests__/js-selectors.test.mjs new file mode 100644 index 000000000..5815f08e6 --- /dev/null +++ b/scripts/__tests__/js-selectors.test.mjs @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest' +import { readSelectorJob, selectedJobs } from './lib/selector.mjs' + +/** + * What the selector jobs of the JavaScript and TypeScript workflows decide for + * a pull request. Each one keeps the paths of the workflow-level filter it + * replaced, so these pin that nothing was lost in the move. + */ + +const WORKFLOWS = { + tests: '.github/workflows/tests.yml', + fta: '.github/workflows/fta-v3.yml', + bench: '.github/workflows/tests-bench.yml', + supplyChain: '.github/workflows/tests-supply-chain.yml', + authChangeset: '.github/workflows/require-auth-npm-changeset.yml', +} + +/** The jobs a pull request runs, without the two that always run. */ +const checks = (relPath, files) => + selectedJobs(relPath, files).filter( + (id) => id !== 'selector' && id !== 'summary', + ) + +describe('the JavaScript and TypeScript workflows', () => { + it.each(Object.values(WORKFLOWS))('%s has a selector job', (relPath) => { + expect(readSelectorJob(relPath)).not.toBeNull() + }) +}) + +describe('the Test JS selector', () => { + const ALL = [ + 'e2e-tests', + 'lint', + 'run-tests', + 'run-tests-bun', + 'wasm-e2e-tests', + ] + + it('runs every job for a change anywhere but the plans', () => { + expect(checks(WORKFLOWS.tests, ['scripts/example.mjs'])).toEqual(ALL) + expect(checks(WORKFLOWS.tests, ['docs/example.md'])).toEqual(ALL) + expect(checks(WORKFLOWS.tests, ['packages/eql/src/v3/a.sql'])).toEqual(ALL) + }) + + it('runs no job for a plan-only change', () => { + expect(checks(WORKFLOWS.tests, ['docs/plans/nested/example.md'])).toEqual( + [], + ) + }) +}) + +describe('the FTA selector', () => { + it('runs the gate for the analysed sources only', () => { + const run = (file) => checks(WORKFLOWS.fta, [file]) + expect( + run('languages/typescript/packages/stack/src/schema/match-defaults.ts'), + ).toEqual(['fta']) + expect( + run('languages/typescript/packages/stack-supabase/src/a.ts'), + ).toEqual(['fta']) + expect(run('languages/typescript/packages/stack/src/index.ts')).toEqual([]) + }) +}) + +describe('the bench selector', () => { + it('runs for the package graph the bench reads', () => { + const run = (file) => checks(WORKFLOWS.bench, [file]) + expect( + run('languages/typescript/packages/stack/src/encryption/index.ts'), + ).toEqual(['tests-bench']) + expect(run('local/docker-compose.postgres.yml')).toEqual(['tests-bench']) + expect(run('languages/typescript/packages/stack-prisma/src/a.ts')).toEqual( + [], + ) + }) +}) + +describe('the supply chain selector', () => { + it('runs for the release workflows and the lint, not for other workflows', () => { + const run = (file) => checks(WORKFLOWS.supplyChain, [file]) + expect(run('.github/workflows/_build-auth-artifacts.yml')).toEqual([ + 'verify-no-caching-in-release-workflows', + ]) + expect(run('scripts/lint-no-workflow-caching.mjs')).toEqual([ + 'verify-no-caching-in-release-workflows', + ]) + expect(run('.github/workflows/tests.yml')).toEqual([]) + }) +}) + +describe('the auth changeset selector', () => { + it('runs for what @cipherstash/auth ships', () => { + const run = (file) => checks(WORKFLOWS.authChangeset, [file]) + expect(run('packages/stack-auth/src/lib.rs')).toEqual([ + 'require-npm-changeset', + ]) + expect(run('languages/typescript/packages/auth/index.js')).toEqual([ + 'require-npm-changeset', + ]) + expect(run('packages/stack-auth/tasks.toml')).toEqual([]) + }) +}) diff --git a/scripts/__tests__/paths-filter.test.mjs b/scripts/__tests__/paths-filter.test.mjs index 5b2da3cd4..ab1451a65 100644 --- a/scripts/__tests__/paths-filter.test.mjs +++ b/scripts/__tests__/paths-filter.test.mjs @@ -4,7 +4,7 @@ import { negationRemovesAll, negationSubtracts, } from './lib/paths-filter.mjs' -import { readWorkflow } from './lib/workflows.mjs' +import { jobTriggerLists } from './lib/selector.mjs' const exact = (entry, input) => entry === input const IN = 'languages/typescript/packages/protect-ffi/crates/**' @@ -92,10 +92,12 @@ describe('negationRemovesAll subtracts only a whole-tree negation', () => { }) }) -describe('tests.yml pull_request filter', () => { - const workflow = readWorkflow('.github/workflows/tests.yml') - // `on:` parses as the boolean `true` under YAML 1.1. - const { paths } = (workflow.on ?? workflow[true]).pull_request +describe('tests.yml selector filter', () => { + // As a `paths:` list. Every job in the workflow reads the same filter. + const [{ paths }] = jobTriggerLists( + '.github/workflows/tests.yml', + 'run-tests', + ) it('skips plan-only pull requests and runs on everything else', () => { expect(filterCovers(paths, 'docs/plans/nested/example.md', exact)).toBe( diff --git a/scripts/__tests__/plan-docs-unread-by-tests.test.mjs b/scripts/__tests__/plan-docs-unread-by-tests.test.mjs index 529099eba..6616426a3 100644 --- a/scripts/__tests__/plan-docs-unread-by-tests.test.mjs +++ b/scripts/__tests__/plan-docs-unread-by-tests.test.mjs @@ -40,7 +40,7 @@ describe('tests.yml skips pull requests that change only docs/plans/', () => { it('no test names a file under docs/plans/', () => { expect( filesNamingAPlan(TESTS), - 'A test that reads a plan is skipped on a plan-only pull request, because `tests.yml` ignores `docs/plans/**`. Move the fact the test needs out of the plan, or drop the negation from the `pull_request` filter.', + 'A test that reads a plan is skipped on a plan-only pull request, because `tests.yml` ignores `docs/plans/**`. Move the fact the test needs out of the plan, or drop the negation from the selector filter.', ).toEqual([]) }) it('the scan finds a plan name when there is one', () => { diff --git a/scripts/__tests__/workflow-paths-filter-parity.test.mjs b/scripts/__tests__/workflow-paths-filter-parity.test.mjs index 93d4facab..4e8799765 100644 --- a/scripts/__tests__/workflow-paths-filter-parity.test.mjs +++ b/scripts/__tests__/workflow-paths-filter-parity.test.mjs @@ -110,15 +110,12 @@ const ASYMMETRIC = CLASSIFIED.filter( * also fire) or the discovery broke. */ const EXPECTED_PARITY_WORKFLOWS = [ - '.github/workflows/fta-v3.yml', '.github/workflows/integration-drizzle.yml', '.github/workflows/integration-prisma-next.yml', '.github/workflows/integration-protect-ffi.yml', '.github/workflows/integration-supabase.yml', '.github/workflows/prisma-example-readme-e2e.yml', '.github/workflows/prisma-next-e2e.yml', - '.github/workflows/tests-bench.yml', - '.github/workflows/tests-supply-chain.yml', ] /** @@ -152,13 +149,6 @@ const EXPECTED_ASYMMETRIES = new Map([ // unchecked — is unreachable when `push` is unfiltered. 'push is deliberately unfiltered (branches: [main]), so main runs a superset of what PRs run', ], - [ - '.github/workflows/tests.yml', - // `push: branches: [main]` has no `paths:`, so main runs everything and only - // pull requests skip `docs/plans/**`. A `push` filter narrower than the - // `pull_request` one is unreachable when `push` is unfiltered. - 'push is deliberately unfiltered (branches: [main]), so main runs a superset of what PRs run', - ], [ '.github/workflows/lint-release.yml', // There is no `push:` trigger at all — this gate exists to catch a broken @@ -169,12 +159,6 @@ const EXPECTED_ASYMMETRIES = new Map([ // release machinery never runs the check) is what the single list IS. 'pull_request is the only trigger; a post-merge copy would report a release-blocking finding too late to act on', ], - [ - '.github/workflows/require-auth-npm-changeset.yml', - // Its list is held to the job's own `git diff` pathspec by - // check-auth-npm-changeset.test.mjs, which is the second copy here. - 'pull_request is the only trigger; the check diffs a pull request against its base, which a push does not have', - ], [ '.github/workflows/bench-eql.yml', // The mirror image of lint-release.yml: `push: branches: [main]` with a diff --git a/scripts/lint-no-workflow-caching.mjs b/scripts/lint-no-workflow-caching.mjs index 2fbb7d88b..7f36ff5b8 100644 --- a/scripts/lint-no-workflow-caching.mjs +++ b/scripts/lint-no-workflow-caching.mjs @@ -169,6 +169,12 @@ const AUDITED_ACTIONS = new Map([ // per-run store with no cross-run key. ['actions/upload-artifact', { cacheInput: null }], ['actions/download-artifact', { cacheInput: null }], + // The selector job's path filter (tests-supply-chain.yml). Read at the + // pinned v3.0.4: on a pull request it lists the changed files through the + // GitHub REST API and matches them with picomatch. Its dependencies are + // @actions/core, @actions/exec, @actions/github and picomatch. No cache, no + // cache input. + ['dorny/paths-filter', { cacheInput: null }], // Supplies zig + cargo-zigbuild (the glibc-pinned gnu builds) and wasm-pack, // all pinned in languages/typescript/packages/protect-ffi/mise.toml. `cache` DEFAULTS TO TRUE, so // an omitted key is a cache restore — and the generic `with.cache` rule below