diff --git a/.github/workflows/playwright-reporter.yml b/.github/workflows/playwright-reporter.yml index cd3dd4de..467bb1a9 100644 --- a/.github/workflows/playwright-reporter.yml +++ b/.github/workflows/playwright-reporter.yml @@ -13,16 +13,18 @@ on: push: paths: - "packages/playwright/**" - - "packages/runtime/src/agent/**" - - "packages/runtime/src/cdn/**" + - "packages/runtime/src/**" + - "packages/runtime/package.json" + - "packages/runtime/tsup.config.ts" - "packages/mcp/test/e2e/app/**" - "package-lock.json" - ".github/workflows/playwright-reporter.yml" pull_request: paths: - "packages/playwright/**" - - "packages/runtime/src/agent/**" - - "packages/runtime/src/cdn/**" + - "packages/runtime/src/**" + - "packages/runtime/package.json" + - "packages/runtime/tsup.config.ts" - "packages/mcp/test/e2e/app/**" - "package-lock.json" - ".github/workflows/playwright-reporter.yml" @@ -110,6 +112,11 @@ jobs: - name: Chromium run: npx playwright install --with-deps chromium + # the unit test that resolves the installed runtime's dist/ by default needs the runtime built, as here + - name: Unit tests with the runtime built + working-directory: packages/playwright + run: npx vitest run test/install.test.ts + - name: The fixture end to end (buggy page flagged, fixed page and latency-only run clean, report rendered) # shell: bash runs with pipefail, so a failing run.mjs fails the step even though tee succeeds shell: bash diff --git a/AGENTS.md b/AGENTS.md index 9dfbda15..a09b7e04 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,7 @@ Default mode is `observe` (report only); `guard` is opt-in; `heal` is experiment | `demos/` | six demo apps with a chaos backend and Playwright trials; `bench/heal/` runs them per mode | | `packages/mcp/` | `@genclass/mcp`: the MCP server for coding agents (bridge on 127.0.0.1:47357, store, explore verdicts, replays, regression-test generation, 8 tools); no runtime deps; `test/e2e/race.e2e.mjs` is the real-browser check (not in CI's main job). `plugin/` is the Claude Code plugin (also an Agent Plugins package); `server.json`, the plugin, `.claude-plugin/`, `.cursor-plugin/`, `gemini-extension.json`, `llms-install.md` and the READMEs pin its version (`npm run sync -w @genclass/mcp` after a version bump; `test/manifests.test.ts` fails otherwise) | | `packages/sv/` | `@genclass/sv`: the Svelte CLI community add-on (`npx sv add @genclass`): adds the runtime, the `/auto` import first in `src/hooks.client`, the dev-only overlay and (default) the telemetry opt-out meta in `src/app.html`, all with the `genclass:init` marker so `remove` undoes it; bundles `@sveltejs/sv-utils`, `sv` is a peer; `test/e2e/` runs sv's add-on harness (pnpm, Chromium) | -| `packages/playwright/` | `@genclass/playwright`: the fixture (`src/fixture.ts`, `install.ts`, `page.ts`, `chaos.ts`, `races.ts`: puts the runtime's script-tag build on every `page` in observe mode with telemetry off, seeded chaos on fetch/XHR responses with `page.route`, witnesses through the agent entry with every hold off, one `genclass-races/1` file per test in `test-results/genclass/`) and the reporter + pure summariser that read those files and write `genclass-report.md` for the PR; peer deps only (`@playwright/test`, `@genclass/runtime`); `test/e2e/run.mjs` is the real-browser check (its own workflow) | +| `packages/playwright/` | `@genclass/playwright`: the fixture (`src/fixture.ts`, `install.ts`, `page.ts`, `chaos.ts`, `races.ts`: puts the runtime's script-tag build on every `page` in observe mode with telemetry off, seeded chaos that holds matched fetch/XHR requests with `page.route` and `route.fallback()`, witnesses through the agent entry with every hold off, one `genclass-races/1` file per test in `test-results/genclass/`) and the reporter + pure summariser that read those files and write `genclass-report.md` for the PR; peer deps only (`@playwright/test`, `@genclass/runtime`); `test/e2e/run.mjs` is the real-browser check (its own workflow) | | `packages/nuxt/`, `packages/astro/` | `@genclass/nuxt` (a Nuxt module, `@nuxt/kit`, built with `@nuxt/module-builder`) and `@genclass/astro` (an Astro integration, `injectScript("page", ...)`): from options `{ mode, token?, telemetry?, devtools? }` they generate the runtime's own one line (`/auto` or `/auto/guard`) plus a `window.GENCLASS_CONFIG` module imported before it; browser only, nothing on the server; `@genclass/runtime` is a peer dependency. The generated code is pure (`src/source.ts`, `src/index.ts`) and the tests check it exactly, then build one fixture app. Workflow: `integrations.yml` (Node 24) | | `docs/adoption/listings/` | how the framework packages and templates get listed (nuxt.com/modules entry and steps, astro.build/integrations, sveltesociety.dev, awesome-vue and Angular lists): research and draft entries; nothing is submitted from the repo | | `bench/visual/` | the visual checks' bench (GitHub Actions only, `.github/workflows/visual-measure.yml`): false reports on the demos, templates, compat apps and about 40 public sites, recall on faults planted from the harness, cost; public sites are counted per check kind only, never named in the repo, its run logs or plain artifacts (per-site results only encrypted with the `VISUAL_BENCH_KEY` secret) | diff --git a/package-lock.json b/package-lock.json index cd992455..ec255d7c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16884,7 +16884,7 @@ "vitest": "^5.0.3" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || >=22.12.0" }, "peerDependencies": { "@genclass/runtime": ">=0.3.0-beta.1", diff --git a/packages/playwright/CHANGELOG.md b/packages/playwright/CHANGELOG.md index daa16710..813102ce 100644 --- a/packages/playwright/CHANGELOG.md +++ b/packages/playwright/CHANGELOG.md @@ -9,26 +9,36 @@ with `page.route`, never from a CDN. The model is skipped by default (`model: "skip"`: no 13 MB download) and the file carries the runtime's witnesses, computed without the model through the agent entry connected with every hold off (`overwrote-newer`, `identical-in-flight`); `model: "load"` adds the model's detections with their - confidence. At the end of each test it writes one `genclass-races/1` file to `test-results/genclass/` (or - `outDir`), named after the test, its id and its seed, with the seed and the chaos settings. `enabled: false` - makes it inert. Peer dependencies only (`@playwright/test`, `@genclass/runtime`, both optional: the reporter - needs neither). -- **Seeded chaos** (`chaos: { seed, delayMs, reorder, match }`): with `page.route` the real response is fetched at - once and its delivery to the page held for a latency derived from the seed and the request's key (method, path, - occurrence), so a run repeats exactly; `reorder: false` keeps the request order (latency only). Only fetch and - XHR requests inside `match` (default: same-origin `/api*` or JSON responses); never the runtime's own files, - documents, scripts or anything outside the filter. Off unless given. With `repeatEach`, the repeat index is - added to the seed, so two repeats give the report its two seeds. + confidence (the model's diagnosis probability, from the runtime's `decide` event). At the end of each test it + writes one `genclass-races/1` file to `genclass` in the project's outputDir (test-results/genclass, emptied by + Playwright at each run; or `outDir`), named after the test, its id, its seed and, for a retry, the attempt, with + the seed, the chaos settings and a `fixture` record (the model option, whether the agent entry connected, how many + requests chaos held, whether a service worker controlled the page, a mode switch it overrode). A test skipped in + its body writes nothing. The runtime is held to observe: a URL or localStorage switch that raised the mode is set + back. The agent link gets a stand-in fetch for its own address, so the fixture adds no console error, failed + request or CSP report to the page. `enabled: false` makes it inert. Peer dependencies only (`@playwright/test`, + `@genclass/runtime`, both optional: the reporter needs neither); `typesVersions` for `moduleResolution: node`; + `engines` `^20.19.0 || >=22.12.0` (CommonJS test projects load the ESM-only fixture with require). +- **Seeded chaos** (`chaos: { seed, delayMs, reorder, match }`): with `page.route` each matched request is held for a + latency derived from the seed and the request's key (method, path, occurrence, assigned when the page sends it), + then passed on with `route.fallback()`, so it still reaches the test's own routes (`context.route`, + `routeFromHAR`) and the browser's network stack, with redirects, cookies, CORS and streaming as they are, and a run + repeats exactly. `reorder: false` sends matched requests in their order, at least 30 ms apart (latency only). Only + fetch and XHR requests inside `match` (default: same-origin `/api*` or an Accept header asking for JSON, decided + from the request); never the runtime's own files, documents, scripts or anything outside the filter. Off unless + given. With `repeatEach`, the repeat index is added to the seed, so two repeats give the report its two seeds. - **Witnesses in the report**: signatures with trigger `witness` get their own group (shown only when there are any), no confidence on the line (a fact that happened, not a probability) and a footer sentence saying what they - are; they are never counted as duplicate submits or stale responses. A model state `off` (no model by choice) - no longer gets the "model was not ready" note. The model-only report is byte for byte what it was. -- **Measured** (`test/e2e`, run by the workflow in Chromium): on the MCP e2e search page with latency 0 to 600 ms - and reordering, the buggy page gets `witness:overwrote-newer` on `GET /api/search` in 5 of 5 seeds, the same - page with latency only in 0 of 5, and the fixed page in 0 of 5; no other signature anywhere. With `model: "load"` - (one run by hand, two seeds) the model loaded through the fixture's path and gave no likely detection on that - page (`stale` at 0.64 in one seed, below the 0.90 report gate; nothing in the other), while the witness was - recorded in both. + are; they are never counted as duplicate submits or stale responses. Files with a `fixture` record get notes + for runs the fixture could not check (no witnesses recorded, a service worker in control) and for chaos that held + no request; a `model: "skip"` file gets no model note. Files without a `fixture` record are reported as before. +- **Measured** (`test/e2e`, run by the workflow in Chromium; six runs of five seeds): on the MCP e2e search page + (its store on the fixture's runtime through the script-tag mapping) with latency 0 to 600 ms and reordering, the + buggy page gets `witness:overwrote-newer` on `GET /api/search` in 30 of 30 seeds, the same page with latency only + in 0 of 30, and the fixed page in 0 of 30; no other signature, console error or failed request anywhere. With + `model: "load"` (one run by hand, two seeds, before chaos moved to requests) the model loaded through the + fixture's path and gave no likely detection on that page (`stale` at 0.64 in one seed, below the 0.90 report + gate; nothing in the other), while the witness was recorded in both. - First version of the reader side: `@genclass/playwright/reporter` (a Playwright reporter by shape: reads `test-results/genclass/*.json` in the `genclass-races/1` format at the end of the run, writes and prints `genclass-report.md`), `@genclass/playwright/summarize` (the pure files-to-markdown function, with diff --git a/packages/playwright/README.md b/packages/playwright/README.md index 01d5b4ed..7875cc8b 100644 --- a/packages/playwright/README.md +++ b/packages/playwright/README.md @@ -5,9 +5,9 @@ GenClass for Playwright, in two parts that share one file format: - **The fixture** (`@genclass/playwright/fixture`) puts [`@genclass/runtime`](../runtime) on every page your tests - open, before the app's first script, in observe mode with telemetry off. With `chaos` it slows and reorders the - app's API responses from a seed, so the orders a real network produces happen in your test, the same way every - run. At the end of each test it writes one JSON file (format `genclass-races/1`) with what the runtime saw, the + open, before the app's first script, in observe mode with telemetry off. With `chaos` it holds the app's API + requests for a seeded time before they leave, so the response orders a slow network produces happen in your test, + the same way every run. At the end of each test it writes one JSON file (format `genclass-races/1`) with what the runtime saw, the seed and the chaos that produced it. - **The reporter** (`@genclass/playwright/reporter`) reads those files when the suite ends, marks what reproduced across seeds, writes one markdown file, `genclass-report.md`, and prints it. A short GitHub Actions step posts that @@ -19,20 +19,23 @@ What the fixture records without a model download: two happens-before facts the (`model: "load"`, a 13 MB download), the file also carries the model's diagnoses with their confidence. Both are things to look at, not verdicts: the report says what happened and leaves the call to you. -Observe only. The runtime never holds, delays or changes anything the app does; the fixture's chaos changes when a -response reaches the page, never what it says; no finding changes a test's outcome. No runtime dependencies: -`@playwright/test` and `@genclass/runtime` are peer dependencies (the reporter alone needs neither). +The runtime only observes: it never holds, delays or changes anything the app does. Chaos, when you turn it on, holds +matched requests before they leave (that is its job) and never changes what they say. No finding changes a test's +outcome, and the fixture logs nothing to the page's console. No runtime dependencies: `@playwright/test` and +`@genclass/runtime` are peer dependencies (the reporter alone needs neither). ## Install Not on npm yet (version 0.1.0 is unreleased). Once published: ```bash -npm install -D @genclass/playwright @genclass/runtime +npm install -D @genclass/playwright @genclass/runtime@beta ``` -Playwright 1.40 or later; `@genclass/runtime` 0.3.0-beta.1 or later (the fixture reads its `dist/` from the -installed package). +Playwright 1.40 or later; `@genclass/runtime` 0.3.0-beta.1 or later, which is on the `beta` dist-tag (`latest` is +0.2.x, which has no agent entry, so the fixture stops with an error naming it). The package is ESM only. A test +project that is CommonJS (no `"type": "module"`) needs Node 20.19+ or 22.12+, where `require` loads ES modules, and, +to type-check its specs, `module` and `moduleResolution` set to `nodenext` (or `bundler`). ## Use @@ -41,13 +44,16 @@ Two lines in the config, one import in a spec: ```ts // playwright.config.ts import { defineConfig } from "@playwright/test"; +import type { GenClassFixtures } from "@genclass/playwright/fixture"; -export default defineConfig({ +export default defineConfig({ use: { genclass: { chaos: { seed: 1, delayMs: [0, 600] } } }, reporter: [["list"], ["@genclass/playwright/reporter"]], }); ``` +The type argument tells TypeScript about the `genclass` option (without it, `defineConfig` rejects `use.genclass`). + ```ts // tests/search.spec.ts import { test, expect } from "@genclass/playwright/fixture"; @@ -60,8 +66,8 @@ test("shows the results of the last query", async ({ page }) => { ``` `test` is Playwright's `test.extend` with GenClass on its `page`; `expect` is Playwright's. Everything else in the -spec stays as it was. The run ends with `test-results/genclass/--s.json` per test and -`genclass-report.md` next to the config. +spec stays as it was. The run ends with `test-results/genclass/--s.json` per test (a retry adds +`-r`, so a failing first attempt is kept) and `genclass-report.md` next to the config. Two seeds, so the report can mark what reproduced: `repeatEach: 2` in the config (or `npx playwright test --repeat-each 2`). The fixture adds the repeat index to the seed, so the repeats run under seeds 1 and 2, and a @@ -78,7 +84,7 @@ Set in `use` in the config, or `test.use({ genclass: {...} })` in a spec or a `d | `model` | `"skip"` | `"skip"`: no model download; the file carries the runtime's witnesses, facts computed without the model, and no model confidences. `"load"`: the runtime loads its model (from `modelUrl`, else the package default on jsDelivr: 13 MB, cached by the browser) and the file also carries its detections with their confidence. | | `modelUrl` | package default | With `model: "load"`: a model directory URL (the runtime's `model.baseUrl`). | | `chaos` | off | Seeded network chaos (below): an object, or `true` for the defaults. Off unless given. | -| `outDir` | `"test-results/genclass"` | Where the files go, relative to the config's directory. The reporter reads the same place by default. | +| `outDir` | `genclass` in the project's `outputDir` | Where the files go, relative to the config's directory. The default (test-results/genclass) is emptied by Playwright at the start of each run, so files of an earlier run are never read as more seeds. The reporter reads it by default. | | `runtimeDir` | the installed package's `dist/` | `@genclass/runtime`'s `dist/` directory, when it is not resolvable from your project. | ### Chaos @@ -91,17 +97,24 @@ use: { genclass: { chaos: { seed: 1, delayMs: [0, 600], reorder: true, match: [" |---|---|---| | `seed` | `1` | Every delay is derived from it. With `repeatEach`, the fixture adds the repeat index. | | `delayMs` | `[0, 600]` | Extra latency per matched response, uniform in `[min, max]` ms, at most 10000. | -| `reorder` | `true` | `false`: matched responses keep their request order (latency only, nothing overtakes). | -| `match` | same-origin `/api*` or JSON | Which requests: a path prefix (`"/api"`), a URL prefix (`"https://api.example.com/"`), a RegExp tested against the full URL, a function of `{ url, method, resourceType, sameOrigin }`, or a list of those. Without it: same-origin requests whose path starts with `/api` or whose response is JSON. | - -How it works: with `page.route`, the fixture fetches the real response at once and holds its delivery to the page -for the planned time, the way a slower network would. Each matched request is keyed by its method, path and -occurrence in the test (`GET /api/search#3`); its delay comes from a small PRNG seeded by `seed` and that key, so -the same test under the same seed gets the same delays whatever the wall clock does, and a finding comes with the -seed that produced it. Nothing is dropped, changed or invented. Only fetch and XHR requests are ever touched, never -documents, scripts, styles, images, WebSockets or the runtime's own files; a request outside `match` is passed -through untouched. One consequence of fetching the response first: a matched response is buffered, so a streaming -response (server-sent events over fetch, NDJSON) on a matched URL does not work; exclude it with `match`. +| `reorder` | `true` | `false`: matched requests leave in the order the page sent them, at least 30 ms apart (latency only: with a server that answers in order, nothing overtakes). | +| `match` | same-origin `/api*` or `Accept: json` | Which requests: a path prefix (`"/api"`), a URL prefix (`"https://api.example.com/"`), a RegExp tested against the full URL, a function of `{ url, method, resourceType, sameOrigin, accept }`, or a list of those. Without it: same-origin requests whose path starts with `/api` or whose Accept header asks for JSON. | + +How it works: with `page.route`, the fixture holds each matched request for its planned time and then passes it on +(`route.fallback()`), the way a slow uplink would. The request then takes its normal path: your own routes +(`page.route`, `context.route`, `routeFromHAR`) and the browser's network stack, with its cookies, redirects, CORS and +streaming as they are. Each matched request is keyed when the page sends it, by its method, path and occurrence in +the test (`GET /api/search#3`; a cross-origin one by host name without the port); its delay comes from a small PRNG +seeded by `seed` and that key, so the same test under the same seed gets the same delays whatever the wall clock or +the server does, and a finding comes with the seed that produced it. Nothing is dropped, changed, buffered or +invented. Only fetch and XHR requests are ever held, never documents, scripts, styles, images, WebSockets or the +runtime's own files; a request outside `match` is passed on at once. + +Two things to know. A route of yours registered after the fixture's (Playwright tries the last one first) that +fulfils a request answers before chaos sees it, so that request gets no delay; call `route.fallback()` in it to let +chaos run. And `page.route` does not see requests a service worker handles; with a service worker controlling the +page, set `serviceWorkers: "block"` in `use`. The race file records how many requests chaos held and whether a +service worker was in control, and the report notes a test where chaos held nothing. ### The model trade-off @@ -117,10 +130,13 @@ store field), with no model involved: A witness is a fact, not a diagnosis: some overwrites are intended, and many are repaired a moment later. The report keeps them in their own group, shows no confidence for them and says so in its footer. They need state the runtime can see: stores registered with the runtime, or found by automatic state discovery (React component state, Redux, -Zustand stores with the devtools middleware), which the fixture turns on. +and Zustand stores whose devtools middleware is enabled, `devtools(..., { enabled: true })`; at its default the +middleware is off outside development builds and the store is not found), which the fixture turns on. -With `model: "load"` the runtime downloads its model (13 MB, from jsDelivr unless `modelUrl` names your own copy; -the browser caches it) and the file also carries the model's **detections** (`delivery:stale`, +With `model: "load"` the runtime downloads its model (about 13 MB with onnxruntime-web's wasm). Each Playwright test +gets a fresh browser context, so in practice every test downloads it again. The model comes from jsDelivr unless +`modelUrl` names your own copy; onnxruntime-web's wasm comes from jsDelivr either way. The file then also carries the +model's **detections** (`delivery:stale`, `request:duplicate`, `failure:failing`, ...) with the model's confidence and the evidence line it printed, grouped by diagnosis in the report. What the model flags, and how often it is wrong on correct apps, was measured on the framework compatibility matrix (the "What to expect" section below). @@ -129,14 +145,22 @@ framework compatibility matrix (the "What to expect" section below). For the runtime to see the app's first requests and state, it has to be there before the app's first script, so the fixture adds one init script per page: the runtime's script-tag build (`dist/genclass.global.js`) with the -page configuration `mode: "observe"`, `telemetry: false`, no visual checks, no devtools overlay, the model as -chosen, and one sink that hands each likely detection to the test runner. The runtime's files are served from disk -on the page's own origin at `/__genclass__/` with `page.route`, never from a CDN. The page then loads the runtime's -agent entry (`dist/agent.js`) and connects it with every hold off (`hold: false`), pointed at a loopback URL the -fixture aborts, so the witnesses are computed and nothing is ever sent to an MCP bridge. Like the runtime's own -observers, the agent entry wraps the page's `setTimeout` and `clearTimeout` (to see pending debounces; it calls the -originals). The page gets two globals, `window.GenClass` (the runtime) and `window.__GENCLASS_PLAYWRIGHT__` (the -fixture's collector), and one exposed function. +page configuration `mode: "observe"`, `telemetry: false`, no visual checks, no devtools overlay and the model as +chosen. If a URL or localStorage switch (`?genclass=heal`, `localStorage.genclass`, say from a saved storage state) +raised the mode, the fixture sets it back to observe and the race file says so. Likely detections reach the test +runner from the runtime's `decide` event. The runtime's files are served from disk on the page's own origin at +`/__genclass__/` with `page.route`, never from a CDN. The page then loads the runtime's agent entry +(`dist/agent.js`) and connects it with every hold off (`hold: false`), so the witnesses are computed. The agent link +is given a stand-in `fetch` for its own address when it connects, so it sends nothing at all: no request, no console +line, no failed request, nothing to a Content-Security-Policy report. Like the runtime's own observers, the agent +entry wraps the page's `setTimeout` and `clearTimeout` (to see pending debounces; it calls the originals). + +What the page gets: the globals `GenClass` (the runtime), `GENCLASS_CONFIG` (its configuration), +`__GENCLASS_AGENT__` (the agent entry) and `__GENCLASS_PLAYWRIGHT__` (the fixture's collector); one function exposed +by Playwright (`__genclassPlaywright`); and, from automatic state discovery, stand-ins for the React and Redux +DevTools hooks (`__REACT_DEVTOOLS_GLOBAL_HOOK__`, `__REDUX_DEVTOOLS_EXTENSION__`, +`__REDUX_DEVTOOLS_EXTENSION_COMPOSE__`) when the page has none. A page route, any route, also turns off the browser's +HTTP cache for that page in Playwright, so cached responses are fetched again. Limits, as they are today: @@ -149,6 +173,12 @@ Limits, as they are today: - The witnesses need writes the runtime can attribute to a response (registered or discovered state). Apps whose state the runtime cannot see produce no witnesses; the model's detections about requests (duplicates, failures) do not depend on state. +- The fixture's runtime does not know your app's own `redact` function (it is set where your app calls + `GenClass.init`). Values in the witness lines are summarised and redacted by the runtime's default rules (field names + that say they hold a secret), not by yours. +- With a service worker controlling the page, `/__genclass__/` and chaos go past `page.route`: set + `serviceWorkers: "block"`. The report notes a test where the witnesses could not be recorded (the agent entry did + not load) or a service worker was in control, instead of calling it clean. ## Measured @@ -156,21 +186,25 @@ The fixture's end-to-end check, [`test/e2e`](test/e2e) (run by [`.github/workflo on the search page of the MCP server's own e2e ([`packages/mcp/test/e2e/app/app.js`](../mcp/test/e2e/app/app.js)): a search box whose results are written by each response, so a slower older answer can land over newer results; the fixed variant keeps only the answer to the latest query. The test types "react" at 80 ms per key; the backend -answers in 40 ms, in order, so without chaos the race never shows. Headless Chromium 153 (Playwright 1.63.0), -2026-10-24, `model: "skip"`, five seeds per test (`repeatEach: 5`): - -| page | chaos | seeds with `witness:overwrote-newer` on `GET /api/search` | any other signature | -|---|---|---:|---:| -| buggy | latency 0 to 600 ms, reorder | 5 of 5 | 0 | -| buggy | latency 0 to 600 ms, latency only (`reorder: false`) | 0 of 5 | 0 | -| fixed | latency 0 to 600 ms, reorder | 0 of 5 | 0 | +answers in 40 ms, in order, so without chaos the race never shows. The page's `app.js` is unchanged, but the check's +server ([`test/e2e/server.mjs`](test/e2e/server.mjs)) maps its `@genclass/runtime` import to the page's script-tag +runtime, so its `rt.atom("search")` store is registered with the fixture's runtime: the script-tag integration case. +The same app with its own ESM runtime gets no witness (the store is on another runtime; see the limits above): a +review run of the unmodified app recorded none in 5 of 5 seeds. Headless Chromium 153 (Playwright 1.63.0), +2026-10-25, `model: "skip"`, six runs of the check, five seeds each (`repeatEach: 5`), so 30 seeds per row: + +| page | chaos | seeds with `witness:overwrote-newer` on `GET /api/search` | any other signature | console errors or failed requests | +|---|---|---:|---:|---:| +| buggy | latency 0 to 600 ms, reorder | 30 of 30 | 0 | 0 | +| buggy | latency 0 to 600 ms, latency only (`reorder: false`) | 0 of 30 | 0 | 0 | +| fixed | latency 0 to 600 ms, reorder | 0 of 30 | 0 | 0 | So on this page the fixture finds the race under every seed, reports nothing on the correct variant (the false report count that goes next to every recall number here), and reports nothing when the chaos cannot reorder, which is the check that the witness comes from the order and not from the latency. -The same buggy page with `model: "load"` (one run by hand, same day, two seeds, not part of the workflow because it -downloads the model): the model loaded through the fixture's path (`genclass-runtime-r17` 2.0.0-rc4t, WASM, in a +The same buggy page with `model: "load"` (one run by hand on 2026-10-24, two seeds, not part of the workflow because +it downloads the model, and before the chaos moved from responses to requests): the model loaded through the fixture's path (`genclass-runtime-r17` 2.0.0-rc4t, WASM, in a worker) and the witness was recorded in 2 of 2 seeds, but the model itself produced no likely detection: in seed 1 its top diagnosis for the late delivery was `stale` at 0.64, below the 0.90 report gate, and in seed 2 it raised no decision. That matches the compatibility matrix below (stale detection is the model's weak case) and is why the @@ -248,6 +282,13 @@ interface GenClassRacesFile { mode?: string; // "observe" model?: { state?: string; name?: string; version?: string; device?: string } | null; // null: GenClass never started chaos?: { latencyMs?: [number, number]; reorder?: boolean; routes?: unknown[] } | null; // the chaos option (latencyMs is delayMs) + fixture?: { // written by the fixture: what it could and could not do on the page + model: "skip" | "load"; // the fixture's model option + agent: string | null; // null: the agent entry connected and witnesses were recorded; else why not + chaosApplied: number | null; // how many requests chaos held (null: chaos off) + serviceWorker: boolean; // a service worker controlled the page + modeOverridden?: string; // a URL or localStorage switch had asked for this mode; set back to observe + }; } ``` @@ -268,9 +309,11 @@ How the report groups them: - **witnesses** (trigger `witness`) get their own group, shown only when there are any, with no confidence on the line and a footer sentence saying what they are; they are never counted as duplicate submits or stale responses; - a file that flagged nothing and whose `model.state` is not `ready` (for example `loading`, or `null` for a page - where GenClass never started) may not have been checked, and the report notes it for that test. State `off` (no - model by choice, the fixture's default) gets no note: the witnesses were still computed. A file with detections - is reported as usual whatever the state: the runtime can unload an idle model after it ran; + where GenClass never started) may not have been checked, and the report notes it for that test. A fixture file + with `model: "skip"` gets no model note; the fixture record decides instead: a file that flagged nothing while the + witnesses could not be recorded, or while a service worker controlled the page, gets a note too, and so does a test + where chaos was on but held no request. A file with detections is reported as usual whatever the state: the runtime + can unload an idle model after it ran; - **nothing about interventions.** The files do not say what an action did to your app, so the report makes no claim about interventions, correct or false. @@ -350,7 +393,7 @@ test; the request ids carry the same invisible character): **Witnesses (facts measured without the model)** - **Reproducible**: 5 of 5 seeds (1, 2, 3, 4, 5) · `GET /api/search` · `witness:overwrote-newer` - > The response of GET /api/search#​1 wrote search.results over what newer GET /api/search#​4 had written 0.26s after GET /api/search#​1 started (2 items \["reac one", "reac two"\] → 2 items \["r one", "r two"\]). + > The response of GET /api/search#​1 wrote search.results over what newer GET /api/search#​4, started 0.26s after GET /api/search#​1, had written (2 items \["reac one", "reac two"\] → 2 items \["r one", "r two"\]). ``` The first line of every report is an HTML comment, invisible on GitHub, that lets a CI step find and update its own @@ -419,9 +462,12 @@ The same with the GitHub CLI only (it updates the bot's comment that starts with ## What to expect from the model With `model: "load"`, the report passes on what the runtime's model flagged. That was measured on the framework -compatibility matrix ([`compat/RESULTS.md`](../../compat/RESULTS.md), run of 2026-10-10: headless Chromium, -10 seeds per cell, model `genclass-runtime-r17` 2.0.0-rc4t, the default of `@genclass/runtime-model` 0.2.0, WASM, -balanced profile, whose report threshold is 0.90 per [`docs/runtime/RESULTS.md`](../../docs/runtime/RESULTS.md)): +compatibility matrix ([`compat/RESULTS.md`](../../compat/RESULTS.md), run of 2026-10-10: `@genclass/runtime` +0.1.0-beta.4, the code released as 0.2.1, not the 0.3.0 beta the fixture needs; headless Chromium, 10 seeds per +cell, model `genclass-runtime-r17` 2.0.0-rc4t, the default of `@genclass/runtime-model` 0.2.0, WASM, balanced +profile, whose report threshold is 0.90 per [`docs/runtime/RESULTS.md`](../../docs/runtime/RESULTS.md)). Three +0.3.0 changes (superseded writes in observe, the body wait, the idle unload) can add likely reports, false ones +included, at a rate not measured (`docs/runtime/RESULTS.md`). - **Double submit** (scenario b, a create form clicked twice): 10 `request:duplicate` detections over 10 seeds in each of the 14 data layers where the scenario applies (React state, TanStack Query, Zustand, Redux, Apollo, @@ -432,8 +478,10 @@ balanced profile, whose report threshold is 0.90 per [`docs/runtime/RESULTS.md`] the middleware at its default, Pinia, Svelte stores, Angular fetch and XHR, plain HTML fetch and push). Stale detection needs state the runtime can discover: React component state, Redux, and Zustand stores whose devtools middleware is enabled (the discovery table of the same page). -- **On correct apps** (scenarios c to h, 880 runs): 49 false findings, `request:overload` ×45 and - `request:duplicate` ×4, all on the polling scenario (g). +- **On correct apps** (scenarios c to h, 880 runs): 319 findings. 270 of them report a failure the scenario injects + on purpose (an HTTP 500, going offline), which the app handles correctly; in your report those appear under Failing + endpoints and Other detections. 49 are false findings, `request:overload` ×45 and `request:duplicate` ×4, all on the + polling scenario (g). So with the model, lead with duplicate submits, expect stale signatures only where that state is discovered, and treat any line as the model's diagnosis to check, not a confirmed bug. The witnesses (the default) were measured @@ -443,8 +491,8 @@ only on the one page above so far. This package reads and writes files on the machine that runs it, serves the runtime's files to the page it tests from disk, and sends nothing anywhere: with `model: "skip"` no network request leaves your test environment -because of it; with `model: "load"` the page downloads the model and onnxruntime-web's wasm from jsDelivr (or -`modelUrl`). The runtime on the page runs with `telemetry: false`. +because of it; with `model: "load"` the page downloads the model (from jsDelivr, or `modelUrl`) and +onnxruntime-web's wasm (from jsDelivr). The runtime on the page runs with `telemetry: false`. The files hold test titles, operation names and the runtime's evidence lines. For a witness that line includes the store field's name and the runtime's short summary of its value before and after (`2 items ["r one", "r two"]`; @@ -466,7 +514,7 @@ The end-to-end check needs both builds and Chromium: ```bash npm run build -w @genclass/runtime && npm run build -w @genclass/playwright npx playwright install --with-deps chromium -node packages/playwright/test/e2e/run.mjs # the table in "Measured"; prints a JSON summary, exit 0 when every count holds +node packages/playwright/test/e2e/run.mjs # one run of the table in "Measured": exit 0 when the buggy page has the witness in at least 1 of 5 seeds and nothing else, and the other two rows have nothing ``` The workflow also runs Playwright itself on `test/playwright-smoke` (no browser: its one test writes race files and diff --git a/packages/playwright/package.json b/packages/playwright/package.json index f3bbb4dc..e587c64f 100644 --- a/packages/playwright/package.json +++ b/packages/playwright/package.json @@ -23,6 +23,13 @@ ], "main": "./dist/index.js", "types": "./dist/index.d.ts", + "typesVersions": { + "*": { + "fixture": ["./dist/fixture.d.ts"], + "reporter": ["./dist/reporter.d.ts"], + "summarize": ["./dist/summarize.d.ts"] + } + }, "exports": { ".": { "types": "./dist/index.d.ts", @@ -83,7 +90,7 @@ "vitest": "^5.0.3" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || >=22.12.0" }, "publishConfig": { "access": "public" diff --git a/packages/playwright/src/chaos.ts b/packages/playwright/src/chaos.ts index 8da8a279..2f2fa545 100644 --- a/packages/playwright/src/chaos.ts +++ b/packages/playwright/src/chaos.ts @@ -1,13 +1,14 @@ // Seeded network chaos for the GenClass Playwright fixture: which responses to slow down, by how much, and whether // they may overtake each other, every value derived from a seed so that a run repeats exactly. Pure: no Playwright, // no I/O, no clock of its own (the scheduler takes its timers from the caller, so tests drive it by hand). The fixture -// (install.ts) applies the plan with page.route: it fetches the real response and holds its delivery for the planned -// time. Nothing is dropped, changed or invented, and requests outside the filter are never touched; the runtime's own -// files (served on RUNTIME_PATH) never are. +// (install.ts) applies the plan with page.route: it holds each matched request for the planned time, then lets it go on +// its way (route.fallback), the way a slow uplink would. The request still goes through the user's own routes, the +// browser's network stack and its cache, so nothing is dropped, changed, buffered or invented; requests outside the +// filter are never held, and the runtime's own files (served on RUNTIME_PATH) never are. // -// Determinism: a matched request is keyed by its method, path and occurrence in the test ("GET /api/search#3"), and -// its delay is read from a small PRNG seeded by the chaos seed and that key. The same test under the same seed gives -// the same delays whatever the wall clock does. +// Determinism: a matched request is keyed, when the page sends it, by its method, path and occurrence in the test +// ("GET /api/search#3"), and its delay is read from a small PRNG seeded by the chaos seed and that key. The same test +// under the same seed gives the same delays whatever the wall clock or the server does. /** Where install.ts serves the runtime's dist/ on the page's origin. Chaos never applies to it. */ export const RUNTIME_PATH = "/__genclass__/"; @@ -17,6 +18,11 @@ export const MAX_DELAY_MS = 10_000; export const DEFAULT_DELAY_MS: readonly [number, number] = [0, 600]; /** The default seed (the fixture adds Playwright's `repeatEachIndex` to it). */ export const DEFAULT_SEED = 1; +/** + * With `reorder: false`: the least time between two releases, so the page has handled one request's response before + * the next request leaves (the runtime's agent schedules use the same 30 ms, DEFAULT_GAP_MS). + */ +export const FIFO_GAP_MS = 30; /** A request as the filter sees it, before its response is fetched. */ export interface ChaosRequest { @@ -26,13 +32,15 @@ export interface ChaosRequest { resourceType: string; /** Same origin as the document that sent it. */ sameOrigin: boolean; + /** The request's Accept header, when it has one. */ + accept?: string; } export type ChaosMatchFn = (r: ChaosRequest) => boolean; /** * Which requests get chaos: a string (a path prefix such as "/api", or a URL prefix such as "https://api.example.com/"), * a RegExp tested against the full URL, a function, or a list of those (any match counts). Unset: same-origin - * requests whose path starts with /api or whose response is JSON. + * requests whose path starts with /api or whose Accept header asks for JSON. */ export type ChaosMatch = string | RegExp | ChaosMatchFn | readonly (string | RegExp | ChaosMatchFn)[]; @@ -41,9 +49,12 @@ export interface ChaosOptions { seed?: number; /** Extra latency per matched response, uniform in [min, max] ms. Default [0, 600]; at most 10000. */ delayMs?: readonly [number, number]; - /** false: matched responses keep their request order (latency only, nothing overtakes). Default true. */ + /** + * false: matched requests leave in the order the page sent them, at least 30 ms apart (latency only: with a server + * that answers in order, nothing overtakes). Default true. + */ reorder?: boolean; - /** Which requests (see ChaosMatch). Default: same-origin, path /api* or a JSON response. */ + /** Which requests (see ChaosMatch). Default: same-origin, path /api* or an Accept header that asks for JSON. */ match?: ChaosMatch; } @@ -141,31 +152,21 @@ function matchesOne(m: string | RegExp | ChaosMatchFn, r: ChaosRequest): boolean } /** - * Whether a request may get chaos, decided before its response is fetched. Only fetch and XHR requests, never the - * runtime's files. With a `match`, the filter decides; without one, every same-origin request is a candidate and - * `confirms` decides from the response. + * Whether a request gets chaos, decided from the request alone before it is sent. Only fetch and XHR requests, never + * the runtime's files. With a `match`, the filter decides; without one, same-origin requests whose path starts with + * /api or whose Accept header asks for JSON. */ export function isCandidate(r: ChaosRequest, match: ChaosMatch | undefined): boolean { if (r.resourceType !== "fetch" && r.resourceType !== "xhr") return false; if (isRuntimeUrl(r.url)) return false; - if (match === undefined) return r.sameOrigin; + if (match === undefined) return r.sameOrigin && (pathOf(r.url).startsWith("/api") || /json/i.test(r.accept ?? "")); const list = Array.isArray(match) ? (match as readonly (string | RegExp | ChaosMatchFn)[]) : [match as string | RegExp | ChaosMatchFn]; return list.some((m) => matchesOne(m, r)); } -/** - * Whether a candidate gets chaos once its response is known. With a `match`, always (the filter already said yes); - * without one, when the path starts with /api or the response is JSON (its content-type names json). - */ -export function confirms(r: ChaosRequest, match: ChaosMatch | undefined, contentType: string | null | undefined): boolean { - if (match !== undefined) return true; - if (pathOf(r.url).startsWith("/api")) return true; - return /json/i.test(contentType ?? ""); -} - /** The filter as written to the race file's `chaos.routes` (functions as "function", RegExps as their source). */ export function describeMatch(match: ChaosMatch | undefined): unknown[] { - if (match === undefined) return ["same-origin /api* or JSON responses"]; + if (match === undefined) return ["same-origin /api* or Accept: json"]; const list = Array.isArray(match) ? (match as readonly (string | RegExp | ChaosMatchFn)[]) : [match as string | RegExp | ChaosMatchFn]; return list.map((m) => (typeof m === "function" ? "function" : m instanceof RegExp ? String(m) : m)); } @@ -182,15 +183,16 @@ const realTimers: ChaosTimers = { }; /** - * Assigns each matched request its key and latency, and holds its delivery: for its own delay, and with `reorder` - * off also until every earlier matched response was released, so nothing overtakes. `dispose()` releases every - * hold at once (the page is gone). Never throws. + * Assigns each matched request its key and latency when the page sends it, and holds it: for its own delay, and with + * `reorder` off also until FIFO_GAP_MS after the earlier matched request was released, so requests leave in the + * order they were sent. `dispose()` releases every hold at once (the page is gone). Never throws. */ export class ChaosScheduler { private readonly counts = new Map(); private last: Promise = Promise.resolve(); private readonly pending = new Map void>(); private disposed = false; + private chained = 0; /** Every plan made so far, in request order (for the race file and tests). */ readonly plans: ChaosPlan[] = []; @@ -199,12 +201,16 @@ export class ChaosScheduler { private readonly timers: ChaosTimers = realTimers, ) {} - /** The key ("GET /api/search#3": method, path, occurrence in this test) and latency of a matched request. */ + /** + * The key ("GET /api/search#3": method, path, occurrence in this test) and latency of a matched request. A + * cross-origin request is keyed by host name and path, without the port, so a backend on a random port keeps its + * delays under the same seed. + */ plan(method: string, url: string, sameOrigin = true): ChaosPlan { let where: string; try { const u = new URL(url); - where = sameOrigin ? u.pathname : `${u.host}${u.pathname}`; + where = sameOrigin ? u.pathname : `${u.hostname}${u.pathname}`; } catch { where = url; } @@ -217,17 +223,19 @@ export class ChaosScheduler { return p; } - /** Resolves when the response may be delivered. */ + /** Resolves when the request may leave. Call it in the order the page sent the requests (route handlers are). */ hold(plan: ChaosPlan): Promise { const own = this.sleep(plan.delayMs); if (this.chaos.reorder) return own; - const after = this.last; + const first = this.chained === 0; + this.chained++; + const after = first ? this.last : this.last.then(() => this.sleep(FIFO_GAP_MS)); const mine = Promise.all([after, own]).then(() => undefined); this.last = mine; return mine; } - /** Release every held response now and stop the timers. */ + /** Release every held request now and stop the timers. */ dispose(): void { this.disposed = true; for (const [id, release] of this.pending) { diff --git a/packages/playwright/src/fixture.ts b/packages/playwright/src/fixture.ts index 4fb1a146..6d7d0992 100644 --- a/packages/playwright/src/fixture.ts +++ b/packages/playwright/src/fixture.ts @@ -9,13 +9,14 @@ // // For each test it puts @genclass/runtime on the page before the app's first script (observe mode, telemetry off, // the model skipped unless asked), applies the seeded chaos when one is given, and at the end writes one -// "genclass-races/1" file to `outDir` (default test-results/genclass next to the config) with what the runtime saw -// and the seed and chaos that produced it, which the reporter turns into the pull-request comment. Observe only: the -// fixture never holds, delays or changes anything the app does; chaos changes when a response arrives, never what -// it says. A test's outcome is never changed by a finding. `enabled: false` makes the fixture inert: the page is -// Playwright's own and nothing is written. +// "genclass-races/1" file to `outDir` (default `genclass` in the project's outputDir, test-results/genclass) with what +// the runtime saw, the seed and chaos that produced it and what the fixture could not do, which the reporter turns +// into the pull-request comment. The runtime only observes: it never holds, delays or changes anything the app does. +// Chaos, when given, holds matched requests before they leave (that is its job), never changes what they say. A +// finding never changes a test's outcome. `enabled: false` makes the fixture inert: the page is Playwright's own and +// nothing is written. -import { test as base, expect } from "@playwright/test"; +import { test as base, expect, type Page, type PlaywrightTestArgs, type PlaywrightTestOptions, type PlaywrightWorkerArgs, type PlaywrightWorkerOptions, type TestType } from "@playwright/test"; import { dirname, isAbsolute, join, relative, sep } from "node:path"; import { DEFAULT_SEED, describeMatch, resolveChaos, type ChaosMatch, type ChaosMatchFn, type ChaosOptions } from "./chaos.js"; import { install, type Collected, type PageLike } from "./install.js"; @@ -38,7 +39,11 @@ export interface GenClassOptions { modelUrl?: string; /** Seeded network chaos (ChaosOptions; `true` for the defaults). Off unless given. */ chaos?: ChaosOptions | true | false; - /** Where the files go, relative to the Playwright config's directory. Default "test-results/genclass". */ + /** + * Where the files go, relative to the Playwright config's directory. Default: `genclass` in the project's + * outputDir (test-results/genclass), which Playwright empties at the start of each run, so no file of an earlier + * run is read as another seed. The reporter reads both places by default. + */ outDir?: string; /** @genclass/runtime's dist/ directory. Default: the installed package's. */ runtimeDir?: string; @@ -54,14 +59,19 @@ export interface TestInfoLike { file: string; testId: string; repeatEachIndex: number; - project: { name: string; testDir: string }; + /** The attempt (0 for the first run); a retry writes a file of its own. */ + retry?: number; + /** The test's status when the fixture finishes ("skipped": no file is written). */ + status?: string; + project: { name: string; testDir: string; outputDir?: string }; config: { configFile?: string; rootDir: string }; } /** The test's name in the file: "[project] file.spec.ts > describe > title" (the project only when it has a name). */ export function testName(info: TestInfoLike): string { const path = [...info.titlePath]; - // Playwright's title path is ["", project, file, ...describes, title]; find the file to be safe about the shape + // TestInfo.titlePath starts with the file ([file, ...describes, title]); a reporter's TestCase.titlePath() adds + // ["", project] in front. Find the file to be safe about either shape const rel = relative(info.project.testDir, info.file).split(sep).join("/"); let at = path.findIndex((t) => t === rel || t.split(sep).join("/") === rel); if (at < 0) at = path.findIndex((t) => t.endsWith(".ts") || t.endsWith(".js") || t.endsWith(".mjs") || t.endsWith(".tsx") || t.endsWith(".jsx")); @@ -72,10 +82,13 @@ export function testName(info: TestInfoLike): string { return project ? `[${project}] ${name}` : name; } -/** The directory the file goes to: `outDir` against the config's directory (else the root dir). */ +/** + * The directory the file goes to: `outDir` against the config's directory (else the root dir); by default `genclass` + * in the project's outputDir (else test-results/genclass next to the config). + */ export function outDirOf(info: TestInfoLike, outDir?: string): string { const base = info.config.configFile ? dirname(info.config.configFile) : info.config.rootDir; - if (!outDir) return join(base, "test-results", "genclass"); + if (!outDir) return info.project.outputDir ? join(info.project.outputDir, "genclass") : join(base, "test-results", "genclass"); return isAbsolute(outDir) ? outDir : join(base, outDir); } @@ -90,7 +103,7 @@ export interface GenClassRun { * Starts the fixture on a page for one test: installs the runtime (and chaos), and returns what to call when the * test ends. Exported for tests and for custom fixtures; `test` below does this on its `page`. */ -export async function startGenClass(page: PageLike, options: GenClassOptions | undefined, info: TestInfoLike): Promise<{ finish(): Promise }> { +export async function startGenClass(page: Page | PageLike, options: GenClassOptions | undefined, info: TestInfoLike): Promise<{ finish(): Promise }> { const o = options ?? {}; const empty: GenClassRun = { collected: null, file: null, path: null }; if (o.enabled === false) return { finish: async () => empty }; @@ -98,10 +111,13 @@ export async function startGenClass(page: PageLike, options: GenClassOptions | u if (o.model !== undefined && o.model !== "skip" && o.model !== "load") throw new Error(`@genclass/playwright: genclass.model must be "skip" or "load" (got ${JSON.stringify(o.model)})`); const chaos = o.chaos ? resolveChaos(o.chaos, info.repeatEachIndex) : null; const seed = chaos ? chaos.seed : DEFAULT_SEED + Math.max(0, Math.floor(info.repeatEachIndex) || 0); - const session = await install(page, { runtimeDir: o.runtimeDir, model: o.model ?? "skip", modelUrl: o.modelUrl, chaos }); + const model = o.model ?? "skip"; + const session = await install(page as unknown as PageLike, { runtimeDir: o.runtimeDir, model, modelUrl: o.modelUrl, chaos }); return { async finish(): Promise { const collected = await session.finish(); + // a test skipped in its body was not run: no file, so the report does not count it as unchecked + if (info.status === "skipped") return { collected, file: null, path: null }; const file = buildRacesFile({ test: testName(info), seed, @@ -110,8 +126,15 @@ export async function startGenClass(page: PageLike, options: GenClassOptions | u chaos: chaos ? { latencyMs: chaos.delayMs, reorder: chaos.reorder, routes: describeMatch(chaos.match) } : null, records: collected.records, witnesses: collected.witnesses, + fixture: { + model, + agent: collected.agent, + chaosApplied: chaos ? collected.plans.length : null, + serviceWorker: collected.serviceWorker, + ...(collected.modeOverridden ? { modeOverridden: collected.modeOverridden } : {}), + }, }); - const path = writeRacesFile(outDirOf(info, o.outDir), racesFileName(file.test, info.testId, seed), file); + const path = writeRacesFile(outDirOf(info, o.outDir), racesFileName(file.test, info.testId, seed, info.retry ?? 0), file); return { collected, file, path }; }, }; @@ -122,10 +145,10 @@ export async function startGenClass(page: PageLike, options: GenClassOptions | u * `test.use({ genclass: {...} })` in a spec. Pages opened another way (`context.newPage()`, popups) are not * instrumented. */ -export const test = base.extend({ +export const test: TestType = base.extend({ genclass: [{}, { option: true }], page: async ({ page, genclass }, use, testInfo) => { - const run = await startGenClass(page as unknown as PageLike, genclass, testInfo); + const run = await startGenClass(page, genclass, testInfo); try { await use(page); } finally { @@ -140,5 +163,5 @@ export const test = base.extend({ }); export { expect }; -export type { ChaosMatch, ChaosMatchFn, ChaosOptions, Collected }; +export type { ChaosMatch, ChaosMatchFn, ChaosOptions, Collected, PageLike }; export { RUNTIME_PATH } from "./chaos.js"; diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index c289b5d7..d8a79bad 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -3,6 +3,7 @@ // Node APIs): the format, its validator and the summariser, the same as `@genclass/playwright/summarize`. The // Playwright reporter is `@genclass/playwright/reporter` and the command `genclass-playwright-report`. // -// This package injects nothing: it does not route requests, start a runtime or touch a page. +// This entry injects nothing: it does not route requests, start a runtime or touch a page. The fixture that does is +// `@genclass/playwright/fixture`. export * from "./summarize.js"; diff --git a/packages/playwright/src/install.ts b/packages/playwright/src/install.ts index 660df113..81199709 100644 --- a/packages/playwright/src/install.ts +++ b/packages/playwright/src/install.ts @@ -10,17 +10,18 @@ // runtime starts before the app's first script, in observe mode with telemetry off and the model as chosen; // - exposes one function (page.exposeFunction) the page pushes each finding through as it happens, so navigations // and an early page close lose nothing; -// - with chaos, routes every fetch/XHR request through the seeded scheduler (chaos.ts): the real response is -// fetched at once and its delivery to the page is held for the planned time. Requests outside the filter, the -// runtime's own files and the agent link's URL are left alone; the agent link's URL is aborted, so the page's -// link never reaches a bridge. -// Observe only: no hold on the page's side, no change to any request or response (chaos changes when a response -// arrives, never what it says). Every handler catches its own errors: a page that closes mid-test is not a failure. +// - with chaos, routes every request through the seeded scheduler (chaos.ts): a matched fetch/XHR request is held +// for its planned time and then passed on with route.fallback(), so it still reaches the user's own routes +// (context.route, routeFromHAR), the browser's network stack and its cache. Anything else is passed on at once. +// The page's agent link never sends a request (page.ts gives it a stand-in fetch); requests to its URL are aborted +// anyway as a safety net. Observe only: no hold on the page's side, no change to any request or response (chaos +// changes when a request leaves, never what it says). Every handler catches its own errors: a page that closes +// mid-test is not a failure. import { existsSync, readFileSync } from "node:fs"; import { createRequire } from "node:module"; import { dirname, extname, join, resolve, sep } from "node:path"; -import { ChaosScheduler, RUNTIME_PATH, confirms, isCandidate, isRuntimeUrl, type ResolvedChaos } from "./chaos.js"; +import { ChaosScheduler, RUNTIME_PATH, isCandidate, isRuntimeUrl, type ResolvedChaos } from "./chaos.js"; import { genclassPageScript, type PageScriptConfig } from "./page.js"; /** The page methods the installer uses (a subset of Playwright's Page, by shape). */ @@ -33,8 +34,7 @@ export interface PageLike { } export interface RouteLike { - fetch(): Promise; - fulfill(options: { response?: ResponseLike; status?: number; headers?: Record; body?: string | Buffer; contentType?: string }): Promise; + fulfill(options: { status?: number; headers?: Record; body?: string | Buffer; contentType?: string }): Promise; fallback(): Promise; abort(errorCode?: string): Promise; } @@ -43,11 +43,8 @@ export interface RequestLike { url(): string; method(): string; resourceType(): string; - frame(): { url(): string }; -} - -export interface ResponseLike { headers(): Record; + frame(): { url(): string }; } export interface InstallOptions { @@ -83,6 +80,10 @@ export interface Collected { agent: string | null; /** The chaos plans applied, in request order. */ plans: { key: string; delayMs: number }[]; + /** A service worker controlled the page when the fixture collected (page.route does not see what it handles). */ + serviceWorker: boolean; + /** The URL or localStorage switch had put the runtime in another mode; the fixture set it back to observe. */ + modeOverridden: string | null; } export interface Session { @@ -139,6 +140,11 @@ export function initScriptContent(runtimeDir: string, cfg: PageScriptConfig): st return `(${genclassPageScript.toString()})(${JSON.stringify(cfg)}, function () {\n${global}\n});\n`; } +/** A URL on the agent link's address (not merely one whose port starts with the same digits). */ +export function isAgentUrl(url: string): boolean { + return url === AGENT_URL || url.startsWith(`${AGENT_URL}/`); +} + function originOf(url: string): string | null { try { return new URL(url).origin; @@ -169,7 +175,7 @@ export function runtimeFile(runtimeDir: string, url: string): string | null { */ export async function install(page: PageLike, o: InstallOptions): Promise { const runtimeDir = resolveRuntimeDir(o.runtimeDir); - const collected: Collected = { records: [], witnesses: [], mode: null, model: null, agent: "not-loaded", plans: [] }; + const collected: Collected = { records: [], witnesses: [], mode: null, model: null, agent: "not-loaded", plans: [], serviceWorker: false, modeOverridden: null }; const scheduler = o.chaos ? new ChaosScheduler(o.chaos) : null; const take = (item: unknown): void => { @@ -180,6 +186,7 @@ export async function install(page: PageLike, o: InstallOptions): Promise { - let candidate = false; - let req: Parameters[0] | null = null; try { const url = request.url(); - if (isRuntimeUrl(url) || url.startsWith(AGENT_URL)) return await route.fallback(); + if (isRuntimeUrl(url) || isAgentUrl(url)) return await route.fallback(); let frameUrl = ""; try { frameUrl = request.frame().url(); @@ -200,28 +205,18 @@ export async function install(page: PageLike, o: InstallOptions): Promise undefined); - let response: ResponseLike; - try { - response = await route.fetch(); - } catch { - // the request itself failed (the server is down, the page is gone): let the browser see the failure - return route.abort("failed").catch(() => undefined); - } - try { - const headers = response.headers(); - const ct = headers["content-type"] ?? headers["Content-Type"] ?? null; - if (confirms(req, chaos.match, ct)) { - await scheduler.hold(scheduler.plan(req.method, req.url, req.sameOrigin)); + let accept = ""; + try { + accept = request.headers()["accept"] ?? ""; + } catch { + /* no headers */ } - await route.fulfill({ response }); + const req = { url, method: request.method(), resourceType: request.resourceType(), sameOrigin: origin !== null && origin === originOf(frameUrl), accept }; + // planned now, in the order the page sent its requests; the request itself is only held, never fetched here + if (isCandidate(req, chaos.match)) await scheduler.hold(scheduler.plan(req.method, req.url, req.sameOrigin)); + await route.fallback(); } catch { - /* the page closed while the response was held */ + /* the page closed while the request was held */ } }); } @@ -236,7 +231,8 @@ export async function install(page: PageLike, o: InstallOptions): Promise route.abort("connectionrefused").catch(() => undefined)); const cfg: PageScriptConfig = { @@ -259,8 +255,10 @@ export async function install(page: PageLike, o: InstallOptions): Promise; if (typeof x.mode === "string") collected.mode = x.mode; if (x.model && typeof x.model === "object") collected.model = x.model as ModelInfo; + // a specific reason the page already pushed (an import error) beats collect()'s generic "not-loaded" if (x.agent === null) collected.agent = null; - else if (typeof x.agent === "string") collected.agent = x.agent; + else if (typeof x.agent === "string" && (collected.agent === null || collected.agent === "not-loaded")) collected.agent = x.agent; + if (x.serviceWorker === true) collected.serviceWorker = true; } } } catch { diff --git a/packages/playwright/src/page.ts b/packages/playwright/src/page.ts index 90a3b04d..da9c3fdf 100644 --- a/packages/playwright/src/page.ts +++ b/packages/playwright/src/page.ts @@ -3,19 +3,26 @@ // and the page's globals). What it does, before any script of the app runs: // // 1. sets window.GENCLASS_CONFIG: observe mode, telemetry off, no visual checks, no devtools, the model as the -// fixture chose, and one sink that hands every likely detection (kind "detection", with its evidence) to Node; +// fixture chose; // 2. runs the runtime's script-tag build (dist/genclass.global.js) with a stand-in for `document.currentScript` // (an init script has no tag, and the build reads `data-base` from its own tag: the stand-in points it at the // fixture's virtual path, so everything it loads later comes from the same place); -// 3. loads the agent entry (dist/agent.js) and connects it with every hold off, so the runtime records the two +// 3. holds the runtime to observe: if the URL or localStorage switch (?genclass=heal, localStorage.genclass) raised +// the mode, it is set back with rt.setMode("observe") and the file says so; +// 4. hands every likely detection to Node from the runtime's "decide" event, synchronously (with the model's +// diagnosis confidence and the console line the runtime printed); +// 5. loads the agent entry (dist/agent.js) and connects it with every hold off, so the runtime records the two // happens-before witnesses (overwrote-newer, identical-in-flight) that need no model, and hands each new one to -// Node as it appears; -// 4. reports the runtime's mode and model status, at start, on every status change, when the page goes away, and -// on request (window.__GENCLASS_PLAYWRIGHT__.collect(), which the fixture calls at the end of the test). +// Node as it appears. connectAgent binds the page's fetch once, when it is called: for that call only, fetch is +// a stand-in that answers the agent's own URL with "null" without a request (so the link stays unconnected and +// sends nothing, no console error, no failed request, no CSP report) and passes every other call to the real one; +// 6. reports the runtime's mode and model status, at start, on every status change, when the page goes away, and +// on request (window.__GENCLASS_PLAYWRIGHT__.collect(), which the fixture calls at the end of the test), with +// whether a service worker controls the page (page.route does not see what it handles). // // Observe only: nothing here holds, delays or changes anything in the page; the agent link is connected with -// `hold: false` and talks to a loopback URL the fixture aborts, so it never reaches a bridge. Top frame only, and only -// on http(s) pages (about:blank and page.setContent get nothing). Never throws into the page. +// `hold: false` and never reaches a bridge. Top frame only, and only on http(s) pages (about:blank and +// page.setContent get nothing). Never throws into the page. /** What install.ts passes to the page script. */ export interface PageScriptConfig { @@ -60,15 +67,7 @@ export function genclassPageScript(cfg: PageScriptConfig, runGlobal: () => void) }; // 1. the runtime's configuration (read by the script-tag build at init, before the app's own scripts run) - const sink = { - kinds: ["detection"], - evidence: true, - possible: false, - send(r: unknown): void { - push({ t: "record", record: r }); - }, - }; - w.GENCLASS_CONFIG = { mode: "observe", telemetry: false, visual: false, analytics: false, devtools: false, report: "silent", model: cfg.model, sinks: [sink] }; + w.GENCLASS_CONFIG = { mode: "observe", telemetry: false, visual: false, analytics: false, devtools: false, report: "silent", model: cfg.model }; // 2. the script-tag build, with a tag of its own to read data-base from let rt: Record | null = null; @@ -97,9 +96,17 @@ export function genclassPageScript(cfg: PageScriptConfig, runGlobal: () => void) if (typeof s.device === "string") m.device = s.device; return m; }; - const report = (): void => push({ t: "status", mode: rt ? String(rt.mode) : null, model: status() }); + let overridden: string | null = null; + const report = (): void => push({ t: "status", mode: rt ? String(rt.mode) : null, model: status(), ...(overridden ? { overridden } : {}) }); + const serviceWorker = (): boolean => { + try { + return !!(navigator.serviceWorker && navigator.serviceWorker.controller); + } catch { + return false; + } + }; - // 3. witnesses through the agent entry (hands each new one over as it appears) + // 5. witnesses through the agent entry (hands each new one over as it appears) let conn: AgentConnection | null = null; let handed = 0; let queued = false; @@ -115,7 +122,7 @@ export function genclassPageScript(cfg: PageScriptConfig, runGlobal: () => void) const api = { collect(): Record { drain(); - return { mode: rt ? String(rt.mode) : null, model: status(), agent: conn ? conn.reason : "not-loaded" }; + return { mode: rt ? String(rt.mode) : null, model: status(), agent: conn ? conn.reason : "not-loaded", serviceWorker: serviceWorker() }; }, }; w.__GENCLASS_PLAYWRIGHT__ = api; @@ -123,8 +130,38 @@ export function genclassPageScript(cfg: PageScriptConfig, runGlobal: () => void) report(); return; } + // 3. observe only, whatever a URL or localStorage switch asked for + try { + if (String(rt.mode) !== "observe" && typeof rt.setMode === "function") { + overridden = String(rt.mode); + (rt.setMode as (m: string) => void).call(rt, "observe"); + } + } catch { + /* ignore */ + } report(); const on = rt.on as ((type: string, fn: (v: unknown) => void) => () => void) | undefined; + // 4. likely detections, with the model's diagnosis confidence (a sink record's `p` is the action's probability) + try { + on?.call(rt, "decide", (v: unknown) => { + try { + const d = v as { id?: string; trigger?: string; diagnosis?: string; diagnosisConfidence?: number; detection?: string; subject?: string }; + if (!d || d.detection !== "likely") return; + let message = ""; + try { + const x = (rt!.explain as (id: string) => { message?: string } | null).call(rt, String(d.id)); + message = x && typeof x.message === "string" ? x.message : ""; + } catch { + /* no explanation */ + } + push({ t: "record", record: { kind: "detection", trigger: d.trigger, diagnosis: d.diagnosis, p: d.diagnosisConfidence, detection: d.detection, evidence: { message, subject: d.subject } } }); + } catch { + /* never into the page */ + } + }); + } catch { + /* an older runtime without this event */ + } try { on?.call(rt, "status", report); // a witness is computed inside the runtime's event dispatch: look after it, once per burst @@ -142,12 +179,29 @@ export function genclassPageScript(cfg: PageScriptConfig, runGlobal: () => void) try { import(/* @vite-ignore */ /* webpackIgnore: true */ `${cfg.base}agent.js`).then( (m: { connectAgent?: (r: unknown, o: Record) => AgentConnection }) => { + // connectAgent binds the page's fetch now: a stand-in for this call answers the agent URL without a request + const realFetch = w.fetch as ((u: unknown, i?: unknown) => Promise) | undefined; + const own = (u: unknown): boolean => { + try { + const s = typeof u === "string" ? u : u instanceof URL ? u.href : String((u as { url?: string })?.url ?? ""); + return s === cfg.agentUrl || s.indexOf(cfg.agentUrl + "/") === 0; + } catch { + return false; + } + }; + const standIn = function (this: unknown, u: unknown, i?: unknown): Promise { + if (own(u)) return Promise.resolve(new Response("null", { status: 200, headers: { "content-type": "application/json" } })); + return (realFetch as (u: unknown, i?: unknown) => Promise).call(this, u, i); + }; try { + if (typeof realFetch === "function") w.fetch = standIn; conn = typeof m.connectAgent === "function" ? m.connectAgent(rt, { url: cfg.agentUrl, hold: false, anyHost: true, dom: false }) : null; push({ t: "agent", ok: !!conn && conn.reason === null, reason: conn ? conn.reason : "no-connectAgent" }); drain(); } catch (e) { push({ t: "agent", ok: false, reason: String((e as Error)?.message ?? e) }); + } finally { + if (w.fetch === standIn) w.fetch = realFetch; } }, (e: unknown) => push({ t: "agent", ok: false, reason: String((e as Error)?.message ?? e) }), @@ -156,7 +210,7 @@ export function genclassPageScript(cfg: PageScriptConfig, runGlobal: () => void) push({ t: "agent", ok: false, reason: String((e as Error)?.message ?? e) }); } - // 4. when the page goes away + // 6. when the page goes away try { addEventListener("pagehide", () => { drain(); diff --git a/packages/playwright/src/races.ts b/packages/playwright/src/races.ts index a351656f..0b81ce82 100644 --- a/packages/playwright/src/races.ts +++ b/packages/playwright/src/races.ts @@ -14,7 +14,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; -import { RACES_SCHEMA, type GenClassRacesFile, type RaceSignature } from "./types.js"; +import { RACES_SCHEMA, type GenClassRacesFile, type RaceSignature, type RacesFixtureInfo } from "./types.js"; /** The `trigger` of a witness signature. */ export const WITNESS_TRIGGER = "witness"; @@ -48,16 +48,21 @@ export interface RacesInput { chaos: GenClassRacesFile["chaos"]; records: readonly unknown[]; witnesses: readonly unknown[]; + /** What the fixture could and could not do on the page (written as the file's `fixture`). */ + fixture?: RacesFixtureInfo; } const METHOD_RE = /\b(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\s+([^\s()?#]+)/; -/** "GET /api/search?q=… (#21)" or "GET /api/search#3~retry2" -> "GET /api/search"; anything else is kept (trimmed). */ +/** + * "GET /api/search?q=… (#21)" or "GET /api/search#3~retry2" -> "GET /api/search". Anything else is kept with the + * runtime's per-run op ids ("#12", "(#12)") taken out, so the same detection gives one signature in every seed. + */ export function opOf(text: string | undefined): string { const t = (text ?? "").replace(/\s+/g, " ").trim(); const m = METHOD_RE.exec(t); if (m) return `${m[1]} ${m[2]}`; - return t.slice(0, 120); + return t.replace(/\s*\(#\d+\)/g, "").replace(/#\d+(~\w+)?/g, "").replace(/\s+/g, " ").trim().slice(0, 120); } function secs(ms: number | undefined): string { @@ -68,9 +73,10 @@ function secs(ms: number | undefined): string { export function witnessLine(w: WitnessRecord): string { const by = w.by ?? "?"; if (w.kind === "overwrote-newer") { - const over = w.overKind === "user" ? `the user (${w.over ?? "?"})` : `newer ${w.over ?? "?"}`; const change = w.change ? ` (${w.change})` : ""; - return `The response of ${by} wrote ${w.subject ?? "?"} over what ${over} had written ${secs(w.gapMs)} after ${by} started${change}.`; + // gapMs: for a newer request, how long after `by` it started; for the user, how long after `by` they wrote + if (w.overKind === "user") return `The response of ${by} wrote ${w.subject ?? "?"} over what the user (${w.over ?? "?"}) had written ${secs(w.gapMs)} after ${by} started${change}.`; + return `The response of ${by} wrote ${w.subject ?? "?"} over what newer ${w.over ?? "?"}, started ${secs(w.gapMs)} after ${by}, had written${change}.`; } if (w.kind === "identical-in-flight") return `${by} was sent while the identical ${w.over ?? "?"} was still in flight (${secs(w.gapMs)} earlier), from a separate user action.`; return `${w.kind ?? "witness"}: ${w.subject ?? ""} by ${by} over ${w.over ?? "?"}`.trim(); @@ -119,11 +125,12 @@ export function buildRacesFile(i: RacesInput): GenClassRacesFile { if (i.mode) file.mode = i.mode; file.model = i.model ?? null; if (i.chaos !== undefined) file.chaos = i.chaos; + if (i.fixture) file.fixture = { ...i.fixture }; return file; } -/** A file name for a test: a slug of its title (at most 80 characters), its id and its seed. */ -export function racesFileName(test: string, id: string, seed: number): string { +/** A file name for a test: a slug of its title (at most 80 characters), its id, its seed and, for a retry, its attempt. */ +export function racesFileName(test: string, id: string, seed: number, retry = 0): string { const slug = test .toLowerCase() .replace(/[^a-z0-9]+/g, "-") @@ -131,7 +138,8 @@ export function racesFileName(test: string, id: string, seed: number): string { .slice(0, 80) .replace(/-+$/, ""); const safeId = id.replace(/[^A-Za-z0-9_-]+/g, ""); - return `${slug || "test"}-${safeId || "0"}-s${seed}.json`; + const attempt = Number.isInteger(retry) && retry > 0 ? `-r${retry}` : ""; + return `${slug || "test"}-${safeId || "0"}-s${seed}${attempt}.json`; } /** Writes the file (creating the directory) and returns its path. */ diff --git a/packages/playwright/src/schema.ts b/packages/playwright/src/schema.ts index 84a05dd6..14e8b199 100644 --- a/packages/playwright/src/schema.ts +++ b/packages/playwright/src/schema.ts @@ -93,6 +93,17 @@ export function validateRaces(value: unknown): RacesParseResult { if (value.model[k] !== undefined && typeof value.model[k] !== "string") errors.push(`model.${k}: expected a string when present, got ${describe(value.model[k])}`); } if (value.chaos !== undefined && value.chaos !== null && !isObject(value.chaos)) errors.push(`chaos: expected an object or null when present, got ${describe(value.chaos)}`); + if (value.fixture !== undefined) { + const x = value.fixture; + if (!isObject(x)) errors.push(`fixture: expected an object when present, got ${describe(x)}`); + else { + if (x.model !== "skip" && x.model !== "load") errors.push(`fixture.model: expected "skip" or "load", got ${describe(x.model)}`); + if (x.agent !== null && typeof x.agent !== "string") errors.push(`fixture.agent: expected a string or null, got ${describe(x.agent)}`); + if (x.chaosApplied !== null && !(isInt(x.chaosApplied) && x.chaosApplied >= 0)) errors.push(`fixture.chaosApplied: expected a whole number or null, got ${describe(x.chaosApplied)}`); + if (typeof x.serviceWorker !== "boolean") errors.push(`fixture.serviceWorker: expected true or false, got ${describe(x.serviceWorker)}`); + if (x.modeOverridden !== undefined && typeof x.modeOverridden !== "string") errors.push(`fixture.modeOverridden: expected a string when present, got ${describe(x.modeOverridden)}`); + } + } if (errors.length) return fail(errors); const file: GenClassRacesFile = { schema: RACES_SCHEMA, test: (value.test as string).trim(), seed: value.seed as number, signatures }; @@ -115,6 +126,11 @@ export function validateRaces(value: unknown): RacesParseResult { if (Array.isArray(c.routes)) out.routes = c.routes.slice(); file.chaos = out; } + if (isObject(value.fixture)) { + const x = value.fixture; + file.fixture = { model: x.model as "skip" | "load", agent: x.agent as string | null, chaosApplied: x.chaosApplied as number | null, serviceWorker: x.serviceWorker as boolean }; + if (typeof x.modeOverridden === "string") file.fixture.modeOverridden = x.modeOverridden; + } return { ok: true, file }; } diff --git a/packages/playwright/src/summarize.ts b/packages/playwright/src/summarize.ts index 6f7f8a37..27067aa9 100644 --- a/packages/playwright/src/summarize.ts +++ b/packages/playwright/src/summarize.ts @@ -108,10 +108,18 @@ export interface TestSummary { /** * Files that flagged nothing and whose model was not "ready" when the fixture collected (null model: "not * started"), so those runs may not have been checked; and their states. Files with detections are not counted, - * whatever the state (the runtime can unload an idle model after it ran), nor files whose state is "off" (no - * model by choice: the fixture's default, which still computes witnesses). + * whatever the state (the runtime can unload an idle model after it ran), nor files whose `fixture.model` is + * "skip" (no model by choice: `unchecked` covers those). */ modelNotReady: { files: number; states: string[] }; + /** + * Files the fixture wrote that flagged nothing although it could not check the page, and why: the agent entry did + * not connect (no witnesses), or a service worker controlled the page (chaos and the runtime's files went past + * page.route). + */ + unchecked: { files: number; reasons: string[] }; + /** Files whose chaos was on but held no request (nothing matched, or a user route or service worker answered first). */ + chaosIdle: number; /** The chaos of the first file (by name) that carries one. */ chaos?: GenClassRacesFile["chaos"]; } @@ -147,6 +155,9 @@ interface Acc { modes: Set; notReady: number; states: Set; + unchecked: number; + reasons: Set; + chaosIdle: number; chaos?: GenClassRacesFile["chaos"]; } @@ -173,7 +184,7 @@ export function buildSummary(inputs: readonly RacesFileInput[], options: Summari } files++; const f = r.file; - const t: Acc = tests.get(f.test) ?? { test: f.test, seeds: new Set(), files: 0, sigs: new Map(), modes: new Set(), notReady: 0, states: new Set() }; + const t: Acc = tests.get(f.test) ?? { test: f.test, seeds: new Set(), files: 0, sigs: new Map(), modes: new Set(), notReady: 0, states: new Set(), unchecked: 0, reasons: new Set(), chaosIdle: 0 }; tests.set(f.test, t); t.files++; t.seeds.add(f.seed); @@ -183,10 +194,21 @@ export function buildSummary(inputs: readonly RacesFileInput[], options: Summari if (!flaggedHere && m === null) { t.notReady++; t.states.add("not started"); - } else if (!flaggedHere && m && m.state !== undefined && m.state !== "ready" && m.state !== "off") { + } else if (!flaggedHere && m && m.state !== undefined && m.state !== "ready" && f.fixture?.model !== "skip") { t.notReady++; t.states.add(m.state); } + const fx = f.fixture; + if (fx) { + const why: string[] = []; + if (fx.agent !== null) why.push(`witnesses not recorded: ${fx.agent}`); + if (fx.serviceWorker) why.push("a service worker controlled the page"); + if (!flaggedHere && why.length) { + t.unchecked++; + for (const w of why) t.reasons.add(w); + } + if (fx.chaosApplied === 0) t.chaosIdle++; + } if (f.chaos && t.chaos === undefined) t.chaos = f.chaos; for (const s of f.signatures) { if (s.diagnosis === "expected") continue; @@ -236,6 +258,8 @@ export function buildSummary(inputs: readonly RacesFileInput[], options: Summari signatures: sigs, modes: [...t.modes].sort(), modelNotReady: { files: t.notReady, states: [...t.states].sort() }, + unchecked: { files: t.unchecked, reasons: [...t.reasons].sort() }, + chaosIdle: t.chaosIdle, }; if (t.chaos !== undefined) ts.chaos = t.chaos; out.push(ts); @@ -320,6 +344,20 @@ function signatureLines(s: SignatureSummary, t: TestSummary, maxEvidence: number return lines; } +/** The notes under a flagged test about what the fixture could not do (none for files without a `fixture`). */ +function fixtureNotes(t: TestSummary): string[] { + const out: string[] = []; + if (t.unchecked.files > 0) + out.push( + `Note: in ${t.unchecked.files} of ${plural(t.files, "file")} for this test nothing was flagged and the fixture could not check the page (${t.unchecked.reasons.map(escapeMarkdown).join("; ")}), so those runs may not have been checked.`, + ); + if (t.chaosIdle > 0) + out.push( + `Note: in ${t.chaosIdle} of ${plural(t.files, "file")} for this test chaos was on but held no request (nothing matched its filter, or a route of the test or a service worker answered first).`, + ); + return out; +} + /** How much of the report is shown: list entries (nothing flagged, files skipped), signatures per test, tests. */ interface Limits { items: number; @@ -443,6 +481,7 @@ function render(s: RaceSummary, options: SummarizeOptions, lim: Limits, maxChars ); L.push(""); } + for (const n of fixtureNotes(t)) L.push(n, ""); } if (flagged.length > lim.tests) { L.push(`_…and ${plural(flagged.length - lim.tests, "more test")} with signatures, not shown._`); @@ -454,7 +493,9 @@ function render(s: RaceSummary, options: SummarizeOptions, lim: Limits, maxChars L.push(""); for (const t of quiet.slice(0, lim.items)) { const notReady = t.modelNotReady.files > 0 ? `; model not ready in ${t.modelNotReady.files} of ${plural(t.files, "file")} (${t.modelNotReady.states.map(escapeMarkdown).join(", ")})` : ""; - L.push(`- ${escapeMarkdown(t.test)} (${plural(t.seeds.length, "seed")}: ${seedList(t.seeds)}${notReady})`); + const unchecked = t.unchecked.files > 0 ? `; not checked in ${t.unchecked.files} of ${plural(t.files, "file")} (${t.unchecked.reasons.map(escapeMarkdown).join("; ")})` : ""; + const idle = t.chaosIdle > 0 ? `; chaos held no request in ${t.chaosIdle} of ${plural(t.files, "file")}` : ""; + L.push(`- ${escapeMarkdown(t.test)} (${plural(t.seeds.length, "seed")}: ${seedList(t.seeds)}${notReady}${unchecked}${idle})`); } if (quiet.length > lim.items) L.push(`- …and ${quiet.length - lim.items} more`); L.push(""); diff --git a/packages/playwright/src/types.ts b/packages/playwright/src/types.ts index 2bb34bd0..d19b8a53 100644 --- a/packages/playwright/src/types.ts +++ b/packages/playwright/src/types.ts @@ -54,12 +54,29 @@ export interface GenClassRacesFile { * Optional: the page's model status when the fixture collected (null: GenClass never started in the page). When a * file flagged nothing and its model was not "ready" (for example "loading"), the report notes that the run may * not have been checked. A file with detections is reported whatever the state: the runtime can unload an idle - * model after it ran ("unloaded"). State "off" (no model by choice: the fixture's default, which still computes - * witnesses) gets no note either. + * model after it ran ("unloaded"). A file whose `fixture.model` is "skip" gets no model note (no model by choice); + * the agent state decides instead. */ model?: { state?: string; name?: string; version?: string; device?: string } | null; /** Optional: the chaos the fixture applied (its `chaos` option: `latencyMs` is its `delayMs`), shown when present. */ chaos?: { latencyMs?: [number, number]; reorder?: boolean; failRate?: number; routes?: unknown[] } | null; + /** + * Optional, written by the fixture: what it could and could not do on the page, so the report can tell a run it + * checked from one it could not. `model`: the fixture's model option. `agent`: null when the agent entry connected + * (witnesses recorded), else why not. `chaosApplied`: how many requests chaos held (null: chaos off). `serviceWorker`: + * a service worker controlled the page (page.route, and so chaos, does not see what it handles). + * `modeOverridden`: the mode a URL or localStorage switch had asked for, set back to observe. + */ + fixture?: RacesFixtureInfo; +} + +/** The fixture's own record in a race file (GenClassRacesFile.fixture). */ +export interface RacesFixtureInfo { + model: "skip" | "load"; + agent: string | null; + chaosApplied: number | null; + serviceWorker: boolean; + modeOverridden?: string; } /** One input of the summariser: a file's name (shown in the report) and its content (JSON text or parsed). */ diff --git a/packages/playwright/test/chaos.test.ts b/packages/playwright/test/chaos.test.ts index 84e70210..729bc62b 100644 --- a/packages/playwright/test/chaos.test.ts +++ b/packages/playwright/test/chaos.test.ts @@ -3,7 +3,7 @@ // overtakes. The scheduler is driven with hand-cranked timers, so nothing here waits on the clock. import { describe, expect, it } from "vitest"; -import { ChaosScheduler, DEFAULT_DELAY_MS, MAX_DELAY_MS, RUNTIME_PATH, confirms, delayFor, describeMatch, hash32, isCandidate, isRuntimeUrl, mulberry32, resolveChaos, type ChaosRequest } from "../src/chaos.js"; +import { ChaosScheduler, DEFAULT_DELAY_MS, FIFO_GAP_MS, MAX_DELAY_MS, RUNTIME_PATH, delayFor, describeMatch, hash32, isCandidate, isRuntimeUrl, mulberry32, resolveChaos, type ChaosRequest } from "../src/chaos.js"; function manualTimers() { const q: { at: number; fn: () => void; id: number }[] = []; @@ -21,15 +21,19 @@ function manualTimers() { if (i >= 0) q.splice(i, 1); }, }, + /** Moves time forward 1 ms at a time, firing what is due and letting promise chains queue new timers. */ async advance(ms: number): Promise { - now += ms; - for (const x of [...q].filter((x) => x.at <= now).sort((a, b) => a.at - b.at)) { - q.splice(q.indexOf(x), 1); - x.fn(); - await Promise.resolve(); + for (let i = 0; i < ms; i++) { + now += 1; + for (const x of [...q].filter((x) => x.at <= now).sort((a, b) => a.at - b.at)) { + q.splice(q.indexOf(x), 1); + x.fn(); + } + for (let k = 0; k < 5; k++) await Promise.resolve(); } await new Promise((r) => setTimeout(r, 0)); }, + now: () => now, pending: () => q.length, }; } @@ -85,15 +89,17 @@ describe("chaos: the filter", () => { expect(isRuntimeUrl("http://a/api/__genclass__/x")).toBe(false); }); - it("by default: same-origin requests, confirmed by an /api path or a JSON response", () => { + it("by default: same-origin requests with an /api path or an Accept header asking for JSON, decided from the request alone", () => { + expect(isCandidate(req(), undefined)).toBe(true); // /api path expect(isCandidate(req({ sameOrigin: false }), undefined)).toBe(false); - expect(confirms(req(), undefined, "text/html")).toBe(true); // /api path - expect(confirms(req({ url: "http://app.test/graphql" }), undefined, "application/json; charset=utf-8")).toBe(true); - expect(confirms(req({ url: "http://app.test/graphql" }), undefined, "text/html")).toBe(false); - expect(confirms(req({ url: "http://app.test/graphql" }), undefined, null)).toBe(false); + expect(isCandidate(req({ url: "http://app.test/graphql", accept: "application/json, text/plain, */*" }), undefined)).toBe(true); + expect(isCandidate(req({ url: "http://app.test/graphql", accept: "*/*" }), undefined)).toBe(false); + // a stream or any other same-origin fetch outside the filter is never held + expect(isCandidate(req({ url: "http://app.test/events", accept: "text/event-stream" }), undefined)).toBe(false); + expect(isCandidate(req({ url: "http://app.test/fragment" }), undefined)).toBe(false); }); - it("with a match: a path prefix, a URL prefix, a RegExp, a function or a list, decided before the fetch", () => { + it("with a match: a path prefix, a URL prefix, a RegExp, a function or a list", () => { expect(isCandidate(req({ url: "http://app.test/graphql" }), "/graphql")).toBe(true); expect(isCandidate(req({ url: "http://app.test/graphql" }), "/api")).toBe(false); expect(isCandidate(req({ url: "https://api.other.com/v1/x", sameOrigin: false }), "https://api.other.com/")).toBe(true); @@ -103,12 +109,10 @@ describe("chaos: the filter", () => { expect(isCandidate(req({ method: "GET" }), (r) => r.method === "POST")).toBe(false); expect(isCandidate(req({ url: "http://app.test/graphql" }), ["/api", "/graphql"])).toBe(true); expect(isCandidate(req({ url: "http://app.test/other" }), ["/api", "/graphql"])).toBe(false); - // the filter already said yes: the response does not matter - expect(confirms(req({ url: "http://app.test/other" }), "/other", "text/plain")).toBe(true); }); it("describeMatch for the race file", () => { - expect(describeMatch(undefined)).toEqual(["same-origin /api* or JSON responses"]); + expect(describeMatch(undefined)).toEqual(["same-origin /api* or Accept: json"]); expect(describeMatch(["/api", /x\/y/, () => true])).toEqual(["/api", "/x\\/y/", "function"]); expect(describeMatch("https://a/")).toEqual(["https://a/"]); }); @@ -130,6 +134,8 @@ describe("chaos: the scheduler", () => { for (const p of pa) expect(p.delayMs >= 0 && p.delayMs <= 600).toBe(true); expect(a.plans).toEqual(pa); expect(a.plan("GET", "https://other.test/api/x", false).key).toBe("GET other.test/api/x#1"); + // a cross-origin backend on another random port keeps its key, so the seed gives it the same delay + expect(b.plan("GET", "http://127.0.0.1:41234/api/x", false).key).toBe(c.plan("GET", "http://127.0.0.1:50999/api/x", false).key); }); it("reorder on: each response waits for its own delay only, so a later one can overtake", async () => { @@ -145,18 +151,31 @@ describe("chaos: the scheduler", () => { expect(order).toEqual(["B", "A"]); }); - it("reorder off: a response is never released before an earlier matched one (latency only)", async () => { + it("reorder off: requests leave in the order they were sent, at least FIFO_GAP_MS apart (latency only)", async () => { const t = manualTimers(); const s = new ChaosScheduler(resolveChaos({ seed: 1, delayMs: [0, 600], reorder: false }), t.timers); - const order: string[] = []; - const a = s.hold({ key: "A", delayMs: 300 }).then(() => order.push("A")); - const b = s.hold({ key: "B", delayMs: 50 }).then(() => order.push("B")); - const c = s.hold({ key: "C", delayMs: 0 }).then(() => order.push("C")); + const at: Record = {}; + const a = s.hold({ key: "A", delayMs: 300 }).then(() => (at.A = t.now())); + const b = s.hold({ key: "B", delayMs: 50 }).then(() => (at.B = t.now())); + const c = s.hold({ key: "C", delayMs: 0 }).then(() => (at.C = t.now())); await t.advance(100); - expect(order).toEqual([]); - await t.advance(300); + expect(at).toEqual({}); + await t.advance(400); await Promise.all([a, b, c]); - expect(order).toEqual(["A", "B", "C"]); + expect(Object.keys(at).sort((x, y) => at[x] - at[y])).toEqual(["A", "B", "C"]); + expect(at.A).toBe(300); + expect(at.B - at.A).toBeGreaterThanOrEqual(FIFO_GAP_MS); + expect(at.C - at.B).toBeGreaterThanOrEqual(FIFO_GAP_MS); + // a request whose own delay is longer than the gap leaves on its own delay + const late = new ChaosScheduler(resolveChaos({ seed: 1, reorder: false }), t.timers); + const t0 = t.now(); + let first = 0; + let second = 0; + void late.hold({ key: "X", delayMs: 0 }).then(() => (first = t.now() - t0)); + void late.hold({ key: "Y", delayMs: 200 }).then(() => (second = t.now() - t0)); + await t.advance(250); + expect(first).toBeLessThanOrEqual(1); // at once (seen at the next tick of the hand-cranked clock) + expect(second).toBe(200); }); it("a zero delay resolves at once; dispose releases every hold and clears the timers", async () => { diff --git a/packages/playwright/test/e2e/run.mjs b/packages/playwright/test/e2e/run.mjs index bc9f4db1..78574bf9 100644 --- a/packages/playwright/test/e2e/run.mjs +++ b/packages/playwright/test/e2e/run.mjs @@ -2,11 +2,13 @@ // @genclass/runtime and this package). It runs Playwright with ./playwright.config.ts (the search page of server.mjs, // the fixture with seeded chaos, five seeds per test, this package's reporter), then checks what was written: // - the buggy page: a `witness|overwrote-newer|GET /api/search` signature in at least one of the five seeds (the -// count is printed: a race shows under some orders, not all); +// count is printed: a race shows under some orders, not all) and no other signature; // - the buggy page with `reorder: false` (latency only, responses keep their order): no signature in any seed, // since nothing overtakes; // - the fixed page: no signature at all in any of the five seeds (the false-report count on the correct variant); -// - every file validates as genclass-races/1 and carries the seed and the chaos settings; +// - every file validates as genclass-races/1 and carries the seed, the chaos settings and a fixture record saying +// the agent connected and chaos held requests; +// - (in search.spec.ts) no page logged a console error or had a failed request; // - genclass-report.md names the buggy test under the witness group and lists the fixed test under "nothing // flagged"; the command renders the same report from the same files. // Exit code 0 when all hold; the summary is printed as JSON. @@ -63,6 +65,8 @@ for (const f of files) { if (!f.file.chaos || !f.file.chaos.latencyMs || f.file.chaos.latencyMs[0] !== 0 || f.file.chaos.latencyMs[1] !== 600 || f.file.chaos.reorder !== reorder) fail(`${f.name}: chaos settings missing or wrong: ${JSON.stringify(f.file.chaos)}`); if (f.file.mode !== "observe") fail(`${f.name}: mode ${f.file.mode}, expected observe`); if (!f.file.model || f.file.model.state !== "off") fail(`${f.name}: model ${JSON.stringify(f.file.model)}, expected state "off" (model skipped)`); + const fx = f.file.fixture; + if (!fx || fx.model !== "skip" || fx.agent !== null || fx.serviceWorker !== false || !(fx.chaosApplied > 0)) fail(`${f.name}: fixture record ${JSON.stringify(fx)}, expected the agent connected and chaos applied`); } const SIG = "witness|overwrote-newer|GET /api/search"; const stale = (f) => f.file.signatures.some((s) => s.sig === SIG); @@ -71,6 +75,7 @@ summary.buggy = { staleSeeds: buggy.filter(stale).map((f) => f.file.seed), other summary.latencyOnly = { signatures: sigs(fifo) }; summary.fixed = { signatures: sigs(fixed) }; if (summary.buggy.staleSeeds.length === 0) fail("no overwrote-newer witness on the buggy page in any seed"); +if (summary.buggy.otherSignatures.length !== 0) fail(`other signatures on the buggy page: ${summary.buggy.otherSignatures.join(", ")}`); if (summary.latencyOnly.signatures.length !== 0) fail(`signatures on the buggy page with reorder off (latency keeps the order, so nothing should overtake): ${summary.latencyOnly.signatures.join(", ")}`); if (summary.fixed.signatures.length !== 0) fail(`signatures on the fixed page (false reports): ${summary.fixed.signatures.join(", ")}`); diff --git a/packages/playwright/test/e2e/search.spec.ts b/packages/playwright/test/e2e/search.spec.ts index f202de50..eb73616a 100644 --- a/packages/playwright/test/e2e/search.spec.ts +++ b/packages/playwright/test/e2e/search.spec.ts @@ -7,6 +7,13 @@ import { expect, test } from "@genclass/playwright/fixture"; async function search(page: import("@playwright/test").Page, path: string): Promise { + // the fixture must leave no trace a suite would fail on: no console error, no failed request + const noise: string[] = []; + page.on("console", (m) => { + if (m.type() === "error") noise.push(`console.error: ${m.text()}`); + }); + page.on("requestfailed", (r) => noise.push(`requestfailed: ${r.url()} ${r.failure()?.errorText ?? ""}`)); + page.on("pageerror", (e) => noise.push(`pageerror: ${e.message}`)); await page.goto(path); await page.getByLabel("Search").pressSequentially("react", { delay: 80 }); // every answer has landed: nothing in flight on the fixture's runtime (its chaos holds each one up to 600 ms) @@ -17,6 +24,7 @@ async function search(page: import("@playwright/test").Page, path: string): Prom }, null, { timeout: 15_000 }); await page.waitForTimeout(300); await expect(page.locator("#results li")).toHaveCount(2); + expect(noise).toEqual([]); } test.describe("search", () => { diff --git a/packages/playwright/test/fixture.test.ts b/packages/playwright/test/fixture.test.ts index 95c615be..a82b3320 100644 --- a/packages/playwright/test/fixture.test.ts +++ b/packages/playwright/test/fixture.test.ts @@ -54,7 +54,7 @@ function info(over: Partial = {}): TestInfoLike { file: join(root, "tests", "checkout.spec.ts"), testId: "a1b2c3-d4e5f6", repeatEachIndex: 0, - project: { name: "chromium", testDir: join(root, "tests") }, + project: { name: "chromium", testDir: join(root, "tests"), outputDir: join(root, "test-results") }, config: { configFile: join(root, "playwright.config.ts"), rootDir: join(root, "tests") }, ...over, }; @@ -68,11 +68,13 @@ describe("fixture: names and places", () => { expect(testName(info({ titlePath: ["weird", "shape"], file: join(root, "tests", "x.spec.ts"), project: { name: "", testDir: join(root, "tests") } }))).toBe("weird > shape"); }); - it("outDirOf: test-results/genclass next to the config by default; a relative outDir against it; an absolute one as is", () => { + it("outDirOf: genclass in the project's outputDir by default; a relative outDir against the config; an absolute one as is", () => { expect(outDirOf(info())).toBe(join(root, "test-results", "genclass")); + expect(outDirOf(info({ project: { name: "p", testDir: join(root, "tests"), outputDir: join(root, "out", "p") } }))).toBe(join(root, "out", "p", "genclass")); + expect(outDirOf(info({ project: { name: "p", testDir: join(root, "tests") } }))).toBe(join(root, "test-results", "genclass")); expect(outDirOf(info(), "out/races")).toBe(join(root, "out", "races")); expect(outDirOf(info(), join(root, "abs"))).toBe(join(root, "abs")); - expect(outDirOf(info({ config: { rootDir: join(root, "tests") } }))).toBe(join(root, "tests", "test-results", "genclass")); + expect(outDirOf(info({ config: { rootDir: join(root, "tests") }, project: { name: "", testDir: join(root, "tests") } }))).toBe(join(root, "tests", "test-results", "genclass")); }); }); @@ -110,10 +112,19 @@ describe("fixture: startGenClass", () => { mode: "observe", model: { state: "off", name: "none" }, chaos: null, + fixture: { model: "skip", agent: null, chaosApplied: null, serviceWorker: false }, }); expect(r.file).toEqual(parsed.ok && parsed.file); }); + it("a retry writes its own file; a test skipped in its body writes none", async () => { + const retry = await startGenClass(new FakePage(), { runtimeDir }, info({ retry: 1 })); + expect((await retry.finish()).path).toMatch(/-s1-r1\.json$/); + const skipped = await startGenClass(new FakePage(), { runtimeDir }, info({ status: "skipped", testId: "skipped-id" })); + expect(await skipped.finish()).toMatchObject({ file: null, path: null }); + expect(readdirSync(join(root, "test-results", "genclass")).some((n) => n.includes("skipped-id"))).toBe(false); + }); + it("with chaos: the seed is the chaos seed plus the repeat index, and the settings are in the file", async () => { const page = new FakePage(); page.evaluateResult = { mode: "observe", model: { state: "off" }, agent: null }; @@ -121,7 +132,7 @@ describe("fixture: startGenClass", () => { expect(page.routes[0]).toBe("**/*"); const r = await run.finish(); expect(r.path).toBe(join(root, "races", "chromium-checkout-spec-ts-checkout-places-one-order-a1b2c3-d4e5f6-s11.json")); - expect(r.file).toMatchObject({ seed: 11, chaos: { latencyMs: [100, 200], reorder: false, routes: ["/api", "/x/"] } }); + expect(r.file).toMatchObject({ seed: 11, chaos: { latencyMs: [100, 200], reorder: false, routes: ["/api", "/x/"] }, fixture: { model: "skip", chaosApplied: 0 } }); expect(parseRaces(readFileSync(r.path!, "utf8")).ok).toBe(true); }); @@ -129,6 +140,6 @@ describe("fixture: startGenClass", () => { const page = new FakePage(); const run = await startGenClass(page, { runtimeDir, chaos: true }, info()); const r = await run.finish(); - expect(r.file).toMatchObject({ seed: 1, chaos: { latencyMs: [0, 600], reorder: true, routes: ["same-origin /api* or JSON responses"] }, model: null, signatures: [] }); + expect(r.file).toMatchObject({ seed: 1, chaos: { latencyMs: [0, 600], reorder: true, routes: ["same-origin /api* or Accept: json"] }, model: null, signatures: [], fixture: { model: "skip", agent: "not-loaded", chaosApplied: 0, serviceWorker: false } }); }); }); diff --git a/packages/playwright/test/install.test.ts b/packages/playwright/test/install.test.ts index 9c7694d0..b96ebf4f 100644 --- a/packages/playwright/test/install.test.ts +++ b/packages/playwright/test/install.test.ts @@ -5,7 +5,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync, existsSync } from "node: import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { AGENT_URL, BINDING, initScriptContent, install, resolveRuntimeDir, runtimeFile, type PageLike, type RequestLike, type ResponseLike, type RouteLike } from "../src/install.js"; +import { AGENT_URL, BINDING, initScriptContent, install, isAgentUrl, resolveRuntimeDir, runtimeFile, type PageLike, type RequestLike, type RouteLike } from "../src/install.js"; import { RUNTIME_PATH, resolveChaos } from "../src/chaos.js"; import { here } from "./helpers.js"; @@ -47,30 +47,25 @@ class FakePage implements PageLike { class FakeRoute implements RouteLike { calls: string[] = []; fulfilled: Parameters[0] | null = null; - constructor( - private readonly response: ResponseLike | Error = { headers: () => ({ "content-type": "application/json" }) }, - ) {} - async fetch(): Promise { - this.calls.push("fetch"); - if (this.response instanceof Error) throw this.response; - return this.response; - } + at = -1; async fulfill(o: Parameters[0]): Promise { this.calls.push("fulfill"); this.fulfilled = o; } async fallback(): Promise { this.calls.push("fallback"); + this.at = Date.now(); } async abort(code?: string): Promise { this.calls.push(`abort:${code ?? ""}`); } } -const request = (url: string, o: { method?: string; resourceType?: string; frameUrl?: string } = {}): RequestLike => ({ +const request = (url: string, o: { method?: string; resourceType?: string; frameUrl?: string; accept?: string } = {}): RequestLike => ({ url: () => url, method: () => o.method ?? "GET", resourceType: () => o.resourceType ?? "fetch", + headers: () => (o.accept ? { accept: o.accept } : {}), frame: () => ({ url: () => o.frameUrl ?? "http://app.test/page" }), }); @@ -130,6 +125,18 @@ describe("install: what goes on the page", () => { expect(script).toContain('telemetry: false'); expect(script).toContain('mode: "observe"'); expect(script).toContain("hold: false"); + // the agent link gets a stand-in fetch for its own URL (no request, no console error), and no sink is configured + expect(script).toContain("standIn"); + expect(script).not.toContain("sinks:"); + expect(script).toContain('"decide"'); + expect(script).toContain('setMode'); + }); + + it("isAgentUrl: the agent's address only, not an app on 127.0.0.1:9xxx", () => { + expect(isAgentUrl(AGENT_URL)).toBe(true); + expect(isAgentUrl(`${AGENT_URL}/v1/hello`)).toBe(true); + expect(isAgentUrl("http://127.0.0.1:9000/api/search")).toBe(false); + expect(isAgentUrl("http://127.0.0.1:9323/")).toBe(false); }); it("the init script is the page function applied to its config and the build, as plain JavaScript", () => { @@ -198,35 +205,44 @@ describe("install: chaos routes", () => { } }); - it("fetches a candidate, delays a confirmed one and fulfils with the real response; a non-JSON non-API response goes through at once", async () => { + it("holds a matched request for its planned time, then passes it on with fallback (never fetches it); unmatched ones go at once", async () => { const page = new FakePage(); - const session = await install(page, { runtimeDir, model: "skip", chaos: resolveChaos({ seed: 1, delayMs: [5, 5] }) }); + const session = await install(page, { runtimeDir, model: "skip", chaos: resolveChaos({ seed: 1, delayMs: [40, 40] }) }); const chaos = page.handler("**/*"); - const api = new FakeRoute({ headers: () => ({ "content-type": "text/html" }) }); const t0 = Date.now(); + const api = new FakeRoute(); await chaos(api, request("http://app.test/api/items?page=2", { resourceType: "xhr" })); - expect(api.calls).toEqual(["fetch", "fulfill"]); - expect(Date.now() - t0).toBeGreaterThanOrEqual(4); - const json = new FakeRoute({ headers: () => ({ "Content-Type": "application/json" }) }); - await chaos(json, request("http://app.test/graphql", { method: "POST" })); - expect(json.calls).toEqual(["fetch", "fulfill"]); - const html = new FakeRoute({ headers: () => ({ "content-type": "text/html" }) }); - await chaos(html, request("http://app.test/fragment")); - expect(html.calls).toEqual(["fetch", "fulfill"]); + expect(api.calls).toEqual(["fallback"]); + expect(api.at - t0).toBeGreaterThanOrEqual(35); + const json = new FakeRoute(); + await chaos(json, request("http://app.test/graphql", { method: "POST", accept: "application/json" })); + expect(json.calls).toEqual(["fallback"]); + const t1 = Date.now(); + const stream = new FakeRoute(); + await chaos(stream, request("http://app.test/events", { accept: "text/event-stream" })); + expect(stream.calls).toEqual(["fallback"]); + expect(stream.at - t1).toBeLessThan(30); const c = await session.finish(); expect(c.plans.map((p) => p.key)).toEqual(["GET /api/items#1", "POST /graphql#1"]); - expect(c.plans.every((p) => p.delayMs === 5)).toBe(true); + expect(c.plans.every((p) => p.delayMs === 40)).toBe(true); + }); + + it("keys requests in the order the page sent them, whatever order they are released in", async () => { + const page = new FakePage(); + const session = await install(page, { runtimeDir, model: "skip", chaos: resolveChaos({ seed: 7, delayMs: [0, 80] }) }); + const chaos = page.handler("**/*"); + const urls = ["a", "b", "c", "d"].map((q) => `http://app.test/api/search?q=${q}`); + await Promise.all(urls.map((u) => chaos(new FakeRoute(), request(u)))); + const c = await session.finish(); + expect(c.plans.map((p) => p.key)).toEqual(["GET /api/search#1", "GET /api/search#2", "GET /api/search#3", "GET /api/search#4"]); }); - it("a request that fails to fetch is aborted so the app sees the failure; a closed page is not an error", async () => { + it("a page that closes while a request is held is not an error", async () => { const page = new FakePage(); await install(page, { runtimeDir, model: "skip", chaos: resolveChaos({ seed: 1, delayMs: [0, 0] }) }); const chaos = page.handler("**/*"); - const down = new FakeRoute(new Error("connection refused")); - await chaos(down, request("http://app.test/api/x")); - expect(down.calls).toEqual(["fetch", "abort:failed"]); - const gone = new FakeRoute({ headers: () => ({ "content-type": "application/json" }) }); - gone.fulfill = async () => { + const gone = new FakeRoute(); + gone.fallback = async () => { throw new Error("Target page, context or browser has been closed"); }; await expect(chaos(gone, request("http://app.test/api/x"))).resolves.toBeUndefined(); @@ -241,12 +257,21 @@ describe("install: collecting", () => { page.push({ t: "record", record: { kind: "detection", diagnosis: "stale" } }); page.push({ t: "witness", witness: { kind: "overwrote-newer" } }); page.push({ t: "agent", ok: true, reason: null }); + page.push({ t: "status", mode: "observe", model: { state: "off", name: "none" }, overridden: "heal" }); page.push({ t: "nonsense" }); page.push(null); - page.evaluateResult = { mode: "observe", model: { state: "off", name: "none", version: "0" }, agent: null }; + page.evaluateResult = { mode: "observe", model: { state: "off", name: "none", version: "0" }, agent: null, serviceWorker: true }; const c = await session.finish(); expect(page.evaluated).toHaveLength(1); - expect(c).toEqual({ records: [{ kind: "detection", diagnosis: "stale" }], witnesses: [{ kind: "overwrote-newer" }], mode: "observe", model: { state: "off", name: "none", version: "0" }, agent: null, plans: [] }); + expect(c).toEqual({ records: [{ kind: "detection", diagnosis: "stale" }], witnesses: [{ kind: "overwrote-newer" }], mode: "observe", model: { state: "off", name: "none", version: "0" }, agent: null, plans: [], serviceWorker: true, modeOverridden: "heal" }); + }); + + it("keeps a specific reason the page pushed over collect()'s generic not-loaded", async () => { + const page = new FakePage(); + const session = await install(page, { runtimeDir, model: "skip", chaos: null }); + page.push({ t: "agent", ok: false, reason: "Failed to fetch dynamically imported module" }); + page.evaluateResult = { mode: "observe", model: { state: "off" }, agent: "not-loaded" }; + expect((await session.finish()).agent).toBe("Failed to fetch dynamically imported module"); }); it("a page that never started GenClass, or that is closed, or whose agent failed", async () => { diff --git a/packages/playwright/test/races.test.ts b/packages/playwright/test/races.test.ts index eeea212b..71383ba8 100644 --- a/packages/playwright/test/races.test.ts +++ b/packages/playwright/test/races.test.ts @@ -36,10 +36,13 @@ describe("races: operations and evidence lines", () => { expect(opOf("write to cart.items")).toBe("write to cart.items"); expect(opOf(undefined)).toBe(""); expect(opOf(`x${"y".repeat(200)}`)).toHaveLength(120); + // per-run op ids come out, so one detection is one signature across seeds + expect(opOf("relation total == sum(items) (#12)")).toBe("relation total == sum(items)"); + expect(opOf("write to cart.items (#7)")).toBe(opOf("write to cart.items (#31)")); }); it("witnessLine says what happened, in the agent's request keys", () => { - expect(witnessLine(overwrote)).toBe("The response of GET /api/search#1 wrote search.results over what newer GET /api/search#4 had written 0.26s after GET /api/search#1 started (2 items → 2 items)."); + expect(witnessLine(overwrote)).toBe("The response of GET /api/search#1 wrote search.results over what newer GET /api/search#4, started 0.26s after GET /api/search#1, had written (2 items → 2 items)."); expect(witnessLine({ ...overwrote, overKind: "user", over: 'type textbox "Title"', change: undefined })).toBe('The response of GET /api/search#1 wrote search.results over what the user (type textbox "Title") had written 0.26s after GET /api/search#1 started.'); expect(witnessLine(identical)).toBe("POST /api/orders#2 was sent while the identical POST /api/orders#1 was still in flight (0.03s earlier), from a separate user action."); expect(witnessLine({ kind: "other", by: "A", over: "B" })).toBe("other: by A over B"); @@ -89,6 +92,21 @@ describe("races: signatures and the file", () => { ]); }); + it("carries the fixture's record and validates it; a bad record is reported, not half-read", () => { + const fixture = { model: "skip" as const, agent: null, chaosApplied: 3, serviceWorker: false, modeOverridden: "heal" }; + const file = buildRacesFile({ test: "t", seed: 1, mode: "observe", model: { state: "off" }, chaos: null, records: [], witnesses: [], fixture }); + const r = validateRaces(file); + expect(r.ok && r.file.fixture).toEqual(fixture); + const bad = validateRaces({ ...file, fixture: { model: "maybe", agent: 1, chaosApplied: -1, serviceWorker: "no" } }); + expect(bad.ok).toBe(false); + if (!bad.ok) expect(bad.errors).toEqual([ + 'fixture.model: expected "skip" or "load", got "maybe"', + "fixture.agent: expected a string or null, got 1", + "fixture.chaosApplied: expected a whole number or null, got -1", + 'fixture.serviceWorker: expected true or false, got "no"', + ]); + }); + it("a page where GenClass never started: model null, no mode, no signatures (still a valid file)", () => { const file = buildRacesFile({ test: "t", seed: 1, mode: null, model: null, chaos: null, records: [], witnesses: [] }); expect(file).toEqual({ schema: "genclass-races/1", test: "t", seed: 1, signatures: [], model: null, chaos: null }); @@ -100,6 +118,9 @@ describe("races: signatures and the file", () => { expect(racesFileName("[chromium] checkout.spec.ts > checkout > places one order", "a1b2-c3d4", 2)).toBe("chromium-checkout-spec-ts-checkout-places-one-order-a1b2-c3d4-s2.json"); expect(racesFileName("x".repeat(200), "id", 1)).toBe(`${"x".repeat(80)}-id-s1.json`); expect(racesFileName("", "../../etc", 1)).toBe("test-etc-s1.json"); + // a retry gets a file of its own, so the failing first attempt is kept + expect(racesFileName("t", "id", 2, 1)).toBe("t-id-s2-r1.json"); + expect(racesFileName("t", "id", 2, 0)).toBe("t-id-s2.json"); }); }); diff --git a/packages/playwright/test/witness-report.test.ts b/packages/playwright/test/witness-report.test.ts index 6dc28e21..f7ce392e 100644 --- a/packages/playwright/test/witness-report.test.ts +++ b/packages/playwright/test/witness-report.test.ts @@ -17,7 +17,8 @@ const witness = (op: string, seed: number, kind = "overwrote-newer"): RaceSignat seeds: [seed], }); const model = (op: string, seed: number): RaceSignature => ({ sig: `delivery|stale|${op}`, trigger: "delivery", diagnosis: "stale", op, confidence: 0.91, evidence: "the model's line", seeds: [seed] }); -const file = (test: string, seed: number, signatures: RaceSignature[], extra: Partial = {}): GenClassRacesFile => ({ schema: "genclass-races/1", test, seed, signatures, mode: "observe", model: { state: "off", name: "none" }, ...extra }); +const fixture = { model: "skip" as const, agent: null, chaosApplied: 4, serviceWorker: false }; +const file = (test: string, seed: number, signatures: RaceSignature[], extra: Partial = {}): GenClassRacesFile => ({ schema: "genclass-races/1", test, seed, signatures, mode: "observe", model: { state: "off", name: "none" }, fixture, ...extra }); describe("witnesses: grouping", () => { it("categoryOf puts trigger witness in its own group whatever the label, and the group is last", () => { @@ -69,15 +70,30 @@ describe("witnesses: the report", () => { expect(footer.split("\n\n")).toHaveLength(3); }); - it("buildSummary: state off is not 'not ready'; loading still is", () => { + it("model notes: a fixture file with model skip gets none; a model-only file with state off still does", () => { const s = buildSummary([ - { name: "a.json", content: file("t", 1, [], { model: { state: "off" } }) }, - { name: "b.json", content: file("t", 2, [], { model: { state: "loading" } }) }, + { name: "a.json", content: file("t", 1, []) }, + { name: "b.json", content: file("t", 2, [], { model: { state: "loading" }, fixture: { ...fixture, model: "load" } }) }, + { name: "c.json", content: { schema: "genclass-races/1", test: "u", seed: 1, signatures: [], model: { state: "off" } } }, ]); - expect(s.tests[0].modelNotReady).toEqual({ files: 1, states: ["loading"] }); + const [t, u] = [s.tests.find((x) => x.test === "t")!, s.tests.find((x) => x.test === "u")!]; + expect(t.modelNotReady).toEqual({ files: 1, states: ["loading"] }); + expect(u.modelNotReady).toEqual({ files: 1, states: ["off"] }); expect(s.totals.byCategory.witness).toEqual({ signatures: 0, reproducible: 0 }); }); + it("notes what the fixture could not do: no agent, a service worker, chaos that held nothing", () => { + const md = summarize([ + { name: "a.json", content: file("quiet", 1, [], { fixture: { ...fixture, agent: "not-loaded" } }) }, + { name: "b.json", content: file("quiet", 2, [], { fixture: { ...fixture, serviceWorker: true, chaosApplied: 0 } }) }, + { name: "c.json", content: file("flagged", 1, [witness("GET /api/search", 1)], { fixture: { ...fixture, chaosApplied: 0 } }) }, + { name: "d.json", content: file("flagged", 2, [], { fixture: { ...fixture, agent: "not-loaded" } }) }, + ]); + expect(md).toContain("- quiet (2 seeds: 1, 2; not checked in 2 of 2 files (a service worker controlled the page; witnesses not recorded: not-loaded); chaos held no request in 1 of 2 files)\n"); + expect(md).toContain("Note: in 1 of 2 files for this test nothing was flagged and the fixture could not check the page (witnesses not recorded: not-loaded), so those runs may not have been checked.\n"); + expect(md).toContain("Note: in 1 of 2 files for this test chaos was on but held no request (nothing matched its filter, or a route of the test or a service worker answered first).\n"); + }); + it("maxChars keeps witness lines like any other signature", () => { const sigs = Array.from({ length: 150 }, (_, i) => witness(`GET /api/r${i}`, 1)); const md = summarize([{ name: "big.json", content: file("big", 1, sigs) }], { maxChars: 4000 });