diff --git a/.github/workflows/playwright-reporter.yml b/.github/workflows/playwright-reporter.yml index 9a6dce28..de5c8d82 100644 --- a/.github/workflows/playwright-reporter.yml +++ b/.github/workflows/playwright-reporter.yml @@ -1,20 +1,29 @@ name: Playwright reporter -# @genclass/playwright (packages/playwright): typecheck, build and the unit tests of the reporter, the summariser and -# the command, then one real Playwright run that loads the reporter by its module name. No browser: the package only -# reads the files the GenClass Playwright fixture writes, and the fixture lives elsewhere. A separate workflow from -# ci.yml so it never blocks the main CI job. It renders the unit-test fixture's report into the job summary to show -# what a pull-request comment would look like; it posts nothing. +# @genclass/playwright (packages/playwright): typecheck, build and the unit tests of the fixture (on a fake page), +# the reporter, the summariser and the command, then one real Playwright run (no browser) that loads the reporter by +# its module name, and in a second job the fixture's end-to-end check in Chromium (packages/playwright/test/e2e): +# the MCP e2e search page with the fixture's seeded chaos, five seeds per test, the buggy page must get the +# overwrote-newer witness, the fixed page and the latency-only run must get nothing, and the reporter must render +# it. A separate workflow from ci.yml so it never blocks the main CI job. It renders the unit-test fixture's report +# into the job summary to show what a pull-request comment would look like; it posts nothing and sends nothing +# anywhere (the app server listens on 127.0.0.1 inside the runner; the page's model is skipped). on: push: paths: - "packages/playwright/**" + - "packages/runtime/src/agent/**" + - "packages/runtime/src/cdn/**" + - "packages/mcp/test/e2e/app/**" - "package-lock.json" - ".github/workflows/playwright-reporter.yml" pull_request: paths: - "packages/playwright/**" + - "packages/runtime/src/agent/**" + - "packages/runtime/src/cdn/**" + - "packages/mcp/test/e2e/app/**" - "package-lock.json" - ".github/workflows/playwright-reporter.yml" workflow_dispatch: @@ -75,3 +84,46 @@ jobs: node ../../bin/genclass-playwright-report.mjs --out - > "$RUNNER_TEMP/expected.md" diff "$RUNNER_TEMP/expected.md" genclass-report.md grep -qF '**1 reproducible**' genclass-report.md + + e2e: + name: "@genclass/playwright: the fixture end to end in Chromium" + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + ONNXRUNTIME_NODE_INSTALL: skip + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - name: Install + run: npm ci --no-audit --no-fund + + - name: Build (the runtime's dist/ is what the fixture serves to the page) + run: | + npm run build -w @genclass/runtime + npm run build -w @genclass/playwright + + - name: Chromium + run: npx playwright install --with-deps chromium + + - name: The fixture end to end (buggy page flagged, fixed page and latency-only run clean, report rendered) + run: node packages/playwright/test/e2e/run.mjs | tee "$RUNNER_TEMP/e2e.log" + + - name: Summary + if: always() + run: | + { + echo '### GenClass fixture, end to end' + echo '```json' + sed -n '/^{/,$p' "$RUNNER_TEMP/e2e.log" | tail -60 + echo '```' + if [ -f packages/playwright/test/e2e/genclass-report.md ]; then + echo + echo '### The report the reporter wrote' + cat packages/playwright/test/e2e/genclass-report.md + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index e8483dec..87769558 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ Default mode is `observe` (report only); `guard` is opt-in; `heal` is experiment | `training/` | vocabulary pruning, curriculum (`curriculum/rt.py` mirrors the runtime's situation renderer), Azure training, ONNX export, `EVAL.md` | | `demos/` | six demo apps with a chaos backend and Playwright trials; `bench/heal/` runs them per mode | | `packages/mcp/` | `@genclass/mcp`: the MCP server for coding agents (bridge on 127.0.0.1:47357, store, explore verdicts, replays, regression-test generation, 8 tools); no runtime deps; `test/e2e/race.e2e.mjs` is the real-browser check (not in CI's main job) | -| `packages/playwright/` | `@genclass/playwright`: Playwright reporter + pure summariser that read the GenClass fixture's `test-results/genclass/*.json` (`genclass-races/1`) and write `genclass-report.md` for the PR; injects nothing, no runtime deps | +| `packages/playwright/` | `@genclass/playwright`: the fixture (`src/fixture.ts`, `install.ts`, `page.ts`, `chaos.ts`, `races.ts`: puts the runtime's script-tag build on every `page` in observe mode with telemetry off, seeded chaos on fetch/XHR responses with `page.route`, witnesses through the agent entry with every hold off, one `genclass-races/1` file per test in `test-results/genclass/`) and the reporter + pure summariser that read those files and write `genclass-report.md` for the PR; peer deps only (`@playwright/test`, `@genclass/runtime`); `test/e2e/run.mjs` is the real-browser check (its own workflow) | | `bench/visual/` | the visual checks' bench (GitHub Actions only, `.github/workflows/visual-measure.yml`): false reports on the demos, templates, compat apps and about 40 public sites, recall on faults planted from the harness, cost; public sites are counted per check kind only, never named in the repo, its run logs or plain artifacts (per-site results only encrypted with the `VISUAL_BENCH_KEY` secret) | | `compat/`, `templates/` | framework compatibility matrix (`RESULTS.md`) and starter templates; evaluation only | | `telemetry-worker/` | the diagnostics collector with the dashboards and accounts (Cloudflare Worker). genclass.dev itself (the site, its admin and the pitch deck) and the page check (`scan-worker/`) live in the private repo genclass-dev/genclass-site, which has this repo as its `lib/` submodule | @@ -46,9 +46,10 @@ NODE_OPTIONS=--expose-gc npx vitest run test/review-perf.test.ts --retry=2 # t `npm run typecheck -w @genclass/mcp && npm run build -w @genclass/mcp && (cd packages/mcp && npx vitest run)`; the e2e needs both packages built: `node packages/mcp/test/e2e/race.e2e.mjs`. -`packages/playwright/` (`@genclass/playwright`, the race-file reporter) has its own workflow, -`playwright-reporter.yml` (typecheck, build, vitest; no browser; never blocks `ci.yml`): -`npm run typecheck -w @genclass/playwright && npm run build -w @genclass/playwright && (cd packages/playwright && npx vitest run)`. +`packages/playwright/` (`@genclass/playwright`, the fixture and the race-file reporter) has its own workflow, +`playwright-reporter.yml` (typecheck, build, vitest; then the fixture's e2e in Chromium; never blocks `ci.yml`): +`npm run typecheck -w @genclass/playwright && npm run build -w @genclass/playwright && (cd packages/playwright && npx vitest run)`; +the e2e needs the runtime built too: `node packages/playwright/test/e2e/run.mjs`. One test: `npx vitest run test/delivery.test.ts -t ""`. The 14 skipped tests are model-parity tests that need `GENCLASS_MODEL_DIR`. CI also runs the `visual-browser` job: the first-load size against the release line (`FIRST_LOAD_BASE`, release/0.3.0 until 0.3.0 is on main; `test/first-load-size.mjs`, fails at 0.6 KB diff --git a/README.md b/README.md index f56c6565..a85ddc05 100644 --- a/README.md +++ b/README.md @@ -254,7 +254,7 @@ All numbers are on held-out data, with each recall reported next to its false-in |---|---| | [`packages/runtime`](packages/runtime) | `@genclass/runtime`, the library. Observers, causality, stores and adapters, learned baselines/relations/profiles, facts, triage, policy gate, actions, model host (Web Worker, ONNX Runtime Web on WebGPU/WASM), devtools overlay. [README](packages/runtime/README.md) · [CHANGELOG](packages/runtime/CHANGELOG.md) | | [`packages/mcp`](packages/mcp) | `@genclass/mcp` (unreleased), the MCP server for coding agents: pages connect through `@genclass/runtime/agent`; the agent gets findings in causal order, replays, explore verdicts and generated regression tests ([AGENT.md](docs/runtime/AGENT.md)). [README](packages/mcp/README.md) | -| [`packages/playwright`](packages/playwright) | `@genclass/playwright` (unreleased), a Playwright reporter: reads what the GenClass fixture flagged per test (`test-results/genclass/*.json`) and writes one markdown report for the pull request, with the evidence line and what reproduced across seeds. [README](packages/playwright/README.md) | +| [`packages/playwright`](packages/playwright) | `@genclass/playwright` (unreleased), GenClass for Playwright: a fixture that puts the runtime on every page under test (observe mode, seeded network chaos with `page.route`, the model skipped by default) and writes one race file per test, and a reporter that turns those files into one markdown report for the pull request, with the evidence line and what reproduced across seeds. [README](packages/playwright/README.md) | | [`packages/runtime-model`](packages/runtime-model) | `@genclass/runtime-model`, the default model package (data only; `files/` is gitignored and filled at release time) and its [model card](packages/runtime-model/MODEL_CARD.md) | | [`sim`](sim) | Training-data simulator: random apps run on the real runtime in a deterministic virtual world, labelled by counterfactual outcomes | | [`realapps`](realapps) | 128 real apps (written for the corpus, plus open-source ones) run with the real runtime in headless Chromium, labelled the same way; the never-worse sweep covered the first 66. [README](realapps/README.md) | @@ -290,9 +290,9 @@ provenance. Not covered by CI: - Playwright browser tests: `npm run test:browser` in `packages/runtime`. -- `@genclass/playwright`: its typecheck, build and unit tests run in their own workflow, - [.github/workflows/playwright-reporter.yml](.github/workflows/playwright-reporter.yml), on pushes and pull - requests that touch the package; it never blocks `ci.yml`. +- `@genclass/playwright`: its typecheck, build, unit tests and the fixture's end-to-end check in Chromium run in + their own workflow, [.github/workflows/playwright-reporter.yml](.github/workflows/playwright-reporter.yml), on + pushes and pull requests that touch the package; it never blocks `ci.yml`. - The npm tarball smoke test: `test/smoke/smoke.sh`. - The sim, the real-app corpus, the demos' evaluation, and training. diff --git a/package-lock.json b/package-lock.json index 46450c87..87f56fe6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,6 +53,7 @@ "os": [ "aix" ], + "peer": true, "engines": { "node": ">=18" } @@ -70,6 +71,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=18" } @@ -87,6 +89,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=18" } @@ -104,6 +107,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=18" } @@ -121,6 +125,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">=18" } @@ -138,6 +143,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">=18" } @@ -155,6 +161,7 @@ "os": [ "freebsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -172,6 +179,7 @@ "os": [ "freebsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -189,6 +197,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -206,6 +215,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -223,6 +233,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -240,6 +251,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -257,6 +269,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -274,6 +287,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -291,6 +305,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -308,6 +323,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -325,6 +341,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -342,6 +359,7 @@ "os": [ "netbsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -359,6 +377,7 @@ "os": [ "netbsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -376,6 +395,7 @@ "os": [ "openbsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -393,6 +413,7 @@ "os": [ "openbsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -410,6 +431,7 @@ "os": [ "openharmony" ], + "peer": true, "engines": { "node": ">=18" } @@ -427,6 +449,7 @@ "os": [ "sunos" ], + "peer": true, "engines": { "node": ">=18" } @@ -444,6 +467,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=18" } @@ -461,6 +485,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=18" } @@ -478,6 +503,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=18" } @@ -3193,6 +3219,7 @@ "genclass-playwright-report": "bin/genclass-playwright-report.mjs" }, "devDependencies": { + "@genclass/runtime": "0.3.0-beta.1", "@playwright/test": "1.63.0", "@types/node": "^22.0.0", "tsup": "^8.5.1", @@ -3201,6 +3228,18 @@ }, "engines": { "node": ">=20" + }, + "peerDependencies": { + "@genclass/runtime": ">=0.3.0-beta.1", + "@playwright/test": ">=1.40.0" + }, + "peerDependenciesMeta": { + "@genclass/runtime": { + "optional": true + }, + "@playwright/test": { + "optional": true + } } }, "packages/runtime": { diff --git a/packages/playwright/.gitignore b/packages/playwright/.gitignore index 8d9a9b9a..f2fd08fe 100644 --- a/packages/playwright/.gitignore +++ b/packages/playwright/.gitignore @@ -1 +1,2 @@ test/playwright-smoke/genclass-report.md +test/e2e/genclass-report.md diff --git a/packages/playwright/CHANGELOG.md b/packages/playwright/CHANGELOG.md index b47e15a8..daa16710 100644 --- a/packages/playwright/CHANGELOG.md +++ b/packages/playwright/CHANGELOG.md @@ -2,14 +2,39 @@ ## Unreleased -- First version, reader only: `@genclass/playwright/reporter` (a Playwright reporter by shape: reads +- **The fixture**, `@genclass/playwright/fixture`: Playwright's `test` (`test.extend`) and `expect`, with a + `genclass` option (`use: { genclass: {...} }`). On every `page` it puts `@genclass/runtime`'s script-tag build + before the app's first script, in observe mode with telemetry off, the visual checks and the devtools off, and + serves the runtime's `dist/` from the installed package (or `runtimeDir`) on the page's origin at `/__genclass__/` + with `page.route`, never from a CDN. The model is skipped by default (`model: "skip"`: no 13 MB download) and the + file carries the runtime's witnesses, computed without the model through the agent entry connected with every + hold off (`overwrote-newer`, `identical-in-flight`); `model: "load"` adds the model's detections with their + confidence. At the end of each test it writes one `genclass-races/1` file to `test-results/genclass/` (or + `outDir`), named after the test, its id and its seed, with the seed and the chaos settings. `enabled: false` + makes it inert. Peer dependencies only (`@playwright/test`, `@genclass/runtime`, both optional: the reporter + needs neither). +- **Seeded chaos** (`chaos: { seed, delayMs, reorder, match }`): with `page.route` the real response is fetched at + once and its delivery to the page held for a latency derived from the seed and the request's key (method, path, + occurrence), so a run repeats exactly; `reorder: false` keeps the request order (latency only). Only fetch and + XHR requests inside `match` (default: same-origin `/api*` or JSON responses); never the runtime's own files, + documents, scripts or anything outside the filter. Off unless given. With `repeatEach`, the repeat index is + added to the seed, so two repeats give the report its two seeds. +- **Witnesses in the report**: signatures with trigger `witness` get their own group (shown only when there are + any), no confidence on the line (a fact that happened, not a probability) and a footer sentence saying what they + are; they are never counted as duplicate submits or stale responses. A model state `off` (no model by choice) + no longer gets the "model was not ready" note. The model-only report is byte for byte what it was. +- **Measured** (`test/e2e`, run by the workflow in Chromium): on the MCP e2e search page with latency 0 to 600 ms + and reordering, the buggy page gets `witness:overwrote-newer` on `GET /api/search` in 5 of 5 seeds, the same + page with latency only in 0 of 5, and the fixed page in 0 of 5; no other signature anywhere. With `model: "load"` + (one run by hand, two seeds) the model loaded through the fixture's path and gave no likely detection on that + page (`stale` at 0.64 in one seed, below the 0.90 report gate; nothing in the other), while the witness was + recorded in both. +- First version of the reader side: `@genclass/playwright/reporter` (a Playwright reporter by shape: reads `test-results/genclass/*.json` in the `genclass-races/1` format at the end of the run, writes and prints `genclass-report.md`), `@genclass/playwright/summarize` (the pure files-to-markdown function, with `buildSummary`, `renderMarkdown`, `parseRaces` and `validateRaces`), and the `genclass-playwright-report` command for sharded runs. Groups by test and by signature, marks signatures flagged in 2 or more seeds as reproducible, lists files that fail validation with the reason, and says nothing about interventions. No runtime dependencies. -- The fixture that writes the files (runtime injection, seeded chaos with `page.route`) is not part of this package; - it is developed separately. - Playwright can load the reporter by name: every export has a `default` condition next to `import`, because Playwright resolves a reporter id with CommonJS `require.resolve`, which does not match `import` (before this, `reporter: [["@genclass/playwright/reporter"]]` stopped the run with `ERR_PACKAGE_PATH_NOT_EXPORTED`). CI now runs diff --git a/packages/playwright/README.md b/packages/playwright/README.md index a9abe84e..01d5b4ed 100644 --- a/packages/playwright/README.md +++ b/packages/playwright/README.md @@ -1,38 +1,192 @@ # @genclass/playwright -### Explain your flaky e2e test: which click submitted twice, which response landed late. +### Explain your flaky e2e test: which response landed over newer data, which click submitted twice. -A Playwright reporter for [GenClass Runtime](https://github.com/genclass-dev/GenClass-lib). When your suite ends, it -reads what GenClass flagged in each test (duplicate submits, stale responses, failing endpoints, each with the -evidence line the runtime printed and the model's confidence), marks what reproduced across seeds, writes one -markdown file, `genclass-report.md`, and prints it. A short GitHub Actions step posts that file as one pull-request -comment that updates in place. +GenClass for Playwright, in two parts that share one file format: -It reads files; it does not inject anything. The files come from the GenClass Playwright fixture, which puts the -runtime on the page under test, adds seeded network chaos with `page.route` and writes one file per test. **The -fixture is being built separately and has not shipped yet.** Until it does, this reporter has nothing to read and -says so. +- **The fixture** (`@genclass/playwright/fixture`) puts [`@genclass/runtime`](../runtime) on every page your tests + open, before the app's first script, in observe mode with telemetry off. With `chaos` it slows and reorders the + app's API responses from a seed, so the orders a real network produces happen in your test, the same way every + run. At the end of each test it writes one JSON file (format `genclass-races/1`) with what the runtime saw, the + seed and the chaos that produced it. +- **The reporter** (`@genclass/playwright/reporter`) reads those files when the suite ends, marks what reproduced + across seeds, writes one markdown file, `genclass-report.md`, and prints it. A short GitHub Actions step posts that + file as one pull-request comment that updates in place. -No runtime dependencies. Playwright is not imported: the reporter implements Playwright's reporter interface by -shape, and the summariser is a pure function you can call from any script. +What the fixture records without a model download: two happens-before facts the runtime computes on its own +(`overwrote-newer`, a response that wrote a field over what a newer request or the user had written; +`identical-in-flight`, two identical requests in flight together from separate user actions). With the model loaded +(`model: "load"`, a 13 MB download), the file also carries the model's diagnoses with their confidence. Both are +things to look at, not verdicts: the report says what happened and leaves the call to you. + +Observe only. The runtime never holds, delays or changes anything the app does; the fixture's chaos changes when a +response reaches the page, never what it says; no finding changes a test's outcome. No runtime dependencies: +`@playwright/test` and `@genclass/runtime` are peer dependencies (the reporter alone needs neither). ## Install Not on npm yet (version 0.1.0 is unreleased). Once published: ```bash -npm install -D @genclass/playwright +npm install -D @genclass/playwright @genclass/runtime ``` +Playwright 1.40 or later; `@genclass/runtime` 0.3.0-beta.1 or later (the fixture reads its `dist/` from the +installed package). + ## Use +Two lines in the config, one import in a spec: + ```ts // playwright.config.ts +import { defineConfig } from "@playwright/test"; + export default defineConfig({ + use: { genclass: { chaos: { seed: 1, delayMs: [0, 600] } } }, reporter: [["list"], ["@genclass/playwright/reporter"]], }); ``` +```ts +// tests/search.spec.ts +import { test, expect } from "@genclass/playwright/fixture"; + +test("shows the results of the last query", async ({ page }) => { + await page.goto("/"); + await page.getByLabel("Search").pressSequentially("react", { delay: 80 }); + await expect(page.locator("#results li").first()).toContainText("react"); +}); +``` + +`test` is Playwright's `test.extend` with GenClass on its `page`; `expect` is Playwright's. Everything else in the +spec stays as it was. The run ends with `test-results/genclass/--s.json` per test and +`genclass-report.md` next to the config. + +Two seeds, so the report can mark what reproduced: `repeatEach: 2` in the config (or `npx playwright test +--repeat-each 2`). The fixture adds the repeat index to the seed, so the repeats run under seeds 1 and 2, and a +signature flagged in both is **Reproducible** in the report. + +### The `genclass` option + +Set in `use` in the config, or `test.use({ genclass: {...} })` in a spec or a `describe` block. + +| key | default | what | +|---|---|---| +| `enabled` | `true` | `false`: the fixture does nothing (the page is Playwright's own, no file is written). | +| `mode` | `"observe"` | The only value. The runtime reports and never holds, delays or changes anything; guard and heal are not for tests. | +| `model` | `"skip"` | `"skip"`: no model download; the file carries the runtime's witnesses, facts computed without the model, and no model confidences. `"load"`: the runtime loads its model (from `modelUrl`, else the package default on jsDelivr: 13 MB, cached by the browser) and the file also carries its detections with their confidence. | +| `modelUrl` | package default | With `model: "load"`: a model directory URL (the runtime's `model.baseUrl`). | +| `chaos` | off | Seeded network chaos (below): an object, or `true` for the defaults. Off unless given. | +| `outDir` | `"test-results/genclass"` | Where the files go, relative to the config's directory. The reporter reads the same place by default. | +| `runtimeDir` | the installed package's `dist/` | `@genclass/runtime`'s `dist/` directory, when it is not resolvable from your project. | + +### Chaos + +```ts +use: { genclass: { chaos: { seed: 1, delayMs: [0, 600], reorder: true, match: ["/api", /\/graphql$/] } } } +``` + +| key | default | what | +|---|---|---| +| `seed` | `1` | Every delay is derived from it. With `repeatEach`, the fixture adds the repeat index. | +| `delayMs` | `[0, 600]` | Extra latency per matched response, uniform in `[min, max]` ms, at most 10000. | +| `reorder` | `true` | `false`: matched responses keep their request order (latency only, nothing overtakes). | +| `match` | same-origin `/api*` or JSON | Which requests: a path prefix (`"/api"`), a URL prefix (`"https://api.example.com/"`), a RegExp tested against the full URL, a function of `{ url, method, resourceType, sameOrigin }`, or a list of those. Without it: same-origin requests whose path starts with `/api` or whose response is JSON. | + +How it works: with `page.route`, the fixture fetches the real response at once and holds its delivery to the page +for the planned time, the way a slower network would. Each matched request is keyed by its method, path and +occurrence in the test (`GET /api/search#3`); its delay comes from a small PRNG seeded by `seed` and that key, so +the same test under the same seed gets the same delays whatever the wall clock does, and a finding comes with the +seed that produced it. Nothing is dropped, changed or invented. Only fetch and XHR requests are ever touched, never +documents, scripts, styles, images, WebSockets or the runtime's own files; a request outside `match` is passed +through untouched. One consequence of fetching the response first: a matched response is buffered, so a streaming +response (server-sent events over fetch, NDJSON) on a matched URL does not work; exclude it with `match`. + +### The model trade-off + +By default (`model: "skip"`) nothing is downloaded and the file carries the runtime's **witnesses**: facts the +runtime computes from what it already traces (the causal chain of every request, the version history of every +store field), with no model involved: + +- `overwrote-newer`: a request's response chain wrote a field that, since the request started, a newer operation + (one that started after it, outside its causal chain) or the user had written. Older data over newer data. +- `identical-in-flight`: two identical requests (same method, URL and body) in flight together, from two separate + user actions. + +A witness is a fact, not a diagnosis: some overwrites are intended, and many are repaired a moment later. The report +keeps them in their own group, shows no confidence for them and says so in its footer. They need state the runtime +can see: stores registered with the runtime, or found by automatic state discovery (React component state, Redux, +Zustand stores with the devtools middleware), which the fixture turns on. + +With `model: "load"` the runtime downloads its model (13 MB, from jsDelivr unless `modelUrl` names your own copy; +the browser caches it) and the file also carries the model's **detections** (`delivery:stale`, +`request:duplicate`, `failure:failing`, ...) with the model's confidence and the evidence line it printed, grouped +by diagnosis in the report. What the model flags, and how often it is wrong on correct apps, was measured on the +framework compatibility matrix (the "What to expect" section below). + +### What the fixture puts on the page + +For the runtime to see the app's first requests and state, it has to be there before the app's first script, so +the fixture adds one init script per page: the runtime's script-tag build (`dist/genclass.global.js`) with the +page configuration `mode: "observe"`, `telemetry: false`, no visual checks, no devtools overlay, the model as +chosen, and one sink that hands each likely detection to the test runner. The runtime's files are served from disk +on the page's own origin at `/__genclass__/` with `page.route`, never from a CDN. The page then loads the runtime's +agent entry (`dist/agent.js`) and connects it with every hold off (`hold: false`), pointed at a loopback URL the +fixture aborts, so the witnesses are computed and nothing is ever sent to an MCP bridge. Like the runtime's own +observers, the agent entry wraps the page's `setTimeout` and `clearTimeout` (to see pending debounces; it calls the +originals). The page gets two globals, `window.GenClass` (the runtime) and `window.__GENCLASS_PLAYWRIGHT__` (the +fixture's collector), and one exposed function. + +Limits, as they are today: + +- Only the `page` fixture is instrumented. Pages from `context.newPage()` or popups are not. +- Top frame only, on http(s) pages. `page.setContent` (about:blank) and iframes get nothing. +- An app that already starts GenClass itself (`GenClass.init()` from the ESM entry) runs a second runtime next to + the fixture's; stores it registered with `rt.atom` or `rt.guard` on its own runtime are not visible to the + fixture's, which sees network, user actions and what automatic discovery finds. An app integrated through the + script tag (`window.GenClass`) shares the fixture's runtime. +- The witnesses need writes the runtime can attribute to a response (registered or discovered state). Apps whose + state the runtime cannot see produce no witnesses; the model's detections about requests (duplicates, failures) + do not depend on state. + +## Measured + +The fixture's end-to-end check, [`test/e2e`](test/e2e) (run by [`.github/workflows/playwright-reporter.yml`](../../.github/workflows/playwright-reporter.yml)), +on the search page of the MCP server's own e2e ([`packages/mcp/test/e2e/app/app.js`](../mcp/test/e2e/app/app.js)): +a search box whose results are written by each response, so a slower older answer can land over newer results; the +fixed variant keeps only the answer to the latest query. The test types "react" at 80 ms per key; the backend +answers in 40 ms, in order, so without chaos the race never shows. Headless Chromium 153 (Playwright 1.63.0), +2026-10-24, `model: "skip"`, five seeds per test (`repeatEach: 5`): + +| page | chaos | seeds with `witness:overwrote-newer` on `GET /api/search` | any other signature | +|---|---|---:|---:| +| buggy | latency 0 to 600 ms, reorder | 5 of 5 | 0 | +| buggy | latency 0 to 600 ms, latency only (`reorder: false`) | 0 of 5 | 0 | +| fixed | latency 0 to 600 ms, reorder | 0 of 5 | 0 | + +So on this page the fixture finds the race under every seed, reports nothing on the correct variant (the false +report count that goes next to every recall number here), and reports nothing when the chaos cannot reorder, which +is the check that the witness comes from the order and not from the latency. + +The same buggy page with `model: "load"` (one run by hand, same day, two seeds, not part of the workflow because it +downloads the model): the model loaded through the fixture's path (`genclass-runtime-r17` 2.0.0-rc4t, WASM, in a +worker) and the witness was recorded in 2 of 2 seeds, but the model itself produced no likely detection: in seed 1 +its top diagnosis for the late delivery was `stale` at 0.64, below the 0.90 report gate, and in seed 2 it raised +no decision. That matches the compatibility matrix below (stale detection is the model's weak case) and is why the +witnesses are the default. + +One page is not a benchmark: the numbers to expect on your app depend on what state the runtime can see (above). +Wider measurements (the six demos, the real-app corpus) are the next step and will go to +[`docs/runtime/RESULTS.md`](../../docs/runtime/RESULTS.md). + +## The reporter + +```ts +// playwright.config.ts +reporter: [["list"], ["@genclass/playwright/reporter"]], +``` + Listed alone (`reporter: [["@genclass/playwright/reporter"]]`) it works too: it prints only the report, once, at the end, so Playwright still adds its own line reporter (dot on CI) for progress and failures, as it does next to the html reporter. Playwright finds the reporter by its module name; `playwright-reporter.yml` checks that with a real @@ -43,12 +197,14 @@ Playwright run (no browser). | `dir` | `test-results/genclass` and `genclass` inside each project's `outputDir` | Where the fixture's files are, relative to the config's directory (a string or a list). | | `outputFile` | `"genclass-report.md"` | The report, relative to the config's directory. `false`: print only. | | `quiet` | `false` | Do not print the report at the end of the run. | -| `minSeeds` | `2` | Seeds a signature must be flagged in, within one test, to be marked **Reproducible** (the fixture's 2-seed rule). | +| `minSeeds` | `2` | Seeds a signature must be flagged in, within one test, to be marked **Reproducible**. | | `title` | `"GenClass race report"` | The report's heading. | | `maxChars` | `60000` | Keep the report under this many characters, so it fits in one GitHub comment (GitHub rejects comment bodies over 65,536). `0`: no limit. | The reporter never changes the run's status and never throws into Playwright: a report it cannot write is one line -on stderr. +on stderr. It reads files; it does not inject anything (the fixture does that), and Playwright is not imported by +it: the reporter implements Playwright's reporter interface by shape, and the summariser is a pure function you can +call from any script. ### Without the reporter, or with shards @@ -71,29 +227,27 @@ const markdown = summarize([{ name: "checkout-1.json", content: jsonText }], { m `buildSummary` returns the same report as data (tests, signatures, seeds, totals, skipped files), and `validateRaces` / `parseRaces` check one file. -## What it reads +### The file format -One JSON file per test (or per test and seed) in `test-results/genclass/`, format `genclass-races/1`, as agreed with -the fixture's authors: +One JSON file per test and seed in `test-results/genclass/`, format `genclass-races/1`: ```ts interface GenClassRacesFile { schema: "genclass-races/1"; - test: string; // the test's title path + test: string; // "[chromium] checkout.spec.ts > checkout > places one order" seed: number; // the chaos seed of this run signatures: { - sig: string; // "trigger|diagnosis|op", ids normalised by the fixture - trigger: string; // the runtime's trigger kind: "request", "delivery", "failure", ... - diagnosis: string; // the model's diagnosis: "duplicate", "stale", "failing", ... + sig: string; // "trigger|diagnosis|op" + trigger: string; // the runtime's trigger kind ("request", "delivery", "failure", ...) or "witness" + diagnosis: string; // the model's diagnosis ("duplicate", "stale", "failing", ...) or the witness kind op: string; // "POST /api/orders" - confidence: number; // the model's probability for that diagnosis, 0 to 1 - evidence: string; // the line the runtime printed for the decision + confidence: number; // the model's probability for that diagnosis, 0 to 1; 1 for a witness (a fact) + evidence: string; // the line the runtime printed, or the witness's line seeds: number[]; // the seeds this signature was seen in }[]; - shadow?: unknown; // carried through, not interpreted - mode?: string; // optional, shown when present - model?: { state?: string; name?: string; version?: string; device?: string } | null; // optional - chaos?: { latencyMs?: [number, number]; reorder?: boolean; failRate?: number; routes?: unknown[] } | null; // optional + mode?: string; // "observe" + model?: { state?: string; name?: string; version?: string; device?: string } | null; // null: GenClass never started + chaos?: { latencyMs?: [number, number]; reorder?: boolean; routes?: unknown[] } | null; // the chaos option (latencyMs is delayMs) } ``` @@ -111,9 +265,12 @@ How the report groups them: - **groups by the model's diagnosis label**: `duplicate` (Duplicate submits), `stale` (Stale responses), `failing` (Failing endpoints) and everything else (Other detections: `transient`, `inconsistent`, `overload`, ...). `expected` is not a detection and is left out. Nothing looks at facts or decides on its own what is a bug; +- **witnesses** (trigger `witness`) get their own group, shown only when there are any, with no confidence on the + line and a footer sentence saying what they are; they are never counted as duplicate submits or stale responses; - a file that flagged nothing and whose `model.state` is not `ready` (for example `loading`, or `null` for a page - where GenClass never started) may not have been checked, and the report notes it for that test. A file with - detections is reported as usual whatever the state: the runtime can unload an idle model after it ran; + where GenClass never started) may not have been checked, and the report notes it for that test. State `off` (no + model by choice, the fixture's default) gets no note: the witnesses were still computed. A file with detections + is reported as usual whatever the state: the runtime can unload an idle model after it ran; - **nothing about interventions.** The files do not say what an action did to your app, so the report makes no claim about interventions, correct or false. @@ -130,13 +287,13 @@ What it does so the comment is safe to post: first), then whole tests from the end. The counts always include everything, and a note under the first line says what was left out. -## What the comment looks like +### What the comment looks like This is the exact output for the unit-test fixture in [`test/fixtures/races`](test/fixtures/races): two tests, two -seeds each, one signature flagged in both seeds and one in a single seed. The fixture is synthetic (the evidence -lines are written for the test, not taken from a real run); `test/package.test.ts` fails if this block and the -summariser drift apart. Each `#` of a request id below is followed by the invisible zero-width space described -above. +seeds each, one signature flagged in both seeds and one in a single seed, both model detections (a run with +`model: "load"`). The fixture is synthetic (the evidence lines are written for the test, not taken from a real run); +`test/package.test.ts` fails if this block and the summariser drift apart. Each `#` of a request id below is +followed by the invisible zero-width space described above. ```markdown @@ -174,7 +331,30 @@ above. Each line is a diagnosis by GenClass's on-device model in the page under test, one per signature (trigger, diagnosis, operation). Reproducible: flagged in 2 or more seeds of the same test. A signature seen in fewer seeds may depend on that seed's timing: re-run with more seeds before acting on it. Confidence is the model's probability for the diagnosis (the highest over the seeds), and the quoted evidence line is from that seed. ``` -The first line is an HTML comment, invisible on GitHub, that lets a CI step find and update its own comment. +A run with the default `model: "skip"` looks like this instead (the e2e check above, shortened to the one flagged +test; the request ids carry the same invisible character): + +```markdown +| | signatures | reproducible | +|---|---:|---:| +| Duplicate submits | 0 | 0 | +| Stale responses | 0 | 0 | +| Failing endpoints | 0 | 0 | +| Other detections | 0 | 0 | +| Witnesses (facts measured without the model) | 1 | 1 | + +### search.spec.ts > search > buggy page: results of the last query + +5 seeds (1, 2, 3, 4, 5) · 1 signature · chaos: latency 0–600 ms, reorder + +**Witnesses (facts measured without the model)** + +- **Reproducible**: 5 of 5 seeds (1, 2, 3, 4, 5) · `GET /api/search` · `witness:overwrote-newer` + > The response of GET /api/search#​1 wrote search.results over what newer GET /api/search#​4 had written 0.26s after GET /api/search#​1 started (2 items \["reac one", "reac two"\] → 2 items \["r one", "r two"\]). +``` + +The first line of every report is an HTML comment, invisible on GitHub, that lets a CI step find and update its own +comment. ## Post it on the pull request @@ -203,7 +383,7 @@ jobs: cache: npm - run: npm ci - run: npx playwright install --with-deps chromium - - run: npx playwright test # the reporter writes genclass-report.md + - run: npx playwright test --repeat-each 2 # the fixture writes the race files, the reporter the report - name: GenClass report in the job summary if: ${{ !cancelled() }} @@ -236,28 +416,10 @@ The same with the GitHub CLI only (it updates the bot's comment that starts with fi ``` -## Pairing it with the fixture - -The fixture is not published yet. The options below are the contract agreed with its authors on 2026-10-13; the -package name and import path are not final, so this is a sketch, not something to copy: - -```ts -// tests/fixtures.ts (sketch) -import { test as base } from "@playwright/test"; -import { genclass } from ""; - -export const test = base.extend( - genclass({ mode: "observe", chaos: { latencyMs: [0, 600], reorder: true, failRate: 0, routes: ["/api/"] }, seeds: 2 }), -); -``` - -Then keep this reporter in `playwright.config.ts` as above. Each test runs under each seed's network timing, the -fixture writes `test-results/genclass/.json`, and the report marks what reproduced in two seeds. - -## What to expect +## What to expect from the model -This reporter only passes on what the runtime flagged. What the runtime flags in observe mode was measured on the -framework compatibility matrix ([`compat/RESULTS.md`](../../compat/RESULTS.md), run of 2026-10-10: headless Chromium, +With `model: "load"`, the report passes on what the runtime's model flagged. That was measured on the framework +compatibility matrix ([`compat/RESULTS.md`](../../compat/RESULTS.md), run of 2026-10-10: headless Chromium, 10 seeds per cell, model `genclass-runtime-r17` 2.0.0-rc4t, the default of `@genclass/runtime-model` 0.2.0, WASM, balanced profile, whose report threshold is 0.90 per [`docs/runtime/RESULTS.md`](../../docs/runtime/RESULTS.md)): @@ -273,20 +435,21 @@ balanced profile, whose report threshold is 0.90 per [`docs/runtime/RESULTS.md`] - **On correct apps** (scenarios c to h, 880 runs): 49 false findings, `request:overload` ×45 and `request:duplicate` ×4, all on the polling scenario (g). -So lead with duplicate submits, expect stale signatures only where that state is discovered, and treat any line as -the model's diagnosis to check, not a confirmed bug. +So with the model, lead with duplicate submits, expect stale signatures only where that state is discovered, and +treat any line as the model's diagnosis to check, not a confirmed bug. The witnesses (the default) were measured +only on the one page above so far. ## Privacy -This package reads files on the machine that runs it and writes one; it makes no network request and sends nothing -anywhere. The files hold test titles, operation names and the runtime's evidence lines (the runtime replaces URL -query values with "…"), and the comment you post contains the same. +This package reads and writes files on the machine that runs it, serves the runtime's files to the page it tests +from disk, and sends nothing anywhere: with `model: "skip"` no network request leaves your test environment +because of it; with `model: "load"` the page downloads the model and onnxruntime-web's wasm from jsDelivr (or +`modelUrl`). The runtime on the page runs with `telemetry: false`. -The runtime's own anonymous diagnostics are off by design on local hosts (`localhost`, `*.localhost`, `127.0.0.1`, -private networks; [`TELEMETRY.md`](../runtime/TELEMETRY.md)), which is where e2e suites usually run. One exception: -`@genclass/runtime` 0.2.1 sent from local hosts by mistake when started with `GenClass.init()`, `/auto` or the -script tag (fixed in 0.2.2). With 0.2.1, or with a test server on a non-local host name, pass `telemetry: false` -to the runtime under test. +The files hold test titles, operation names and the runtime's evidence lines. For a witness that line includes the +store field's name and the runtime's short summary of its value before and after (`2 items ["r one", "r two"]`; +fields whose name says they hold a secret are redacted by the runtime, as elsewhere). The comment you post contains +the same, so read a report before posting it from a repository whose test data is private. ## Development @@ -295,12 +458,20 @@ From the repository root (CI runs these in `.github/workflows/playwright-reporte ```bash npm run typecheck -w @genclass/playwright npm run build -w @genclass/playwright -cd packages/playwright && npx vitest run # schema, summariser (exact output), reporter, command, package invariants +cd packages/playwright && npx vitest run # chaos, race files, the installer on a fake page, schema, summariser (exact output), reporter, command, package invariants +``` + +The end-to-end check needs both builds and Chromium: + +```bash +npm run build -w @genclass/runtime && npm run build -w @genclass/playwright +npx playwright install --with-deps chromium +node packages/playwright/test/e2e/run.mjs # the table in "Measured"; prints a JSON summary, exit 0 when every count holds ``` -The workflow then runs Playwright itself on `test/playwright-smoke` (no browser: its one test writes the fixture's -race files and uses no page), which loads `@genclass/playwright/reporter` by name the way your config does, and -checks that the reporter's file equals the command's rendering of the same files. The module-resolution test in +The workflow also runs Playwright itself on `test/playwright-smoke` (no browser: its one test writes race files and +uses no page), which loads `@genclass/playwright/reporter` by name the way your config does, and checks that the +reporter's file equals the command's rendering of the same files. The module-resolution test in `test/package.test.ts` needs `dist/` (build first); without it, outside CI, it is skipped. Apache-2.0. diff --git a/packages/playwright/package.json b/packages/playwright/package.json index 4da67466..f3bbb4dc 100644 --- a/packages/playwright/package.json +++ b/packages/playwright/package.json @@ -1,7 +1,7 @@ { "name": "@genclass/playwright", "version": "0.1.0", - "description": "A Playwright reporter for GenClass race files: reads what the GenClass fixture flagged in each test (duplicate submits, stale responses, failing endpoints, with the evidence line and the model's confidence) and writes one markdown report your CI can post on the pull request. No runtime dependencies.", + "description": "GenClass for Playwright: a fixture that puts @genclass/runtime on every page under test (observe mode, seeded network chaos with page.route) and writes one race file per test, and a reporter that turns those files into one markdown report your CI posts on the pull request. Peer dependencies only.", "type": "module", "license": "Apache-2.0", "repository": { @@ -39,6 +39,11 @@ "import": "./dist/reporter.js", "default": "./dist/reporter.js" }, + "./fixture": { + "types": "./dist/fixture.d.ts", + "import": "./dist/fixture.js", + "default": "./dist/fixture.js" + }, "./package.json": "./package.json" }, "bin": { @@ -57,7 +62,20 @@ "typecheck": "tsc -p tsconfig.json --noEmit", "test": "vitest run" }, + "peerDependencies": { + "@genclass/runtime": ">=0.3.0-beta.1", + "@playwright/test": ">=1.40.0" + }, + "peerDependenciesMeta": { + "@genclass/runtime": { + "optional": true + }, + "@playwright/test": { + "optional": true + } + }, "devDependencies": { + "@genclass/runtime": "0.3.0-beta.1", "@playwright/test": "1.63.0", "@types/node": "^22.0.0", "tsup": "^8.5.1", diff --git a/packages/playwright/src/chaos.ts b/packages/playwright/src/chaos.ts new file mode 100644 index 00000000..8da8a279 --- /dev/null +++ b/packages/playwright/src/chaos.ts @@ -0,0 +1,260 @@ +// Seeded network chaos for the GenClass Playwright fixture: which responses to slow down, by how much, and whether +// they may overtake each other, every value derived from a seed so that a run repeats exactly. Pure: no Playwright, +// no I/O, no clock of its own (the scheduler takes its timers from the caller, so tests drive it by hand). The fixture +// (install.ts) applies the plan with page.route: it fetches the real response and holds its delivery for the planned +// time. Nothing is dropped, changed or invented, and requests outside the filter are never touched; the runtime's own +// files (served on RUNTIME_PATH) never are. +// +// Determinism: a matched request is keyed by its method, path and occurrence in the test ("GET /api/search#3"), and +// its delay is read from a small PRNG seeded by the chaos seed and that key. The same test under the same seed gives +// the same delays whatever the wall clock does. + +/** Where install.ts serves the runtime's dist/ on the page's origin. Chaos never applies to it. */ +export const RUNTIME_PATH = "/__genclass__/"; +/** The largest extra latency the fixture applies, in ms (a hold must stay bounded). */ +export const MAX_DELAY_MS = 10_000; +/** The default `delayMs` when `chaos` is given without one. */ +export const DEFAULT_DELAY_MS: readonly [number, number] = [0, 600]; +/** The default seed (the fixture adds Playwright's `repeatEachIndex` to it). */ +export const DEFAULT_SEED = 1; + +/** A request as the filter sees it, before its response is fetched. */ +export interface ChaosRequest { + url: string; + method: string; + /** Playwright's resource type; only "fetch" and "xhr" are ever candidates. */ + resourceType: string; + /** Same origin as the document that sent it. */ + sameOrigin: boolean; +} + +export type ChaosMatchFn = (r: ChaosRequest) => boolean; +/** + * Which requests get chaos: a string (a path prefix such as "/api", or a URL prefix such as "https://api.example.com/"), + * a RegExp tested against the full URL, a function, or a list of those (any match counts). Unset: same-origin + * requests whose path starts with /api or whose response is JSON. + */ +export type ChaosMatch = string | RegExp | ChaosMatchFn | readonly (string | RegExp | ChaosMatchFn)[]; + +export interface ChaosOptions { + /** Every delay is derived from it. Default 1; with `repeatEach`, the fixture adds the repeat index. */ + seed?: number; + /** Extra latency per matched response, uniform in [min, max] ms. Default [0, 600]; at most 10000. */ + delayMs?: readonly [number, number]; + /** false: matched responses keep their request order (latency only, nothing overtakes). Default true. */ + reorder?: boolean; + /** Which requests (see ChaosMatch). Default: same-origin, path /api* or a JSON response. */ + match?: ChaosMatch; +} + +export interface ResolvedChaos { + seed: number; + delayMs: [number, number]; + reorder: boolean; + match: ChaosMatch | undefined; +} + +/** One matched request: its key and the latency the seed gives it. */ +export interface ChaosPlan { + key: string; + delayMs: number; +} + +/** 32-bit FNV-1a with a murmur3 finaliser (the same hash the runtime's agent schedules use). */ +export function hash32(s: string): number { + let h = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i); + h = Math.imul(h, 0x01000193) >>> 0; + } + h ^= h >>> 16; + h = Math.imul(h, 0x85ebca6b) >>> 0; + h ^= h >>> 13; + h = Math.imul(h, 0xc2b2ae35) >>> 0; + h ^= h >>> 16; + return h >>> 0; +} + +/** mulberry32: a small seeded PRNG, uniform in [0, 1). */ +export function mulberry32(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** The latency for one key under a seed: uniform in [min, max], the same for the same seed and key. */ +export function delayFor(seed: number, key: string, delayMs: readonly [number, number]): number { + const [min, max] = delayMs; + if (max <= min) return min; + const r = mulberry32(hash32(`${seed}|${key}`))(); + return Math.min(max, min + Math.floor(r * (max - min + 1))); +} + +function int(v: unknown, fallback: number): number { + return typeof v === "number" && Number.isFinite(v) && v >= 0 ? Math.floor(v) : fallback; +} + +/** Fills in the defaults and bounds the values. `true` means the defaults. Throws on a `match` of the wrong type. */ +export function resolveChaos(o: ChaosOptions | true | undefined | null, seedOffset = 0): ResolvedChaos { + const c: ChaosOptions = o === true || !o ? {} : o; + const d = c.delayMs; + let min = DEFAULT_DELAY_MS[0]; + let max = DEFAULT_DELAY_MS[1]; + if (Array.isArray(d) && d.length === 2) { + min = Math.min(MAX_DELAY_MS, int(d[0], 0)); + max = Math.min(MAX_DELAY_MS, Math.max(min, int(d[1], min))); + } else if (d !== undefined) throw new TypeError(`@genclass/playwright: chaos.delayMs must be [min, max] in ms, got ${JSON.stringify(d)}`); + if (c.match !== undefined && !isMatch(c.match)) throw new TypeError("@genclass/playwright: chaos.match must be a string, a RegExp, a function, or a list of those"); + return { seed: int(c.seed, DEFAULT_SEED) + Math.max(0, int(seedOffset, 0)), delayMs: [min, max], reorder: c.reorder !== false, match: c.match }; +} + +function isMatch(m: unknown): m is ChaosMatch { + if (Array.isArray(m)) return m.every((x) => typeof x === "string" || x instanceof RegExp || typeof x === "function"); + return typeof m === "string" || m instanceof RegExp || typeof m === "function"; +} + +function pathOf(url: string): string { + try { + return new URL(url).pathname; + } catch { + return url; + } +} + +/** True for the runtime's own files (never chaos, whatever the filter says). */ +export function isRuntimeUrl(url: string): boolean { + return pathOf(url).startsWith(RUNTIME_PATH); +} + +function matchesOne(m: string | RegExp | ChaosMatchFn, r: ChaosRequest): boolean { + if (typeof m === "function") return m(r) === true; + if (m instanceof RegExp) { + m.lastIndex = 0; + return m.test(r.url); + } + return m.startsWith("/") ? pathOf(r.url).startsWith(m) : r.url.startsWith(m); +} + +/** + * Whether a request may get chaos, decided before its response is fetched. Only fetch and XHR requests, never the + * runtime's files. With a `match`, the filter decides; without one, every same-origin request is a candidate and + * `confirms` decides from the response. + */ +export function isCandidate(r: ChaosRequest, match: ChaosMatch | undefined): boolean { + if (r.resourceType !== "fetch" && r.resourceType !== "xhr") return false; + if (isRuntimeUrl(r.url)) return false; + if (match === undefined) return r.sameOrigin; + const list = Array.isArray(match) ? (match as readonly (string | RegExp | ChaosMatchFn)[]) : [match as string | RegExp | ChaosMatchFn]; + return list.some((m) => matchesOne(m, r)); +} + +/** + * Whether a candidate gets chaos once its response is known. With a `match`, always (the filter already said yes); + * without one, when the path starts with /api or the response is JSON (its content-type names json). + */ +export function confirms(r: ChaosRequest, match: ChaosMatch | undefined, contentType: string | null | undefined): boolean { + if (match !== undefined) return true; + if (pathOf(r.url).startsWith("/api")) return true; + return /json/i.test(contentType ?? ""); +} + +/** The filter as written to the race file's `chaos.routes` (functions as "function", RegExps as their source). */ +export function describeMatch(match: ChaosMatch | undefined): unknown[] { + if (match === undefined) return ["same-origin /api* or JSON responses"]; + const list = Array.isArray(match) ? (match as readonly (string | RegExp | ChaosMatchFn)[]) : [match as string | RegExp | ChaosMatchFn]; + return list.map((m) => (typeof m === "function" ? "function" : m instanceof RegExp ? String(m) : m)); +} + +/** The timers the scheduler holds with (the globals by default; a test passes its own). */ +export interface ChaosTimers { + setTimeout(fn: () => void, ms: number): unknown; + clearTimeout(id: unknown): void; +} + +const realTimers: ChaosTimers = { + setTimeout: (fn, ms) => setTimeout(fn, ms), + clearTimeout: (id) => clearTimeout(id as ReturnType), +}; + +/** + * Assigns each matched request its key and latency, and holds its delivery: for its own delay, and with `reorder` + * off also until every earlier matched response was released, so nothing overtakes. `dispose()` releases every + * hold at once (the page is gone). Never throws. + */ +export class ChaosScheduler { + private readonly counts = new Map(); + private last: Promise = Promise.resolve(); + private readonly pending = new Map void>(); + private disposed = false; + /** Every plan made so far, in request order (for the race file and tests). */ + readonly plans: ChaosPlan[] = []; + + constructor( + readonly chaos: ResolvedChaos, + private readonly timers: ChaosTimers = realTimers, + ) {} + + /** The key ("GET /api/search#3": method, path, occurrence in this test) and latency of a matched request. */ + plan(method: string, url: string, sameOrigin = true): ChaosPlan { + let where: string; + try { + const u = new URL(url); + where = sameOrigin ? u.pathname : `${u.host}${u.pathname}`; + } catch { + where = url; + } + const sig = `${method.toUpperCase()} ${where}`; + const n = (this.counts.get(sig) ?? 0) + 1; + this.counts.set(sig, n); + const key = `${sig}#${n}`; + const p = { key, delayMs: delayFor(this.chaos.seed, key, this.chaos.delayMs) }; + this.plans.push(p); + return p; + } + + /** Resolves when the response may be delivered. */ + hold(plan: ChaosPlan): Promise { + const own = this.sleep(plan.delayMs); + if (this.chaos.reorder) return own; + const after = this.last; + const mine = Promise.all([after, own]).then(() => undefined); + this.last = mine; + return mine; + } + + /** Release every held response now and stop the timers. */ + dispose(): void { + this.disposed = true; + for (const [id, release] of this.pending) { + try { + this.timers.clearTimeout(id); + } catch { + /* ignore */ + } + release(); + } + this.pending.clear(); + } + + private sleep(ms: number): Promise { + if (this.disposed || ms <= 0) return Promise.resolve(); + return new Promise((resolve) => { + let id: unknown = null; + const release = () => { + if (id !== null) this.pending.delete(id); + resolve(); + }; + try { + id = this.timers.setTimeout(release, ms); + this.pending.set(id, release); + } catch { + resolve(); + } + }); + } +} diff --git a/packages/playwright/src/fixture.ts b/packages/playwright/src/fixture.ts new file mode 100644 index 00000000..4fb1a146 --- /dev/null +++ b/packages/playwright/src/fixture.ts @@ -0,0 +1,144 @@ +// `@genclass/playwright/fixture`: Playwright's `test` with GenClass on every page, and the `expect` that goes with it. +// +// // playwright.config.ts +// use: { genclass: { chaos: { seed: 1, delayMs: [0, 600] } } }, +// reporter: [["list"], ["@genclass/playwright/reporter"]], +// +// // a spec +// import { test, expect } from "@genclass/playwright/fixture"; +// +// For each test it puts @genclass/runtime on the page before the app's first script (observe mode, telemetry off, +// the model skipped unless asked), applies the seeded chaos when one is given, and at the end writes one +// "genclass-races/1" file to `outDir` (default test-results/genclass next to the config) with what the runtime saw +// and the seed and chaos that produced it, which the reporter turns into the pull-request comment. Observe only: the +// fixture never holds, delays or changes anything the app does; chaos changes when a response arrives, never what +// it says. A test's outcome is never changed by a finding. `enabled: false` makes the fixture inert: the page is +// Playwright's own and nothing is written. + +import { test as base, expect } from "@playwright/test"; +import { dirname, isAbsolute, join, relative, sep } from "node:path"; +import { DEFAULT_SEED, describeMatch, resolveChaos, type ChaosMatch, type ChaosMatchFn, type ChaosOptions } from "./chaos.js"; +import { install, type Collected, type PageLike } from "./install.js"; +import { buildRacesFile, racesFileName, writeRacesFile } from "./races.js"; +import type { GenClassRacesFile } from "./types.js"; + +export interface GenClassOptions { + /** false: the fixture does nothing (no runtime on the page, no file). Default true. */ + enabled?: boolean; + /** Only "observe": the runtime reports and never holds, delays or changes anything. Guard and heal are not for tests. */ + mode?: "observe"; + /** + * "skip" (default): no model download (13 MB), so the file carries the runtime's witnesses (overwrote-newer, + * identical-in-flight), facts computed without the model, and no model diagnoses or confidences. "load": the + * runtime loads its model (from `modelUrl`, else the package default on jsDelivr) and the file also carries its + * detections with their confidence. + */ + model?: "skip" | "load"; + /** With `model: "load"`: a model directory URL (the runtime's `model.baseUrl`). */ + modelUrl?: string; + /** Seeded network chaos (ChaosOptions; `true` for the defaults). Off unless given. */ + chaos?: ChaosOptions | true | false; + /** Where the files go, relative to the Playwright config's directory. Default "test-results/genclass". */ + outDir?: string; + /** @genclass/runtime's dist/ directory. Default: the installed package's. */ + runtimeDir?: string; +} + +export interface GenClassFixtures { + genclass: GenClassOptions; +} + +/** The parts of Playwright's TestInfo the fixture reads. */ +export interface TestInfoLike { + titlePath: readonly string[]; + file: string; + testId: string; + repeatEachIndex: number; + project: { name: string; testDir: string }; + config: { configFile?: string; rootDir: string }; +} + +/** The test's name in the file: "[project] file.spec.ts > describe > title" (the project only when it has a name). */ +export function testName(info: TestInfoLike): string { + const path = [...info.titlePath]; + // Playwright's title path is ["", project, file, ...describes, title]; find the file to be safe about the shape + const rel = relative(info.project.testDir, info.file).split(sep).join("/"); + let at = path.findIndex((t) => t === rel || t.split(sep).join("/") === rel); + if (at < 0) at = path.findIndex((t) => t.endsWith(".ts") || t.endsWith(".js") || t.endsWith(".mjs") || t.endsWith(".tsx") || t.endsWith(".jsx")); + const titles = at >= 0 ? path.slice(at) : path.filter(Boolean); + if (at >= 0 && titles[0] !== rel && rel) titles[0] = rel; + const name = titles.join(" > ") || info.testId; + const project = info.project.name; + return project ? `[${project}] ${name}` : name; +} + +/** The directory the file goes to: `outDir` against the config's directory (else the root dir). */ +export function outDirOf(info: TestInfoLike, outDir?: string): string { + const base = info.config.configFile ? dirname(info.config.configFile) : info.config.rootDir; + if (!outDir) return join(base, "test-results", "genclass"); + return isAbsolute(outDir) ? outDir : join(base, outDir); +} + +export interface GenClassRun { + /** What the page reported and the file written for it (null while the test runs or when disabled). */ + readonly collected: Collected | null; + readonly file: GenClassRacesFile | null; + readonly path: string | null; +} + +/** + * Starts the fixture on a page for one test: installs the runtime (and chaos), and returns what to call when the + * test ends. Exported for tests and for custom fixtures; `test` below does this on its `page`. + */ +export async function startGenClass(page: PageLike, options: GenClassOptions | undefined, info: TestInfoLike): Promise<{ finish(): Promise }> { + const o = options ?? {}; + const empty: GenClassRun = { collected: null, file: null, path: null }; + if (o.enabled === false) return { finish: async () => empty }; + if (o.mode !== undefined && o.mode !== "observe") throw new Error(`@genclass/playwright: genclass.mode must be "observe" (got ${JSON.stringify(o.mode)}): guard and heal are not for tests`); + if (o.model !== undefined && o.model !== "skip" && o.model !== "load") throw new Error(`@genclass/playwright: genclass.model must be "skip" or "load" (got ${JSON.stringify(o.model)})`); + const chaos = o.chaos ? resolveChaos(o.chaos, info.repeatEachIndex) : null; + const seed = chaos ? chaos.seed : DEFAULT_SEED + Math.max(0, Math.floor(info.repeatEachIndex) || 0); + const session = await install(page, { runtimeDir: o.runtimeDir, model: o.model ?? "skip", modelUrl: o.modelUrl, chaos }); + return { + async finish(): Promise { + const collected = await session.finish(); + const file = buildRacesFile({ + test: testName(info), + seed, + mode: collected.mode, + model: collected.model, + chaos: chaos ? { latencyMs: chaos.delayMs, reorder: chaos.reorder, routes: describeMatch(chaos.match) } : null, + records: collected.records, + witnesses: collected.witnesses, + }); + const path = writeRacesFile(outDirOf(info, o.outDir), racesFileName(file.test, info.testId, seed), file); + return { collected, file, path }; + }, + }; +} + +/** + * Playwright's `test` with GenClass on its `page`. Options come from `use: { genclass: {...} }` in the config or + * `test.use({ genclass: {...} })` in a spec. Pages opened another way (`context.newPage()`, popups) are not + * instrumented. + */ +export const test = base.extend({ + genclass: [{}, { option: true }], + page: async ({ page, genclass }, use, testInfo) => { + const run = await startGenClass(page as unknown as PageLike, genclass, testInfo); + try { + await use(page); + } finally { + try { + await run.finish(); + } catch (e) { + // a file that cannot be written is one line, never a failed test + process.stderr.write(`[@genclass/playwright] could not write the race file: ${e instanceof Error ? e.message : String(e)}\n`); + } + } + }, +}); + +export { expect }; +export type { ChaosMatch, ChaosMatchFn, ChaosOptions, Collected }; +export { RUNTIME_PATH } from "./chaos.js"; diff --git a/packages/playwright/src/install.ts b/packages/playwright/src/install.ts new file mode 100644 index 00000000..660df113 --- /dev/null +++ b/packages/playwright/src/install.ts @@ -0,0 +1,274 @@ +// Puts GenClass on a Playwright page without touching the app under test, and collects what it saw. Node only, but +// Playwright-agnostic: it takes the few page methods it uses as an interface (PageLike), so unit tests drive it with +// a fake page and the fixture (fixture.ts) passes the real one. +// +// What it does to the page: +// - serves @genclass/runtime's dist/ (resolved from the installed package, or `runtimeDir`) on the virtual +// same-origin path RUNTIME_PATH ("/__genclass__/") with page.route, so the script-tag build, the agent entry and +// anything the runtime loads later come from disk, never from a CDN; +// - adds one init script (page.addInitScript): the page-side function of page.ts plus the script-tag build, so the +// runtime starts before the app's first script, in observe mode with telemetry off and the model as chosen; +// - exposes one function (page.exposeFunction) the page pushes each finding through as it happens, so navigations +// and an early page close lose nothing; +// - with chaos, routes every fetch/XHR request through the seeded scheduler (chaos.ts): the real response is +// fetched at once and its delivery to the page is held for the planned time. Requests outside the filter, the +// runtime's own files and the agent link's URL are left alone; the agent link's URL is aborted, so the page's +// link never reaches a bridge. +// Observe only: no hold on the page's side, no change to any request or response (chaos changes when a response +// arrives, never what it says). Every handler catches its own errors: a page that closes mid-test is not a failure. + +import { existsSync, readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, extname, join, resolve, sep } from "node:path"; +import { ChaosScheduler, RUNTIME_PATH, confirms, isCandidate, isRuntimeUrl, type ResolvedChaos } from "./chaos.js"; +import { genclassPageScript, type PageScriptConfig } from "./page.js"; + +/** The page methods the installer uses (a subset of Playwright's Page, by shape). */ +export interface PageLike { + route(url: string, handler: (route: RouteLike, request: RequestLike) => Promise | void): Promise; + addInitScript(script: { content: string }): Promise; + exposeFunction(name: string, fn: (item: unknown) => unknown): Promise; + evaluate(expression: string): Promise; + isClosed?(): boolean; +} + +export interface RouteLike { + fetch(): Promise; + fulfill(options: { response?: ResponseLike; status?: number; headers?: Record; body?: string | Buffer; contentType?: string }): Promise; + fallback(): Promise; + abort(errorCode?: string): Promise; +} + +export interface RequestLike { + url(): string; + method(): string; + resourceType(): string; + frame(): { url(): string }; +} + +export interface ResponseLike { + headers(): Record; +} + +export interface InstallOptions { + /** The runtime's dist/ directory. Default: the installed @genclass/runtime package's. */ + runtimeDir?: string; + /** "skip": no model (the file carries witnesses only). "load": the runtime loads its model. */ + model: "skip" | "load"; + /** With `model: "load"`: a model directory URL (the runtime's `model.baseUrl`). */ + modelUrl?: string; + /** Resolved chaos, or null for none. */ + chaos: ResolvedChaos | null; +} + +/** The runtime's model status as the page reported it last. */ +export interface ModelInfo { + state?: string; + name?: string; + version?: string; + device?: string; +} + +/** What one test collected. */ +export interface Collected { + /** Sink records of kind "detection" (likely tier), as the runtime sent them. */ + records: unknown[]; + /** The agent entry's witnesses, in order of appearance. */ + witnesses: unknown[]; + /** The runtime's mode on the page (null: it never started). */ + mode: string | null; + /** Its model status (null: it never started). */ + model: ModelInfo | null; + /** The agent entry's state: null when connected, else why not ("not-loaded", an error message). */ + agent: string | null; + /** The chaos plans applied, in request order. */ + plans: { key: string; delayMs: number }[]; +} + +export interface Session { + /** Collect what the page still holds, release every chaos hold, and return everything seen. */ + finish(): Promise; +} + +/** The name of the function exposed on the page. */ +export const BINDING = "__genclassPlaywright"; +/** The loopback URL the page's agent link is pointed at; the fixture aborts every request to it. */ +export const AGENT_URL = "http://127.0.0.1:9"; + +const require = createRequire(import.meta.url); +const CONTENT_TYPES: Record = { + ".js": "text/javascript; charset=utf-8", + ".mjs": "text/javascript; charset=utf-8", + ".json": "application/json; charset=utf-8", + ".map": "application/json; charset=utf-8", + ".wasm": "application/wasm", +}; +const fileCache = new Map(); + +/** The installed @genclass/runtime's dist/ directory (its package.json export), or `given`. Throws when neither exists. */ +export function resolveRuntimeDir(given?: string): string { + if (given) { + const dir = resolve(given); + if (!existsSync(join(dir, "genclass.global.js"))) throw new Error(`@genclass/playwright: runtimeDir ${dir} has no genclass.global.js (point it at @genclass/runtime's dist/)`); + return dir; + } + let pkg: string; + try { + pkg = require.resolve("@genclass/runtime/package.json"); + } catch { + throw new Error("@genclass/playwright: @genclass/runtime is not installed (npm install -D @genclass/runtime), or pass genclass.runtimeDir"); + } + const dir = join(dirname(pkg), "dist"); + if (!existsSync(join(dir, "genclass.global.js"))) throw new Error(`@genclass/playwright: ${dir} has no genclass.global.js: the installed @genclass/runtime has no build`); + if (!existsSync(join(dir, "agent.js"))) throw new Error(`@genclass/playwright: ${dir} has no agent.js: the fixture needs @genclass/runtime 0.3.0-beta.1 or later`); + return dir; +} + +function readCached(path: string): Buffer { + let b = fileCache.get(path); + if (!b) { + b = readFileSync(path); + fileCache.set(path, b); + } + return b; +} + +/** The init script's source: the page function applied to its config and to the script-tag build wrapped in a function. */ +export function initScriptContent(runtimeDir: string, cfg: PageScriptConfig): string { + const global = readCached(join(runtimeDir, "genclass.global.js")).toString("utf8"); + return `(${genclassPageScript.toString()})(${JSON.stringify(cfg)}, function () {\n${global}\n});\n`; +} + +function originOf(url: string): string | null { + try { + return new URL(url).origin; + } catch { + return null; + } +} + +/** The file inside dist/ a request for RUNTIME_PATH names, or null when it is not a plain file path inside it. */ +export function runtimeFile(runtimeDir: string, url: string): string | null { + let pathname: string; + try { + pathname = decodeURIComponent(new URL(url).pathname); + } catch { + return null; + } + if (!pathname.startsWith(RUNTIME_PATH)) return null; + const rel = pathname.slice(RUNTIME_PATH.length); + if (!/^[A-Za-z0-9_.-]+(\/[A-Za-z0-9_.-]+)*$/.test(rel) || rel.split("/").some((s) => s === "." || s === "..")) return null; + const abs = resolve(runtimeDir, ...rel.split("/")); + if (!abs.startsWith(resolve(runtimeDir) + sep)) return null; + return abs; +} + +/** + * Installs GenClass on the page and returns the session to finish at the end of the test. Throws only before touching + * the page (a runtime that cannot be found); everything after that is best effort. + */ +export async function install(page: PageLike, o: InstallOptions): Promise { + const runtimeDir = resolveRuntimeDir(o.runtimeDir); + const collected: Collected = { records: [], witnesses: [], mode: null, model: null, agent: "not-loaded", plans: [] }; + const scheduler = o.chaos ? new ChaosScheduler(o.chaos) : null; + + const take = (item: unknown): void => { + if (!item || typeof item !== "object") return; + const x = item as Record; + if (x.t === "record" && x.record) collected.records.push(x.record); + else if (x.t === "witness" && x.witness) collected.witnesses.push(x.witness); + else if (x.t === "status") { + collected.mode = typeof x.mode === "string" ? x.mode : null; + collected.model = x.model && typeof x.model === "object" ? (x.model as ModelInfo) : null; + } else if (x.t === "agent") collected.agent = x.ok === true ? null : String(x.reason ?? "unknown"); + }; + await page.exposeFunction(BINDING, take); + + // chaos first: the routes below are matched before it (Playwright tries the last registered handler first) + if (scheduler) { + const chaos = scheduler.chaos; + await page.route("**/*", async (route, request) => { + let candidate = false; + let req: Parameters[0] | null = null; + try { + const url = request.url(); + if (isRuntimeUrl(url) || url.startsWith(AGENT_URL)) return await route.fallback(); + let frameUrl = ""; + try { + frameUrl = request.frame().url(); + } catch { + /* a detached frame */ + } + const origin = originOf(url); + req = { url, method: request.method(), resourceType: request.resourceType(), sameOrigin: origin !== null && origin === originOf(frameUrl) }; + candidate = isCandidate(req, chaos.match); + } catch { + candidate = false; + } + if (!candidate || !req) return route.fallback().catch(() => undefined); + let response: ResponseLike; + try { + response = await route.fetch(); + } catch { + // the request itself failed (the server is down, the page is gone): let the browser see the failure + return route.abort("failed").catch(() => undefined); + } + try { + const headers = response.headers(); + const ct = headers["content-type"] ?? headers["Content-Type"] ?? null; + if (confirms(req, chaos.match, ct)) { + await scheduler.hold(scheduler.plan(req.method, req.url, req.sameOrigin)); + } + await route.fulfill({ response }); + } catch { + /* the page closed while the response was held */ + } + }); + } + + // the runtime's files, from disk + await page.route(`**${RUNTIME_PATH}**`, async (route, request) => { + try { + const file = runtimeFile(runtimeDir, request.url()); + if (!file || !existsSync(file)) return await route.fulfill({ status: 404, contentType: "text/plain", body: "not found" }); + await route.fulfill({ status: 200, headers: { "content-type": CONTENT_TYPES[extname(file)] ?? "application/octet-stream", "cache-control": "no-store" }, body: readCached(file) }); + } catch { + /* the page closed */ + } + }); + // the page's agent link talks to this URL: never let it reach a bridge + await page.route(`${AGENT_URL}/**`, (route) => route.abort("connectionrefused").catch(() => undefined)); + + const cfg: PageScriptConfig = { + base: RUNTIME_PATH, + model: o.model === "load" ? (o.modelUrl ? { baseUrl: o.modelUrl } : {}) : false, + agentUrl: AGENT_URL, + binding: BINDING, + }; + await page.addInitScript({ content: initScriptContent(runtimeDir, cfg) }); + + return { + async finish(): Promise { + try { + if (!page.isClosed?.()) { + const r = await Promise.race([ + page.evaluate(`(function () { var a = window.__GENCLASS_PLAYWRIGHT__; return a ? a.collect() : null; })()`), + new Promise((res) => setTimeout(() => res(null), 2000).unref?.()), + ]); + if (r && typeof r === "object") { + const x = r as Record; + if (typeof x.mode === "string") collected.mode = x.mode; + if (x.model && typeof x.model === "object") collected.model = x.model as ModelInfo; + if (x.agent === null) collected.agent = null; + else if (typeof x.agent === "string") collected.agent = x.agent; + } + } + } catch { + /* the page is gone: what it pushed so far is what there is */ + } + scheduler?.dispose(); + collected.plans = scheduler ? scheduler.plans.slice() : []; + return collected; + }, + }; +} diff --git a/packages/playwright/src/page.ts b/packages/playwright/src/page.ts new file mode 100644 index 00000000..90a3b04d --- /dev/null +++ b/packages/playwright/src/page.ts @@ -0,0 +1,168 @@ +// The part of the fixture that runs inside the page (install.ts puts it in page.addInitScript, serialised with +// Function.prototype.toString, so this file imports nothing and the function below references only its parameters +// and the page's globals). What it does, before any script of the app runs: +// +// 1. sets window.GENCLASS_CONFIG: observe mode, telemetry off, no visual checks, no devtools, the model as the +// fixture chose, and one sink that hands every likely detection (kind "detection", with its evidence) to Node; +// 2. runs the runtime's script-tag build (dist/genclass.global.js) with a stand-in for `document.currentScript` +// (an init script has no tag, and the build reads `data-base` from its own tag: the stand-in points it at the +// fixture's virtual path, so everything it loads later comes from the same place); +// 3. loads the agent entry (dist/agent.js) and connects it with every hold off, so the runtime records the two +// happens-before witnesses (overwrote-newer, identical-in-flight) that need no model, and hands each new one to +// Node as it appears; +// 4. reports the runtime's mode and model status, at start, on every status change, when the page goes away, and +// on request (window.__GENCLASS_PLAYWRIGHT__.collect(), which the fixture calls at the end of the test). +// +// Observe only: nothing here holds, delays or changes anything in the page; the agent link is connected with +// `hold: false` and talks to a loopback URL the fixture aborts, so it never reaches a bridge. Top frame only, and only +// on http(s) pages (about:blank and page.setContent get nothing). Never throws into the page. + +/** What install.ts passes to the page script. */ +export interface PageScriptConfig { + /** The virtual path the runtime's dist/ is served on ("/__genclass__/"). */ + base: string; + /** The runtime's `model` option: false (skip) or its options (load). */ + model: false | { baseUrl?: string }; + /** The loopback URL the agent link is pointed at (the fixture aborts requests to it). */ + agentUrl: string; + /** The name of the function Node exposed (page.exposeFunction) that receives every item. */ + binding: string; +} + +/** The agent entry's connection, as far as this script uses it. */ +interface AgentConnection { + reason: string | null; + witnesses(): unknown[]; +} + +/** + * The page script. `runGlobal` is the runtime's script-tag build wrapped in a function (install.ts builds the + * source text). Serialised with toString(): keep it free of imports, closures over module scope and TypeScript-only + * runtime features. + */ +export function genclassPageScript(cfg: PageScriptConfig, runGlobal: () => void): void { + const w = window as unknown as Record; + try { + if (window.top !== window) return; + } catch { + return; + } + if (!/^https?:$/.test(location.protocol)) return; + if (w.__GENCLASS_PLAYWRIGHT__) return; + + const push = (item: Record): void => { + try { + const f = w[cfg.binding]; + if (typeof f === "function") void (f as (x: unknown) => unknown)(item); + } catch { + /* never into the page */ + } + }; + + // 1. the runtime's configuration (read by the script-tag build at init, before the app's own scripts run) + const sink = { + kinds: ["detection"], + evidence: true, + possible: false, + send(r: unknown): void { + push({ t: "record", record: r }); + }, + }; + w.GENCLASS_CONFIG = { mode: "observe", telemetry: false, visual: false, analytics: false, devtools: false, report: "silent", model: cfg.model, sinks: [sink] }; + + // 2. the script-tag build, with a tag of its own to read data-base from + let rt: Record | null = null; + try { + const tag = document.createElement("script"); + tag.dataset.base = cfg.base; + Object.defineProperty(document, "currentScript", { configurable: true, get: () => tag }); + try { + runGlobal(); + } finally { + delete (document as unknown as Record).currentScript; + } + const g = w.GenClass as { runtime?: Record | null } | undefined; + rt = g && g.runtime ? g.runtime : null; + } catch (e) { + push({ t: "status", mode: null, model: null, error: String((e as Error)?.message ?? e) }); + return; + } + + const status = (): Record | null => { + if (!rt) return null; + const s = (rt.status ?? {}) as Record; + const m: Record = { state: String(s.state ?? "off") }; + if (typeof s.model === "string") m.name = s.model; + if (typeof s.version === "string") m.version = s.version; + if (typeof s.device === "string") m.device = s.device; + return m; + }; + const report = (): void => push({ t: "status", mode: rt ? String(rt.mode) : null, model: status() }); + + // 3. witnesses through the agent entry (hands each new one over as it appears) + let conn: AgentConnection | null = null; + let handed = 0; + let queued = false; + const drain = (): void => { + if (!conn) return; + try { + const list = conn.witnesses(); + for (; handed < list.length; handed++) push({ t: "witness", witness: list[handed] }); + } catch { + /* ignore */ + } + }; + const api = { + collect(): Record { + drain(); + return { mode: rt ? String(rt.mode) : null, model: status(), agent: conn ? conn.reason : "not-loaded" }; + }, + }; + w.__GENCLASS_PLAYWRIGHT__ = api; + if (!rt) { + report(); + return; + } + report(); + const on = rt.on as ((type: string, fn: (v: unknown) => void) => () => void) | undefined; + try { + on?.call(rt, "status", report); + // a witness is computed inside the runtime's event dispatch: look after it, once per burst + on?.call(rt, "event", () => { + if (queued) return; + queued = true; + queueMicrotask(() => { + queued = false; + drain(); + }); + }); + } catch { + /* an older runtime without these events */ + } + try { + import(/* @vite-ignore */ /* webpackIgnore: true */ `${cfg.base}agent.js`).then( + (m: { connectAgent?: (r: unknown, o: Record) => AgentConnection }) => { + try { + conn = typeof m.connectAgent === "function" ? m.connectAgent(rt, { url: cfg.agentUrl, hold: false, anyHost: true, dom: false }) : null; + push({ t: "agent", ok: !!conn && conn.reason === null, reason: conn ? conn.reason : "no-connectAgent" }); + drain(); + } catch (e) { + push({ t: "agent", ok: false, reason: String((e as Error)?.message ?? e) }); + } + }, + (e: unknown) => push({ t: "agent", ok: false, reason: String((e as Error)?.message ?? e) }), + ); + } catch (e) { + push({ t: "agent", ok: false, reason: String((e as Error)?.message ?? e) }); + } + + // 4. when the page goes away + try { + addEventListener("pagehide", () => { + drain(); + report(); + }); + } catch { + /* ignore */ + } +} diff --git a/packages/playwright/src/races.ts b/packages/playwright/src/races.ts new file mode 100644 index 00000000..a351656f --- /dev/null +++ b/packages/playwright/src/races.ts @@ -0,0 +1,143 @@ +// What one test collected -> one "genclass-races/1" file (the format in types.ts, which the reporter and the +// summariser read). Pure: no I/O here except the one writer at the bottom, no clock, no randomness; the same input +// gives the same bytes. +// +// Two sources, told apart by `trigger`: +// - the runtime's likely detections (sink records of kind "detection"): trigger is the runtime's trigger kind, +// diagnosis and confidence are the model's, evidence is the console line the runtime printed. Only with a model +// on the page (`model: "load"`). +// - the agent entry's witnesses (overwrote-newer, identical-in-flight): trigger "witness", diagnosis the witness +// kind, confidence 1 (a fact that happened, not a probability; the summariser shows no confidence for them) and +// an evidence line written here from the witness's fields. These need no model, so they are what the default +// (`model: "skip"`) carries. A witness is a fact, never a diagnosis: nothing here maps one to "stale" or +// "duplicate", and the report keeps them in their own group. + +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { RACES_SCHEMA, type GenClassRacesFile, type RaceSignature } from "./types.js"; + +/** The `trigger` of a witness signature. */ +export const WITNESS_TRIGGER = "witness"; + +/** A sink record of kind "detection", as far as the file reads it (the runtime's SinkRecord, by shape). */ +export interface DetectionRecord { + kind?: string; + trigger?: string; + diagnosis?: string; + p?: number; + detection?: string; + evidence?: { message?: string; subject?: string }; +} + +/** An agent witness, as far as the file reads it (the runtime's agent Witness, by shape). */ +export interface WitnessRecord { + kind?: string; + subject?: string; + by?: string; + over?: string; + overKind?: string; + gapMs?: number; + change?: string; +} + +export interface RacesInput { + test: string; + seed: number; + mode: string | null; + model: GenClassRacesFile["model"]; + chaos: GenClassRacesFile["chaos"]; + records: readonly unknown[]; + witnesses: readonly unknown[]; +} + +const METHOD_RE = /\b(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\s+([^\s()?#]+)/; + +/** "GET /api/search?q=… (#21)" or "GET /api/search#3~retry2" -> "GET /api/search"; anything else is kept (trimmed). */ +export function opOf(text: string | undefined): string { + const t = (text ?? "").replace(/\s+/g, " ").trim(); + const m = METHOD_RE.exec(t); + if (m) return `${m[1]} ${m[2]}`; + return t.slice(0, 120); +} + +function secs(ms: number | undefined): string { + return `${((ms ?? 0) / 1000).toFixed(2)}s`; +} + +/** The evidence line for a witness, from its fields (request keys as the agent names them: "GET /api/search#3"). */ +export function witnessLine(w: WitnessRecord): string { + const by = w.by ?? "?"; + if (w.kind === "overwrote-newer") { + const over = w.overKind === "user" ? `the user (${w.over ?? "?"})` : `newer ${w.over ?? "?"}`; + const change = w.change ? ` (${w.change})` : ""; + return `The response of ${by} wrote ${w.subject ?? "?"} over what ${over} had written ${secs(w.gapMs)} after ${by} started${change}.`; + } + if (w.kind === "identical-in-flight") return `${by} was sent while the identical ${w.over ?? "?"} was still in flight (${secs(w.gapMs)} earlier), from a separate user action.`; + return `${w.kind ?? "witness"}: ${w.subject ?? ""} by ${by} over ${w.over ?? "?"}`.trim(); +} + +function clamp01(n: unknown): number { + return typeof n === "number" && Number.isFinite(n) ? Math.min(1, Math.max(0, n)) : 0; +} + +function add(map: Map, s: RaceSignature): void { + const prev = map.get(s.sig); + if (!prev) map.set(s.sig, s); + else if (s.confidence > prev.confidence) { + prev.confidence = s.confidence; + prev.evidence = s.evidence; + } +} + +/** The distinct signatures of a test, sorted by `sig`: detections (likely, not `expected`) and witnesses. */ +export function signaturesOf(records: readonly unknown[], witnesses: readonly unknown[], seed: number): RaceSignature[] { + const out = new Map(); + for (const x of records) { + if (!x || typeof x !== "object") continue; + const r = x as DetectionRecord; + if (r.kind !== undefined && r.kind !== "detection") continue; + const diagnosis = String(r.diagnosis ?? "").trim(); + if (!diagnosis || diagnosis === "expected") continue; + const trigger = String(r.trigger ?? "unknown").trim() || "unknown"; + const op = opOf(r.evidence?.subject); + add(out, { sig: `${trigger}|${diagnosis}|${op}`, trigger, diagnosis, op, confidence: clamp01(r.p), evidence: String(r.evidence?.message ?? "").replace(/\s+/g, " ").trim(), seeds: [seed] }); + } + for (const x of witnesses) { + if (!x || typeof x !== "object") continue; + const w = x as WitnessRecord; + const diagnosis = String(w.kind ?? "").trim(); + if (!diagnosis) continue; + const op = opOf(w.by); + add(out, { sig: `${WITNESS_TRIGGER}|${diagnosis}|${op}`, trigger: WITNESS_TRIGGER, diagnosis, op, confidence: 1, evidence: witnessLine(w), seeds: [seed] }); + } + return [...out.values()].sort((a, b) => (a.sig < b.sig ? -1 : a.sig > b.sig ? 1 : 0)); +} + +/** The file for one test run. Validates against the reporter's schema by construction. */ +export function buildRacesFile(i: RacesInput): GenClassRacesFile { + const file: GenClassRacesFile = { schema: RACES_SCHEMA, test: i.test, seed: i.seed, signatures: signaturesOf(i.records, i.witnesses, i.seed) }; + if (i.mode) file.mode = i.mode; + file.model = i.model ?? null; + if (i.chaos !== undefined) file.chaos = i.chaos; + return file; +} + +/** A file name for a test: a slug of its title (at most 80 characters), its id and its seed. */ +export function racesFileName(test: string, id: string, seed: number): string { + const slug = test + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 80) + .replace(/-+$/, ""); + const safeId = id.replace(/[^A-Za-z0-9_-]+/g, ""); + return `${slug || "test"}-${safeId || "0"}-s${seed}.json`; +} + +/** Writes the file (creating the directory) and returns its path. */ +export function writeRacesFile(dir: string, name: string, file: GenClassRacesFile): string { + mkdirSync(dir, { recursive: true }); + const path = join(dir, name); + writeFileSync(path, `${JSON.stringify(file, null, 2)}\n`); + return path; +} diff --git a/packages/playwright/src/summarize.ts b/packages/playwright/src/summarize.ts index 881f9ea8..6f7f8a37 100644 --- a/packages/playwright/src/summarize.ts +++ b/packages/playwright/src/summarize.ts @@ -7,6 +7,10 @@ // failing endpoints and other detections, with the evidence line the runtime printed and the model's // confidence. The grouping is by the model's diagnosis label only; nothing here looks at facts or decides what // is a bug. +// - signatures whose trigger is "witness" (the fixture's happens-before facts, computed by the runtime without the +// model: overwrote-newer, identical-in-flight) get a group of their own, shown only when there are any, with no +// confidence (a fact happened or not) and a footer sentence that says what they are. They are never counted as +// duplicate submits or stale responses: whether one is a bug is for the reader. // - a signature flagged in at least `minSeeds` seeds of the same test (default 2, the fixture's 2-seed rule) is // marked reproducible; one seen in fewer may depend on that seed's timing. // - totals per group. @@ -21,18 +25,22 @@ import { parseRaces, validateRaces } from "./schema.js"; import type { GenClassRacesFile, RacesFileInput, RacesParseResult } from "./types.js"; -/** The report groups, in report order. */ -export type RaceCategory = "duplicate" | "stale" | "failing" | "other"; +/** The report groups, in report order. "witness" holds the fixture's model-less facts (trigger "witness"). */ +export type RaceCategory = "duplicate" | "stale" | "failing" | "other" | "witness"; -export const CATEGORIES: readonly RaceCategory[] = ["duplicate", "stale", "failing", "other"]; +export const CATEGORIES: readonly RaceCategory[] = ["duplicate", "stale", "failing", "other", "witness"]; export const CATEGORY_TITLES: Readonly> = { duplicate: "Duplicate submits", stale: "Stale responses", failing: "Failing endpoints", other: "Other detections", + witness: "Witnesses (facts measured without the model)", }; +/** The trigger the fixture gives a witness signature. */ +export const WITNESS_TRIGGER = "witness"; + /** The default number of seeds a signature must be seen in to count as reproducible. */ export const DEFAULT_MIN_SEEDS = 2; /** The marker the report starts with, so a CI step can find and update its own comment. */ @@ -45,9 +53,11 @@ export const MIN_MAX_CHARS = 1000; /** * The group of a diagnosis label: "duplicate", "stale" and "failing" have their own; everything else (transient, * inconsistent, conflict, overload, slow, unusual, a plugin's own labels) is "other". "expected" is not a detection - * and is left out of the report. + * and is left out of the report. With the trigger given, a witness (trigger "witness") is "witness" whatever its + * label says. */ -export function categoryOf(diagnosis: string): RaceCategory { +export function categoryOf(diagnosis: string, trigger?: string): RaceCategory { + if (trigger === WITNESS_TRIGGER) return "witness"; return diagnosis === "duplicate" || diagnosis === "stale" || diagnosis === "failing" ? diagnosis : "other"; } @@ -98,7 +108,8 @@ export interface TestSummary { /** * Files that flagged nothing and whose model was not "ready" when the fixture collected (null model: "not * started"), so those runs may not have been checked; and their states. Files with detections are not counted, - * whatever the state (the runtime can unload an idle model after it ran). + * whatever the state (the runtime can unload an idle model after it ran), nor files whose state is "off" (no + * model by choice: the fixture's default, which still computes witnesses). */ modelNotReady: { files: number; states: string[] }; /** The chaos of the first file (by name) that carries one. */ @@ -172,7 +183,7 @@ export function buildSummary(inputs: readonly RacesFileInput[], options: Summari if (!flaggedHere && m === null) { t.notReady++; t.states.add("not started"); - } else if (!flaggedHere && m && m.state !== undefined && m.state !== "ready") { + } else if (!flaggedHere && m && m.state !== undefined && m.state !== "ready" && m.state !== "off") { t.notReady++; t.states.add(m.state); } @@ -183,7 +194,7 @@ export function buildSummary(inputs: readonly RacesFileInput[], options: Summari for (const n of seeds) t.seeds.add(n); const prev = t.sigs.get(s.sig); if (!prev) { - t.sigs.set(s.sig, { sig: s.sig, trigger: s.trigger, diagnosis: s.diagnosis, op: s.op, category: categoryOf(s.diagnosis), confidence: s.confidence, evidence: s.evidence, seeds: [...seeds], reproducible: false }); + t.sigs.set(s.sig, { sig: s.sig, trigger: s.trigger, diagnosis: s.diagnosis, op: s.op, category: categoryOf(s.diagnosis, s.trigger), confidence: s.confidence, evidence: s.evidence, seeds: [...seeds], reproducible: false }); continue; } prev.seeds = [...new Set([...prev.seeds, ...seeds])]; @@ -300,7 +311,9 @@ function chaosText(c: NonNullable): string { function signatureLines(s: SignatureSummary, t: TestSummary, maxEvidence: number): string[] { const seen = `${s.seeds.length} of ${plural(t.seeds.length, "seed")} (${seedList(s.seeds)})`; - const head = `${s.reproducible ? "**Reproducible**: " : ""}${seen} · ${code(s.op || s.sig)} · ${code(`${s.trigger}:${s.diagnosis}`)} · confidence ${s.confidence.toFixed(2)}`; + // a witness is a fact that happened: it has no confidence to show + const conf = s.category === "witness" ? "" : ` · confidence ${s.confidence.toFixed(2)}`; + const head = `${s.reproducible ? "**Reproducible**: " : ""}${seen} · ${code(s.op || s.sig)} · ${code(`${s.trigger}:${s.diagnosis}`)}${conf}`; const lines = [`- ${head}`]; const ev = clip(s.evidence, maxEvidence); lines.push(ev ? ` > ${escapeMarkdown(ev)}` : " > (no evidence line in the file)"); @@ -393,10 +406,12 @@ function render(s: RaceSummary, options: SummarizeOptions, lim: Limits, maxChars const note = cutNote(s, lim, maxChars); if (note) L.push(note, ""); + const witnesses = T.byCategory.witness.signatures > 0; if (T.signatures > 0) { L.push("| | signatures | reproducible |"); L.push("|---|---:|---:|"); - for (const c of CATEGORIES) L.push(`| ${CATEGORY_TITLES[c]} | ${T.byCategory[c].signatures} | ${T.byCategory[c].reproducible} |`); + // the witness row only when there are any (a reporter-only setup never has them) + for (const c of CATEGORIES) if (c !== "witness" || witnesses) L.push(`| ${CATEGORY_TITLES[c]} | ${T.byCategory[c].signatures} | ${T.byCategory[c].reproducible} |`); L.push(""); } @@ -458,12 +473,25 @@ function render(s: RaceSummary, options: SummarizeOptions, lim: Limits, maxChars if (T.signatures > 0) { L.push("---"); L.push(""); - L.push( - `Each line is a diagnosis by GenClass's on-device model in the page under test, one per signature (trigger, diagnosis, operation). ` + - `Reproducible: flagged in ${rule} of the same test. ` + - `A signature seen in fewer seeds may depend on that seed's timing: re-run with more seeds before acting on it. ` + - `Confidence is the model's probability for the diagnosis (the highest over the seeds), and the quoted evidence line is from that seed.`, - ); + const parts: string[] = []; + if (T.signatures > T.byCategory.witness.signatures) { + parts.push( + `Each line is a diagnosis by GenClass's on-device model in the page under test, one per signature (trigger, diagnosis, operation). ` + + `Reproducible: flagged in ${rule} of the same test. ` + + `A signature seen in fewer seeds may depend on that seed's timing: re-run with more seeds before acting on it. ` + + `Confidence is the model's probability for the diagnosis (the highest over the seeds), and the quoted evidence line is from that seed.`, + ); + } + if (witnesses) { + parts.push( + `Lines under "${CATEGORY_TITLES.witness}" are happens-before facts the runtime measured in the page without the model, one per signature (witness kind, operation): ` + + `overwrote-newer, a response wrote a field over what a newer request or the user had written since the request started; ` + + `identical-in-flight, two identical requests in flight together from separate user actions. ` + + `They carry no confidence: a fact is listed when it happened. Reproducible: seen in ${rule} of the same test. ` + + `Whether a fact is a bug is for you to decide (some overwrites are intended; many are repaired a moment later).`, + ); + } + L.push(parts.join("\n\n")); } while (L.length && L[L.length - 1] === "") L.pop(); return L.join("\n") + "\n"; diff --git a/packages/playwright/src/types.ts b/packages/playwright/src/types.ts index 0f878729..2bb34bd0 100644 --- a/packages/playwright/src/types.ts +++ b/packages/playwright/src/types.ts @@ -1,12 +1,14 @@ -// The files this package reads, defined locally (no import from the fixture or from @genclass/runtime): the -// GenClass Playwright fixture writes one JSON file per test to `test-results/genclass/.json` in the format -// "genclass-races/1" (contract agreed with the fixture's owners on 2026-10-13). This package only reads them; it -// never injects anything into a page, never routes a request and never starts a runtime. +// The files this package reads and writes, defined locally (no import from @genclass/runtime): the fixture +// (`@genclass/playwright/fixture`, src/fixture.ts) writes one JSON file per test to `test-results/genclass/` in the +// format "genclass-races/1", and the reporter, the summariser and the command read them. The reading side never +// injects anything into a page, never routes a request and never starts a runtime. // // Wording follows @genclass/runtime (src/types.ts): `trigger` is a TriggerKind ("request", "delivery", "failure", // ...), `diagnosis` is the model's diagnosis label ("duplicate", "stale", "failing", ...) and `confidence` is the // probability the model gave that diagnosis (Decision.diagnosisConfidence). `evidence` is the line the runtime -// printed for the decision. +// printed for the decision. One more trigger is the fixture's own: "witness", for the two happens-before facts the +// runtime's agent entry computes without the model (diagnosis "overwrote-newer" or "identical-in-flight", +// confidence 1: a fact, not a probability). /** The value of the `schema` field of every file this package reads. */ export const RACES_SCHEMA = "genclass-races/1"; @@ -15,13 +17,16 @@ export const RACES_SCHEMA = "genclass-races/1"; export interface RaceSignature { /** "trigger|diagnosis|op", e.g. "request|duplicate|POST /api/orders". The grouping key. */ sig: string; - /** The trigger kind of the decision (runtime TriggerKind: "request", "delivery", "failure", ...). */ + /** + * The trigger kind of the decision (runtime TriggerKind: "request", "delivery", "failure", ...), or "witness" for + * a fact the runtime measured without the model. + */ trigger: string; - /** The model's diagnosis ("duplicate", "stale", "failing", "transient", "inconsistent", ...). */ + /** The model's diagnosis ("duplicate", "stale", "failing", "transient", "inconsistent", ...), or the witness kind. */ diagnosis: string; /** The operation the decision was about, ids normalised ("POST /api/orders", "GET /api/items/:id"). */ op: string; - /** The model's probability for `diagnosis`, in [0, 1] (the runtime's Decision.diagnosisConfidence). */ + /** The model's probability for `diagnosis`, in [0, 1] (the runtime's Decision.diagnosisConfidence); 1 for a witness. */ confidence: number; /** The evidence line the runtime printed for this decision (one line, query values already replaced by "…"). */ evidence: string; @@ -49,10 +54,11 @@ export interface GenClassRacesFile { * Optional: the page's model status when the fixture collected (null: GenClass never started in the page). When a * file flagged nothing and its model was not "ready" (for example "loading"), the report notes that the run may * not have been checked. A file with detections is reported whatever the state: the runtime can unload an idle - * model after it ran ("unloaded"). + * model after it ran ("unloaded"). State "off" (no model by choice: the fixture's default, which still computes + * witnesses) gets no note either. */ model?: { state?: string; name?: string; version?: string; device?: string } | null; - /** Optional: the chaos the fixture applied (its `chaos` option), shown when present. */ + /** Optional: the chaos the fixture applied (its `chaos` option: `latencyMs` is its `delayMs`), shown when present. */ chaos?: { latencyMs?: [number, number]; reorder?: boolean; failRate?: number; routes?: unknown[] } | null; } diff --git a/packages/playwright/test/chaos.test.ts b/packages/playwright/test/chaos.test.ts new file mode 100644 index 00000000..84e70210 --- /dev/null +++ b/packages/playwright/test/chaos.test.ts @@ -0,0 +1,180 @@ +// The seeded chaos (src/chaos.ts): the same seed gives the same delays, another seed gives others; the filter +// decides what is touched (never anything outside it, never the runtime's own files); with reorder off nothing +// overtakes. The scheduler is driven with hand-cranked timers, so nothing here waits on the clock. + +import { describe, expect, it } from "vitest"; +import { ChaosScheduler, DEFAULT_DELAY_MS, MAX_DELAY_MS, RUNTIME_PATH, confirms, delayFor, describeMatch, hash32, isCandidate, isRuntimeUrl, mulberry32, resolveChaos, type ChaosRequest } from "../src/chaos.js"; + +function manualTimers() { + const q: { at: number; fn: () => void; id: number }[] = []; + let now = 0; + let n = 0; + return { + timers: { + setTimeout: (fn: () => void, ms: number) => { + const id = ++n; + q.push({ at: now + ms, fn, id }); + return id; + }, + clearTimeout: (id: unknown) => { + const i = q.findIndex((x) => x.id === id); + if (i >= 0) q.splice(i, 1); + }, + }, + async advance(ms: number): Promise { + now += ms; + for (const x of [...q].filter((x) => x.at <= now).sort((a, b) => a.at - b.at)) { + q.splice(q.indexOf(x), 1); + x.fn(); + await Promise.resolve(); + } + await new Promise((r) => setTimeout(r, 0)); + }, + pending: () => q.length, + }; +} + +const req = (p: Partial = {}): ChaosRequest => ({ url: "http://app.test/api/items", method: "GET", resourceType: "fetch", sameOrigin: true, ...p }); + +describe("chaos: seeded values", () => { + it("hash32 and mulberry32 are deterministic and spread", () => { + expect(hash32("GET /api/search#1")).toBe(hash32("GET /api/search#1")); + expect(hash32("GET /api/search#1")).not.toBe(hash32("GET /api/search#2")); + const r = mulberry32(42); + const a = [r(), r(), r()]; + const b = mulberry32(42); + expect([b(), b(), b()]).toEqual(a); + for (const x of a) expect(x >= 0 && x < 1).toBe(true); + }); + + it("delayFor: within [min, max], the same for the same seed and key, different across seeds for most keys", () => { + const keys = Array.from({ length: 50 }, (_, i) => `GET /api/x#${i + 1}`); + let differ = 0; + for (const k of keys) { + const d = delayFor(7, k, [100, 600]); + expect(d).toBe(delayFor(7, k, [100, 600])); + expect(d >= 100 && d <= 600).toBe(true); + if (d !== delayFor(8, k, [100, 600])) differ++; + } + expect(differ).toBeGreaterThan(40); + expect(delayFor(1, "k", [250, 250])).toBe(250); + expect(delayFor(1, "k", [0, 0])).toBe(0); + }); + + it("resolveChaos: defaults, bounds, the repeat offset, and bad values", () => { + expect(resolveChaos(true)).toEqual({ seed: 1, delayMs: [...DEFAULT_DELAY_MS], reorder: true, match: undefined }); + expect(resolveChaos({})).toEqual(resolveChaos(true)); + expect(resolveChaos({ seed: 5 }, 2).seed).toBe(7); + expect(resolveChaos({ delayMs: [50, 20] }).delayMs).toEqual([50, 50]); + expect(resolveChaos({ delayMs: [0, 99999] }).delayMs).toEqual([0, MAX_DELAY_MS]); + expect(resolveChaos({ reorder: false, match: "/api" })).toMatchObject({ reorder: false, match: "/api" }); + expect(() => resolveChaos({ delayMs: [1] as unknown as [number, number] })).toThrow(/delayMs/); + expect(() => resolveChaos({ match: 5 as unknown as string })).toThrow(/match/); + }); +}); + +describe("chaos: the filter", () => { + it("only fetch and XHR requests, never the runtime's own files", () => { + expect(isCandidate(req(), undefined)).toBe(true); + expect(isCandidate(req({ resourceType: "xhr" }), undefined)).toBe(true); + for (const t of ["document", "script", "stylesheet", "image", "font", "websocket", "other"]) expect(isCandidate(req({ resourceType: t }), undefined), t).toBe(false); + expect(isCandidate(req({ url: `http://app.test${RUNTIME_PATH}genclass.global.js` }), undefined)).toBe(false); + expect(isCandidate(req({ url: `http://app.test${RUNTIME_PATH}agent.js` }), "/")).toBe(false); + expect(isCandidate(req({ url: `http://app.test${RUNTIME_PATH}x.js` }), () => true)).toBe(false); + expect(isRuntimeUrl("http://a/__genclass__/x")).toBe(true); + expect(isRuntimeUrl("http://a/api/__genclass__/x")).toBe(false); + }); + + it("by default: same-origin requests, confirmed by an /api path or a JSON response", () => { + expect(isCandidate(req({ sameOrigin: false }), undefined)).toBe(false); + expect(confirms(req(), undefined, "text/html")).toBe(true); // /api path + expect(confirms(req({ url: "http://app.test/graphql" }), undefined, "application/json; charset=utf-8")).toBe(true); + expect(confirms(req({ url: "http://app.test/graphql" }), undefined, "text/html")).toBe(false); + expect(confirms(req({ url: "http://app.test/graphql" }), undefined, null)).toBe(false); + }); + + it("with a match: a path prefix, a URL prefix, a RegExp, a function or a list, decided before the fetch", () => { + expect(isCandidate(req({ url: "http://app.test/graphql" }), "/graphql")).toBe(true); + expect(isCandidate(req({ url: "http://app.test/graphql" }), "/api")).toBe(false); + expect(isCandidate(req({ url: "https://api.other.com/v1/x", sameOrigin: false }), "https://api.other.com/")).toBe(true); + expect(isCandidate(req({ url: "https://api.other.com/v1/x", sameOrigin: false }), /\/v1\//)).toBe(true); + expect(isCandidate(req({ url: "https://api.other.com/v2/x", sameOrigin: false }), /\/v1\//g)).toBe(false); + expect(isCandidate(req({ method: "POST" }), (r) => r.method === "POST")).toBe(true); + expect(isCandidate(req({ method: "GET" }), (r) => r.method === "POST")).toBe(false); + expect(isCandidate(req({ url: "http://app.test/graphql" }), ["/api", "/graphql"])).toBe(true); + expect(isCandidate(req({ url: "http://app.test/other" }), ["/api", "/graphql"])).toBe(false); + // the filter already said yes: the response does not matter + expect(confirms(req({ url: "http://app.test/other" }), "/other", "text/plain")).toBe(true); + }); + + it("describeMatch for the race file", () => { + expect(describeMatch(undefined)).toEqual(["same-origin /api* or JSON responses"]); + expect(describeMatch(["/api", /x\/y/, () => true])).toEqual(["/api", "/x\\/y/", "function"]); + expect(describeMatch("https://a/")).toEqual(["https://a/"]); + }); +}); + +describe("chaos: the scheduler", () => { + it("keys requests by method, path and occurrence; the same seed plans the same delays, another seed others", () => { + const a = new ChaosScheduler(resolveChaos({ seed: 3, delayMs: [0, 600] })); + const b = new ChaosScheduler(resolveChaos({ seed: 3, delayMs: [0, 600] })); + const c = new ChaosScheduler(resolveChaos({ seed: 4, delayMs: [0, 600] })); + const urls = ["http://a/api/search?q=r", "http://a/api/search?q=re", "http://a/api/items/7", "http://a/api/search?q=rea"]; + const pa = urls.map((u) => a.plan("get", u)); + const pb = urls.map((u) => b.plan("GET", u)); + const pc = urls.map((u) => c.plan("GET", u)); + expect(pa.map((p) => p.key)).toEqual(["GET /api/search#1", "GET /api/search#2", "GET /api/items/7#1", "GET /api/search#3"]); + expect(pb).toEqual(pa); + expect(pc.map((p) => p.key)).toEqual(pa.map((p) => p.key)); + expect(pc.map((p) => p.delayMs)).not.toEqual(pa.map((p) => p.delayMs)); + for (const p of pa) expect(p.delayMs >= 0 && p.delayMs <= 600).toBe(true); + expect(a.plans).toEqual(pa); + expect(a.plan("GET", "https://other.test/api/x", false).key).toBe("GET other.test/api/x#1"); + }); + + it("reorder on: each response waits for its own delay only, so a later one can overtake", async () => { + const t = manualTimers(); + const s = new ChaosScheduler(resolveChaos({ seed: 1, delayMs: [0, 600] }), t.timers); + const order: string[] = []; + const first = s.hold({ key: "A", delayMs: 300 }).then(() => order.push("A")); + const second = s.hold({ key: "B", delayMs: 50 }).then(() => order.push("B")); + await t.advance(100); + expect(order).toEqual(["B"]); + await t.advance(300); + await Promise.all([first, second]); + expect(order).toEqual(["B", "A"]); + }); + + it("reorder off: a response is never released before an earlier matched one (latency only)", async () => { + const t = manualTimers(); + const s = new ChaosScheduler(resolveChaos({ seed: 1, delayMs: [0, 600], reorder: false }), t.timers); + const order: string[] = []; + const a = s.hold({ key: "A", delayMs: 300 }).then(() => order.push("A")); + const b = s.hold({ key: "B", delayMs: 50 }).then(() => order.push("B")); + const c = s.hold({ key: "C", delayMs: 0 }).then(() => order.push("C")); + await t.advance(100); + expect(order).toEqual([]); + await t.advance(300); + await Promise.all([a, b, c]); + expect(order).toEqual(["A", "B", "C"]); + }); + + it("a zero delay resolves at once; dispose releases every hold and clears the timers", async () => { + const t = manualTimers(); + const s = new ChaosScheduler(resolveChaos({ seed: 1 }), t.timers); + let now = false; + void s.hold({ key: "Z", delayMs: 0 }).then(() => (now = true)); + await Promise.resolve(); + expect(now).toBe(true); + const held = [s.hold({ key: "A", delayMs: 5000 }), s.hold({ key: "B", delayMs: 9000 })]; + expect(t.pending()).toBe(2); + s.dispose(); + expect(t.pending()).toBe(0); + await Promise.all(held); + // after dispose nothing is held any more + let late = false; + void s.hold({ key: "C", delayMs: 5000 }).then(() => (late = true)); + await Promise.resolve(); + expect(late).toBe(true); + }); +}); diff --git a/packages/playwright/test/e2e/playwright.config.ts b/packages/playwright/test/e2e/playwright.config.ts new file mode 100644 index 00000000..5d5aeab4 --- /dev/null +++ b/packages/playwright/test/e2e/playwright.config.ts @@ -0,0 +1,30 @@ +// The fixture's end-to-end check (run.mjs drives it): the search page from server.mjs, the fixture with seeded chaos, +// five seeds per test (repeatEach: the fixture adds the repeat index to the seed), and this package's reporter, so +// the run ends with test-results/genclass/*.json and genclass-report.md in this directory. Chromium only. +import { defineConfig } from "@playwright/test"; + +const port = Number(process.env.GENCLASS_E2E_PORT ?? 47431); + +export default defineConfig({ + testDir: ".", + testMatch: /.*\.spec\.ts/, + outputDir: "test-results", + workers: 1, + repeatEach: 5, + retries: 0, + timeout: 30_000, + reporter: [["list"], ["@genclass/playwright/reporter"]], + webServer: { + command: `node server.mjs ${port}`, + url: `http://127.0.0.1:${port}/`, + reuseExistingServer: false, + timeout: 30_000, + }, + use: { + baseURL: `http://127.0.0.1:${port}`, + headless: true, + genclass: { + chaos: { seed: 1, delayMs: [0, 600], reorder: true }, + }, + }, +}); diff --git a/packages/playwright/test/e2e/run.mjs b/packages/playwright/test/e2e/run.mjs new file mode 100644 index 00000000..bc9f4db1 --- /dev/null +++ b/packages/playwright/test/e2e/run.mjs @@ -0,0 +1,89 @@ +// End-to-end: the fixture on a real page in headless Chromium (playwright-reporter.yml runs it after building +// @genclass/runtime and this package). It runs Playwright with ./playwright.config.ts (the search page of server.mjs, +// the fixture with seeded chaos, five seeds per test, this package's reporter), then checks what was written: +// - the buggy page: a `witness|overwrote-newer|GET /api/search` signature in at least one of the five seeds (the +// count is printed: a race shows under some orders, not all); +// - the buggy page with `reorder: false` (latency only, responses keep their order): no signature in any seed, +// since nothing overtakes; +// - the fixed page: no signature at all in any of the five seeds (the false-report count on the correct variant); +// - every file validates as genclass-races/1 and carries the seed and the chaos settings; +// - genclass-report.md names the buggy test under the witness group and lists the fixed test under "nothing +// flagged"; the command renders the same report from the same files. +// Exit code 0 when all hold; the summary is printed as JSON. +// +// npm run build -w @genclass/runtime && npm run build -w @genclass/playwright +// node packages/playwright/test/e2e/run.mjs + +import { spawnSync } from "node:child_process"; +import { existsSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseRaces, summarize } from "../../dist/summarize.js"; + +const here = dirname(fileURLToPath(import.meta.url)); +const racesDir = join(here, "test-results", "genclass"); +const reportPath = join(here, "genclass-report.md"); +const summary = {}; +const fail = (msg) => { + summary.failed = msg; + console.log(JSON.stringify(summary, null, 2)); + process.exit(1); +}; + +for (const p of [join(here, "test-results"), reportPath]) rmSync(p, { recursive: true, force: true }); +if (!existsSync(resolve(here, "../../../runtime/dist/genclass.global.js"))) fail("build @genclass/runtime first"); +if (!existsSync(resolve(here, "../../dist/fixture.js"))) fail("build @genclass/playwright first"); + +const cli = resolve(here, "../../../../node_modules/@playwright/test/cli.js"); +const run = spawnSync(process.execPath, [cli, "test", "--config", join(here, "playwright.config.ts")], { cwd: here, stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, CI: "1" } }); +const out = `${run.stdout}${run.stderr}`; +console.log(out.slice(-4000)); +summary.playwrightStatus = run.status; +if (run.status !== 0) fail("the Playwright run failed"); + +// the files +const names = existsSync(racesDir) ? readdirSync(racesDir).filter((n) => n.endsWith(".json")).sort() : []; +summary.files = names.length; +if (names.length !== 15) fail(`expected 15 race files (3 tests × 5 seeds), found ${names.length}`); +const files = []; +for (const n of names) { + const r = parseRaces(readFileSync(join(racesDir, n), "utf8")); + if (!r.ok) fail(`${n} does not validate: ${r.errors.join("; ")}`); + files.push({ name: n, file: r.file }); +} +const buggy = files.filter((f) => f.file.test.includes("buggy page: results")); +const fifo = files.filter((f) => f.file.test.includes("buggy page, latency only")); +const fixed = files.filter((f) => f.file.test.includes("fixed page")); +if (buggy.length !== 5 || fifo.length !== 5 || fixed.length !== 5) fail(`expected 5 files per test, found buggy ${buggy.length}, latency-only ${fifo.length}, fixed ${fixed.length}`); +const seedsOf = (list) => list.map((f) => f.file.seed).sort((a, b) => a - b); +summary.seeds = { buggy: seedsOf(buggy), latencyOnly: seedsOf(fifo), fixed: seedsOf(fixed) }; +for (const s of Object.values(summary.seeds)) if (s.join() !== "1,2,3,4,5") fail("expected seeds 1 to 5 for each test"); +for (const f of files) { + const reorder = !fifo.includes(f); + if (!f.file.chaos || !f.file.chaos.latencyMs || f.file.chaos.latencyMs[0] !== 0 || f.file.chaos.latencyMs[1] !== 600 || f.file.chaos.reorder !== reorder) fail(`${f.name}: chaos settings missing or wrong: ${JSON.stringify(f.file.chaos)}`); + if (f.file.mode !== "observe") fail(`${f.name}: mode ${f.file.mode}, expected observe`); + if (!f.file.model || f.file.model.state !== "off") fail(`${f.name}: model ${JSON.stringify(f.file.model)}, expected state "off" (model skipped)`); +} +const SIG = "witness|overwrote-newer|GET /api/search"; +const stale = (f) => f.file.signatures.some((s) => s.sig === SIG); +const sigs = (list) => list.flatMap((f) => f.file.signatures.map((s) => `seed ${f.file.seed}: ${s.sig}`)); +summary.buggy = { staleSeeds: buggy.filter(stale).map((f) => f.file.seed), otherSignatures: sigs(buggy).filter((s) => !s.endsWith(SIG)), evidence: buggy.filter(stale).map((f) => f.file.signatures.find((s) => s.sig === SIG).evidence) }; +summary.latencyOnly = { signatures: sigs(fifo) }; +summary.fixed = { signatures: sigs(fixed) }; +if (summary.buggy.staleSeeds.length === 0) fail("no overwrote-newer witness on the buggy page in any seed"); +if (summary.latencyOnly.signatures.length !== 0) fail(`signatures on the buggy page with reorder off (latency keeps the order, so nothing should overtake): ${summary.latencyOnly.signatures.join(", ")}`); +if (summary.fixed.signatures.length !== 0) fail(`signatures on the fixed page (false reports): ${summary.fixed.signatures.join(", ")}`); + +// the report +if (!existsSync(reportPath)) fail("the reporter wrote no genclass-report.md"); +const report = readFileSync(reportPath, "utf8"); +const expected = summarize(names.map((n) => ({ name: `test-results/genclass/${n}`, content: readFileSync(join(racesDir, n), "utf8") })), { sources: ["test-results/genclass"] }); +if (report !== expected) fail("genclass-report.md differs from the command's rendering of the same files"); +if (!report.includes("| Witnesses (facts measured without the model) |")) fail("the report has no witness row"); +if (!report.includes("search.spec.ts > search > buggy page: results of the last query")) fail("the report does not name the buggy test"); +if (!/`GET \/api\/search` · `witness:overwrote-newer`/.test(report)) fail("the report has no overwrote-newer line for GET /api/search"); +if (!report.includes("2 tests with nothing flagged") || !report.includes("fixed page: results of the last query (5 seeds: 1, 2, 3, 4, 5)")) fail("the report does not list the fixed test under nothing flagged"); +if (report.includes("model was not ready")) fail("the report notes a model that was not ready (the model was off by choice)"); +summary.reportFirstLines = report.split("\n").slice(0, 4); +summary.ok = true; +console.log(JSON.stringify(summary, null, 2)); diff --git a/packages/playwright/test/e2e/search.spec.ts b/packages/playwright/test/e2e/search.spec.ts new file mode 100644 index 00000000..f202de50 --- /dev/null +++ b/packages/playwright/test/e2e/search.spec.ts @@ -0,0 +1,40 @@ +// Three tests on the e2e search page (server.mjs), all under the fixture's seeded chaos: the buggy page, where a +// slower older answer can land over the newer results; the same page with latency only (reorder off: nothing +// overtakes, so the bug cannot show); and the fixed page, which keeps only the latest answer. The tests themselves +// pass either way: what run.mjs checks is what the fixture wrote for each seed (a witness on the buggy page under +// reordering, none in the other two) and the report the reporter made of it. +// the built package, by name, as a user's spec imports it (run.mjs checks dist/ exists first) +import { expect, test } from "@genclass/playwright/fixture"; + +async function search(page: import("@playwright/test").Page, path: string): Promise { + await page.goto(path); + await page.getByLabel("Search").pressSequentially("react", { delay: 80 }); + // every answer has landed: nothing in flight on the fixture's runtime (its chaos holds each one up to 600 ms) + await page.waitForTimeout(200); + await page.waitForFunction(() => { + const g = (window as unknown as { GenClass?: { runtime?: { inflight(): unknown[] } | null } }).GenClass; + return !g?.runtime || g.runtime.inflight().length === 0; + }, null, { timeout: 15_000 }); + await page.waitForTimeout(300); + await expect(page.locator("#results li")).toHaveCount(2); +} + +test.describe("search", () => { + test("buggy page: results of the last query", async ({ page }) => { + await search(page, "/"); + }); + + test("fixed page: results of the last query", async ({ page }) => { + await search(page, "/?fixed=1"); + await expect(page.locator("#for")).toHaveText("Results for react"); + }); + + test.describe("latency only", () => { + test.use({ genclass: { chaos: { seed: 1, delayMs: [0, 600], reorder: false } } }); + + test("buggy page, latency only: results of the last query", async ({ page }) => { + await search(page, "/"); + await expect(page.locator("#for")).toHaveText("Results for react"); + }); + }); +}); diff --git a/packages/playwright/test/e2e/server.mjs b/packages/playwright/test/e2e/server.mjs new file mode 100644 index 00000000..2f9a5531 --- /dev/null +++ b/packages/playwright/test/e2e/server.mjs @@ -0,0 +1,67 @@ +// The app server of the fixture's end-to-end check (run.mjs): the buggy search page of the MCP server's e2e +// (packages/mcp/test/e2e/app/app.js), served the way race.e2e.mjs serves it, with two differences that this check +// needs and says so here: +// - the import map resolves `@genclass/runtime` to the page's own GenClass, the script-tag build the fixture +// injected (window.GenClass): the app's `GenClass.init(...)` then returns the fixture's runtime and its +// `rt.atom("search", ...)` registers the store there, the way an app integrated through the script tag does. A +// real ESM runtime next to the fixture's would be a second runtime whose registered stores the fixture does not +// see (the fixture sees what automatic discovery finds: React, Redux and Zustand state). +// - `@genclass/runtime/agent` resolves to a stub whose connectAgent does nothing: it is the link to the MCP bridge, +// which this check does not run (the fixture connects the agent entry itself, with every hold off). +// The page's own logic (app.js) is untouched. `?fixed=1` serves the fixed variant (window.__FIXED__). The backend +// answers /api/search in 40 ms, in order: on its own the race never shows; the fixture's chaos is what reorders. +// +// Usage: node server.mjs [port] (prints "listening " on stdout; port 0 picks a free one) + +import { createServer } from "node:http"; +import { readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const appJs = resolve(here, "../../../mcp/test/e2e/app/app.js"); +const port = Number(process.argv[2] ?? process.env.GENCLASS_E2E_PORT ?? 0); + +const SHIM_RUNTIME = `// the page's GenClass: the script-tag build the fixture injected before this module ran +export const GenClass = { + init: (o) => window.GenClass.init(o), + get runtime() { return window.GenClass.runtime; }, +}; +`; +const SHIM_AGENT = `// the MCP bridge link is not part of this check (the fixture connects the agent entry itself) +export function connectAgent() { return null; } +`; + +const server = createServer((req, res) => { + const url = new URL(req.url, "http://localhost"); + if (url.pathname === "/api/search") { + const q = url.searchParams.get("q") ?? ""; + setTimeout(() => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ items: [`${q} one`, `${q} two`] })); + }, 40); + return; + } + if (url.pathname === "/" || url.pathname === "/index.html") { + const fixed = url.searchParams.get("fixed") === "1"; + res.writeHead(200, { "content-type": "text/html" }); + res.end(`Search + +${fixed ? "" : ""} +

    +`); + return; + } + const js = (body) => { + res.writeHead(200, { "content-type": "text/javascript" }); + res.end(body); + }; + if (url.pathname === "/app.js") return js(readFileSync(appJs)); + if (url.pathname === "/rt/index.js") return js(SHIM_RUNTIME); + if (url.pathname === "/rt/agent.js") return js(SHIM_AGENT); + res.writeHead(404).end(); +}); + +server.listen(port, "127.0.0.1", () => { + console.log(`listening http://127.0.0.1:${server.address().port}`); +}); diff --git a/packages/playwright/test/fixture.test.ts b/packages/playwright/test/fixture.test.ts new file mode 100644 index 00000000..95c615be --- /dev/null +++ b/packages/playwright/test/fixture.test.ts @@ -0,0 +1,134 @@ +// The fixture entry (src/fixture.ts) without a browser: startGenClass on a fake page and a fake TestInfo. Inert when +// disabled; otherwise it installs, and at the end writes one genclass-races/1 file where the reporter reads it, named +// after the test, with the seed and chaos that produced it. + +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { outDirOf, startGenClass, testName, type TestInfoLike } from "../src/fixture.js"; +import type { PageLike } from "../src/install.js"; +import { parseRaces } from "../src/schema.js"; + +class FakePage implements PageLike { + routes: string[] = []; + initScripts: string[] = []; + exposed: string[] = []; + evaluateResult: unknown = null; + async route(pattern: string): Promise { + this.routes.push(pattern); + } + async addInitScript(s: { content: string }): Promise { + this.initScripts.push(s.content); + } + async exposeFunction(name: string): Promise { + this.exposed.push(name); + } + async evaluate(): Promise { + return this.evaluateResult; + } + isClosed(): boolean { + return false; + } + untouched(): boolean { + return this.routes.length === 0 && this.initScripts.length === 0 && this.exposed.length === 0; + } +} + +let root = ""; +let runtimeDir = ""; +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "genclass-fixture-")); + runtimeDir = join(root, "rt"); + mkdirSync(runtimeDir, { recursive: true }); + writeFileSync(join(runtimeDir, "genclass.global.js"), "/* build */\n"); + writeFileSync(join(runtimeDir, "agent.js"), "export function connectAgent() {}\n"); +}); +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +function info(over: Partial = {}): TestInfoLike { + return { + titlePath: ["", "chromium", "checkout.spec.ts", "checkout", "places one order"], + file: join(root, "tests", "checkout.spec.ts"), + testId: "a1b2c3-d4e5f6", + repeatEachIndex: 0, + project: { name: "chromium", testDir: join(root, "tests") }, + config: { configFile: join(root, "playwright.config.ts"), rootDir: join(root, "tests") }, + ...over, + }; +} + +describe("fixture: names and places", () => { + it("testName: [project] file > describe > title; no project prefix without a project name; odd shapes still name the test", () => { + expect(testName(info())).toBe("[chromium] checkout.spec.ts > checkout > places one order"); + expect(testName(info({ project: { name: "", testDir: join(root, "tests") }, titlePath: ["", "", "checkout.spec.ts", "checkout", "places one order"] }))).toBe("checkout.spec.ts > checkout > places one order"); + expect(testName(info({ titlePath: ["", "chromium", "sub/other.spec.ts", "t"], file: join(root, "tests", "sub", "other.spec.ts") }))).toBe("[chromium] sub/other.spec.ts > t"); + expect(testName(info({ titlePath: ["weird", "shape"], file: join(root, "tests", "x.spec.ts"), project: { name: "", testDir: join(root, "tests") } }))).toBe("weird > shape"); + }); + + it("outDirOf: test-results/genclass next to the config by default; a relative outDir against it; an absolute one as is", () => { + expect(outDirOf(info())).toBe(join(root, "test-results", "genclass")); + expect(outDirOf(info(), "out/races")).toBe(join(root, "out", "races")); + expect(outDirOf(info(), join(root, "abs"))).toBe(join(root, "abs")); + expect(outDirOf(info({ config: { rootDir: join(root, "tests") } }))).toBe(join(root, "tests", "test-results", "genclass")); + }); +}); + +describe("fixture: startGenClass", () => { + it("enabled false: the page is untouched and nothing is written", async () => { + const page = new FakePage(); + const run = await startGenClass(page, { enabled: false, chaos: true, runtimeDir }, info()); + const r = await run.finish(); + expect(page.untouched()).toBe(true); + expect(r).toEqual({ collected: null, file: null, path: null }); + expect(existsSync(join(root, "test-results"))).toBe(false); + }); + + it("refuses a mode other than observe and an unknown model value, before touching the page", async () => { + const page = new FakePage(); + await expect(startGenClass(page, { mode: "guard" as unknown as "observe", runtimeDir }, info())).rejects.toThrow(/must be "observe"/); + await expect(startGenClass(page, { model: "cdn" as unknown as "skip", runtimeDir }, info())).rejects.toThrow(/must be "skip" or "load"/); + expect(page.untouched()).toBe(true); + }); + + it("writes one valid file per test, named after it, with the seed from the repeat index when there is no chaos", async () => { + const page = new FakePage(); + page.evaluateResult = { mode: "observe", model: { state: "off", name: "none" }, agent: null }; + const run = await startGenClass(page, { runtimeDir }, info({ repeatEachIndex: 2 })); + expect(page.initScripts).toHaveLength(1); + const r = await run.finish(); + expect(r.path).toBe(join(root, "test-results", "genclass", "chromium-checkout-spec-ts-checkout-places-one-order-a1b2c3-d4e5f6-s3.json")); + expect(readdirSync(join(root, "test-results", "genclass"))).toEqual(["chromium-checkout-spec-ts-checkout-places-one-order-a1b2c3-d4e5f6-s3.json"]); + const parsed = parseRaces(readFileSync(r.path!, "utf8")); + expect(parsed.ok && parsed.file).toEqual({ + schema: "genclass-races/1", + test: "[chromium] checkout.spec.ts > checkout > places one order", + seed: 3, + signatures: [], + mode: "observe", + model: { state: "off", name: "none" }, + chaos: null, + }); + expect(r.file).toEqual(parsed.ok && parsed.file); + }); + + it("with chaos: the seed is the chaos seed plus the repeat index, and the settings are in the file", async () => { + const page = new FakePage(); + page.evaluateResult = { mode: "observe", model: { state: "off" }, agent: null }; + const run = await startGenClass(page, { runtimeDir, chaos: { seed: 10, delayMs: [100, 200], reorder: false, match: ["/api", /x/] }, outDir: "races" }, info({ repeatEachIndex: 1 })); + expect(page.routes[0]).toBe("**/*"); + const r = await run.finish(); + expect(r.path).toBe(join(root, "races", "chromium-checkout-spec-ts-checkout-places-one-order-a1b2c3-d4e5f6-s11.json")); + expect(r.file).toMatchObject({ seed: 11, chaos: { latencyMs: [100, 200], reorder: false, routes: ["/api", "/x/"] } }); + expect(parseRaces(readFileSync(r.path!, "utf8")).ok).toBe(true); + }); + + it("chaos true means the defaults", async () => { + const page = new FakePage(); + const run = await startGenClass(page, { runtimeDir, chaos: true }, info()); + const r = await run.finish(); + expect(r.file).toMatchObject({ seed: 1, chaos: { latencyMs: [0, 600], reorder: true, routes: ["same-origin /api* or JSON responses"] }, model: null, signatures: [] }); + }); +}); diff --git a/packages/playwright/test/install.test.ts b/packages/playwright/test/install.test.ts new file mode 100644 index 00000000..9c7694d0 --- /dev/null +++ b/packages/playwright/test/install.test.ts @@ -0,0 +1,268 @@ +// The installer (src/install.ts) on a fake page: what it registers, what the init script carries, how the runtime's +// files are served (and what is refused), how chaos routes behave, and what finish() collects. No browser. + +import { mkdirSync, mkdtempSync, rmSync, writeFileSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { AGENT_URL, BINDING, initScriptContent, install, resolveRuntimeDir, runtimeFile, type PageLike, type RequestLike, type ResponseLike, type RouteLike } from "../src/install.js"; +import { RUNTIME_PATH, resolveChaos } from "../src/chaos.js"; +import { here } from "./helpers.js"; + +type Handler = (route: RouteLike, request: RequestLike) => Promise | void; + +class FakePage implements PageLike { + routes: { pattern: string; handler: Handler }[] = []; + initScripts: string[] = []; + exposed = new Map unknown>(); + evaluateResult: unknown = null; + evaluated: string[] = []; + closed = false; + async route(pattern: string, handler: Handler): Promise { + this.routes.push({ pattern, handler }); + } + async addInitScript(s: { content: string }): Promise { + this.initScripts.push(s.content); + } + async exposeFunction(name: string, fn: (item: unknown) => unknown): Promise { + this.exposed.set(name, fn); + } + async evaluate(expression: string): Promise { + this.evaluated.push(expression); + return this.evaluateResult; + } + isClosed(): boolean { + return this.closed; + } + handler(pattern: string): Handler { + const r = this.routes.find((x) => x.pattern === pattern); + if (!r) throw new Error(`no route ${pattern}`); + return r.handler; + } + push(item: unknown): void { + this.exposed.get(BINDING)!(item); + } +} + +class FakeRoute implements RouteLike { + calls: string[] = []; + fulfilled: Parameters[0] | null = null; + constructor( + private readonly response: ResponseLike | Error = { headers: () => ({ "content-type": "application/json" }) }, + ) {} + async fetch(): Promise { + this.calls.push("fetch"); + if (this.response instanceof Error) throw this.response; + return this.response; + } + async fulfill(o: Parameters[0]): Promise { + this.calls.push("fulfill"); + this.fulfilled = o; + } + async fallback(): Promise { + this.calls.push("fallback"); + } + async abort(code?: string): Promise { + this.calls.push(`abort:${code ?? ""}`); + } +} + +const request = (url: string, o: { method?: string; resourceType?: string; frameUrl?: string } = {}): RequestLike => ({ + url: () => url, + method: () => o.method ?? "GET", + resourceType: () => o.resourceType ?? "fetch", + frame: () => ({ url: () => o.frameUrl ?? "http://app.test/page" }), +}); + +let runtimeDir = ""; +beforeEach(() => { + runtimeDir = mkdtempSync(join(tmpdir(), "genclass-rt-")); + writeFileSync(join(runtimeDir, "genclass.global.js"), "/* the script-tag build */ window.__fakeGlobalRan = true;\n"); + writeFileSync(join(runtimeDir, "agent.js"), "export function connectAgent() {}\n"); + mkdirSync(join(runtimeDir, "cdn")); + writeFileSync(join(runtimeDir, "cdn", "worker.js"), "// worker\n"); +}); +afterEach(() => { + rmSync(runtimeDir, { recursive: true, force: true }); +}); + +describe("install: the runtime directory", () => { + it("takes a given dist/ and refuses one without the script-tag build", () => { + expect(resolveRuntimeDir(runtimeDir)).toBe(runtimeDir); + const empty = mkdtempSync(join(tmpdir(), "genclass-empty-")); + expect(() => resolveRuntimeDir(empty)).toThrow(/no genclass\.global\.js/); + rmSync(empty, { recursive: true, force: true }); + }); + + const built = existsSync(join(here, "..", "..", "runtime", "dist", "genclass.global.js")); + it.runIf(built)("by default finds the installed @genclass/runtime's dist/", () => { + const dir = resolveRuntimeDir(); + expect(dir.endsWith(join("runtime", "dist"))).toBe(true); + expect(existsSync(join(dir, "agent.js"))).toBe(true); + }); + + it("maps a request on the virtual path to a file inside dist/, and nothing else", () => { + expect(runtimeFile(runtimeDir, `http://app.test${RUNTIME_PATH}genclass.global.js`)).toBe(join(runtimeDir, "genclass.global.js")); + expect(runtimeFile(runtimeDir, `http://app.test${RUNTIME_PATH}cdn/worker.js?x=1`)).toBe(join(runtimeDir, "cdn", "worker.js")); + expect(runtimeFile(runtimeDir, `http://app.test${RUNTIME_PATH}../package.json`)).toBeNull(); + expect(runtimeFile(runtimeDir, `http://app.test${RUNTIME_PATH}cdn/..%2F..%2Fetc/passwd`)).toBeNull(); + expect(runtimeFile(runtimeDir, `http://app.test${RUNTIME_PATH}`)).toBeNull(); + expect(runtimeFile(runtimeDir, `http://app.test${RUNTIME_PATH}a b.js`)).toBeNull(); + expect(runtimeFile(runtimeDir, "http://app.test/other/agent.js")).toBeNull(); + expect(runtimeFile(runtimeDir, "not a url")).toBeNull(); + }); +}); + +describe("install: what goes on the page", () => { + it("one exposed function, the runtime route, the agent abort route and one init script; no chaos route without chaos", async () => { + const page = new FakePage(); + await install(page, { runtimeDir, model: "skip", chaos: null }); + expect([...page.exposed.keys()]).toEqual([BINDING]); + expect(page.routes.map((r) => r.pattern)).toEqual([`**${RUNTIME_PATH}**`, `${AGENT_URL}/**`]); + expect(page.initScripts).toHaveLength(1); + const script = page.initScripts[0]; + expect(script).toContain("/* the script-tag build */"); + expect(script).toContain(`"base":"${RUNTIME_PATH}"`); + expect(script).toContain('"model":false'); + expect(script).toContain(`"agentUrl":"${AGENT_URL}"`); + expect(script).toContain(`"binding":"${BINDING}"`); + expect(script).toContain("__GENCLASS_PLAYWRIGHT__"); + expect(script).toContain('telemetry: false'); + expect(script).toContain('mode: "observe"'); + expect(script).toContain("hold: false"); + }); + + it("the init script is the page function applied to its config and the build, as plain JavaScript", () => { + const content = initScriptContent(runtimeDir, { base: RUNTIME_PATH, model: { baseUrl: "/models/" }, agentUrl: AGENT_URL, binding: BINDING }); + expect(content.startsWith("(function genclassPageScript(cfg, runGlobal)")).toBe(true); + expect(content).toContain('"model":{"baseUrl":"/models/"}'); + // it parses as a script (no TypeScript left in it) + expect(() => new Function(content)).not.toThrow(); + }); + + it("model load without a URL passes an empty model options object", async () => { + const page = new FakePage(); + await install(page, { runtimeDir, model: "load", chaos: null }); + expect(page.initScripts[0]).toContain('"model":{}'); + }); + + it("serves the runtime's files from disk with the right content type, and 404 for anything else", async () => { + const page = new FakePage(); + await install(page, { runtimeDir, model: "skip", chaos: null }); + const serve = page.handler(`**${RUNTIME_PATH}**`); + const ok = new FakeRoute(); + await serve(ok, request(`http://app.test${RUNTIME_PATH}genclass.global.js`, { resourceType: "script" })); + expect(ok.calls).toEqual(["fulfill"]); + expect(ok.fulfilled).toMatchObject({ status: 200, headers: { "content-type": "text/javascript; charset=utf-8", "cache-control": "no-store" } }); + expect(String(ok.fulfilled!.body)).toContain("the script-tag build"); + const worker = new FakeRoute(); + await serve(worker, request(`http://app.test${RUNTIME_PATH}cdn/worker.js`, { resourceType: "other" })); + expect(worker.fulfilled).toMatchObject({ status: 200 }); + for (const u of [`http://app.test${RUNTIME_PATH}missing.js`, `http://app.test${RUNTIME_PATH}../package.json`]) { + const r = new FakeRoute(); + await serve(r, request(u)); + expect(r.fulfilled, u).toMatchObject({ status: 404 }); + } + }); + + it("aborts the page's agent link so it never reaches a bridge", async () => { + const page = new FakePage(); + await install(page, { runtimeDir, model: "skip", chaos: null }); + const r = new FakeRoute(); + await page.handler(`${AGENT_URL}/**`)(r, request(`${AGENT_URL}/v1/hello`, { method: "POST" })); + expect(r.calls).toEqual(["abort:connectionrefused"]); + }); +}); + +describe("install: chaos routes", () => { + it("registers the chaos route first, so the runtime's files and the agent URL are matched before it", async () => { + const page = new FakePage(); + await install(page, { runtimeDir, model: "skip", chaos: resolveChaos({ seed: 1, delayMs: [0, 0] }) }); + expect(page.routes.map((r) => r.pattern)).toEqual(["**/*", `**${RUNTIME_PATH}**`, `${AGENT_URL}/**`]); + }); + + it("leaves alone what is outside the filter: documents, scripts, cross-origin requests, the runtime's files, the agent URL", async () => { + const page = new FakePage(); + await install(page, { runtimeDir, model: "skip", chaos: resolveChaos({ seed: 1, delayMs: [0, 0] }) }); + const chaos = page.handler("**/*"); + for (const [url, o] of [ + ["http://app.test/page", { resourceType: "document" }], + ["http://app.test/app.js", { resourceType: "script" }], + ["https://api.other.test/api/x", {}], + [`http://app.test${RUNTIME_PATH}agent.js`, {}], + [`${AGENT_URL}/v1/hello`, { method: "POST" }], + ] as const) { + const r = new FakeRoute(); + await chaos(r, request(url, o)); + expect(r.calls, url).toEqual(["fallback"]); + } + }); + + it("fetches a candidate, delays a confirmed one and fulfils with the real response; a non-JSON non-API response goes through at once", async () => { + const page = new FakePage(); + const session = await install(page, { runtimeDir, model: "skip", chaos: resolveChaos({ seed: 1, delayMs: [5, 5] }) }); + const chaos = page.handler("**/*"); + const api = new FakeRoute({ headers: () => ({ "content-type": "text/html" }) }); + const t0 = Date.now(); + await chaos(api, request("http://app.test/api/items?page=2", { resourceType: "xhr" })); + expect(api.calls).toEqual(["fetch", "fulfill"]); + expect(Date.now() - t0).toBeGreaterThanOrEqual(4); + const json = new FakeRoute({ headers: () => ({ "Content-Type": "application/json" }) }); + await chaos(json, request("http://app.test/graphql", { method: "POST" })); + expect(json.calls).toEqual(["fetch", "fulfill"]); + const html = new FakeRoute({ headers: () => ({ "content-type": "text/html" }) }); + await chaos(html, request("http://app.test/fragment")); + expect(html.calls).toEqual(["fetch", "fulfill"]); + const c = await session.finish(); + expect(c.plans.map((p) => p.key)).toEqual(["GET /api/items#1", "POST /graphql#1"]); + expect(c.plans.every((p) => p.delayMs === 5)).toBe(true); + }); + + it("a request that fails to fetch is aborted so the app sees the failure; a closed page is not an error", async () => { + const page = new FakePage(); + await install(page, { runtimeDir, model: "skip", chaos: resolveChaos({ seed: 1, delayMs: [0, 0] }) }); + const chaos = page.handler("**/*"); + const down = new FakeRoute(new Error("connection refused")); + await chaos(down, request("http://app.test/api/x")); + expect(down.calls).toEqual(["fetch", "abort:failed"]); + const gone = new FakeRoute({ headers: () => ({ "content-type": "application/json" }) }); + gone.fulfill = async () => { + throw new Error("Target page, context or browser has been closed"); + }; + await expect(chaos(gone, request("http://app.test/api/x"))).resolves.toBeUndefined(); + }); +}); + +describe("install: collecting", () => { + it("keeps what the page pushes through the binding and what collect() returns at the end", async () => { + const page = new FakePage(); + const session = await install(page, { runtimeDir, model: "skip", chaos: null }); + page.push({ t: "status", mode: "observe", model: { state: "off", name: "none" } }); + page.push({ t: "record", record: { kind: "detection", diagnosis: "stale" } }); + page.push({ t: "witness", witness: { kind: "overwrote-newer" } }); + page.push({ t: "agent", ok: true, reason: null }); + page.push({ t: "nonsense" }); + page.push(null); + page.evaluateResult = { mode: "observe", model: { state: "off", name: "none", version: "0" }, agent: null }; + const c = await session.finish(); + expect(page.evaluated).toHaveLength(1); + expect(c).toEqual({ records: [{ kind: "detection", diagnosis: "stale" }], witnesses: [{ kind: "overwrote-newer" }], mode: "observe", model: { state: "off", name: "none", version: "0" }, agent: null, plans: [] }); + }); + + it("a page that never started GenClass, or that is closed, or whose agent failed", async () => { + const page = new FakePage(); + const session = await install(page, { runtimeDir, model: "skip", chaos: null }); + page.push({ t: "status", mode: null, model: null, error: "boom" }); + page.push({ t: "agent", ok: false, reason: "not-dev-host" }); + page.closed = true; + const c = await session.finish(); + expect(page.evaluated).toEqual([]); + expect(c).toMatchObject({ records: [], witnesses: [], mode: null, model: null, agent: "not-dev-host" }); + const failing = new FakePage(); + const s2 = await install(failing, { runtimeDir, model: "skip", chaos: null }); + failing.evaluate = async () => { + throw new Error("Execution context was destroyed"); + }; + await expect(s2.finish()).resolves.toMatchObject({ agent: "not-loaded" }); + }); +}); diff --git a/packages/playwright/test/package.test.ts b/packages/playwright/test/package.test.ts index 350ee8a4..1c2e9f91 100644 --- a/packages/playwright/test/package.test.ts +++ b/packages/playwright/test/package.test.ts @@ -1,5 +1,7 @@ -// Package invariants: no runtime dependencies, nothing imported from Playwright or @genclass/runtime, the pure -// entries use no Node APIs, the exports map names what tsup builds, the README shows the fixture's exact report. +// Package invariants: no runtime dependencies (the fixture's needs, @playwright/test and @genclass/runtime, are +// optional peer dependencies), nothing but the fixture imports Playwright, the pure entries use no Node APIs, the +// page script imports nothing at all, the exports map names what tsup builds, the README shows the fixture's exact +// report. import { existsSync, readFileSync, realpathSync } from "node:fs"; import { createRequire } from "node:module"; @@ -16,16 +18,21 @@ const src = (f: string) => readFileSync(join(pkgDir, "src", f), "utf8"); const imports = (text: string) => [...text.matchAll(/^\s*(?:import|export)\s[^;]*?from\s+"([^"]+)"/gm)].map((m) => m[1]); describe("package: dependencies and imports", () => { - it("has no runtime or peer dependencies", () => { + it("has no runtime dependencies; Playwright and the runtime are optional peer dependencies (the fixture's)", () => { expect(pkg.dependencies).toBeUndefined(); - expect(pkg.peerDependencies).toBeUndefined(); expect(pkg.optionalDependencies).toBeUndefined(); + expect(pkg.peerDependencies).toEqual({ "@genclass/runtime": ">=0.3.0-beta.1", "@playwright/test": ">=1.40.0" }); + expect(pkg.peerDependenciesMeta).toEqual({ "@genclass/runtime": { optional: true }, "@playwright/test": { optional: true } }); expect(pkg.license).toBe("Apache-2.0"); }); - it("imports only its own modules and Node built-ins (and the pure entries not even those)", () => { - for (const f of ["index.ts", "summarize.ts", "schema.ts", "types.ts"]) for (const i of imports(src(f))) expect(i, `${f}: ${i}`).toMatch(/^\.\/[a-z]+\.js$/); - for (const f of ["reporter.ts", "files.ts", "cli.ts"]) for (const i of imports(src(f))) expect(i, `${f}: ${i}`).toMatch(/^(\.\/[a-z]+\.js|node:[a-z]+)$/); + it("imports only its own modules and Node built-ins (the pure entries not even those); only the fixture imports Playwright", () => { + for (const f of ["index.ts", "summarize.ts", "schema.ts", "types.ts", "chaos.ts", "page.ts"]) for (const i of imports(src(f))) expect(i, `${f}: ${i}`).toMatch(/^\.\/[a-z]+\.js$/); + expect(imports(src("page.ts"))).toEqual([]); + for (const f of ["reporter.ts", "files.ts", "cli.ts", "install.ts", "races.ts"]) for (const i of imports(src(f))) expect(i, `${f}: ${i}`).toMatch(/^(\.\/[a-z]+\.js|node:[a-z]+)$/); + for (const i of imports(src("fixture.ts"))) expect(i, `fixture.ts: ${i}`).toMatch(/^(\.\/[a-z]+\.js|node:[a-z]+|@playwright\/test)$/); + // the runtime is never imported: the fixture reads its dist/ from disk and the page loads it from there + for (const f of ["fixture.ts", "install.ts", "page.ts", "races.ts", "chaos.ts"]) expect(src(f)).not.toMatch(/from "@genclass\/runtime/); }); it("the exports map and the bin name what tsup builds; every entry also has a default condition", () => { @@ -34,11 +41,13 @@ describe("package: dependencies and imports", () => { ".": { types: "./dist/index.d.ts", import: "./dist/index.js", default: "./dist/index.js" }, "./summarize": { types: "./dist/summarize.d.ts", import: "./dist/summarize.js", default: "./dist/summarize.js" }, "./reporter": { types: "./dist/reporter.d.ts", import: "./dist/reporter.js", default: "./dist/reporter.js" }, + "./fixture": { types: "./dist/fixture.d.ts", import: "./dist/fixture.js", default: "./dist/fixture.js" }, "./package.json": "./package.json", }); expect(pkg.bin).toEqual({ "genclass-playwright-report": "bin/genclass-playwright-report.mjs" }); const tsup = readFileSync(join(pkgDir, "tsup.config.ts"), "utf8"); - expect(tsup).toContain('entry: { index: "src/index.ts", summarize: "src/summarize.ts", reporter: "src/reporter.ts", cli: "src/cli.ts" }'); + expect(tsup).toContain('entry: { index: "src/index.ts", summarize: "src/summarize.ts", reporter: "src/reporter.ts", fixture: "src/fixture.ts", cli: "src/cli.ts" }'); + expect(tsup).toContain('external: ["@playwright/test", "@genclass/runtime"]'); }); it("the root entry and /summarize export the same pure API", () => { @@ -54,6 +63,8 @@ describe("package: dependencies and imports", () => { expect(entry.version).toBe(pkg.version); expect(entry.dependencies).toBeUndefined(); expect(entry.devDependencies).toEqual(pkg.devDependencies); + expect(entry.peerDependencies).toEqual(pkg.peerDependencies); + expect(entry.peerDependenciesMeta).toEqual(pkg.peerDependenciesMeta); expect(entry.bin).toEqual(pkg.bin); }); }); @@ -64,7 +75,7 @@ describe("package: dependencies and imports", () => { // a real Playwright run that loads the reporter this way (test/playwright-smoke). const built = existsSync(join(pkgDir, "dist", "reporter.js")); -describe("package: how Playwright finds the reporter", () => { +describe("package: how Playwright finds the reporter and the fixture", () => { it.runIf(built || Boolean(process.env.CI))("require.resolve from the repository root finds dist/reporter.js, whose default export is the reporter", async () => { const require = createRequire(import.meta.url); const repoRoot = join(pkgDir, "..", ".."); @@ -75,7 +86,18 @@ describe("package: how Playwright finds the reporter", () => { const reporter = new (mod.default as new (o: object) => { printsToStdio(): boolean; onEnd(): void })({ quiet: true, outputFile: false }); expect(reporter.printsToStdio()).toBe(false); expect(typeof reporter.onEnd).toBe("function"); - for (const id of ["@genclass/playwright", "@genclass/playwright/summarize"]) expect(realpathSync(require.resolve(id, { paths: [repoRoot] }))).toMatch(/[\\/]dist[\\/](index|summarize)\.js$/); + for (const id of ["@genclass/playwright", "@genclass/playwright/summarize", "@genclass/playwright/fixture"]) expect(realpathSync(require.resolve(id, { paths: [repoRoot] }))).toMatch(/[\\/]dist[\\/](index|summarize|fixture)\.js$/); + }); + + it.runIf(built || Boolean(process.env.CI))("dist/fixture.js exports test and expect, and carries the page script as source text", async () => { + const mod = (await import(pathToFileURL(join(pkgDir, "dist", "fixture.js")).href)) as Record; + expect(typeof mod.test).toBe("function"); + expect(typeof mod.expect).toBe("function"); + expect(typeof mod.startGenClass).toBe("function"); + const text = readFileSync(join(pkgDir, "dist", "fixture.js"), "utf8"); + // the page script is serialised with toString() at run time: the bundle must keep it a plain function + expect(text).toContain("function genclassPageScript(cfg, runGlobal)"); + expect(text).toContain("__GENCLASS_PLAYWRIGHT__"); }); }); diff --git a/packages/playwright/test/playwright-smoke/races.spec.ts b/packages/playwright/test/playwright-smoke/races.spec.ts index 01ae3e9e..9c6f7a85 100644 --- a/packages/playwright/test/playwright-smoke/races.spec.ts +++ b/packages/playwright/test/playwright-smoke/races.spec.ts @@ -1,5 +1,6 @@ -// Stands in for the GenClass fixture (which is developed elsewhere): it writes the unit-test fixture's -// genclass-races/1 files to /genclass/ during the run. No page, so no browser is launched. +// Stands in for the GenClass fixture (src/fixture.ts, which needs a browser; test/e2e runs it in Chromium): it +// writes the unit-test fixture's genclass-races/1 files to /genclass/ during the run. No page, so no +// browser is launched, and the reporter is still loaded by Playwright the way a user's config loads it. import { copyFileSync, mkdirSync, readdirSync } from "node:fs"; import { join } from "node:path"; import { expect, test } from "@playwright/test"; diff --git a/packages/playwright/test/races.test.ts b/packages/playwright/test/races.test.ts new file mode 100644 index 00000000..eeea212b --- /dev/null +++ b/packages/playwright/test/races.test.ts @@ -0,0 +1,124 @@ +// The race-file builder (src/races.ts): what the fixture collected -> a "genclass-races/1" file the reporter's own +// validator accepts, with the signatures the summariser groups on. + +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { buildRacesFile, opOf, racesFileName, signaturesOf, witnessLine, writeRacesFile } from "../src/races.js"; +import { parseRaces, validateRaces } from "../src/schema.js"; +import { buildSummary } from "../src/summarize.js"; + +const detection = (over: Record = {}) => ({ + schema: 1, + kind: "detection", + id: "s1", + ts: 1, + sessionId: "x", + tags: {}, + mode: "observe", + sampled: true, + trigger: "delivery", + diagnosis: "stale", + p: 0.91, + detection: "likely", + evidence: { message: "The response of GET /api/search (#21) arrives after a newer GET /api/search (#23) was answered.", subject: "response to GET /api/search?q=… (#21)" }, + ...over, +}); +const overwrote = { kind: "overwrote-newer", t: 1200, subject: "search.results", by: "GET /api/search#1", over: "GET /api/search#4", overKind: "request", gapMs: 260, change: "2 items → 2 items" }; +const identical = { kind: "identical-in-flight", t: 900, subject: "POST /api/orders", by: "POST /api/orders#2", over: "POST /api/orders#1", overKind: "request", gapMs: 30 }; + +describe("races: operations and evidence lines", () => { + it("opOf keeps the method and path, drops the query, the id and the suffix", () => { + expect(opOf("response to GET /api/search?q=… (#21)")).toBe("GET /api/search"); + expect(opOf("GET /api/search#3~retry2")).toBe("GET /api/search"); + expect(opOf("POST /api/orders (network error)")).toBe("POST /api/orders"); + expect(opOf("write to cart.items")).toBe("write to cart.items"); + expect(opOf(undefined)).toBe(""); + expect(opOf(`x${"y".repeat(200)}`)).toHaveLength(120); + }); + + it("witnessLine says what happened, in the agent's request keys", () => { + expect(witnessLine(overwrote)).toBe("The response of GET /api/search#1 wrote search.results over what newer GET /api/search#4 had written 0.26s after GET /api/search#1 started (2 items → 2 items)."); + expect(witnessLine({ ...overwrote, overKind: "user", over: 'type textbox "Title"', change: undefined })).toBe('The response of GET /api/search#1 wrote search.results over what the user (type textbox "Title") had written 0.26s after GET /api/search#1 started.'); + expect(witnessLine(identical)).toBe("POST /api/orders#2 was sent while the identical POST /api/orders#1 was still in flight (0.03s earlier), from a separate user action."); + expect(witnessLine({ kind: "other", by: "A", over: "B" })).toBe("other: by A over B"); + }); +}); + +describe("races: signatures and the file", () => { + it("detections become signatures with the model's confidence; witnesses get trigger witness and confidence 1", () => { + const sigs = signaturesOf([detection()], [overwrote, identical], 3); + expect(sigs.map((s) => s.sig)).toEqual(["delivery|stale|GET /api/search", "witness|identical-in-flight|POST /api/orders", "witness|overwrote-newer|GET /api/search"]); + expect(sigs[0]).toMatchObject({ trigger: "delivery", diagnosis: "stale", op: "GET /api/search", confidence: 0.91, seeds: [3] }); + expect(sigs[0].evidence).toContain("arrives after a newer"); + expect(sigs[2]).toMatchObject({ trigger: "witness", diagnosis: "overwrote-newer", op: "GET /api/search", confidence: 1, seeds: [3] }); + }); + + it("drops expected diagnoses, records of other kinds and junk; merges repeats keeping the most confident evidence", () => { + const sigs = signaturesOf( + [detection({ diagnosis: "expected" }), detection({ kind: "intervention" }), null, 5, detection({ p: 0.5, evidence: { message: "low", subject: "GET /api/search" } }), detection({ p: 0.95, evidence: { message: "high", subject: "GET /api/search" } }), detection({ p: 2 })], + [overwrote, overwrote, {}, "x"], + 1, + ); + expect(sigs).toHaveLength(2); + expect(sigs[0]).toMatchObject({ sig: "delivery|stale|GET /api/search", confidence: 1, evidence: "The response of GET /api/search (#21) arrives after a newer GET /api/search (#23) was answered." }); + expect(sigs[1].sig).toBe("witness|overwrote-newer|GET /api/search"); + }); + + it("buildRacesFile validates against the reporter's schema and carries mode, model, chaos and the seed", () => { + const file = buildRacesFile({ + test: "search.spec.ts > search > results of the last query", + seed: 2, + mode: "observe", + model: { state: "off", name: "none" }, + chaos: { latencyMs: [0, 600], reorder: true, routes: ["same-origin /api* or JSON responses"] }, + records: [detection()], + witnesses: [overwrote], + }); + const r = validateRaces(file); + expect(r.ok).toBe(true); + if (!r.ok) return; + expect(r.file).toEqual(file); + expect(file.model).toEqual({ state: "off", name: "none" }); + expect(file.chaos).toEqual({ latencyMs: [0, 600], reorder: true, routes: ["same-origin /api* or JSON responses"] }); + const s = buildSummary([{ name: "a.json", content: file }]); + expect(s.tests[0].signatures.map((x) => [x.category, x.sig])).toEqual([ + ["stale", "delivery|stale|GET /api/search"], + ["witness", "witness|overwrote-newer|GET /api/search"], + ]); + }); + + it("a page where GenClass never started: model null, no mode, no signatures (still a valid file)", () => { + const file = buildRacesFile({ test: "t", seed: 1, mode: null, model: null, chaos: null, records: [], witnesses: [] }); + expect(file).toEqual({ schema: "genclass-races/1", test: "t", seed: 1, signatures: [], model: null, chaos: null }); + expect(validateRaces(file).ok).toBe(true); + expect(buildSummary([{ name: "a.json", content: file }]).tests[0].modelNotReady).toEqual({ files: 1, states: ["not started"] }); + }); + + it("racesFileName: a slug of the title, the test id and the seed", () => { + expect(racesFileName("[chromium] checkout.spec.ts > checkout > places one order", "a1b2-c3d4", 2)).toBe("chromium-checkout-spec-ts-checkout-places-one-order-a1b2-c3d4-s2.json"); + expect(racesFileName("x".repeat(200), "id", 1)).toBe(`${"x".repeat(80)}-id-s1.json`); + expect(racesFileName("", "../../etc", 1)).toBe("test-etc-s1.json"); + }); +}); + +describe("races: writing", () => { + let dir = ""; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "genclass-races-")); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("writes a file the reporter reads, creating the directory", () => { + const file = buildRacesFile({ test: "t", seed: 4, mode: "observe", model: { state: "off" }, chaos: null, records: [], witnesses: [identical] }); + const path = writeRacesFile(join(dir, "test-results", "genclass"), racesFileName(file.test, "id", 4), file); + expect(path).toBe(join(dir, "test-results", "genclass", "t-id-s4.json")); + const text = readFileSync(path, "utf8"); + expect(text.endsWith("\n")).toBe(true); + const r = parseRaces(text); + expect(r.ok && r.file).toEqual(file); + }); +}); diff --git a/packages/playwright/test/summarize.test.ts b/packages/playwright/test/summarize.test.ts index a18b1bbc..69936c6c 100644 --- a/packages/playwright/test/summarize.test.ts +++ b/packages/playwright/test/summarize.test.ts @@ -46,7 +46,7 @@ describe("summarize: the fixture (two tests, two seeds)", () => { testsWithSignatures: 2, signatures: 2, reproducible: 1, - byCategory: { duplicate: { signatures: 1, reproducible: 1 }, stale: { signatures: 1, reproducible: 0 }, failing: { signatures: 0, reproducible: 0 }, other: { signatures: 0, reproducible: 0 } }, + byCategory: { duplicate: { signatures: 1, reproducible: 1 }, stale: { signatures: 1, reproducible: 0 }, failing: { signatures: 0, reproducible: 0 }, other: { signatures: 0, reproducible: 0 }, witness: { signatures: 0, reproducible: 0 } }, }); }); diff --git a/packages/playwright/test/witness-report.test.ts b/packages/playwright/test/witness-report.test.ts new file mode 100644 index 00000000..6dc28e21 --- /dev/null +++ b/packages/playwright/test/witness-report.test.ts @@ -0,0 +1,89 @@ +// The report with the fixture's witness signatures (trigger "witness"): their own group, shown only when there are +// any, no confidence on their lines, a footer that says what they are, and no "model not ready" note for a model +// that was off by choice. The model-only report (test/summarize.test.ts, README) is unchanged by all of it. + +import { describe, expect, it } from "vitest"; +import { CATEGORIES, CATEGORY_TITLES, WITNESS_TRIGGER, buildSummary, categoryOf, summarize } from "../src/summarize.js"; +import type { GenClassRacesFile, RaceSignature } from "../src/types.js"; +import { FIXTURE_REPORT, fixtureInputs } from "./helpers.js"; + +const witness = (op: string, seed: number, kind = "overwrote-newer"): RaceSignature => ({ + sig: `witness|${kind}|${op}`, + trigger: WITNESS_TRIGGER, + diagnosis: kind, + op, + confidence: 1, + evidence: `The response of ${op}#1 wrote results over what newer ${op}#4 had written 0.26s after ${op}#1 started.`, + seeds: [seed], +}); +const model = (op: string, seed: number): RaceSignature => ({ sig: `delivery|stale|${op}`, trigger: "delivery", diagnosis: "stale", op, confidence: 0.91, evidence: "the model's line", seeds: [seed] }); +const file = (test: string, seed: number, signatures: RaceSignature[], extra: Partial = {}): GenClassRacesFile => ({ schema: "genclass-races/1", test, seed, signatures, mode: "observe", model: { state: "off", name: "none" }, ...extra }); + +describe("witnesses: grouping", () => { + it("categoryOf puts trigger witness in its own group whatever the label, and the group is last", () => { + expect(categoryOf("overwrote-newer", "witness")).toBe("witness"); + expect(categoryOf("stale", "witness")).toBe("witness"); + expect(categoryOf("stale")).toBe("stale"); + expect(categoryOf("overwrote-newer")).toBe("other"); + expect(CATEGORIES[CATEGORIES.length - 1]).toBe("witness"); + expect(CATEGORY_TITLES.witness).toBe("Witnesses (facts measured without the model)"); + }); + + it("the model-only report is byte for byte what it was: no witness row, no witness sentence", () => { + const md = summarize(fixtureInputs(), { sources: ["test-results/genclass"] }); + expect(md).toBe(FIXTURE_REPORT); + expect(md).not.toContain("Witnesses"); + }); +}); + +describe("witnesses: the report", () => { + it("a witness-only report: the row, the group, no confidence, reproducible across seeds, the witness footer only", () => { + const md = summarize([ + { name: "a-1.json", content: file("search.spec.ts > search", 1, [witness("GET /api/search", 1)]) }, + { name: "a-2.json", content: file("search.spec.ts > search", 2, [witness("GET /api/search", 2), witness("POST /api/orders", 2, "identical-in-flight")]) }, + { name: "b-1.json", content: file("quiet.spec.ts > quiet", 1, []) }, + ]); + expect(md).toContain("**2 signatures** in 1 test, **1 reproducible** (flagged in 2 or more seeds of the same test) · 3 files read · mode observe\n"); + expect(md).toContain("| Other detections | 0 | 0 |\n| Witnesses (facts measured without the model) | 2 | 1 |\n"); + expect(md).toContain("**Witnesses (facts measured without the model)**\n\n- **Reproducible**: 2 of 2 seeds (1, 2) · `GET /api/search` · `witness:overwrote-newer`\n > The response of GET /api/search#​1 wrote results"); + expect(md).toContain("- 1 of 2 seeds (2) · `POST /api/orders` · `witness:identical-in-flight`\n"); + expect(md).not.toContain("· confidence"); + expect(md).not.toContain("diagnosis by GenClass's on-device model"); + expect(md).toContain('\n---\n\nLines under "Witnesses (facts measured without the model)" are happens-before facts the runtime measured in the page without the model'); + expect(md).toContain("They carry no confidence: a fact is listed when it happened. Reproducible: seen in 2 or more seeds of the same test."); + // the model was off by choice: no "not ready" note for the quiet test + expect(md).toContain("- quiet.spec.ts > quiet (1 seed: 1)\n"); + expect(md).not.toContain("not ready"); + }); + + it("model detections and witnesses together: both groups, both footer paragraphs, in that order", () => { + const md = summarize([{ name: "a.json", content: file("t", 1, [witness("GET /api/search", 1), model("GET /api/search", 1)], { model: { state: "ready" } }) }]); + expect(md).toContain("| Stale responses | 1 | 0 |"); + expect(md).toContain("| Witnesses (facts measured without the model) | 1 | 0 |"); + expect(md.indexOf("**Stale responses**")).toBeLessThan(md.indexOf("**Witnesses (facts measured without the model)**")); + expect(md).toContain("`delivery:stale` · confidence 0.91\n"); + expect(md).toContain("`witness:overwrote-newer`\n"); + const footer = md.slice(md.lastIndexOf("\n---\n")); + expect(footer.indexOf("Each line is a diagnosis by GenClass's on-device model")).toBeGreaterThan(0); + expect(footer.indexOf("Lines under")).toBeGreaterThan(footer.indexOf("Each line is a diagnosis")); + expect(footer.split("\n\n")).toHaveLength(3); + }); + + it("buildSummary: state off is not 'not ready'; loading still is", () => { + const s = buildSummary([ + { name: "a.json", content: file("t", 1, [], { model: { state: "off" } }) }, + { name: "b.json", content: file("t", 2, [], { model: { state: "loading" } }) }, + ]); + expect(s.tests[0].modelNotReady).toEqual({ files: 1, states: ["loading"] }); + expect(s.totals.byCategory.witness).toEqual({ signatures: 0, reproducible: 0 }); + }); + + it("maxChars keeps witness lines like any other signature", () => { + const sigs = Array.from({ length: 150 }, (_, i) => witness(`GET /api/r${i}`, 1)); + const md = summarize([{ name: "big.json", content: file("big", 1, sigs) }], { maxChars: 4000 }); + expect(md.length).toBeLessThanOrEqual(4000); + expect(md).toContain("**150 signatures** in 1 test"); + expect(md).toContain("_Shortened to fit 4000 characters: "); + expect(md).toContain("`witness:overwrote-newer`"); + }); +}); diff --git a/packages/playwright/tsup.config.ts b/packages/playwright/tsup.config.ts index 564a4e0a..6c000a31 100644 --- a/packages/playwright/tsup.config.ts +++ b/packages/playwright/tsup.config.ts @@ -1,11 +1,13 @@ import { defineConfig } from "tsup"; -// Four ESM entries: the pure summariser (`@genclass/playwright` and `@genclass/playwright/summarize`), the -// Playwright reporter (`@genclass/playwright/reporter`, loaded by Playwright by module name, default export) and the -// command (`bin/genclass-playwright-report.mjs` imports dist/cli.js). No splitting: each entry is one file. Nothing -// is external because nothing is imported besides Node built-ins. +// Five ESM entries: the pure summariser (`@genclass/playwright` and `@genclass/playwright/summarize`), the +// Playwright reporter (`@genclass/playwright/reporter`, loaded by Playwright by module name, default export), the +// fixture (`@genclass/playwright/fixture`: `test` and `expect`; it imports @playwright/test, a peer dependency, and +// reads @genclass/runtime's dist/ from disk at run time) and the command (`bin/genclass-playwright-report.mjs` +// imports dist/cli.js). No splitting: each entry is one file. Only the peer dependency is external; nothing else is +// imported besides Node built-ins. export default defineConfig({ - entry: { index: "src/index.ts", summarize: "src/summarize.ts", reporter: "src/reporter.ts", cli: "src/cli.ts" }, + entry: { index: "src/index.ts", summarize: "src/summarize.ts", reporter: "src/reporter.ts", fixture: "src/fixture.ts", cli: "src/cli.ts" }, format: ["esm"], platform: "node", target: "node20", @@ -15,4 +17,5 @@ export default defineConfig({ splitting: false, minify: false, treeshake: true, + external: ["@playwright/test", "@genclass/runtime"], });