From 79d113d604568dd2c82ee656e1e4fa3ea23f6ebf Mon Sep 17 00:00:00 2001 From: Karan Date: Mon, 26 Oct 2026 19:13:22 -0400 Subject: [PATCH 1/6] telemetry-worker: delete a project (Project Settings, Delete project) An account's project could not be deleted on its own, only with the whole account. DELETE /api/account/projects/ { confirm: "delete" } (signed-in browser only, the site's Origin) deletes an owned app with its stats, visits, the agent's reports and log, its integrations and export copies; its key stops counting at once (forgetToken), and its private link and agent link stop working. The dashboard's Project Settings gets a Delete project section (and a Danger zone entry in its menu): the first click asks, the second deletes and opens the Projects page with a notice. Tests: the API (400 without the confirm, 403 for another Origin or a CLI session, 404 for another account, every table emptied, the key no longer counted, the other app untouched, a claimed app's links stop working) and the page script in happy-dom (asks once, then one DELETE; no section on a private-link dashboard). Co-Authored-By: Claude Opus 5.5 --- telemetry-worker/README.md | 1 + telemetry-worker/src/auth.ts | 28 +++++++++++++++ telemetry-worker/src/pages.ts | 30 ++++++++++++++-- telemetry-worker/test/auth.test.ts | 56 +++++++++++++++++++++++++++++ telemetry-worker/test/pages.test.ts | 19 ++++++++++ 5 files changed, 131 insertions(+), 3 deletions(-) diff --git a/telemetry-worker/README.md b/telemetry-worker/README.md index c9a93b1b..32d7c6cd 100644 --- a/telemetry-worker/README.md +++ b/telemetry-worker/README.md @@ -370,6 +370,7 @@ minutes in a single-use row named by the `gc_pending` cookie). | `GET /api/account/projects//stats?range=` | cookie | the dashboard stats (same shape as `/api/projects/`), owned apps only; 402 `{ locked: true, upgrade_url }` for a plan without the dashboard (none today); 403 for a bearer token | | `POST /api/account/projects//link` | cookie + Origin | a new private dashboard link for an owned app, the MCP view's agent link (what @genclass/mcp reads as `GENCLASS_DASHBOARD`): 201 `{ dashboardUrl }`, shown once (only its hash is stored, so any earlier link stops working); 403 for a bearer token, 404 for an app of another account | | `DELETE /api/account/projects//link` | cookie + Origin | turn off the app's private dashboard link (its hash is replaced by one nobody holds); 204 | +| `DELETE /api/account/projects/` | cookie + Origin | `{ confirm: "delete" }`: delete an owned app (Project Settings, Delete project) with its stats, visits, the agent's reports and log, its integrations and export copies; its key stops counting at once and its links stop working; 204, 400 without the confirm, 404 for an app of another account, 403 for a bearer token | | `DELETE /api/account/sessions/current` | bearer (CLI logout) or cookie + Origin | 204 | | `DELETE /api/account/sessions/` | cookie + Origin | revoke one of this account's sessions; 204, 404 otherwise (403 for a bearer token) | | `DELETE /api/account/sessions/others` | cookie + Origin | revoke every session of the account except this one; 204 | diff --git a/telemetry-worker/src/auth.ts b/telemetry-worker/src/auth.ts index 5e5db499..8281832f 100644 --- a/telemetry-worker/src/auth.ts +++ b/telemetry-worker/src/auth.ts @@ -1574,6 +1574,31 @@ async function turnOffLink(c: C, id: string): Promise { return noContent(); } +/** + * DELETE /api/account/projects/ { confirm: "delete" }: delete an owned app with everything kept for it (its stats, + * visits, the agent's reports and log, its paid features, its export copies) and the app itself. Its key stops counting + * at once: batches the app still sends are stored as batches without a known key are (R2 only) and reach no dashboard. + * Its private link and agent link stop working. Only the signed-in browser may do it. + */ +async function deleteProject(c: C, id: string): Promise { + const a = await browserWriter(c, "a project can be deleted"); + if (a instanceof Response) return a; + const body = await readJson(c, 1024); + if (body instanceof Response) return body; + if (body.confirm !== "delete") return fail(400, 'send { "confirm": "delete" } to delete this project'); + const p = await ownedProject(c, a.account.id, id); + if (!p) return fail(404, "not found"); + await c.db.batch([ + c.db.prepare("DELETE FROM integrations WHERE project_id = ?").bind(id), + c.db.prepare("DELETE FROM forward_log WHERE project_id = ?").bind(id), + ...["daily", "hourly", "recent", "sessions", "visits", "agent_reports", "agent_log"].map((t) => c.db.prepare(`DELETE FROM ${t} WHERE token = ?`).bind(p.token)), + c.db.prepare("DELETE FROM projects WHERE id = ? AND account_id = ?").bind(id, a.account.id), + ]); + forgetToken(p.token); + await forgetExports(c.db, c.env.BUCKET, [p.token]).catch(() => undefined); + return noContent(); +} + /** * DELETE /api/account/sessions/. A CLI session may end only itself ("current", the CLI's logout); the browser * may end any of the account's sessions, or every session but itself ("others"). @@ -1905,6 +1930,8 @@ export async function routeAuth(request: Request, url: URL, path: string, env: E } const link = /^\/api\/account\/projects\/([^/]+)\/link$/.exec(path); if (link) return m === "DELETE" ? turnOffLink(c, link[1]!) : post ? makeLink(c, link[1]!) : method(); + const proj = /^\/api\/account\/projects\/([^/]+)$/.exec(path); + if (proj) return m === "DELETE" ? deleteProject(c, proj[1]!) : method(); const ses = /^\/api\/account\/sessions\/([^/]+)$/.exec(path); if (ses) return m === "DELETE" ? revokeSession(c, ses[1]!) : method(); const cred = /^\/api\/account\/credentials\/([^/]+)$/.exec(path); @@ -1972,6 +1999,7 @@ function accountNotice(url: URL): AccountView["notice"] { if (q.get("linked") === "github") return { kind: "ok", text: "GitHub is linked. You can now sign in with it." }; if (q.get("password") === "changed") return { kind: "ok", text: "Your password is changed. Every other browser and terminal is signed out." }; if (q.get("password") === "added") return { kind: "ok", text: "Your password is set. You can now sign in with your email and password." }; + if (q.get("deleted") === "project") return { kind: "ok", text: "The project is deleted, with its stats and its links." }; if (q.get("error") === "google_taken") return { kind: "bad", text: "That Google account is already linked to another GenClass account, so it was not linked here." }; if (q.get("error") === "github_taken") return { kind: "bad", text: "That GitHub account is already linked to another GenClass account, so it was not linked here." }; return undefined; diff --git a/telemetry-worker/src/pages.ts b/telemetry-worker/src/pages.ts index 4167afd9..fe4b24ad 100644 --- a/telemetry-worker/src/pages.ts +++ b/telemetry-worker/src/pages.ts @@ -1642,6 +1642,20 @@ const DASH_JS = ` if (b) { b.disabled = false; b.textContent = "Make a new agent link"; b.setAttribute("data-on", "1"); b.removeAttribute("data-armed"); } }); } + document.addEventListener("click", function(e){ + var b = e.target && e.target.closest ? e.target.closest("[data-delproj]") : null; if (!b || !P.account) return; + var err = b.parentNode && b.parentNode.nextElementSibling; + // the first click asks, the second (within 6 s) deletes the project and opens the Projects page + if (b.getAttribute("data-armed") !== "1") { + b.setAttribute("data-armed", "1"); b.textContent = "Confirm: delete this project and its data"; + setTimeout(function(){ if (b.getAttribute("data-armed") === "1" && !b.disabled) { b.removeAttribute("data-armed"); b.textContent = "Delete project"; } }, 6000); + return; + } + b.disabled = true; if (err) err.textContent = ""; + fetch("/api/account/projects/" + encodeURIComponent(P.id), { method: "DELETE", credentials: "same-origin", headers: { accept: "application/json", "content-type": "application/json" }, body: JSON.stringify({ confirm: "delete" }) }) + .then(function(r){ if (r.ok) { location.href = "/account?deleted=project"; return; } return r.json().catch(function(){ return {}; }).then(function(j){ throw new Error((j && j.error) || "HTTP " + r.status); }); }) + .catch(function(x){ if (err) err.textContent = "Not deleted: " + x.message + "."; b.disabled = false; b.removeAttribute("data-armed"); b.textContent = "Delete project"; }); + }); document.addEventListener("click", function(e){ var b = e.target && e.target.closest ? e.target.closest("[data-mklink]") : null; if (!b || !P.account) return; var box = b.closest(".alink"), err = box ? box.querySelector(".err") : null; @@ -1692,7 +1706,7 @@ const DASH_JS = ` (function(){ var VIEW = { top: "overview", trend: "overview", usage: "overview", advisor: "overview", getc: "overview", found: "problems", "possible-s": "problems", where: "pages", did: "pages", "recent-s": "activity", visits: "visitors", "ix-sources": "visitors", "ix-devices": "visitors", "ix-sessions": "visitors", "ix-entry": "visitors", "ix-funnels": "visitors", clicks: "visitors", moves: "visitors", - "health-s": "health", adv: "health", loop: "loop", mcp: "mcp", "mcp-tools": "mcp", "mcp-json": "mcp", settings: "settings", keys: "settings", forwarding: "forwarding", "export": "export" }; + "health-s": "health", adv: "health", loop: "loop", mcp: "mcp", "mcp-tools": "mcp", "mcp-json": "mcp", settings: "settings", keys: "settings", delproj: "settings", forwarding: "forwarding", "export": "export" }; var TITLE = { problems: ["Problems", "Each kind of problem GenClass found in this period: where it happened, how to fix it, and a prompt for your coding agent."], pages: ["Pages and actions", "The pages with the most problems, and what GenClass did about them."], @@ -1701,11 +1715,11 @@ const DASH_JS = ` health: ["Health", "Whether GenClass could watch your pages at all: model loads, decision time, what was not checked."], loop: ["Loop", "What broke on this app, and what your coding agent does with each problem."], mcp: ["MCP", "Let your coding agent read this project: production problems through @genclass/mcp, and what happens inside your pages while you develop."], - settings: ["Project Settings", "This project's name, key and agent access."], + settings: ["Project Settings", "This project's name, key and agent access, and deleting it."], forwarding: ["Integrations", "Send each new detection where you already look."], "export": ["Export", "Every event GenClass received for this app, as a file or through the API."] }; - var PARENT = { "export": "settings" }, SCROLL = { did: 1, "possible-s": 1, "ix-sources": 1, "ix-devices": 1, "ix-sessions": 1, "ix-entry": 1, "ix-funnels": 1, clicks: 1, moves: 1, keys: 1, usage: 1, advisor: 1, getc: 1, adv: 1, "mcp-tools": 1, "mcp-json": 1 }; + var PARENT = { "export": "settings" }, SCROLL = { did: 1, "possible-s": 1, "ix-sources": 1, "ix-devices": 1, "ix-sessions": 1, "ix-entry": 1, "ix-funnels": 1, clicks: 1, moves: 1, keys: 1, usage: 1, advisor: 1, getc: 1, adv: 1, "mcp-tools": 1, "mcp-json": 1, delproj: 1 }; var links = [].slice.call(document.querySelectorAll(".snav a, .sb-pm a")); var parts = [].slice.call(document.querySelectorAll("[data-v]")); var current = null; @@ -2147,6 +2161,15 @@ export function dashboardPage(

Agent access

For @genclass/mcp
${agentAccess(!!acct, !!opts.privateLink)} +${ + acct && opts.id + ? `
+

Delete project

Cannot be undone
+

Deletes ${esc(project.name)} with its stats, visitors, the agent's log and its export. Its key stops counting at once: batches the app still sends reach no dashboard, so take the GenClass lines out of its code too. Its agent link and any private dashboard link stop working.

+
+
` + : "" +} `; // the rail (Studio's project navigation), the product menus of the views with sub-pages, the command menu's extras const rail: RailItem[][] = [ @@ -2200,6 +2223,7 @@ export function dashboardPage( groups: [ { label: "Configuration", items: [{ href: "#settings", label: "General" }, { href: "#keys", label: "API keys" }] }, ...(opts.extra ? [{ label: "Data", items: [{ href: "#forwarding", label: "Integrations" }, { href: "#export", label: "Export" }] }] : []), + ...(acct && opts.id ? [{ label: "Danger zone", items: [{ href: "#delproj", label: "Delete project" }] }] : []), ], }, ]; diff --git a/telemetry-worker/test/auth.test.ts b/telemetry-worker/test/auth.test.ts index 42c0e6eb..c6a5cc7e 100644 --- a/telemetry-worker/test/auth.test.ts +++ b/telemetry-worker/test/auth.test.ts @@ -1755,6 +1755,62 @@ describe("accounts: deleting an account", () => { }); }); +describe("accounts: deleting a project (Project Settings, Delete project)", () => { + it("removes the app with its stats, visits, agent rows and links; its key stops counting; the account's other apps stay", async () => { + const { env, sql } = setup(); + const { cookie } = await signedIn(env); + await givePro(env, cookie); // the fixture's free plan has no dashboard; every plan has one in production + const made = ((await (await call(env, "POST", "/api/account/projects", { body: { name: "Rehearsal" }, cookie })).json()) as any).project; + const kept = ((await (await call(env, "POST", "/api/account/projects", { body: { name: "Kept" }, cookie })).json()) as any).project; + await sendBatch(env, made.token); + await sendBatch(env, kept.token, "dddddddddddd"); + const at = clock.toISOString(); + sql.db.prepare("INSERT INTO agent_reports (token, finding, kind, route, updated) VALUES (?, 'f1', 'stale', '/', ?)").run(made.token, at); + sql.db.prepare("INSERT INTO agent_log (token, at, t, received) VALUES (?, ?, 'found', ?)").run(made.token, at, at); + const count = (t: string, token: string) => Number(sql.rows(`SELECT COUNT(*) AS n FROM ${t} WHERE token = ?`, token)[0]!.n); + expect(count("daily", made.token)).toBeGreaterThan(0); + // the dashboard of an owned app offers it; a private-link dashboard does not + const page = await (await call(env, "GET", `/account/apps/${made.id}`, { cookie })).text(); + expect(page).toContain("data-delproj"); + expect(page).toContain('href="#delproj"'); + + const path = `/api/account/projects/${made.id}`; + expect((await call(env, "DELETE", path, { body: {}, cookie })).status).toBe(400); + expect((await call(env, "DELETE", path, { body: { confirm: "delete" }, cookie, origin: "https://evil.example" })).status).toBe(403); + expect((await call(env, "DELETE", path, { body: { confirm: "delete" } })).status).toBe(401); + expect((await call(env, "DELETE", path, { body: { confirm: "delete" }, bearer: await cliToken(env, cookie), origin: null })).status).toBe(403); + const bob = await signedIn(env, "bob@example.com"); + expect((await call(env, "DELETE", path, { body: { confirm: "delete" }, cookie: bob.cookie })).status).toBe(404); + expect((await call(env, "GET", path, { cookie })).status).toBe(405); + expect(count("daily", made.token)).toBeGreaterThan(0); + + expect((await call(env, "DELETE", path, { body: { confirm: "delete" }, cookie })).status).toBe(204); + for (const t of ["projects", "daily", "hourly", "recent", "sessions", "visits", "agent_reports", "agent_log", "event_batches"]) expect(count(t, made.token), t).toBe(0); + // its key stops counting at once; the other app is untouched + await sendBatch(env, made.token); + expect(count("daily", made.token)).toBe(0); + expect(count("daily", kept.token)).toBeGreaterThan(0); + const list = (await (await call(env, "GET", "/api/account/projects", { cookie })).json()) as any; + expect(list.projects.map((p: { name: string }) => p.name)).toEqual(["Kept"]); + expect((await call(env, "GET", `/account/apps/${made.id}`, { cookie })).status).toBe(404); + expect((await call(env, "DELETE", path, { body: { confirm: "delete" }, cookie })).status).toBe(404); + expect(await (await call(env, "GET", "/account?deleted=project", { cookie })).text()).toContain("The project is deleted, with its stats and its links."); + }); + + it("a claimed app's private link and the agent link stop working with it", async () => { + const { env } = setup(); + const { cookie } = await signedIn(env); + const anon = (await (await call(env, "POST", "/api/projects", { body: { name: "Old app" }, origin: null })).json()) as any; + const { project } = (await (await call(env, "POST", "/api/account/projects/claim", { body: { dashboard_url: anon.dashboardUrl }, cookie })).json()) as any; + const agent = ((await (await call(env, "POST", `/api/account/projects/${project.id}/link`, { cookie })).json()) as any).dashboardUrl; + expect((await handle(new Request(agent), env, { now: () => clock })).status).toBe(200); + expect(await (await handle(new Request(agent), env, { now: () => clock })).text()).not.toContain("data-delproj"); + expect((await call(env, "DELETE", `/api/account/projects/${project.id}`, { body: { confirm: "delete" }, cookie })).status).toBe(204); + expect((await handle(new Request(agent), env, { now: () => clock })).status).toBe(404); + expect((await handle(new Request(anon.dashboardUrl), env, { now: () => clock })).status).toBe(404); + }); +}); + describe("accounts: private links of owned apps", () => { it("a claimed app's link keeps working until the owner turns it off; then it opens and claims nothing; the dashboard says which", async () => { const { env, sql } = setup(); diff --git a/telemetry-worker/test/pages.test.ts b/telemetry-worker/test/pages.test.ts index 4b77720e..231d153f 100644 --- a/telemetry-worker/test/pages.test.ts +++ b/telemetry-worker/test/pages.test.ts @@ -967,6 +967,25 @@ describe("dashboard page: the Supabase Studio frame (src/studio.ts): header, rai await r.close(); }); + it("Delete project in an account: Project Settings' danger zone asks once, then sends DELETE with the confirm; a private link has none", async () => { + const r = await render(sampleStats("7d", NOW), { account, id: "0123456789abcdef", firstEvent: "2026-10-12T09:00:00.000Z", runtimeReady: true }); + expect([...r.doc.querySelectorAll(".sb-pm a")].some((a) => a.getAttribute("href") === "#delproj" && a.textContent === "Delete project")).toBe(true); + const b = r.doc.querySelector("#settings #delproj [data-delproj]") as unknown as HTMLElement; + expect(b.textContent).toBe("Delete project"); + b.click(); + expect(b.textContent).toBe("Confirm: delete this project and its data"); + expect(r.writes).toEqual([]); + b.click(); + await new Promise((x) => setTimeout(x, 10)); + expect(r.writes).toEqual(["DELETE /api/account/projects/0123456789abcdef"]); + expect(r.text("#delproj .err")).toBe(""); + await r.close(); + const s = await render(sampleStats("7d", NOW)); + expect(s.doc.querySelector("[data-delproj]")).toBeNull(); + expect(s.doc.querySelector('a[href="#delproj"]')).toBeNull(); + await s.close(); + }); + it("Copy as Markdown: the period as a report a person or an agent can read (totals, problems with pages and fixes, latest detections)", async () => { const st = sampleStats("7d", NOW); const copied: string[] = []; From 09a0c9231edd9f98c7b8fce3c8c32b2fc4f4eb7a Mon Sep 17 00:00:00 2001 From: Karan Date: Mon, 26 Oct 2026 19:16:59 -0400 Subject: [PATCH 2/6] telemetry-worker: Project Settings mentions deleting only where it can be done The subtitle said "and deleting it" on private-link dashboards too, which have no Delete project section; it now says so in account mode only. The page test checks both. Co-Authored-By: Claude Opus 5.5 --- telemetry-worker/src/pages.ts | 2 +- telemetry-worker/test/pages.test.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/telemetry-worker/src/pages.ts b/telemetry-worker/src/pages.ts index fe4b24ad..a7ab3662 100644 --- a/telemetry-worker/src/pages.ts +++ b/telemetry-worker/src/pages.ts @@ -1715,7 +1715,7 @@ const DASH_JS = ` health: ["Health", "Whether GenClass could watch your pages at all: model loads, decision time, what was not checked."], loop: ["Loop", "What broke on this app, and what your coding agent does with each problem."], mcp: ["MCP", "Let your coding agent read this project: production problems through @genclass/mcp, and what happens inside your pages while you develop."], - settings: ["Project Settings", "This project's name, key and agent access, and deleting it."], + settings: ["Project Settings", P.account ? "This project's name, key and agent access, and deleting it." : "This project's name, key and agent access."], forwarding: ["Integrations", "Send each new detection where you already look."], "export": ["Export", "Every event GenClass received for this app, as a file or through the API."] }; diff --git a/telemetry-worker/test/pages.test.ts b/telemetry-worker/test/pages.test.ts index 231d153f..ff7fcd99 100644 --- a/telemetry-worker/test/pages.test.ts +++ b/telemetry-worker/test/pages.test.ts @@ -980,9 +980,13 @@ describe("dashboard page: the Supabase Studio frame (src/studio.ts): header, rai expect(r.writes).toEqual(["DELETE /api/account/projects/0123456789abcdef"]); expect(r.text("#delproj .err")).toBe(""); await r.close(); - const s = await render(sampleStats("7d", NOW)); + const a = await render(sampleStats("7d", NOW), { account, id: "0123456789abcdef", firstEvent: "2026-10-12T09:00:00.000Z", runtimeReady: true }, { url: "https://genclass.dev/account/apps/0123456789abcdef#settings" }); + expect(a.text("#vdesc")).toBe("This project's name, key and agent access, and deleting it."); + await a.close(); + const s = await render(sampleStats("7d", NOW), {}, { url: "https://genclass.dev/dashboard/sample#settings" }); expect(s.doc.querySelector("[data-delproj]")).toBeNull(); expect(s.doc.querySelector('a[href="#delproj"]')).toBeNull(); + expect(s.text("#vdesc")).toBe("This project's name, key and agent access."); await s.close(); }); From 1266bf8ce4035002a70e8fd92c449c814c4b179d Mon Sep 17 00:00:00 2001 From: Karan Date: Mon, 26 Oct 2026 19:57:41 -0400 Subject: [PATCH 3/6] telemetry-worker: onboarding by clicks and copied prompts, nothing to type - Connect dialog: choices above the prompt (Guard, Eager, Badge on the live site, Connect my coding agent to production, Deploy when done). The page runs installPrompt's own source (installPromptSource), so the copied prompt is the server's text for those choices; ticking the agent choice makes the agent link (one makeAgentLink for it and the MCP page). - installPrompt: a site with no build step is covered by the prompt alone (the script tag with data-mode, data-visual, data-devtools; GENCLASS_CONFIG for what a tag cannot carry; the agent link from the CDN on local hosts); values one response writes together go in one atom; with the agent link, claude mcp add with GENCLASS_DASHBOARD and GENCLASS_REPORT=on in the agent's own settings; deploy with the project's own command when chosen. - Problems and the overview: Fix them all copies one prompt for every kind of problem and the dev page's checks; every fix prompt has the agent say what to do on the page and wait for the loads itself (sleep 10 between checks, up to 5 minutes), so nobody types done. - /account/new?name= fills in the project's name, through sign-in (the page check's Fix it with GenClass names the checked site). Tests: the choices produce exactly installPrompt's text in the page, the agent choice posts once and names the link, the prompt's no-build path and atom step, Fix them all, the name prefill (escaped, carried through /login). Co-Authored-By: Claude Opus 5.5 --- telemetry-worker/README.md | 4 +- telemetry-worker/src/account-pages.ts | 4 +- telemetry-worker/src/auth.ts | 6 +- telemetry-worker/src/pages.ts | 194 ++++++++++++++++++++++---- telemetry-worker/test/auth.test.ts | 13 ++ telemetry-worker/test/pages.test.ts | 75 +++++++++- 6 files changed, 264 insertions(+), 32 deletions(-) diff --git a/telemetry-worker/README.md b/telemetry-worker/README.md index 32d7c6cd..6702f45a 100644 --- a/telemetry-worker/README.md +++ b/telemetry-worker/README.md @@ -81,7 +81,7 @@ genclass.dev/dashboard/ --> dashboard page --GET /api/projects/? | `PUT /api/projects//funnels` | body `{ funnels: [{ name, steps }] }` (application/json, at most 4 KB): replaces the app's funnels (at most 5, 2 to 6 routes each; a step ending in `*` matches every route under it). 200 `{ ok, funnels }`, 400 with the reason, 403 for the sample. Account mode: `PUT /api/account/projects//funnels` with the browser session and the site's Origin | | `GET /start` | "Get a token" page | | `POST /api/projects//agent` | the MCP server's log of the closed loop, mirrored, when the developer sets `GENCLASS_REPORT` for `@genclass/mcp` (docs/runtime/AGENT.md section 4.5): `{ schema: "genclass-agent/1", events: [{ t: "found" \| "explain" \| "armed" \| "replay" \| "test" \| "explore" \| "cleared", at, id?, kind?, route?, reproduced?, runs?, races?, what? }] }`, at most 50 events and 16 KB a request, rate limited per token (`SESSION_LIMIT`). Every accepted event is one row of `agent_log` (at most 500 per app, the oldest go first; unknown types are dropped); the ones about a finding also update `agent_reports` (kind, normalised route, found, explained, last replay and verdict, how many replays reproduced it, tested, proven: a replay that stops reproducing after one that did). 90 days; the stats carry them as `agent` (`findings`, `counts`, `stages`: found, explained, reproduced, fixed, closed, open, continued; `log`, the last 50) and the dashboard's Closed-loop development view shows them. The sample and unknown secrets 404; other methods 405. Ids, kinds, routes, verdicts, counts and times only, never page text | -| `GET /dashboard/` | the dashboard (for an app with no account too: it is on Free), in the Supabase Studio frame (`src/studio.ts`), in views the rail switches between: Project overview (the name and status, the summary, four stats, the problems chart, usage cards per hour or day, Get connected, the problems that need attention with a Copy prompt each, and Getting started until the first event), Problems (each kind with its pages, its fix and a Copy prompt for a coding agent), Pages and Actions, Activity, Visitors (without interaction data, under the copy-prompt button, the runtime analytics need, `ANALYTICS_RUNTIME` 0.3.0-beta.1, with the install line and the app's older versions from the `runtime` breakdown; the Connect dialog says the same), Health, MCP (the @genclass/mcp setup per coding agent, with GENCLASS_DASHBOARD set to the agent link, a Copy prompt, the tools, the JSON address), Closed loop (Closed-loop development: Fig. 3, every issue's path, found, reproduced, fix proven, closed, or round again, with the period's counts from the agent's reports; Table 15, the agent's log mirrored from `@genclass/mcp`; Fig. 6, the app's problems and what the agent does with each), Integrations (forwarding, account mode) and Project Settings (general, the key, agent access; Export in account mode). Problems, Visitors and Project Settings have Studio's product menu. The header's Connect opens the Connect dialog (also `#connect`); the hash names the view (`#found`, `#recent-s`, `#mcp`); every data view copies as Markdown; and without the script every section shows. On a private link the page's own address is the agent link. With a plan file that left the dashboard out, the locked page (sign in, add the app to the account, see the plans; `src/locked.ts`). 404 page for an unknown secret; `GET /dashboard` redirects to `/start`. `/dashboard/sample` stays open | +| `GET /dashboard/` | the dashboard (for an app with no account too: it is on Free), in the Supabase Studio frame (`src/studio.ts`), in views the rail switches between: Project overview (the name and status, the summary, four stats, the problems chart, usage cards per hour or day, Get connected, the problems that need attention with a Copy prompt each and Fix them all, and Getting started until the first event), Problems (each kind with its pages, its fix and a Copy prompt for a coding agent; Fix them all copies one prompt for every kind and the dev page's checks, in which the agent says what to do on the page and waits for the loads itself), Pages and Actions, Activity, Visitors (without interaction data, under the copy-prompt button, the runtime analytics need, `ANALYTICS_RUNTIME` 0.3.0-beta.1, with the install line and the app's older versions from the `runtime` breakdown; the Connect dialog says the same), Health, MCP (the @genclass/mcp setup per coding agent, with GENCLASS_DASHBOARD set to the agent link, a Copy prompt, the tools, the JSON address), Closed loop (Closed-loop development: Fig. 3, every issue's path, found, reproduced, fix proven, closed, or round again, with the period's counts from the agent's reports; Table 15, the agent's log mirrored from `@genclass/mcp`; Fig. 6, the app's problems and what the agent does with each), Integrations (forwarding, account mode) and Project Settings (general, the key, agent access; Export in account mode). Problems, Visitors and Project Settings have Studio's product menu. The header's Connect opens the Connect dialog (also `#connect`); its Framework tab's prompt (`installPrompt`: the runtime, or the script tags of a site with no build step, the state in one atom per response, the agent link, the MCP server) follows the choices above it, Guard (and Eager), Badge on the live site, Connect my coding agent to production (makes the agent link, or names a private link) and Deploy when done, by running `installPrompt`'s own source in the page; the hash names the view (`#found`, `#recent-s`, `#mcp`); every data view copies as Markdown; and without the script every section shows. On a private link the page's own address is the agent link. With a plan file that left the dashboard out, the locked page (sign in, add the app to the account, see the plans; `src/locked.ts`). 404 page for an unknown secret; `GET /dashboard` redirects to `/start`. `/dashboard/sample` stays open | Rate limits on batches: `LIMIT`, 1200 per 60 s per client IP (a ceiling for offices and campuses behind one address; since runtime 0.2.2 a page stops sending after 3 failed requests in a row, so a low per-IP limit would silence every @@ -348,7 +348,7 @@ minutes in a single-use row named by the `gc_pending` cookie). | `GET /login/passkey` | `__Host-gc_mfa` | the passkey step after a password, Google or GitHub for an account with a passkey; 400 page when the step expired or is missing | | `POST /logout` | cookie + Origin | revokes the session, clears the cookie, 303 to `/login?signed_out=1` | | `GET /account` | cookie | the organization's Projects page in the Studio frame (302 to `/login?next=/account` without one): a search box, New project, a card per project (its key, its status, Copy key, Turn off its private link), Import a dashboard from a private link (and "Your plan has no dashboard" with See plans, for a plan file that left it out); `#settings`: Account settings, with Security (sign-in methods, "Require a passkey at sign-in", Add a passkey), sessions and Delete account. `?claim=1` returns to a locked private link this tab kept | -| `GET /account/new` | cookie | Create a new project (Studio's form: the organization, the project's name, what it comes with); it posts to `POST /api/account/projects` and opens the new project with its Connect dialog (302 to `/login?next=/account/new` without a session) | +| `GET /account/new` | cookie | Create a new project (Studio's form: the organization, the project's name, what it comes with); it posts to `POST /api/account/projects` and opens the new project with its Connect dialog (302 to `/login?next=/account/new` without a session). `?name=` fills in the name, through sign-in (the page check's Fix it with GenClass names the checked site) | | `GET /account/apps/` | cookie | an owned app's dashboard (the project switcher lists the account's projects; the Connect dialog has the terminal's login); for a plan without the dashboard (none today), the locked page (the key, the Connect section, See plans, no stats); 404 for an app of another account | | `GET /login/device?code=ABCD-EFGH` | cookie optional | signed in: the code and the request, Approve and Deny; signed out: the code and the ways to sign in, whether or not the code is live | | `GET /auth/google`, `GET /auth/google/callback` | - | Google sign-in (404 page when not configured); `?link=1` needs a session | diff --git a/telemetry-worker/src/account-pages.ts b/telemetry-worker/src/account-pages.ts index b7f05c90..53f7c6ef 100644 --- a/telemetry-worker/src/account-pages.ts +++ b/telemetry-worker/src/account-pages.ts @@ -941,6 +941,8 @@ export interface NewProjectView { /** The account's projects (the header's switcher) and its plan's name (the organization's badge). */ projects: { id: string; name: string }[]; plan?: string; + /** The name filled in (?name=, from the page check's "Fix it with GenClass"); empty for none. */ + suggested?: string; } /** @@ -954,7 +956,7 @@ export function newProjectPage(nonce: string, v: NewProjectView): string {

Create a new project

A project is one web app. It gets its own key, its own dashboard of what breaks for its users, and access for your coding agent through the MCP server. The dashboard is free.

Your account. Every project in it is on the ${esc(v.plan ?? "Free")} plan.

-

Shown on the dashboard and in the project switcher. Up to 80 characters.

+

Shown on the dashboard and in the project switcher. Up to 80 characters.

Comes with
  • ${icon("key", 15)}A public project key for the app's code (the runtime's token)
  • ${icon("chart", 15)}Project overview, problems, pages, activity, visitors and health
    • diff --git a/telemetry-worker/src/auth.ts b/telemetry-worker/src/auth.ts index 8281832f..f5fae7f0 100644 --- a/telemetry-worker/src/auth.ts +++ b/telemetry-worker/src/auth.ts @@ -1812,9 +1812,11 @@ export async function routeAuth(request: Request, url: URL, path: string, env: E // Studio's "Create a new project" (the form posts to /api/account/projects, as the CLI does) if (!get) return method(); const a = await webSession(c); - if (!a) return redirect("/login?next=%2Faccount%2Fnew"); + // ?name= fills in the project's name (the page check's "Fix it with GenClass" names the site it checked) + const suggested = (url.searchParams.get("name") ?? "").replace(/[\u0000-\u001f]/g, "").trim().slice(0, 80); + if (!a) return redirect(`/login?next=${encodeURIComponent(`/account/new${suggested ? `?name=${encodeURIComponent(suggested)}` : ""}`)}`); const view = await accountView(c, a); - return page(200, (n) => newProjectPage(n, { account: view.account, projects: view.projects.map((p) => ({ id: p.id, name: p.name })), plan: view.plan })); + return page(200, (n) => newProjectPage(n, { account: view.account, projects: view.projects.map((p) => ({ id: p.id, name: p.name })), plan: view.plan, suggested })); } case "/login/device": { if (!get) return method(); diff --git a/telemetry-worker/src/pages.ts b/telemetry-worker/src/pages.ts index a7ab3662..dd60f1ec 100644 --- a/telemetry-worker/src/pages.ts +++ b/telemetry-worker/src/pages.ts @@ -63,6 +63,30 @@ export const LOGO = /** The MCP server for coding agents, pinned as its own install blocks pin it (packages/mcp/src/clients.ts). */ export const MCP_PKG = "@genclass/mcp@0.2.0"; +/** The package names and URLs installPrompt names; passed in, so the page script runs the same function. */ +export interface PromptNames { + pkg: string; + cdn: string; + /** the agent link's module on the CDN, for pages without a bundler */ + agent: string; + mcp: string; +} +export const PROMPT_NAMES: PromptNames = { pkg: RUNTIME_PKG, cdn: CDN_SCRIPT, agent: `${CDN_SCRIPT}/dist/agent.js`, mcp: MCP_PKG }; + +/** What the Connect dialog's choices add to installPrompt; all off is observe, the overlay in development only. */ +export interface InstallChoices { + /** guard: GenClass may act when it is sure (drop a stale response, fix a CSS bug on the page), reversibly */ + guard?: boolean; + /** with guard: act on smaller signals (aggressiveness "eager") */ + eager?: boolean; + /** the devtools overlay on every host, the deployed site too (for demos and staging) */ + badge?: boolean; + /** the agent link (a private dashboard link): the MCP server's production numbers and the Loop view */ + agentLink?: string | null; + /** deploy with the project's own deploy command at the end */ + deploy?: boolean; +} + // The palette, type, buttons, header and focus ring of genclass.dev (site/src/css/base.css and components.css), with // the site's values: black ground, bone text, the GenClass mark's green and red-orange, square corners, 1px rules, // Satoshi for text and JetBrains Mono for code. Like the site, every page here is dark only, whatever the visitor's @@ -542,6 +566,14 @@ details.more summary:hover{color:var(--ink)} .kn .lnk,.fine .lnk{color:var(--ink2);text-decoration:underline;text-underline-offset:3px} .kn .lnk:hover{color:var(--ink)} .cprompt{padding:8px 0 4px;display:grid;justify-items:start;gap:10px} +/* the Connect dialog's choices: each a checkbox and one line of what it adds to the prompt */ +.copts{display:grid;gap:6px;margin:4px 0 12px;max-width:72ch} +.copt{display:flex;gap:10px;align-items:flex-start;padding:8px 10px;border:1px solid var(--rule2);border-radius:6px;cursor:pointer;font-size:13px;color:var(--ink3);line-height:1.45} +.copt b{color:var(--ink);font-weight:500} +.copt input{margin-top:3px;accent-color:var(--accent)} +.copt.sub{margin-left:26px} +.copt:has(input:disabled){opacity:.55;cursor:default} +.copt:has(input:checked){border-color:var(--ink3)} .cprompt .cp-sub{font-size:13px;color:var(--ink3);max-width:64ch;line-height:1.5} .cprompt textarea.cp-text{width:100%;max-width:72ch;min-height:150px;padding:10px 12px;font:12.5px/1.5 var(--mono);color:var(--ink);background:var(--ctl);border:1px solid var(--rule2);border-radius:6px;resize:vertical} .connect .cana{padding:0} @@ -840,6 +872,8 @@ const DASH_JS = ` return Object.keys(m).map(function(r){ return { route: r, count: m[r] }; }).sort(function(a, b){ return b.count - a.count; }); } function lower(s){ return s.charAt(0).toLowerCase() + s.slice(1); } + // page loads the agent needs: it says what to do, then waits for them itself, so nobody has to type "done" + var WAIT = "When a step needs page loads on the dev page, tell me in one line exactly what to do there (for example: reload it and repeat the search fast), then wait for the loads yourself: check every 10 seconds (sleep 10 in a shell in between), for up to 5 minutes, instead of ending your turn."; // what a coding agent gets from Copy prompt for one kind of problem: what, where, the usual fix, the MCP steps function fixPrompt(k, n, where){ var t = T(k), pages = where.slice(0, 4).map(function(w){ return w.route + " (" + fmt(w.count) + ")"; }).join(", "); @@ -848,11 +882,33 @@ const DASH_JS = ` if (t.fix) out.push("How it is usually fixed: " + t.fix); out.push("", "Use the GenClass MCP server (" + MCP + "; the project's MCP page in the GenClass dashboard says how to add it):", "1. genclass_findings, then genclass_explain on the finding that matches. " + stepFor(k, where.length ? where[0].route : "the page"), + " " + WAIT, "2. Find the code behind it and fix it; keep the regression test genclass_explain generates.", "3. Replay the recorded order again with genclass_replay: it must say not reproduced.", "4. Tell me what you changed, the test you added and the replay's verdict."); return out.join("\\n"); } + // one prompt for every kind of problem in the period, and the checks on the dev page: the loop for each (reproduce, + // fix, prove), from one paste + function fixAllPrompt(S){ + var list = (S.breakdowns.detect_diagnosis || []).filter(function(x){ return x.key !== "expected"; }); + var out = ["Fix the problems GenClass found in this web app (" + P.name + ") and prove each fix, with the GenClass MCP server (" + MCP + "; the project's MCP page in the GenClass dashboard says how to add it).", "", "In production, " + lower(RANGE[range]) + ":"]; + list.forEach(function(x){ + var t = T(x.key), w = whereOf(S, x.key); + out.push("- " + t.name + ", " + plural(x.count, "time", "times") + (w.length ? ", on " + w.slice(0, 3).map(function(v){ return v.route; }).join(", ") : "") + "." + (t.fix ? " Usual fix: " + t.fix : "")); + }); + out.push("- And every check genclass_findings lists from the dev page (unreadable text, a scroll lock left on, a page that stops taking clicks).", "", + "1. genclass_status: the dev page must be connected; if it is not, tell me what to open.", + "2. For each problem: genclass_findings, then genclass_explain; reproduce it with genclass_explore or genclass_replay. " + WAIT, + "3. Fix the cause in the code, small and idiomatic; keep the regression test genclass_explain generates.", + "4. Replay the recorded order again: it must say not reproduced.", + "5. Sum up in one line per problem: what it was, the proof, the fix."); + return out.join("\\n"); + } + function fixAll(S){ + var any = (S.breakdowns.detect_diagnosis || []).some(function(x){ return x.key !== "expected"; }); + ["fixall", "fixall2"].forEach(function(id){ var b = document.getElementById(id); if (!b) return; b.classList.toggle("hidden", !any); b.setAttribute("data-copy", any ? fixAllPrompt(S) : ""); }); + } function promptBtn(k, n, where, cls){ var b = el("button", cls || "btn sm ghost", "Copy prompt"); b.type = "button"; b.setAttribute("data-copy", fixPrompt(k, n, where)); b.setAttribute("aria-label", "Copy a prompt to fix " + lower(T(k).name)); @@ -1666,11 +1722,37 @@ const DASH_JS = ` return; } b.disabled = true; if (err) err.textContent = ""; - fetch("/api/account/projects/" + encodeURIComponent(P.id) + "/link", { method: "POST", credentials: "same-origin", headers: { accept: "application/json", "content-type": "application/json" }, body: "{}" }) + makeAgentLink().catch(function(x){ if (err) err.textContent = "Not created: " + x.message + "."; b.disabled = false; b.removeAttribute("data-armed"); }); + }); + // a new agent link (auth.ts makeLink): shown on the MCP page and in Project Settings, and named by the Connect prompt + function makeAgentLink(){ + return fetch("/api/account/projects/" + encodeURIComponent(P.id) + "/link", { method: "POST", credentials: "same-origin", headers: { accept: "application/json", "content-type": "application/json" }, body: "{}" }) .then(function(r){ return r.json().catch(function(){ return {}; }).then(function(j){ if (!r.ok) throw new Error((j && j.error) || "HTTP " + r.status); return j; }); }) - .then(function(j){ agentLink = j.dashboardUrl; mcpOpt.link = true; alinkShow(); mcpAll(); }) - .catch(function(x){ if (err) err.textContent = "Not created: " + x.message + "."; b.disabled = false; b.removeAttribute("data-armed"); }); + .then(function(j){ agentLink = j.dashboardUrl; mcpOpt.link = true; alinkShow(); mcpAll(); promptNow(); }); + } + // ---------- the Connect dialog's choices: its prompt is installPrompt, the server's own function, with them + var installPrompt = ${installPromptSource()}; + var NAMES = ${JSON.stringify(PROMPT_NAMES).replace(/`; + const local = '["localhost", "127.0.0.1"].includes(location.hostname)'; + const out = [ + "Add GenClass to this web app with its project key, so the app's GenClass dashboard shows what breaks for its users in production (double submits, stale responses, failing endpoints, write races), and so you, the coding agent, can see what happens on the dev pages, replay it and prove fixes. Do every step yourself.", "", - `1. Install the runtime with the project's package manager: npm install ${RUNTIME_PKG} (or pnpm add / yarn add / bun add).`, - `2. Start it as early as possible in the browser entry (main.ts, the root layout or the root component), with this project's key ${token}:`, + `1. Install the runtime with the project's package manager: npm install ${n.pkg} (or pnpm add / yarn add / bun add). A site with no build step (plain HTML pages, no bundler) installs nothing: it uses the script tags below.`, + `2. Start it as early as possible, with this project's key ${token}. With a bundler, in the browser entry (main.ts, the root layout or the root component):`, ' import { GenClass } from "@genclass/runtime";', - ` const rt = GenClass.init({ token: "${token}"${ep} });`, - ` Without a bundler, put this in the HTML head instead: `, - ` The key is public: it can only add telemetry to this app, so it belongs in front-end code. (npx ${RUNTIME_PKG} init --token ${token} --yes does steps 1 and 2 and shows the diff.)`, - "3. Keep the default mode, observe: GenClass reports and never changes what the app does. In development builds also mount the devtools overlay, which shows what GenClass sees on the page: if (import.meta.env.DEV) import(\"@genclass/runtime/devtools\").then((d) => d.mountDevtools(rt)); (Next.js: process.env.NODE_ENV === \"development\").", - '4. Connect the state that responses write (search results, carts, lists, forms), so GenClass sees each write with the response or click that caused it. React, Redux and Zustand state is found automatically; otherwise create it with rt.atom("name", initial) or wrap the store you have with rt.guard("name", { get, set }), and write through the handle.', + ` const rt = GenClass.init({ ${init.join(", ")} });`, + " Without a bundler, in the of every HTML page, before the page's own scripts:", + ...(cfg.length ? [` `] : []), + ` ${tag}`, + " and in the page's scripts the handle (rt below) is GenClass.runtime.", + ` The key is public: it can only add telemetry to this app, so it belongs in front-end code. (With a bundler, npx ${n.pkg} init --token ${token} --yes does steps 1 and 2 and shows the diff.)`, + (guard + ? `3. Mode guard, as chosen on the dashboard: when GenClass is sure, it acts with minimal, reversible steps (it drops a stale response or fixes a CSS bug on the page) and never changes a response${eager ? '; aggressiveness "eager" lets it act on smaller signals' : ""}.` + : "3. Keep the default mode, observe: GenClass reports and never changes what the app does.") + + (badge + ? ' Mount the devtools overlay on every host, the deployed site too, as chosen on the dashboard, so visitors see what GenClass does: import("@genclass/runtime/devtools").then((d) => d.mountDevtools(rt)); (the script tag\'s data-devtools does the same).' + : ' In development builds also mount the devtools overlay, which shows what GenClass sees on the page: if (import.meta.env.DEV) import("@genclass/runtime/devtools").then((d) => d.mountDevtools(rt)); (Next.js: process.env.NODE_ENV === "development"; the script tag\'s data-devtools="local" does the same).'), + '4. Connect the state that responses write (search results, carts, lists, forms), so GenClass sees each write with the response or click that caused it. React, Redux and Zustand state is found automatically; otherwise create it with rt.atom("name", initial) or wrap the store you have with rt.guard("name", { get, set }), and write through the handle. Values one response writes together (a search\'s query and its results) go in one atom, written once per response, so GenClass can drop a stale response as one write: const search = rt.atom("search", { query: "", results: [] }); then search.set((s) => ({ ...s, query: data.q, results: data.results })) and render from search.subscribe.', '5. Development builds only, add the agent link: import { connectAgent } from "@genclass/runtime/agent"; if (import.meta.env.DEV) connectAgent(rt); (Next.js: process.env.NODE_ENV === "development"). It records what happens on local pages for the GenClass MCP server and is never part of a production build.', - `6. Add the GenClass MCP server to yourself: npx -y ${MCP_PKG} init writes .mcp.json for Claude Code (npx -y ${MCP_PKG} config prints the block for Cursor, VS Code, Codex, Gemini CLI and others); then reload your MCP servers. Its tools: genclass_status, genclass_findings, genclass_explain, genclass_replay, genclass_explore, genclass_scan.`, - "7. Run the dev server and load a page to check that the app builds and runs without errors; genclass_status should then list the page. Local hosts (localhost, private networks) send nothing to the dashboard by default, so the first production events come from the deployed site.", - "8. Tell me what you changed.", - ].join("\n"); + ` Without a bundler, after the runtime's script tag: `, + link + ? `6. Add the GenClass MCP server to yourself with this project's agent link, in your own settings rather than a file in the repository (anyone with the link can read this app's stats). Claude Code: claude mcp add genclass -e GENCLASS_DASHBOARD=${link} -e GENCLASS_REPORT=on -- npx -y ${n.mcp} (other agents: npx -y ${n.mcp} config prints the block; add those two env values to it). GENCLASS_REPORT=on mirrors what you find, replay and prove to the dashboard's Loop view. Its tools: genclass_status, genclass_findings, genclass_explain, genclass_replay, genclass_explore, genclass_scan, genclass_analytics.` + : `6. Add the GenClass MCP server to yourself: npx -y ${n.mcp} init writes .mcp.json for Claude Code (npx -y ${n.mcp} config prints the block for Cursor, VS Code, Codex, Gemini CLI and others); then reload your MCP servers. Its tools: genclass_status, genclass_findings, genclass_explain, genclass_replay, genclass_explore, genclass_scan.`, + "7. Run the dev server and load a page to check that the app builds and runs without errors; genclass_status should then list the page (a new MCP server loads in your next conversation). Local hosts (localhost, private networks) send nothing to the dashboard by default, so the first production events come from the deployed site.", + ...(c.deploy + ? ["8. Deploy it, as chosen on the dashboard, with the command this project already uses (a deploy script in package.json, wrangler deploy for a Cloudflare Worker, vercel deploy --prod, and so on), then check that the deployed site serves the change."] + : []), + `${c.deploy ? 9 : 8}. Tell me what you changed${c.deploy ? " and where it is deployed" : ""}, and that a new conversation with you has the GenClass tools.`, + ]; + return out.join("\n"); +} + +/** installPrompt's own source for the page script, safe inside its ')}`, { cookie })).text(); + expect(odd).toContain('value=""><script>x</script>">'); + expect(odd).not.toContain(""); + expect(await (await call(env, "GET", "/account/new", { cookie })).text()).toContain("required autofocus>"); + }); }); describe("accounts: cookies in development", () => { diff --git a/telemetry-worker/test/pages.test.ts b/telemetry-worker/test/pages.test.ts index ff7fcd99..0d758b68 100644 --- a/telemetry-worker/test/pages.test.ts +++ b/telemetry-worker/test/pages.test.ts @@ -875,7 +875,7 @@ describe("dashboard page: the Supabase Studio frame (src/studio.ts): header, rai const p = installPrompt("gc_TestToken0123456789abc"); expect(p).toContain(`npm install ${RUNTIME_PKG}`); expect(p).toContain('GenClass.init({ token: "gc_TestToken0123456789abc" });'); - expect(p).toContain(``); + expect(p).toContain(``); expect(p).toContain("observe"); expect(p).toContain("Run the dev server"); // the whole flow from one paste: the state, the agent link, the MCP server in the agent, the status check @@ -887,6 +887,29 @@ describe("dashboard page: the Supabase Studio frame (src/studio.ts): header, rai expect(p).toContain("genclass_status should then list the page"); expect(p).toContain(`npx ${RUNTIME_PKG} init --token gc_TestToken0123456789abc --yes`); expect(installPrompt("gc_x", "https://c.example/v1/events")).toContain('GenClass.init({ token: "gc_x", telemetry: { endpoint: "https://c.example/v1/events" } });'); + // a site with no build step gets everything from the prompt alone: the script tag, the agent link from the CDN on local hosts + expect(p).toContain("A site with no build step (plain HTML pages, no bundler) installs nothing"); + expect(p).toContain(`import("${CDN_SCRIPT}/dist/agent.js")`); + expect(p).toContain('["localhost", "127.0.0.1"].includes(location.hostname)'); + // values one response writes together go in one atom + expect(p).toContain('rt.atom("search", { query: "", results: [] })'); + expect(p).not.toContain("Deploy it"); + }); + + it("Connect prompt choices: guard (and eager), the badge on every host, the agent link with the Loop view, deploy", () => { + const t = "gc_TestToken0123456789abc"; + const g = installPrompt(t, null, { guard: true, eager: true, badge: true, deploy: true, agentLink: "https://genclass.dev/dashboard/LINK" }); + expect(g).toContain(`const rt = GenClass.init({ token: "${t}", mode: "guard", aggressiveness: "eager", visual: { on: true, fix: true } });`); + expect(g).toContain(''); + expect(g).toContain(``); + expect(g).toContain("3. Mode guard, as chosen on the dashboard"); + expect(g).toContain("Mount the devtools overlay on every host, the deployed site too"); + expect(g).toContain(`claude mcp add genclass -e GENCLASS_DASHBOARD=https://genclass.dev/dashboard/LINK -e GENCLASS_REPORT=on -- npx -y ${MCP_PKG}`); + expect(g).not.toContain(`npx -y ${MCP_PKG} init`); + expect(g).toContain("8. Deploy it, as chosen on the dashboard"); + expect(g).toContain("9. Tell me what you changed and where it is deployed"); + // eager only with guard + expect(installPrompt(t, null, { eager: true })).not.toContain("eager"); }); it("Project overview: usage cards from the period's totals, the three problems that need attention first, each with its prompt", async () => { @@ -967,6 +990,56 @@ describe("dashboard page: the Supabase Studio frame (src/studio.ts): header, rai await r.close(); }); + it("Fix them all: one prompt for every kind of problem, and the dev page's checks; the agent waits for page loads itself", async () => { + const st = sampleStats("7d", NOW); + const r = await render(st); + const b = r.doc.getElementById("fixall")!, b2 = r.doc.getElementById("fixall2")!; + expect(b.classList.contains("hidden")).toBe(false); + const p = b.getAttribute("data-copy")!; + expect(b2.getAttribute("data-copy")).toBe(p); + for (const x of st.breakdowns.detect_diagnosis!.filter((x) => x.key !== "expected")) expect(p).toMatch(new RegExp(`\\n- [^\\n]*${nf.format(x.count)} times?`)); + expect(p).toContain("every check genclass_findings lists from the dev page"); + expect(p).toContain("then wait for the loads yourself: check every 10 seconds (sleep 10 in a shell in between)"); + expect(p).toContain("4. Replay the recorded order again: it must say not reproduced."); + // each problem's own prompt waits the same way + expect(r.doc.querySelector("#issues [data-copy]")!.getAttribute("data-copy")).toContain("then wait for the loads yourself"); + await r.close(); + }); + + it("Connect choices: ticking them rewrites the copied prompt with the server's own installPrompt; the agent choice makes the link", async () => { + const st = sampleStats("7d", NOW); + const r = await render(st, { account, id: "0123456789abcdef", firstEvent: "2026-10-12T09:00:00.000Z", runtimeReady: true }); + const box = (k: string) => r.doc.querySelector(`#cchoices [data-choice="${k}"]`) as unknown as HTMLInputElement; + const copied = () => r.doc.getElementById("cpinstall")!.getAttribute("data-copy"); + expect(copied()).toBe(installPrompt(st.token)); + expect(box("eager").disabled).toBe(true); + box("guard").click(); + expect(box("eager").disabled).toBe(false); + box("eager").click(); + box("badge").click(); + box("deploy").click(); + expect(copied()).toBe(installPrompt(st.token, null, { guard: true, eager: true, badge: true, deploy: true })); + expect((r.doc.getElementById("cpinstallt") as unknown as HTMLTextAreaElement).value).toBe(copied()); + // the agent choice makes the agent link (POST .../link), which the prompt then names, as the MCP page does + box("agent").click(); + await new Promise((x) => setTimeout(x, 10)); + const link = `https://genclass.dev/dashboard/${"S".repeat(32)}`; + expect(r.writes).toEqual(["POST /api/account/projects/0123456789abcdef/link"]); + expect(copied()).toBe(installPrompt(st.token, null, { guard: true, eager: true, badge: true, deploy: true, agentLink: link })); + expect(r.doc.querySelector("#mcp .alink-out code")!.textContent).toBe(link); + // unticking guard unticks eager + box("guard").click(); + expect(box("eager").checked).toBe(false); + expect(copied()).toBe(installPrompt(st.token, null, { badge: true, deploy: true, agentLink: link })); + await r.close(); + // a private link names itself; no request + const s = await render(st); + (s.doc.querySelector('#cchoices [data-choice="agent"]') as unknown as HTMLInputElement).click(); + expect(s.doc.getElementById("cpinstall")!.getAttribute("data-copy")).toBe(installPrompt(st.token, null, { agentLink: "sample" })); + expect(s.writes).toEqual([]); + await s.close(); + }); + it("Delete project in an account: Project Settings' danger zone asks once, then sends DELETE with the confirm; a private link has none", async () => { const r = await render(sampleStats("7d", NOW), { account, id: "0123456789abcdef", firstEvent: "2026-10-12T09:00:00.000Z", runtimeReady: true }); expect([...r.doc.querySelectorAll(".sb-pm a")].some((a) => a.getAttribute("href") === "#delproj" && a.textContent === "Delete project")).toBe(true); From 1d5b42ed56fa72a9780665009f1f311d10ba2279 Mon Sep 17 00:00:00 2001 From: Mehar Khanna Date: Tue, 27 Oct 2026 00:38:08 -0400 Subject: [PATCH 4/6] runtime: the measured visual checks' counts in telemetry (VISUAL.md section 10) Approved by the owner on 2026-10-27, counts only. The session event gains `visual` ("off" | "local" | "on") and `fix`; once the checks' chunk has run, every summary event carries counts.checks, counts.checksByRoute (at most 20 normalised paths), counts.fixes { applied, undone } and runtime.visual { passes, ms, partial, tier }. The summary counts are added by the chunk (src/visual/bridge.ts withChecks), so first load grows only by the two session fields. Never a selector, element text, colour, URL, size or a per-finding event. With telemetry on, the chunk prints one console notice saying what of the checks is sent. TELEMETRY.md, both READMEs, PRIVACY.md, VISUAL.md section 10 and the CHANGELOG updated. Tests: test/telemetry-visual.test.ts. No model-visible text changed (situation-v2.3). Co-Authored-By: Claude Opus 5.5 --- PRIVACY.md | 7 +- README.md | 2 +- docs/runtime/VISUAL.md | 63 +++++-- packages/runtime/CHANGELOG.md | 22 +++ packages/runtime/README.md | 7 +- packages/runtime/TELEMETRY.md | 30 +++- packages/runtime/src/telemetry/client.ts | 5 + packages/runtime/src/visual/bridge.ts | 60 ++++++- packages/runtime/src/visual/start.ts | 3 + .../runtime/test/telemetry-visual.test.ts | 159 ++++++++++++++++++ packages/runtime/test/visual-bridge.test.ts | 2 +- 11 files changed, 325 insertions(+), 35 deletions(-) create mode 100644 packages/runtime/test/telemetry-visual.test.ts diff --git a/PRIVACY.md b/PRIVACY.md index 449b239c..09987b83 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -30,6 +30,11 @@ object; version 0.2.1 sent from those hosts by default, by mistake, fixed in 0.2 their users. - **Technical details:** GenClass and model versions, settings, device type (WebGPU or WASM), load times and error counts, and the website's hostname. +- **Counts of the measured visual checks, from runtime 0.3.0-beta.4, only where they run** (off on public sites + unless the website's developer turned them on): whether the checks run on the page and whether their fixes are on, + how many problems each check found (such as text in its background's colour, or a page that stopped scrolling), the + same per page path, and how many fixes GenClass applied on the page and undid. Never which element, its text, its + colours or any address. - **The page's path**, with id-like segments (and, from runtime 0.3.0, segments containing an `@`, such as an e-mail address) replaced by `:id` and no query string or fragment (for example `/orders/:id`), on each page load and each decision. Other words in the path, such as a username, are sent as they are. @@ -73,7 +78,7 @@ works on their own site. We do not sell the data, use it for advertising, or try ## Dashboards for website owners A website owner can get a private dashboard link for their site's token. Whoever holds that link can see aggregated diagnostics for that site (counts of decisions, detections, possible detections and actions, the site's hostnames, -routes, the names the developer gave to protected functions, model speed, versions; with interaction analytics on, +routes, the names the developer gave to protected functions, model speed, versions, the measured visual checks' counts per check and page and their fixes; with interaction analytics on, page views, the paths between pages, time on page, scroll depth and clicks per page and element, visits per day, pages per visit, where visits came from, devices, countries, new or returning, entry and exit pages and funnels) and a list of recent detections (time, route or function name, diagnosis, action). Dashboards do not show the situation text. Data from browsers that opted out or diff --git a/README.md b/README.md index e52639d2..43a63cae 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ export const submitOrder = protect("checkout submit", (cart) => fetch("/api/orde - **Decisions are local.** Situations are built and decided in the browser by the local model; nothing is sent anywhere to make a decision. - **Anonymous diagnostics are on by default** with `GenClass.init()` in a browser, except on local and private hosts (`localhost`, `*.localhost`, `*.local`, `*.test`, `127.0.0.1`, `::1`, private IP ranges) (`0.2.1` sends from local hosts too, by mistake; fixed in `0.2.2`, see the CHANGELOG), in Node/SSR, and in `createRuntime()`. They go to the GenClass maintainers' collector ([`telemetry-worker/`](telemetry-worker), a Cloudflare Worker writing to a private R2 bucket) and are used to improve the model. - **By default no text from your page is sent.** Sent: a random per-page session id, versions, mode, device class, the app's hostname and id-normalised path (from `0.3.0` a segment with an `@`, such as an e-mail address, is `:id` too), and per decision (every detection and every decision whose diagnosis is not `expected`, every action and veto; of the routine ones, one in four and at most 200 per page load; counts are exact; `0.2.2` and earlier send every decision) the trigger, the diagnosis and its confidence, the action and gate, the outcome and latency. The collector stores no IP address or user agent. One console notice per page says it is on. -- **The measured visual checks send nothing** (new in `0.3.0`): their findings stay in the page (the console, `rt.checks()`, your own sinks). +- **The measured visual checks send counts only** (from `0.3.0-beta.4`): whether they run on the page and whether their runtime fixes are on (`session`), and in each `summary` the findings per check, per check and page path (at most 20 paths), the fixes applied and undone, and their cost. Never a selector, element text, colour, URL or one event per finding; the findings themselves stay in the page (the console, `rt.checks()`, your own sinks). They are off on public hosts unless you turn them on. - **The situation text the model read is sent only if you opt in** with `GenClass.init({ telemetry: { include: { situation: true } } })`. It is redacted by field meaning, but can still contain route names with query strings, short error messages and short field values; read [TELEMETRY.md](packages/runtime/TELEMETRY.md) before enabling it. - **Opt out:** `GenClass.init({ telemetry: false })`, `?genclass=no-telemetry`, `localStorage.setItem("genclass.telemetry", "off")`; browsers sending Global Privacy Control are never collected. - **Interaction analytics are off unless the app turns them on** (`analytics: true` with the app's token, new in `0.3.0-beta.1`): per page view the route, engaged time, scroll depth, and clicks, rage clicks and dead clicks per element, the element named from markup, never its text; and, within one page load, how each page view was reached, the route before it and the time between, the first element used on it, and where the page load ended with its routes in order. No visitor id; one `localStorage` entry (the visit flag, which links the page loads of a visit and says whether the browser visited before; `runtime.analytics.stop()` removes it), every opt-out above applies, and they never feed a decision. Where analytics need consent (EU, EEA, UK), start them after consent with `runtime.analytics.start()`. See [TELEMETRY.md](packages/runtime/TELEMETRY.md#interaction-analytics). diff --git a/docs/runtime/VISUAL.md b/docs/runtime/VISUAL.md index e7caa429..6ee92f08 100644 --- a/docs/runtime/VISUAL.md +++ b/docs/runtime/VISUAL.md @@ -82,7 +82,8 @@ What follows from that: sampling hashes the session id; no `Math.random`. - **Rule 7.** No new dependency. axe-core is not added; the contrast pipeline below reimplements the parts of its approach that matter (stack at text rects, walk to the first opaque layer, abstain on anything complex). -- **Rule 8.** No visual telemetry ships until the owner approves the exact fields (section 10). +- **Rule 8.** No visual telemetry shipped until the owner approved the exact fields (section 10): counts only, + approved on 2026-10-27 and sent from `0.3.0-beta.4`. ## 2. Shared machinery @@ -996,8 +997,9 @@ Why off on public hosts by default: 1. **The visitor pays the CPU.** A layout pass costs main-thread time on every page view, and on a low-end phone that time is 4 to 6 times larger (estimate, to be measured). The visitor gets nothing back unless a developer reads the reports. -2. **Nobody reads production consoles.** Until telemetry for checks is approved (section 10), production reports - reach only an app's own `report` function or sinks. An app that wants them sets `visual: true` and wires a sink. +2. **Nobody reads production consoles.** Production findings reach only an app's own `report` function or sinks; + since `0.3.0-beta.4` their counts also reach the app's dashboard through telemetry (section 10). An app that wants + them sets `visual: true`. 3. **Visitors' environments cause false reports.** Dark-mode extensions, page translation, zoom, forced colours and user font settings change what is measured. The abstain list covers the known ones; the bench (section 15) has to show the rest is rare before a production default is reconsidered. @@ -1182,26 +1184,54 @@ receives every report. ## 10. Telemetry -Nothing about visual checks is sent until the owner approves the exact fields. Rule 8 says widening what is sent -needs the owner's OK, and the request of 2026-10-13 did not mention telemetry. Until then the telemetry client has no -visual code at all (which also keeps its first-load bytes unchanged). (The first implementation added a `visual` -field to the session and summary events; the review of 2026-10-13 removed both, section 19.) - -Proposed fields, counts only: +**Approved by the owner on 2026-10-27; sent from `0.3.0-beta.4`.** Rule 8 says widening what is sent needs the +owner's OK, and the request of 2026-10-13 did not mention telemetry, so until then nothing about the checks was sent. +(The first implementation added a `visual` field to the session and summary events; the review of 2026-10-13 removed +both, section 19.) The owner approved these fields, counts only, on 2026-10-27, after a production site full of stuck +scroll locks and frozen clicks (each measured and fixed in its visitors' consoles) looked clean on its dashboard: | field | where | content | |---|---|---| | `counts.checks` | summary event | `{ [kind]: n }` | -| `counts.checksByRoute` | summary event | `{ [normalisedRoute]: { [kind]: n } }`, at most 20 routes; the route is the same `normalizePath(pathname)` already sent with decisions | +| `counts.checksByRoute` | summary event | `{ [normalisedRoute]: { [kind]: n } }`, at most 20 routes; the route is the same `normalizePath(pathname)` already sent with decisions (a segment holding an `@` as `:id`) | | `counts.fixes` | summary event | `{ applied, undone }` | | `runtime.visual` | summary event | `{ passes, ms, partial, tier }` | | `visual`, `fix` | session event | `"off" \| "local" \| "on"` and a boolean, so counts can be normalised by coverage | -Never sent: selectors, element text, colours, URLs, sizes, per-finding events. Route keys would be cut as telemetry's -`route()` cuts other routes. When approved: update `TELEMETRY.md`, both READMEs, the CHANGELOG and the console notice -check. Coverage note: telemetry is off on local hosts and visual is off on public hosts by default, so counts would -come from apps that set `visual: true` in production, from visitors who open a link with `?genclass-visual=on`, and -from pages where the devtools overlay mounted or `rt.checkNow()` ran. +Never sent: selectors, element text, colours, URLs, sizes, per-finding events, and the rest of +`SessionSummary.visual` (`elements`, `maxSliceMs`, `abstained`, `unsupported`, `stopped`). + +As built: + +- **Session event** (`src/telemetry/client.ts`): `visual` is the first-load stub's setting (`rt.vis.setting`, `"off"` + without a stub) and `fix` is the stub's `fix`: `visual.on === true` and `visual.fix` is `true` or a non-empty list, + the condition under which the engine's `resolveCfg` allows any fix kind. Whether a fix can run also needs guard or + heal, which the same event's `effectiveMode` says. These two fields are the only first-load bytes. +- **Summary event**: the chunk adds the counts (`src/visual/bridge.ts`, `withChecks`), so first load carries no code + for them. `follow()` replaces the telemetry client's `push` with one that adds them to every `summary` event before + it is queued; the client queues its own events through `this.push`, so the own property sees periodic, hidden, + pagehide and destroy summaries alike (teardowns run in order, so the destroy summary still reads the engine). A page + where the chunk never loaded (`"local"` on a public host) sends summaries exactly as before. The counts are + cumulative for the page load, like every other summary count; the collector counts them exactly through its + per-page-load high-water marks. +- **Console notice**: when the chunk starts on a page with telemetry on, it prints once per page `[GenClass] The + diagnostics also carry the measured visual checks' counts: findings per check and page, runtime fixes applied and + undone, never selectors, page text, colours or URLs.` (`VISUAL_TELEMETRY_NOTICE`). The telemetry notice itself is + unchanged ("decisions and counts, no page text"). +- **Collector and dashboard** (`telemetry-worker/src/dashboard.ts`, `pages.ts`): counters `checks`, `check_kind`, + `check_issue` (kind TAB route), `check_fix` (applied, undone), `check_loads`, `check_tier`, `check_partial` and + `check_ms` from summaries, `visual_setting` and `visual_fix` from session events; no migration (the counters tables + hold any metric). The dashboard shows checks as their own kind (a "Measured check" badge, a hatched slot in the + problems chart, cards with what was measured, where, the usual fix and a Copy prompt that uses `genclass_scan`), counts + them in the app's status and in Fix them all, and lists their fixes in What GenClass did. They are never counted as + model detections. +- Tests: `packages/runtime/test/telemetry-visual.test.ts`, `telemetry-worker/test/dashboard.test.ts` and + `pages.test.ts` ("measured visual checks"). + +Coverage: telemetry is off on local hosts and visual is off on public hosts by default, so counts come from apps +that set `visual: true` in production (`data-visual="on"`, or `data-visual="fix"` with the runtime fixes), from +visitors who open a link with `?genclass-visual=on`, and from pages where the devtools overlay mounted or +`rt.checkNow()` ran. The dashboard's Health view says on how many page loads the checks ran. ## 11. Runtime fixes and undo @@ -1473,7 +1503,8 @@ Owner decisions: 1. **Defaults**: `visual: "local"` (on for development hosts and with devtools mounted, off on public hosts unless `visual: true`), and whether `?genclass-visual=on` may enable report-only checks on public hosts. 2. **Fixes**: explicit `visual.fix` plus `guard` or `heal`, and the three v1 fix kinds. -3. **Telemetry**: the fields in section 10 (counts per kind and normalised route), under rule 8. +3. **Telemetry**: the fields in section 10 (counts per kind and normalised route), under rule 8. Approved on + 2026-10-27, sent from `0.3.0-beta.4`. 4. **Sinks**: `"check"` records only for sinks that list it in `kinds`. 5. **Gates**: the numbers in section 15. Measured on 2026-10-13 (RESULTS.md section 14): the false-report and fix gates are met; the 4x-throttling gate is not reliably met (pass 1 inside 100 ms on 78.1% of clean local pages in runs 1 diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index 29ce78a9..5e08968f 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## Unreleased (0.3.0-beta.4) + +- **The measured visual checks' counts reach the dashboard** (owner approval 2026-10-27; VISUAL.md section 10, + TELEMETRY.md). A site full of stuck scroll locks and frozen clicks, each measured and fixed in its visitors' + consoles, looked clean on genclass.dev, because telemetry sent nothing about the checks. Now, counts only: the + `session` event says whether the checks run on the page (`visual`: `off`, `local` or `on`) and whether the app + turned their runtime fixes on (`fix`); once the checks have run, every `summary` event carries `counts.checks` (per + kind), `counts.checksByRoute` (per normalised path, at most 20), `counts.fixes` (`applied`, `undone`) and + `runtime.visual` (`passes`, `ms`, `partial`, `tier`). Never a selector, element text, colour, URL, size or one event + per finding. The summary counts are added by the checks' own file (`src/visual/bridge.ts`), so first load grows only + by the two session fields. With telemetry on, the console says once when the checks start what of them is sent. + The checks stay off on public hosts unless the app turns them on (`visual: true`, `data-visual="on"`, or + `data-visual="fix"` in guard mode for the fixes). Tests: `telemetry-visual.test.ts`. No model-visible text changed + (`situation-v2.3`). +- **The dashboard shows them** (telemetry-worker): checks are their own kind ("Measured check", never a model + detection), with a card per check (what was measured, the pages, the usual fix, a Copy prompt that measures the page + again with `genclass_scan`), a hatched slot in the problems chart, their pages in the Pages table, their runtime + fixes in What GenClass did, and their coverage in Health. They count in the app's status (Critical, Needs attention, + Healthy) and in Fix them all; a check fix still in place counts as fixed live. The sample dashboard shows simulated + checks. The Snow Day Predictor's demo estimate (`src/demo.ts`) yields to the measured counts as soon as that project + receives its first page load with measured checks. + ## 0.3.0-beta.3 (2026-10-26) - **A colour fix stays while its element is out of view** (`src/visual/engine.ts`, VISUAL.md 11.2). The periodic diff --git a/packages/runtime/README.md b/packages/runtime/README.md index afba33d3..316a8cb5 100644 --- a/packages/runtime/README.md +++ b/packages/runtime/README.md @@ -820,8 +820,9 @@ opened the link; `visual: false` or `?genclass-visual=off` turns them off. in a visible tab, and all their page reads (passes, the 2 s read of `html` and `body`, the click, hover, focus and scroll checks) run in idle callbacks: slices of 8 ms, at most 100 ms per pass and 1 s per 10 minutes. One step can still take longer than a slice when the browser has to lay out a heavy page first (measured in RESULTS.md section 14). -Low-end devices run only the near-free checks (failed assets, CSP, layout shifts, the interaction checks). Nothing -about the checks is sent anywhere. +Low-end devices run only the near-free checks (failed assets, CSP, layout shifts, the interaction checks). With +telemetry on, only their counts are sent (per check and page, fixes applied and undone, from `0.3.0-beta.4`; see +[Privacy and telemetry](#privacy-and-telemetry)): never a selector, element text, colour or URL. **Runtime fixes (opt-in).** With `GenClass.init({ mode: "guard", visual: { on: true, fix: true } })`, GenClass can apply three narrow fixes: a readable text colour (one rule in its own stylesheet), and removing a leaked inline @@ -1339,7 +1340,7 @@ GenClass.init({ model: { baseUrl: "/genclass-model/", ortWasmPaths: "/genclass-m - **Decisions are local.** Situations are built and decided in the browser by the local model; nothing is sent anywhere to make a decision. - **Anonymous diagnostics are on by default** with `GenClass.init()` in a browser, except on local and private hosts (`localhost`, `*.localhost`, `*.local`, `*.test`, `127.0.0.1`, `::1`, private IP ranges) (`0.2.1` sends from local hosts too, by mistake; fixed in `0.2.2`, see the CHANGELOG), in Node/SSR, and in `createRuntime()`. They go to the GenClass maintainers' collector ([`telemetry-worker/`](https://github.com/genclass-dev/GenClass-lib/tree/main/telemetry-worker), a Cloudflare Worker writing to a private R2 bucket) and are used to improve the model. - **By default no text from your page is sent.** Sent: a random per-page session id, versions, mode, device class, the app's hostname and id-normalised path (from `0.3.0` a segment with an `@`, such as an e-mail address, is `:id` too), and per decision (every detection and every decision whose diagnosis is not `expected`, every action and veto; of the routine ones, one in four and at most 200 per page load; counts are exact; `0.2.2` and earlier send every decision) the trigger, the diagnosis and its confidence, the action and gate, the outcome and latency. The collector stores no IP address or user agent. One console notice per page says it is on. -- **The measured visual checks send nothing** (new in `0.3.0`): their findings stay in the page (the console, `rt.checks()`, your own sinks). +- **The measured visual checks send counts only** (from `0.3.0-beta.4`): whether they run on the page and whether their runtime fixes are on (`session`), and in each `summary` the findings per check, per check and page path (at most 20 paths), the fixes applied and undone, and their cost. Never a selector, element text, colour, URL or one event per finding; the findings themselves stay in the page (the console, `rt.checks()`, your own sinks). They are off on public hosts unless you turn them on. - **Interaction analytics are off unless you turn them on** (new in `0.3.0-beta.1`): `GenClass.init({ token, analytics: true })` also sends, per page view, the route, engaged time, scroll depth and clicks, rage clicks and dead clicks per element, and within one page load how each view was reached, the route before it with the time between, the first element used on it, and where the page load ended with its routes in order (at most 10), for your dashboard. Elements are named from markup (`data-genclass-name`, `data-testid`, an id, a link's target, class names, a short path), never their text; the one thing stored on the device is the visit flag in `localStorage` (it links the page loads of a visit and says whether the browser visited before; `runtime.analytics.stop()` removes it), and no visitor id exists. They need the token and telemetry on, every opt-out above turns them off, and they never feed a decision or the model. Where analytics need consent (the EU, EEA and UK), leave them off and call `runtime.analytics.start()` after consent. Details: [TELEMETRY.md](https://github.com/genclass-dev/GenClass-lib/blob/main/packages/runtime/TELEMETRY.md#interaction-analytics). - **The situation text the model read is sent only if you opt in** with `GenClass.init({ telemetry: { include: { situation: true } } })`. It is redacted by field meaning, but can still contain route names with query strings, short error messages and short field values; read [TELEMETRY.md](https://github.com/genclass-dev/GenClass-lib/blob/main/packages/runtime/TELEMETRY.md) before enabling it. - **Opt out:** `GenClass.init({ telemetry: false })`, `?genclass=no-telemetry`, `localStorage.setItem("genclass.telemetry", "off")`; browsers sending Global Privacy Control are never collected. diff --git a/packages/runtime/TELEMETRY.md b/packages/runtime/TELEMETRY.md index 392d9843..91733c5e 100644 --- a/packages/runtime/TELEMETRY.md +++ b/packages/runtime/TELEMETRY.md @@ -63,15 +63,25 @@ segment holding an `@`, such as an e-mail address, is `:id` too). random visit id kept 30 minutes, where the visit came from (the referring host, utm values), the device class, browser and OS families and whether the browser visited before (a flag in `localStorage`). See [Interaction analytics](#interaction-analytics). -- **Measured visual checks** ([VISUAL.md](../../docs/runtime/VISUAL.md); new in `0.3.0`): nothing about - them is sent. Their findings stay in the page (the console, `rt.checks()`, your own sinks). Counts per check kind - are a proposal that needs the maintainers' approval of the exact fields first; this file will list them if that - happens. +- **Measured visual checks** ([VISUAL.md](../../docs/runtime/VISUAL.md) section 10; counts sent from `0.3.0-beta.4`, + approved by the maintainers on 2026-10-27): **counts only.** The `session` event says whether the checks run on the + page (`visual`: `off`, `local` or `on`) and whether the app turned their runtime fixes on (`fix`). Once the checks + have run on the page, every `summary` event carries how many findings each check made (`counts.checks`), the same + per page path (`counts.checksByRoute`, the path normalised as on `session`, at most 20 paths), how many runtime + fixes were applied and undone (`counts.fixes`), and how many passes ran, their check time in ms, how many ran out + of their budget and the tier (`runtime.visual`). The findings themselves stay in the page (the console, + `rt.checks()`, your own sinks): never a selector, element text, colour, URL, size or one event per finding. The + checks are off on public hosts unless the app turns them on (`visual: true`), so most pages send `visual: "local"` + and no counts. When they start with telemetry on, the console says so once: + + ``` + [GenClass] The diagnostics also carry the measured visual checks' counts: findings per check and page, runtime fixes applied and undone, never selectors, page text, colours or URLs. + ``` - **Never, in either setting:** typed values of password, payment (`cc-*`), one-time-code or secret-named inputs (the runtime never records them in the first place), cookies, request or response headers, request or response bodies, storage contents, the sentences GenClass prints about what it changed, the measured visual checks' - selectors, element text, colours, URLs and messages, IP addresses or user agents (the collector does not store - them). + selectors, element text, colours, URLs, sizes and messages (only their counts, above), IP addresses or user agents + (the collector does not store them). ### Batch envelope (schema `genclass-telemetry/1`) @@ -93,7 +103,7 @@ Every event has `t` (type), `seq` (0, 1, 2, … per page) and `at` (ms since sta | `t` | when | fields | |---|---|---| -| `session` | once, at init | `runtime`, `host` (the page's hostname, which identifies the app using GenClass), `route` (the page path with id-like segments replaced by `:id`, and from 0.3.0 any segment holding an `@` or `%40`, such as an e-mail address; no query or fragment), `mode`, `effectiveMode`, `aggressiveness`, `sampled`, `model` (`local` / `custom` / `off`), `modelState`, `modelVersion`, `triage`, `shadow`, `holdWrites`, `device` (`webgpu` available, `cores`, `memoryGB` (the browser's coarse deviceMemory), `crossOriginIsolated`, `effectiveType`, `saveData`), `sample`, `situation` (whether situation text is included). Not the page's `decisionSample` or `maxDecisions` setting: `summary.telemetry` says how many records were sent and withheld ([Volume](#volume)) | +| `session` | once, at init | `runtime`, `host` (the page's hostname, which identifies the app using GenClass), `route` (the page path with id-like segments replaced by `:id`, and from 0.3.0 any segment holding an `@` or `%40`, such as an e-mail address; no query or fragment), `mode`, `effectiveMode`, `aggressiveness`, `sampled`, `model` (`local` / `custom` / `off`), `modelState`, `modelVersion`, `triage`, `shadow`, `holdWrites`, `device` (`webgpu` available, `cores`, `memoryGB` (the browser's coarse deviceMemory), `crossOriginIsolated`, `effectiveType`, `saveData`), `sample`, `situation` (whether situation text is included), `visual` (`0.3.0-beta.4`: the measured visual checks `off`, `local` (development hosts, the devtools overlay, `rt.checkNow()`) or `on`) and `fix` (the app turned their runtime fixes on, `visual: { on: true, fix }`; they still run only in guard or heal). Not the page's `decisionSample` or `maxDecisions` setting: `summary.telemetry` says how many records were sent and withheld ([Volume](#volume)) | | `model` | model state changes, and once right after `session` when the model was skipped or failed before telemetry started (the device floor: `state` `skipped`, `reason` `device-memory` / `save-data` / `loadIf`; with `model.inlineFallback: false` and no worker, `reason` `worker-unavailable`) | `state`, `version`, `model`, `variant`, `device` (webgpu/wasm), `threads`, `worker`, `workerError` (why the model worker was not used, URLs replaced by ``), `gpu` (WebGPU probe summary), `ort`, `loadMs`, `warmupMs`, `fromCache`, `bytes`, `phase`, `reason`, `error` (GenClass model-host error text, ≤ 200 chars, URLs replaced by ``), `attempts` (variant, device and error per load attempt, URLs replaced by ``) | | `status` | mode, aggressiveness or breaker changes | `mode`, `effectiveMode`, `aggressiveness`, `breaker` | | `decision` | a model decision: every one whose top diagnosis is not `expected` (detections and near-misses), that acted or was vetoed, always; a `decisionSample` share (default 1 in 4) of the routine ones (top diagnosis `expected`, nothing acted), at most `maxDecisions` (default 200) routine ones per page load ([Volume](#volume)); all are counted in `summary` | `id`, `trigger`, `route` (the page path, normalised; for a decision deferred until the model loaded or the tab was visible again, 0.3.0, the path at its trigger), `model`, `latencyMs`, `held` (the subject waited for the answer; false = decided in the background), `situation` (text, see above), `autoState` (true when discovered state was recorded on the page), `budget`, `compact`, `questions` (question ids), `answers` (per question: calibrated `probabilities` per label, `choice`, `confidence`; or `p` / `score`), `diagnosis`, `diagnosisConfidence`, `action` (chosen), `confidence`, `tier`, `candidate`, `ran`, `executed`, `acted`, `reason` (why the passive action ran), `mass`, `gateKind`, `threshold` / `gain` / `margin`, `thresholdSource`, `gates` (report/guard/heal thresholds, aggressiveness, level and sources), `effectiveMode`, `shadow`, `fn` (only on batches with a token: the name the app gave the `protect()`ed function the decision's subject ran inside, the outermost one; omitted otherwise) | @@ -102,7 +112,7 @@ Every event has `t` (type), `seq` (0, 1, 2, … per page) and `at` (ms since sta | `veto` | `onBeforeAction` vetoed an action (every one) | `decision`, `action`, `enforced` | | `breaker` | the breaker tripped or reset | `tripped`, `reason`, `undos`, `errors`, `decisions` (ids) | | `model-error` | model errors (first 20 per page; all are counted) | `code`, `message` (≤ 200 chars, URLs replaced by ``: a model host's message can quote the browser's worker error, which names the script and the page origin) | -| `summary` | about once a minute (≥ 60 s apart) while decisions continue, never capped; when the tab is hidden, on pagehide and on destroy | `reason`, `counts` (decisions, acted, detections (likely ones), `possibleDetections` (0.3.0: the possible ones, counted apart), actions applied/failed, late reverts, undos, vetoes, breaker trips, per trigger / diagnosis / action ran, detections per diagnosis, action limits hit, model errors per code, fail-opens per reason and trigger), `runtime` (model p50/p95 latency, decisions, dropped, hidden-tab skips, held time, denied and shadow counts, error count), `telemetry` (queued, dropped, send failures, events sent, `decisionRecords` (decisions whose records were sent), `decisionsSampledOut`, `decisionsCapped`) | +| `summary` | about once a minute (≥ 60 s apart) while decisions continue, never capped; when the tab is hidden, on pagehide and on destroy | `reason`, `counts` (decisions, acted, detections (likely ones), `possibleDetections` (0.3.0: the possible ones, counted apart), actions applied/failed, late reverts, undos, vetoes, breaker trips, per trigger / diagnosis / action ran, detections per diagnosis, action limits hit, model errors per code, fail-opens per reason and trigger), `runtime` (model p50/p95 latency, decisions, dropped, hidden-tab skips, held time, denied and shadow counts, error count), and from `0.3.0-beta.4`, once the measured visual checks have run on the page (their own file adds them, so first load carries no code for them): `counts.checks` (findings per check kind), `counts.checksByRoute` (`{ [path]: { [kind]: n } }`, at most 20 paths, normalised as on `session`), `counts.fixes` (`applied`, `undone`) and `runtime.visual` (`passes`, `ms` of check time, `partial` passes, `tier` `full` or `light`), `telemetry` (queued, dropped, send failures, events sent, `decisionRecords` (decisions whose records were sent), `decisionsSampledOut`, `decisionsCapped`) | | `interactions` | only with `analytics: true` and a token (or a custom endpoint): when a page view ends (a route change, page exit), when the tab is hidden, and before a `summary`, if anything changed ([Interaction analytics](#interaction-analytics)) | `view` (page view number in this page load), `route` (normalised as on `session`), `start` (the view's first event: a page view), `engagedMs`, `scroll` (milestones 25 / 50 / 75 / 100 first reached since the previous event), `clicks`, `rage`, `dead` (counts per element key, at most 50 / 20 / 20 keys, the rest under `(other)`), `end` (the view ended). Counts are deltas since the view's previous event. Unreleased, on a view's first event only: `visit` (16 hex characters, random, see [Visits](#visits)); and when that view opened the visit, `entry: true`, `ref` (the referring site's host, `www.` dropped, when it is another site; `(other)` when the host looks like data), `utm` (`source`, `medium`, `campaign`: lower case, letters, digits, `. _ + -` and spaces, words that look like ids dropped, at most 40 characters each), `device` (`desktop` / `mobile` / `tablet`), `browser` (`Chrome`, `Safari`, `Firefox`, `Edge`, `Opera`, `Samsung Internet`, `Other`), `os` (`Windows`, `macOS`, `iOS`, `Android`, `Linux`, `ChromeOS`, `Other`), `returning` (whether this browser had a visit before; absent when storage is blocked). Navigation paths (Unreleased): on the view's first event `nav` (how it was reached: `load`, `reload`, `back_forward`, `push`, `back`, `forward`, `pop`) and, from a page load's second view on, `from` (the previous view's route) and `fromMs`; once per view `first` (the element key of the first click, or of the first focus on a form field after a key press) and `firstMs`; on the page load's last event (pagehide) `exit: true` and `path` (its routes in order, at most 10, then `"(more)"`) | Not sent: `expectation` reports (a learned step that did not happen, `docs/runtime/EXPECTATIONS.md`) and anything @@ -268,7 +278,9 @@ New in `0.3.0` (`0.2.x` ignores a token and sends neither `token` nor `fn`). An detections (the `detect` events with `tier: "possible"`, shown apart), actions, the app's hostnames, top routes and protected functions, model load, backend and latency, versions; with interaction analytics on, page views, visits, their sources, devices, countries, entry and exit pages and the app's funnels) and its recent detections. Dashboard - data is also deleted after 90 days (visits after 90 days on every plan). + data is also deleted after 90 days (visits after 90 days on every plan). From runtime `0.3.0-beta.4` it also shows + the measured visual checks' counts per check and page and their runtime fixes, as their own kind ("measured + check"), never as model detections. - A token changes nothing about when data is collected: with telemetry off (any opt-out, or Global Privacy Control, or a local host) nothing is sent and the dashboard gets nothing from that page. On a local host the console says so once when a token is set; otherwise only with `debug: true`. diff --git a/packages/runtime/src/telemetry/client.ts b/packages/runtime/src/telemetry/client.ts index d92c9b92..9a337b94 100644 --- a/packages/runtime/src/telemetry/client.ts +++ b/packages/runtime/src/telemetry/client.ts @@ -274,6 +274,11 @@ export class TelemetryClient implements TelemetryStatus { }), sample: this.cfg.sample, situation: this.cfg.situation, + // the measured visual checks on this page (VISUAL.md section 10): off, "local" (development hosts, the overlay, + // checkNow()) or on, and whether the app turned their runtime fixes on, so the summary's check counts can be read + // against coverage. The counts themselves are added to `summary` by the checks' own chunk (src/visual/bridge.ts) + visual: this.rt.vis?.setting ?? "off", + fix: !!this.rt.vis?.fix, // decisionSample / maxDecisions are not sent (no owner OK for the page's config values, AGENTS.md rule 8): a // collector scales from summary.telemetry (decisionRecords, decisionsSampledOut, decisionsCapped) }); diff --git a/packages/runtime/src/visual/bridge.ts b/packages/runtime/src/visual/bridge.ts index 939ee309..b2e616f5 100644 --- a/packages/runtime/src/visual/bridge.ts +++ b/packages/runtime/src/visual/bridge.ts @@ -9,13 +9,16 @@ // "interventions" and a function behave as for every other report), and a sink record for sinks whose `kinds` // lists "check"; // - write an audit entry of kind "fix"; -// - follow status changes (mode, enabled, breaker) and the runtime's teardown. +// - follow status changes (mode, enabled, breaker) and the runtime's teardown; +// - add the counts VISUAL.md section 10 lists to telemetry's `summary` events (checks per kind and per route, fixes +// applied and undone, the passes' cost and tier), so first load carries no code for them. Never a selector, +// element text, colour, URL, size or a per-finding event. // It never pushes RtEvents, facts or situations and never touches the gate, the queue or the model (AGENTS.md rule 2 // exception). Some members it reads are private in TypeScript only (cadence, reporter, sinks, fire, pushAudit, auditBase, // sessionId, sessionTags, sampled, reportMode, _aggr): the RtLike interface below names them, and // test/visual-bridge.test.ts fails when one of them is renamed. -import type { AuditEntry, CheckFinding, Clock, ModeOrOff, Report, SessionSummary, SinkRecord } from "../types.js"; +import type { AuditEntry, CheckFinding, Clock, ModeOrOff, Report, SessionSummary, SinkRecord, VisualSummary } from "../types.js"; import type { VisualHost } from "./host.js"; import type { VisualStub } from "./start.js"; import { RequestCounter, type RequestsRt } from "./requests.js"; @@ -28,6 +31,8 @@ export interface RtLike { readonly base: { latency(sig: string): { p95: number } | undefined } & RequestsRt["base"]; readonly isPaused: boolean; vis: VisualStub | null; + /** The telemetry client (src/telemetry/client.ts) when telemetry is on: `push` queues an event. */ + telemetry?: { push?: (e: Record) => void }; effectiveMode(): ModeOrOff; summary(): SessionSummary; on(type: "status", fn: () => void): () => void; @@ -117,12 +122,36 @@ export function hostFor(rt0: unknown, describe: (el: Element) => string, normali }; } +/** Printed once per page when telemetry is on and the checks start: what of them telemetry sends (TELEMETRY.md). */ +export const VISUAL_TELEMETRY_NOTICE = + "[GenClass] The diagnostics also carry the measured visual checks' counts: findings per check and page, runtime fixes applied and undone, never selectors, page text, colours or URLs."; +let noticeShown = false; + +/** Tests only: print the notice again. */ +export function resetVisualNotice(): void { + noticeShown = false; +} + +/** + * The counts telemetry's `summary` event carries for the checks (VISUAL.md section 10, approved 2026-10-27), added to + * the event in place: `counts.checks` (per kind), `counts.checksByRoute` (per normalised route, at most 20, as the + * engine keeps them), `counts.fixes` (applied, undone) and `runtime.visual` (passes, check time in ms, partial passes, + * tier). Nothing else of VisualSummary (elements, slices, abstentions, unsupported kinds) is sent. + */ +export function withChecks(e: Record, v: VisualSummary | undefined): void { + if (!v) return; + const counts = e.counts as Record | undefined; + if (counts) Object.assign(counts, { checks: v.checks, checksByRoute: v.checksByRoute, fixes: { applied: v.fixes.applied, undone: v.fixes.undone } }); + const run = e.runtime as Record | undefined; + if (run) run.visual = { passes: v.passes, ms: v.ms, partial: v.partial, tier: v.tier }; +} + /** * Follow the runtime: status changes re-check the fix gate, teardown stops the checks, and rt.summary() of this * runtime gains its `requests` key (src/visual/requests.ts) and, once the checks have run, its `visual` key (added - * here, so first load carries no code for either). + * here, so first load carries no code for either); telemetry's summary events gain the check counts (withChecks). */ -export function follow(rt0: unknown, ctl: { gateFixes(): void; stop(): void; summary(): object | null }): void { +export function follow(rt0: unknown, ctl: { gateFixes(): void; stop(): void; summary(): { visual: VisualSummary } | null }): void { const rt = rt0 as RtLike & { summary: () => SessionSummary }; const reqs = new RequestCounter(rt); rt.on("status", () => ctl.gateFixes()); @@ -138,4 +167,27 @@ export function follow(rt0: unknown, ctl: { gateFixes(): void; stop(): void; sum const q = r.summary?.requests; push(q ? { ...r, summary: { ...r.summary!, requests: { ...q, hosts: {} } } } : r); }; + // the client calls this.push for its own events too, so an own property here sees every summary it queues + const tel = rt.telemetry; + if (typeof tel?.push === "function") { + const queue = tel.push.bind(tel); + tel.push = (e) => { + if (e.t === "summary") { + try { + withChecks(e, ctl.summary()?.visual); + } catch { + /* the summary goes out without them */ + } + } + queue(e); + }; + if (!noticeShown) { + noticeShown = true; + try { + ((rt.global.console as Console | undefined) ?? (globalThis as { console?: Console }).console)?.info?.(VISUAL_TELEMETRY_NOTICE); + } catch { + /* ignore */ + } + } + } } diff --git a/packages/runtime/src/visual/start.ts b/packages/runtime/src/visual/start.ts index e115be6d..ea16b341 100644 --- a/packages/runtime/src/visual/start.ts +++ b/packages/runtime/src/visual/start.ts @@ -20,6 +20,8 @@ import { visualRegistry } from "./registry.js"; export interface VisualStub { /** "on": the checks start after load; "local": only when the devtools overlay mounts or checkNow() is called. */ readonly setting: "local" | "on"; + /** The app turned runtime fixes on (`visual: { on: true, fix }`); they run only in guard or heal. Telemetry's session event. */ + readonly fix: boolean; /** The findings so far (rt.checks()), at most 200. */ readonly b: CheckFinding[]; /** The chunk's controller once it has loaded. */ @@ -71,6 +73,7 @@ export function startVisual(rt: RuntimeImpl, opt: InitOptions["visual"], init: b )); const v: VisualStub = { setting: base === true || local || q == "on" ? "on" : "local", + fix: o.on === true && (o.fix === true || (Array.isArray(o.fix) && o.fix.length > 0)), b: [], now: () => load().then((c) => c?.now() ?? []), start: load, diff --git a/packages/runtime/test/telemetry-visual.test.ts b/packages/runtime/test/telemetry-visual.test.ts new file mode 100644 index 00000000..a957bfdc --- /dev/null +++ b/packages/runtime/test/telemetry-visual.test.ts @@ -0,0 +1,159 @@ +// Telemetry's counts for the measured visual checks (docs/runtime/VISUAL.md section 10, approved by the owner on +// 2026-10-27; TELEMETRY.md): the session event says whether the checks run on the page and whether the app turned +// their runtime fixes on; once the checks' chunk has run, every summary event carries the findings per kind and per +// route, the fixes applied and undone, and the passes' cost and tier, added by the chunk (src/visual/bridge.ts), so +// first load carries no code for them. Never a selector, element text, colour, URL, size or a per-finding event. +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { RuntimeImpl } from "../src/runtime.js"; +import type { TelemetryClient } from "../src/telemetry/client.js"; +import { resetTelemetryNotice } from "../src/telemetry/index.js"; +import type { TelemetryTransport, VisualSummary } from "../src/types.js"; +import { follow, hostFor, resetVisualNotice, VISUAL_TELEMETRY_NOTICE, withChecks } from "../src/visual/bridge.js"; +import { toFinding } from "../src/visual/report.js"; +import { setup } from "./helpers.js"; + +type Ev = Record; + +function recorder(): TelemetryTransport & { events: Ev[] } { + const events: Ev[] = []; + return { + events, + send(_url, body) { + events.push(...(JSON.parse(body) as { events: Ev[] }).events); + return Promise.resolve(); + }, + }; +} + +/** A document the first-load stub accepts (it only listens); readyState "loading", so the chunk is never imported. */ +const doc = () => ({ readyState: "loading", visibilityState: "visible", addEventListener: () => undefined, removeEventListener: () => undefined }); + +const rts: RuntimeImpl[] = []; +function tsetup(opts: Parameters[0] = {}) { + const transport = recorder(); + const s = setup({ mode: undefined, ...opts, telemetry: { transport }, extraGlobal: { crypto: globalThis.crypto, ...(opts.extraGlobal ?? {}) } }); + rts.push(s.rt); + return { ...s, transport, tel: s.rt.telemetry as unknown as TelemetryClient }; +} + +const VS: VisualSummary = { + checks: { "scroll-lock-stuck": 2, "text-invisible": 1 }, + checksByRoute: { "/": { "scroll-lock-stuck": 2 }, "/items/:id": { "text-invisible": 1 } }, + fixes: { applied: 2, undone: 1 }, + passes: 3, + elements: 412, + ms: 41.3, + maxSliceMs: 7.9, + partial: 1, + abstained: 5, + unsupported: ["image-css-broken"], + tier: "full", + stopped: true, +}; + +const ctl = (v: () => VisualSummary | null) => ({ gateFixes: () => undefined, stop: () => undefined, summary: () => (v() ? { visual: v()! } : null) }); + +afterEach(() => { + for (const rt of rts.splice(0)) rt.destroy(); + resetVisualNotice(); + resetTelemetryNotice(); + vi.restoreAllMocks(); +}); + +describe("telemetry: the measured visual checks' counts (VISUAL.md section 10)", () => { + it("the session event: visual off and fix false without the checks; the stub's setting and the fix option with them", async () => { + const session = async (opts: Parameters[0]) => { + const { tel, transport } = tsetup(opts); + await tel.flush(); + return transport.events.find((e) => e.t === "session")!; + }; + expect(await session({})).toMatchObject({ visual: "off", fix: false }); + expect(await session({ visual: true, extraGlobal: { document: doc() } })).toMatchObject({ visual: "on", fix: false }); + expect(await session({ visual: { on: true, fix: true }, mode: "guard", extraGlobal: { document: doc() } })).toMatchObject({ visual: "on", fix: true, mode: "guard" }); + expect(await session({ visual: { on: true, fix: ["scroll-lock-stuck"] }, extraGlobal: { document: doc() } })).toMatchObject({ visual: "on", fix: true }); + expect(await session({ visual: { on: true, fix: [] }, extraGlobal: { document: doc() } })).toMatchObject({ visual: "on", fix: false }); + // "local" never fixes (the engine's resolveCfg needs on: true), so the session says so + expect(await session({ visual: { on: "local", fix: true }, extraGlobal: { document: doc() } })).toMatchObject({ visual: "local", fix: false }); + }); + + it("a summary carries counts.checks, counts.checksByRoute, counts.fixes and runtime.visual once the checks ran, and nothing else of them", async () => { + const { rt, tel, transport } = tsetup(); + let v: VisualSummary | null = null; + follow(rt, ctl(() => v)); + tel.summary("hidden"); + await tel.flush(); + const before = transport.events.find((e) => e.t === "summary")!; + expect(before.counts).not.toHaveProperty("checks"); + expect(before.counts).not.toHaveProperty("fixes"); + expect(before.runtime).not.toHaveProperty("visual"); + + v = VS; + tel.summary("hidden"); + await tel.flush(); + const s = transport.events.filter((e) => e.t === "summary").at(-1)!; + expect(s.counts.checks).toEqual({ "scroll-lock-stuck": 2, "text-invisible": 1 }); + expect(s.counts.checksByRoute).toEqual({ "/": { "scroll-lock-stuck": 2 }, "/items/:id": { "text-invisible": 1 } }); + expect(s.counts.fixes).toEqual({ applied: 2, undone: 1 }); + expect(s.runtime.visual).toEqual({ passes: 3, ms: 41.3, partial: 1, tier: "full" }); + // the model's counts are unchanged beside them + expect(s.counts).toMatchObject({ decisions: 0, detections: 0, byDiagnosis: {} }); + const text = JSON.stringify(transport.events); + for (const never of ["elements", "maxSliceMs", "abstained", "unsupported", "stopped", "image-css-broken"]) expect(text).not.toContain(never); + // no per-finding event: only the event types telemetry already had + expect(new Set(transport.events.map((e) => e.t))).toEqual(new Set(["session", "summary"])); + }); + + it("page exit and destroy send them too; a finding reported through the host adds no event of its own", async () => { + const { rt, tel, transport } = tsetup(); + follow(rt, ctl(() => VS)); + const f = toFinding( + { + kind: "scroll-lock-stuck", + el: null, + key: "scroll-lock-stuck|body", + severity: "high", + measured: { overflow: "hidden" }, + threshold: { value: 1000, unit: "ms", source: "VISUAL.md 3.20" }, + text: "the page cannot scroll: body has overflow: hidden.", + }, + { id: "c1", at: 1, route: "/", scheme: "light", viewport: { w: 1280, h: 800 }, coarse: false, tier: "full", target: "body", selector: "body.locked-for-test", structural: false }, + ); + hostFor(rt, () => "body", (p) => p).report(f, false); + rt.destroy(); + await tel.settled(); + const s = transport.events.filter((e) => e.t === "summary").at(-1)!; + expect(s).toMatchObject({ reason: "destroy", counts: { checks: VS.checks, fixes: VS.fixes } }); + expect(JSON.stringify(transport.events)).not.toContain("locked-for-test"); + expect(transport.events.some((e) => e.t === "check")).toBe(false); + }); + + it("withChecks: a summary without counts or runtime (rt.summary() threw) stays as it was", () => { + const e: Ev = { t: "summary", counts: { decisions: 1 } }; + withChecks(e, VS); + expect(e).toEqual({ t: "summary", counts: { decisions: 1, checks: VS.checks, checksByRoute: VS.checksByRoute, fixes: VS.fixes } }); + const bare: Ev = { t: "summary" }; + withChecks(bare, VS); + expect(bare).toEqual({ t: "summary" }); + withChecks(e, undefined); + expect(e.counts.checks).toEqual(VS.checks); + }); + + it("the console says once per page what telemetry sends of the checks, only when telemetry is on", () => { + const info = vi.spyOn(console, "info").mockImplementation(() => undefined); + const off = setup({ mode: undefined }); + rts.push(off.rt); + follow(off.rt, ctl(() => null)); + expect(info.mock.calls.flat()).not.toContain(VISUAL_TELEMETRY_NOTICE); + const { rt } = tsetup(); + follow(rt, ctl(() => null)); + const { rt: rt2 } = tsetup(); + follow(rt2, ctl(() => null)); + expect(info.mock.calls.flat().filter((m) => m === VISUAL_TELEMETRY_NOTICE)).toHaveLength(1); + }); + + it("the bridge reads the runtime's telemetry by this name (renaming it fails here)", () => { + const { rt } = tsetup(); + expect("telemetry" in rt).toBe(true); + expect(typeof (rt.telemetry as unknown as { push?: unknown }).push).toBe("function"); + }); +}); diff --git a/packages/runtime/test/visual-bridge.test.ts b/packages/runtime/test/visual-bridge.test.ts index 9e7bd772..7d03f931 100644 --- a/packages/runtime/test/visual-bridge.test.ts +++ b/packages/runtime/test/visual-bridge.test.ts @@ -29,7 +29,7 @@ const finding = (id = "c1"): CheckFinding => ); function stub(rt: RuntimeImpl): VisualStub { - const v: VisualStub = { setting: "on", b: [], now: () => Promise.resolve([]), start: () => undefined }; + const v: VisualStub = { setting: "on", fix: false, b: [], now: () => Promise.resolve([]), start: () => undefined }; rt.vis = v; return v; } From 6a46d53df39bb16f7825a71d24b483ea208bccb8 Mon Sep 17 00:00:00 2001 From: Mehar Khanna Date: Tue, 27 Oct 2026 00:38:08 -0400 Subject: [PATCH 5/6] telemetry-worker: measured checks on the dashboard, their own kind The collector counts the runtime's visual check counts exactly per page load (checks, check_kind, check_issue, check_fix, check_loads, check_tier, check_partial, check_ms from summaries; visual_setting, visual_fix from session events); no migration. stats() gains series.checks, checkSeries and previous.checks. The dashboard shows them as measured checks, never model findings: a card per check with a "Measured check" badge (what was measured, where, the usual fix, a Copy prompt that measures the page again with genclass_scan), a hatched slot in Fig. 1, their pages in the Pages table, their fixes in What GenClass did, their coverage in Health. They count in the app's status and in Fix them all. The sample shows simulated checks. The Snow Day demo estimate yields to the measured counts once its project gets a page load with measured checks. Co-Authored-By: Claude Opus 5.5 --- telemetry-worker/README.md | 6 +- telemetry-worker/src/dashboard.ts | 107 ++++++++-- telemetry-worker/src/demo.ts | 7 +- telemetry-worker/src/pages.ts | 265 ++++++++++++++++++------ telemetry-worker/src/sample.ts | 42 +++- telemetry-worker/test/dashboard.test.ts | 92 ++++++++ telemetry-worker/test/pages.test.ts | 163 ++++++++++++++- 7 files changed, 584 insertions(+), 98 deletions(-) diff --git a/telemetry-worker/README.md b/telemetry-worker/README.md index 6702f45a..87797adc 100644 --- a/telemetry-worker/README.md +++ b/telemetry-worker/README.md @@ -77,7 +77,7 @@ genclass.dev/dashboard/ --> dashboard page --GET /api/projects/? | `OPTIONS /v1/events` | CORS preflight (`access-control-allow-origin: *`) | | `GET /v1/health` | `{ ok: true, schema }` | | `POST /api/projects` | body `{ name? }` (JSON object or empty; name sanitised, at most 80 chars, default "Untitled app"). 201 `{ token, dashboardUrl, name, created }`, `dashboardUrl` = `https://genclass.dev/dashboard/<32 base62>`. CORS `*`, no credentials. 429 JSON when rate-limited, 400 bad JSON, 413 over 4 KB. | -| `GET /api/projects/?range=24h\|7d\|30d` | dashboard stats JSON (every plan includes the dashboard; a plan file that left it out would answer 402 `{ ok:false, error, locked:true }`) (default `7d`): `name, token, created, lastEvent, range, from, generated, totals, breakdowns{metric: [{key,count}]}` (top 25 each; 200 for `detect_issue` and `possible_issue`), `distinct{metric: n}` (distinct keys before that cut, e.g. pages with a problem), `series` (24 hourly or 7/30 daily buckets: sessions, decisions, detections, acted, possible, ix_views, ix_sessions), `recent` (last 50 detections/interventions in the window; possible ones are not listed). `totals` also has `latencyAvgMs`, `modelLoadAvgMs`, `failOpens`, `modelErrors`, `interventions`, `actionLimits`, `detectionsWithPage` and `possibleWithPage`. Apps that turned on interaction analytics (runtime `interactions` events, [docs/runtime/ANALYTICS.md](../docs/runtime/ANALYTICS.md)) also get `ix_views`, `ix_engaged_ms`, `ix_click`, `ix_rage` and `ix_dead` totals and breakdowns (by route, or route TAB element key; `ix_scroll` by route TAB milestone; top 200 each), counted with their own per-batch budget; and per visit `ix_sessions`, `ix_ref`, `ix_utm_source`, `ix_utm_medium`, `ix_utm_campaign`, `ix_device`, `ix_browser`, `ix_os`, `ix_country`, `ix_returning`, plus `visits` (`sessions, pages, bounces, entry[], exit[]` over the visits that started in the period) and `funnels` (`[{ name, steps, counts }]`; docs/runtime/ANALYTICS.md section 3.1); with navigation paths (ANALYTICS.md section 11) also `ix_entry` and `ix_exit` (totals and by route), `ix_nav` and `ix_nav_ms` (from TAB to), `ix_arrive` (by `nav`, and route TAB `nav`), `ix_first` (route TAB element key), `ix_first_n` and `ix_first_ms` (by route) and `ix_path` (the first 5 routes of a page load, joined with ` > `). 404 `{ ok:false, error:"not found" }` for an unknown or malformed secret. Same-origin only (no CORS). | +| `GET /api/projects/?range=24h\|7d\|30d` | dashboard stats JSON (every plan includes the dashboard; a plan file that left it out would answer 402 `{ ok:false, error, locked:true }`) (default `7d`): `name, token, created, lastEvent, range, from, generated, totals, breakdowns{metric: [{key,count}]}` (top 25 each; 200 for `detect_issue`, `possible_issue` and `check_issue`), `distinct{metric: n}` (distinct keys before that cut, e.g. pages with a problem), `series` (24 hourly or 7/30 daily buckets: sessions, decisions, detections, acted, possible, checks, ix_views, ix_sessions), `typeSeries` and `checkSeries` (per bucket, detections by diagnosis and measured checks by kind), `recent` (last 50 detections/interventions in the window; possible ones are not listed). `totals` also has `latencyAvgMs`, `modelLoadAvgMs`, `failOpens`, `modelErrors`, `interventions`, `actionLimits`, `detectionsWithPage`, `possibleWithPage` and `checksWithPage`. Apps that turned on interaction analytics (runtime `interactions` events, [docs/runtime/ANALYTICS.md](../docs/runtime/ANALYTICS.md)) also get `ix_views`, `ix_engaged_ms`, `ix_click`, `ix_rage` and `ix_dead` totals and breakdowns (by route, or route TAB element key; `ix_scroll` by route TAB milestone; top 200 each), counted with their own per-batch budget; and per visit `ix_sessions`, `ix_ref`, `ix_utm_source`, `ix_utm_medium`, `ix_utm_campaign`, `ix_device`, `ix_browser`, `ix_os`, `ix_country`, `ix_returning`, plus `visits` (`sessions, pages, bounces, entry[], exit[]` over the visits that started in the period) and `funnels` (`[{ name, steps, counts }]`; docs/runtime/ANALYTICS.md section 3.1); with navigation paths (ANALYTICS.md section 11) also `ix_entry` and `ix_exit` (totals and by route), `ix_nav` and `ix_nav_ms` (from TAB to), `ix_arrive` (by `nav`, and route TAB `nav`), `ix_first` (route TAB element key), `ix_first_n` and `ix_first_ms` (by route) and `ix_path` (the first 5 routes of a page load, joined with ` > `). 404 `{ ok:false, error:"not found" }` for an unknown or malformed secret. Same-origin only (no CORS). | | `PUT /api/projects//funnels` | body `{ funnels: [{ name, steps }] }` (application/json, at most 4 KB): replaces the app's funnels (at most 5, 2 to 6 routes each; a step ending in `*` matches every route under it). 200 `{ ok, funnels }`, 400 with the reason, 403 for the sample. Account mode: `PUT /api/account/projects//funnels` with the browser session and the site's Origin | | `GET /start` | "Get a token" page | | `POST /api/projects//agent` | the MCP server's log of the closed loop, mirrored, when the developer sets `GENCLASS_REPORT` for `@genclass/mcp` (docs/runtime/AGENT.md section 4.5): `{ schema: "genclass-agent/1", events: [{ t: "found" \| "explain" \| "armed" \| "replay" \| "test" \| "explore" \| "cleared", at, id?, kind?, route?, reproduced?, runs?, races?, what? }] }`, at most 50 events and 16 KB a request, rate limited per token (`SESSION_LIMIT`). Every accepted event is one row of `agent_log` (at most 500 per app, the oldest go first; unknown types are dropped); the ones about a finding also update `agent_reports` (kind, normalised route, found, explained, last replay and verdict, how many replays reproduced it, tested, proven: a replay that stops reproducing after one that did). 90 days; the stats carry them as `agent` (`findings`, `counts`, `stages`: found, explained, reproduced, fixed, closed, open, continued; `log`, the last 50) and the dashboard's Closed-loop development view shows them. The sample and unknown secrets 404; other methods 405. Ids, kinds, routes, verdicts, counts and times only, never page text | @@ -121,13 +121,13 @@ Metrics (`daily.metric`/`hourly.metric`; `key` in brackets): | from event | metrics | |---|---| -| `session` | `sessions`\*, `host` [hostname], `page_route` [route], `runtime` [version], `model_version` [version], `mode` [effectiveMode], `model_kind` [local/custom/off], `webgpu_available` [yes/no] | +| `session` | `sessions`\*, `host` [hostname], `page_route` [route], `runtime` [version], `model_version` [version], `mode` [effectiveMode], `model_kind` [local/custom/off], `webgpu_available` [yes/no], `visual_setting` [off/local/on] and `visual_fix` (runtime 0.3.0-beta.4: the measured visual checks' coverage) | | `model` | `model_state` [state], `model_ready`, `backend` [webgpu/wasm], `model_load_ms_sum` / `model_load_ms_n`, `model_from_cache`, `model_load_failed` | | `decision` | `decisions`\*, `trigger`\*, `route`, `fn`, `diagnosis`\*, `decision_mode`, `executed`, `acted`\*, `acted_action` [ran], `fn_acted`, `latency_ms_sum` / `latency_ms_n`, `latency_bucket` | | `detect` | `detections`\*, `detect_diagnosis`\*, `detect_trigger`, `detect_route`, `detect_issue` [diagnosis, tab, route], `fn_detect` (route/fn from the decision in the same batch). With `tier: "possible"` (runtime 0.3.0) instead: `possible`\*, `possible_diagnosis`\* (exact from the events; the summary has no per-diagnosis count for the tier), `possible_route`, `possible_issue` [diagnosis, tab, route], `possible_fn` | | `action` | `action_outcome`\* [applied/failed/undone], `intervention` [action, applied only] | | `veto`, `breaker`, `model-error` | `vetoes`\*, `breaker_trips`\*, `model_error_events` [code] | -| `summary` only | `fail_open` [reason:trigger], `model_errors` [code], `action_limits` [kind], `hidden_skipped` and `model_dropped` (`runtime.model.hiddenSkipped`, `.dropped`), `telemetry_dropped`, `records_sampled_out`, `records_capped` (`telemetry.dropped`, `.decisionsSampledOut`, `.decisionsCapped`) | +| `summary` only | `fail_open` [reason:trigger], `model_errors` [code], `action_limits` [kind], `hidden_skipped` and `model_dropped` (`runtime.model.hiddenSkipped`, `.dropped`), `telemetry_dropped`, `records_sampled_out`, `records_capped` (`telemetry.dropped`, `.decisionsSampledOut`, `.decisionsCapped`); the measured visual checks (runtime 0.3.0-beta.4, docs/runtime/VISUAL.md section 10; never detections): `checks`, `check_kind` [kind], `check_issue` [kind, tab, route] (`counts.checks`, `.checksByRoute`), `check_fix` [applied/undone] (`counts.fixes`), `check_loads` (page loads whose checks ran), `check_tier` [full/light], `check_partial`, `check_ms` (`runtime.visual`) | \* **Exact totals.** These metrics, and the `summary`-only ones, count each page load once, however its batches arrive. A page load's `summary` carries them cumulatively (`counts.decisions`, `.acted`, `.detections`, `.byTrigger`, diff --git a/telemetry-worker/src/dashboard.ts b/telemetry-worker/src/dashboard.ts index 70d73349..24859822 100644 --- a/telemetry-worker/src/dashboard.ts +++ b/telemetry-worker/src/dashboard.ts @@ -12,6 +12,13 @@ // `summary.counts.possibleDetections`. Their own metrics (possible, possible_*), exact like detections, never in // detections or detect_diagnosis. The worker never derives the tier from decision records: what the dashboard lists // is what the page's console listed. +// - Measured visual checks (runtime 0.3.0-beta.4, docs/runtime/VISUAL.md section 10): deterministic measurements with +// fixed thresholds (a scroll lock left on, a page that stops taking clicks, invisible text, ads over content), never a +// model finding, so never in detections. The runtime sends counts only, cumulative in each `summary` +// (`counts.checks` per kind, `counts.checksByRoute`, `counts.fixes`, `runtime.visual`), counted exactly like the +// detections: checks, check_kind, check_issue (kind TAB page), check_fix (applied, undone), check_loads (page +// loads whose checks ran), check_tier, check_partial, check_ms. The session event's `visual` and `fix` give +// visual_setting (off, local, on) and visual_fix, the coverage the counts are read against. // - Interaction analytics (runtime `interactions` events, docs/runtime/ANALYTICS.md; only from apps that turned them on): // page views, engaged time, scroll milestones, and clicks, rage clicks and dead clicks per page and element key, as // ix_* counters with their own per-batch budget (MAX_IX_COUNTERS), so they never crowd out the diagnostics counters. @@ -345,9 +352,52 @@ export function summaryValues(e: Ev): SummaryValues { one("telemetry_dropped", tel.dropped); one("records_sampled_out", tel.decisionsSampledOut); one("records_capped", tel.decisionsCapped); + checkValues(values, counts, obj(e.runtime).visual); return { seq: seqOf(e.seq) ?? 0, at: num(e.at) ?? 0, values }; } +/** A check kind as the runtime names it (CheckKind: lower-case words joined by "-"); anything else is not counted. */ +const CHECK_KIND_RE = /^[a-z][a-z0-9]*(?:-[a-z0-9]+){0,5}$/; +/** Routes of one summary's `checksByRoute` the collector reads (the runtime keeps at most 20). */ +const CHECK_ROUTES = 20; + +/** + * The measured visual checks' values in a `summary` (VISUAL.md section 10; src/visual/bridge.ts withChecks), all + * cumulative for the page load: findings per kind and per kind x page (the key cut like every exact key, so a long + * route is shortened, the same way on every summary), fixes applied and undone, and from `runtime.visual` that the + * checks ran on this page load, their tier, partial passes and check time. Only summaries that carry them set them. + */ +function checkValues(values: Exact, counts: Ev, visual: unknown): void { + if (counts.checks !== null && typeof counts.checks === "object" && !Array.isArray(counts.checks)) { + const kinds: Record = {}; + for (const [k, n] of Object.entries(countMap(counts.checks))) if (CHECK_KIND_RE.test(k)) kinds[k] = n; + values.checks = { "": Object.values(kinds).reduce((a, b) => a + b, 0) }; + values.check_kind = kinds; + // kind TAB page, as detect_issue keys detections (the route as the runtime normalised it for its decisions) + const issue: Record = {}; + for (const [r0, row] of Object.entries(obj(counts.checksByRoute)).slice(0, CHECK_ROUTES)) { + const route = str(r0)?.startsWith("/") ? str(r0)! : "(unknown)"; + for (const [k, n] of Object.entries(countMap(row))) { + const key = cutKey(`${k}\t${route}`); + if (!CHECK_KIND_RE.test(k) || (!(key in issue) && Object.keys(issue).length >= EXACT_KEYS)) continue; + issue[key] = (issue[key] ?? 0) + n; + } + } + values.check_issue = issue; + } + const fixes = obj(counts.fixes); + if (num(fixes.applied) !== undefined || num(fixes.undone) !== undefined) values.check_fix = countMap({ applied: fixes.applied, undone: fixes.undone }); + const v = obj(visual); + if (visual !== undefined && Object.keys(v).length) { + values.check_loads = { "": 1 }; + if (v.tier === "full" || v.tier === "light") values.check_tier = { [v.tier]: 1 }; + const partial = num(v.partial); + if (partial !== undefined && partial >= 0) values.check_partial = { "": partial }; + const ms = num(v.ms); + if (ms !== undefined && ms >= 0) values.check_ms = { "": Math.round(ms) }; + } +} + // Interaction analytics (`interactions` events): what one event may add. The runtime sends at most 50 element keys // for clicks and 20 each for rage and dead clicks; anything beyond these bounds is not a real page. const IX_KEYS = 60; @@ -521,6 +571,9 @@ export function summarize(batch: AggBatch): { c.add("model_kind", str(e.model) ?? "(unknown)"); const dev = e.device as Ev | undefined; if (dev && typeof dev === "object") c.add("webgpu_available", dev.webgpu ? "yes" : "no"); + // the measured visual checks' coverage (runtime 0.3.0-beta.4): whether they run here, and whether fixes may + if (e.visual === "off" || e.visual === "local" || e.visual === "on") c.add("visual_setting", e.visual); + if (e.fix === true) c.add("visual_fix"); break; } case "model": { @@ -891,7 +944,7 @@ export async function aggregate(db: D1Like, token: string, batch: AggBatch, now: /** 90d and 1y: account dashboards on a plan with that much history (src/plans.ts); link dashboards use RANGES. */ export type Range = "24h" | "7d" | "30d" | "90d" | "1y"; export const RANGES: Range[] = ["24h", "7d", "30d"]; -const SERIES_METRICS = ["sessions", "decisions", "detections", "acted", "possible", "ix_views", "ix_sessions"]; +const SERIES_METRICS = ["sessions", "decisions", "detections", "acted", "possible", "checks", "ix_views", "ix_sessions"]; export interface Stats { name: string; @@ -902,16 +955,22 @@ export interface Stats { from: string; generated: string; totals: Record; - /** Top 25 keys per metric (200 for the type x page metrics `detect_issue` and `possible_issue`, and the ix_* ones). */ + /** Top 25 keys per metric (200 for the type x page metrics `detect_issue`, `possible_issue` and `check_issue`, and the ix_* ones). */ breakdowns: Record; /** Distinct keys per metric in the period, before the top-N cut (e.g. how many pages had a problem). */ distinct: Record; - /** Per bucket; `possible` is the lower-confidence tier, never part of `detections`; `ix_views` the page views and `ix_sessions` the visits counted by interaction analytics. */ - series: { bucket: string; sessions: number; decisions: number; detections: number; acted: number; possible: number; ix_views: number; ix_sessions: number }[]; + /** + * Per bucket; `possible` is the lower-confidence tier, never part of `detections`; `checks` the measured visual + * checks' findings, never part of `detections` either; `ix_views` the page views and `ix_sessions` the visits counted + * by interaction analytics. + */ + series: { bucket: string; sessions: number; decisions: number; detections: number; acted: number; possible: number; checks: number; ix_views: number; ix_sessions: number }[]; /** Detections per bucket by diagnosis (the dashboard's stacked timeline). */ typeSeries: { bucket: string; counts: Record }[]; + /** Measured check findings per bucket by check kind (their cards' sparklines). */ + checkSeries: { bucket: string; counts: Record }[]; /** The same totals over the period just before this one (deltas). */ - previous: { sessions: number; detections: number; acted: number; ix_views: number; ix_sessions: number }; + previous: { sessions: number; detections: number; acted: number; checks: number; ix_views: number; ix_sessions: number }; recent: Record[]; /** Interaction analytics: the visits that started in the period (visits are kept 90 days whatever the plan). */ visits: VisitStats; @@ -919,6 +978,8 @@ export interface Stats { funnels: FunnelStats[]; /** What the developer's coding agent reported through @genclass/mcp in the period (agentStats); off unless the developer turned reporting on. */ agent: AgentStats; + /** The recorded demo's estimate stands in for the measured counts (src/demo.ts). */ + demo?: boolean; } // --------------------------------------------------------------------------------------------- visits and funnels @@ -1293,7 +1354,7 @@ export async function stats(db: D1Like, p: Project, range: Range, now: Date): Pr const prevStart = new Date(Date.parse(hourly ? `${from}:00:00.000Z` : `${from}T00:00:00.000Z`) - span * (hourly ? 3_600_000 : 86_400_000)); const prevFrom = hourly ? hourKey(prevStart) : dayKey(prevStart); - const [agg, series, recent, types, prev, fun] = await db.batch([ + const [agg, series, recent, types, prev, fun, checkTypes] = await db.batch([ db.prepare(`SELECT metric, key, SUM(count) AS c FROM ${table} WHERE token = ? AND ${col} >= ? GROUP BY metric, key`).bind(p.token, from), db .prepare(`SELECT ${col} AS b, metric, SUM(count) AS c FROM ${table} WHERE token = ? AND ${col} >= ? AND key = '' AND metric IN (${SERIES_METRICS.map(() => "?").join(", ")}) GROUP BY ${col}, metric`) @@ -1303,9 +1364,10 @@ export async function stats(db: D1Like, p: Project, range: Range, now: Date): Pr .bind(p.token, hourly ? new Date(now.getTime() - 86_400_000).toISOString() : `${from}T00:00:00.000Z`), db.prepare(`SELECT ${col} AS b, key, SUM(count) AS c FROM ${table} WHERE token = ? AND ${col} >= ? AND metric = 'detect_diagnosis' GROUP BY ${col}, key`).bind(p.token, from), db - .prepare(`SELECT metric, SUM(count) AS c FROM ${table} WHERE token = ? AND ${col} >= ? AND ${col} < ? AND key = '' AND metric IN ('sessions', 'detections', 'acted', 'ix_views', 'ix_sessions') GROUP BY metric`) + .prepare(`SELECT metric, SUM(count) AS c FROM ${table} WHERE token = ? AND ${col} >= ? AND ${col} < ? AND key = '' AND metric IN ('sessions', 'detections', 'acted', 'checks', 'ix_views', 'ix_sessions') GROUP BY metric`) .bind(p.token, prevFrom, from), db.prepare("SELECT funnels FROM projects WHERE token = ?").bind(p.token), + db.prepare(`SELECT ${col} AS b, key, SUM(count) AS c FROM ${table} WHERE token = ? AND ${col} >= ? AND metric = 'check_kind' GROUP BY ${col}, key`).bind(p.token, from), ]); const rows = (r: unknown) => ((r as { results?: Record[] })?.results ?? []) as Record[]; @@ -1324,7 +1386,7 @@ export async function stats(db: D1Like, p: Project, range: Range, now: Date): Pr distinct[metric] = list.length; sums[metric] = list.reduce((s, x) => s + x.count, 0); list.sort((a, b) => b.count - a.count || a.key.localeCompare(b.key)); - list.splice(metric === "detect_issue" || metric === "possible_issue" || metric.startsWith("ix_") ? 200 : 25); + list.splice(metric === "detect_issue" || metric === "possible_issue" || metric === "check_issue" || metric.startsWith("ix_") ? 200 : 25); } const sumOf = (m: string) => sums[m] ?? 0; totals.latencyAvgMs = totals.latency_ms_n ? Math.round(totals.latency_ms_sum! / totals.latency_ms_n) : 0; @@ -1336,23 +1398,28 @@ export async function stats(db: D1Like, p: Project, range: Range, now: Date): Pr // detections (and possible ones) whose page is known (their decision record came in the same batch); the rest have no route totals.detectionsWithPage = sumOf("detect_route"); totals.possibleWithPage = sumOf("possible_route"); + // measured check findings whose page is known (a page load sends its first 20 routes' counts) + totals.checksWithPage = sumOf("check_issue"); if (breakdowns.latency_bucket) { const order = LATENCY_BUCKETS.map(([, l]) => l); breakdowns.latency_bucket.sort((a, b) => order.indexOf(a.key) - order.indexOf(b.key)); } - const byBucket = new Map(buckets.map((b) => [b, { bucket: b, sessions: 0, decisions: 0, detections: 0, acted: 0, possible: 0, ix_views: 0, ix_sessions: 0 }])); + const byBucket = new Map(buckets.map((b) => [b, { bucket: b, sessions: 0, decisions: 0, detections: 0, acted: 0, possible: 0, checks: 0, ix_views: 0, ix_sessions: 0 }])); for (const r of rows(series)) { const s = byBucket.get(String(r.b)); if (s && SERIES_METRICS.includes(String(r.metric))) (s as Record)[String(r.metric)] = Number(r.c) || 0; } - const byType = new Map(buckets.map((b) => [b, { bucket: b, counts: {} as Record }])); - for (const r of rows(types)) { - const s = byType.get(String(r.b)); - if (s) s.counts[String(r.key ?? "")] = (s.counts[String(r.key ?? "")] ?? 0) + (Number(r.c) || 0); - } - const previous = { sessions: 0, detections: 0, acted: 0, ix_views: 0, ix_sessions: 0 }; + const perBucket = (r0: unknown) => { + const m = new Map(buckets.map((b) => [b, { bucket: b, counts: {} as Record }])); + for (const r of rows(r0)) { + const s = m.get(String(r.b)); + if (s) s.counts[String(r.key ?? "")] = (s.counts[String(r.key ?? "")] ?? 0) + (Number(r.c) || 0); + } + return [...m.values()]; + }; + const previous = { sessions: 0, detections: 0, acted: 0, checks: 0, ix_views: 0, ix_sessions: 0 }; for (const r of rows(prev)) { const m = String(r.metric) as keyof typeof previous; if (m in previous) previous[m] = Number(r.c) || 0; @@ -1372,16 +1439,20 @@ export async function stats(db: D1Like, p: Project, range: Range, now: Date): Pr breakdowns, distinct, series: [...byBucket.values()], - typeSeries: [...byType.values()], + typeSeries: perBucket(types), + checkSeries: perBucket(checkTypes), previous, recent: rows(recent).map((r) => ({ ...r, executed: r.executed === null ? null : !!r.executed, acted: r.acted === null ? null : !!r.acted })), agent: await agentStats(db, p.token, fromIso), visits: vs.visits, funnels: vs.funnels, }; - // the recorded demo's estimate for its project (src/demo.ts); every other project gets its measured counts + // the recorded demo's estimate for its project (src/demo.ts), until that project's first page load with measured + // checks arrives (runtime 0.3.0-beta.4 on the live site): from then on, in every period, its measured counts const demo = demoData(p.name); - return demo && p.last_event ? applyDemo(out, demo, now) : out; + if (!demo || !p.last_event) return out; + const measured = await db.prepare("SELECT 1 AS ok FROM daily WHERE token = ? AND metric = 'check_loads' LIMIT 1").bind(p.token).first(); + return measured ? out : applyDemo(out, demo, now); } // --------------------------------------------------------------------------------------------- retention diff --git a/telemetry-worker/src/demo.ts b/telemetry-worker/src/demo.ts index f8d65a32..6947c915 100644 --- a/telemetry-worker/src/demo.ts +++ b/telemetry-worker/src/demo.ts @@ -4,7 +4,9 @@ // project stats() answers with this estimate instead of the measured counts: the week since GenClass was installed, // 2,582 visitors (Search Console), and the 1,037 problems and 458 live fixes the owner counted on the site. The // dashboard, Copy as Markdown and the MCP server (GET /api/projects/) all read it from there; stored data, -// exports, funnels and the agent log stay measured. Remove the entry after the recording. +// exports, funnels and the agent log stay measured. The estimate yields to the measured counts as soon as the project +// receives its first page load with measured checks (runtime 0.3.0-beta.4, which sends their counts: stats() in +// src/dashboard.ts); remove the entry after the recording. import type { Stats } from "./dashboard.js"; @@ -341,13 +343,14 @@ export function applyDemo(s: Stats, d: DemoData, now: Date): Stats { return { ...s, + demo: true, lastEvent: recent[0]?.t ?? s.lastEvent, totals, breakdowns, distinct: { ...s.distinct, detect_route: Object.keys(routes).length, page_route: Object.keys(pages).length }, series, typeSeries, - previous: { sessions: 0, detections: 0, acted: 0, ix_views: 0, ix_sessions: 0 }, + previous: { sessions: 0, detections: 0, acted: 0, checks: 0, ix_views: 0, ix_sessions: 0 }, recent, visits: { ...s.visits, sessions: visitors, pages: loads, bounces: n(d.bounces), entry: kv(sc(d.entry)), exit: kv(sc(d.exit)) }, }; diff --git a/telemetry-worker/src/pages.ts b/telemetry-worker/src/pages.ts index a8d50bc2..aac45859 100644 --- a/telemetry-worker/src/pages.ts +++ b/telemetry-worker/src/pages.ts @@ -378,13 +378,19 @@ export function notFoundPage(nonce: string): string { // (test/pages.test.ts measures them). // A bar draws only the types it has, so slots that are not palette neighbours can touch; the 2px canvas gap between // segments, the legend, the hover breakdown and the table view carry identity there. -// Five problem types get a slot, the rest share "Other". Possible issues (the lower-confidence tier) are drawn hollow: +// Five problem types get a slot, the rest share "Other". The measured visual checks (never model findings) get one more +// slot of their own, hatched in the muted ink rather than a hue, so they never read as a model type. Possible issues +// (the lower-confidence tier) are drawn hollow: // outlined bars in the muted ink (Fig. 2, one series, so no legend) and ring keys in the type's colour; never in the // confirmed counts. // Every dynamic string goes through textContent. export const DASH_CSS = ` :root{--t1:#3987E5;--t2:#E8552F;--t3:#2AA3C7;--t4:#C98500;--t5:#B964C9;--t0:#88897F;--grid:#262626} +/* the measured checks' key: hatched in the muted ink, never a problem type's hue */ +:root{--chk:repeating-linear-gradient(135deg,var(--ink2) 0 2px,transparent 2px 4px)} +.find h3 .bdg,.adv h3 .bdg{margin-left:2px} +table.dt td .bdg{margin-left:8px;vertical-align:middle} /* the page header: the project's name on the overview, the view's title elsewhere; the period and the tools */ .phead{display:flex;flex-wrap:wrap;gap:16px 24px;align-items:flex-start;justify-content:space-between;margin-bottom:28px} .phead h1{font-size:30px;line-height:1.2;letter-spacing:-.02em;overflow-wrap:anywhere} @@ -696,12 +702,14 @@ const DASH_JS = ` what: "An operation did something different from what it normally does, such as writing fewer fields than usual.", fix: "Compare the operation's result with what it usually returns." } }; - var SLOTS = ["duplicate", "stale", "failing", "slow", "conflict"]; + var SLOTS = ["duplicate", "stale", "failing", "slow", "conflict"], BASE_SLOTS = SLOTS; var ACTIONS = { coalesce: "Merged a repeated request", discard: "Dropped a stale write", defer: "Waited for related requests", delay: "Delayed a request", - retry: "Retried a failed request", serve_cached: "Answered from cache", block: "Blocked a request", hedge: "Sent a backup request", rollback: "Rolled back a write", resync: "Re-synced state" }; + retry: "Retried a failed request", serve_cached: "Answered from cache", block: "Blocked a request", hedge: "Sent a backup request", rollback: "Rolled back a write", resync: "Re-synced state", + check_fix: "Fixed a measured check on the visitor's screen (a reversible CSS change)" }; function T(k){ return TYPES[k] || { name: k && k !== "(unknown)" ? k.charAt(0).toUpperCase() + k.slice(1) : "Other", c: "--t0", what: "", fix: "" }; } function slotOf(k){ return SLOTS.indexOf(k) >= 0 ? k : "other"; } - function colorOf(slot){ return "var(" + (slot === "other" ? "--t0" : TYPES[slot].c) + ")"; } + function colorOf(slot){ return slot === "checks" ? CHK : "var(" + (slot === "other" ? "--t0" : TYPES[slot].c) + ")"; } + function slotName(slot){ return slot === "other" ? "Other" : slot === "checks" ? "Measured checks" : TYPES[slot].name; } function key(color){ var s = el("span", "key"); s.style.background = color; return s; } function ago(iso){ var s = Math.max(0, (Date.now() - Date.parse(iso)) / 1000); @@ -734,19 +742,23 @@ const DASH_JS = ` // ---------- summary and tiles function summary(S){ var T0 = S.totals, B = S.breakdowns, host = document.getElementById("summary"); host.textContent = ""; - var pages = count(S, "detect_route"), modes = map(B.mode), acting = (modes.guard || 0) + (modes.heal || 0) > 0, poss = T0.possible || 0; + var pages = problemPages(S), modes = map(B.mode), acting = (modes.guard || 0) + (modes.heal || 0) > 0, poss = T0.possible || 0; + var checks = T0.checks || 0, found = problemsOf(S), fixed = checkFixed(S); function put(t, bold){ host.appendChild(bold ? el("b", null, t) : document.createTextNode(t)); } if (!T0.sessions) { put("No page loads in this period yet."); return; } put(RANGE[S.range] + " GenClass watched "); put(plural(T0.sessions, "page load", "page loads"), true); - if (P.demo && T0.ix_sessions) { put(" from "); put(plural(T0.ix_sessions, "visitor", "visitors"), true); } - if (!T0.detections && poss) { put(" and found no problems. It flagged "); put(plural(poss, "possible issue", "possible issues"), true); put(" at lower confidence, listed separately below."); return; } - if (!T0.detections) { put(" and found nothing wrong. It only reports situations that stand out, so a quiet stretch is normal."); return; } - put(" and found "); put(plural(T0.detections, "problem", "problems"), true); + if (S.demo && T0.ix_sessions) { put(" from "); put(plural(T0.ix_sessions, "visitor", "visitors"), true); } + if (!found && poss) { put(" and found no problems. It flagged "); put(plural(poss, "possible issue", "possible issues"), true); put(" at lower confidence, listed separately below."); return; } + if (!found) { put(" and found nothing wrong. It only reports situations that stand out, so a quiet stretch is normal."); return; } + put(" and found "); put(plural(found, "problem", "problems"), true); + // the measured checks are their own kind: said apart from the model's findings + if (checks) put(checks === found ? " (all measured checks)" : " (" + plural(checks, "measured check", "measured checks") + ")"); if (pages) { put(" on "); put(plural(pages, "page", "pages"), true); } put(". "); if (acting && T0.interventions) { put("It stepped in "); put(plural(T0.interventions, "time", "times"), true); var u = map(B.action_outcome).undone || 0; put(u ? ", and " + plural(u, "change was", "changes were") + " undone." : ", and nothing it changed was undone."); } - else if (acting) put("It is allowed to step in and did not need to."); - else put("It is in observe mode, so it reported them and changed nothing."); + else if (acting && !fixed) put("It is allowed to step in and did not need to."); + else if (!acting) put("It is in observe mode, so it reported them and changed nothing."); + if (fixed) { put((acting && !T0.interventions ? "" : " ") + "It fixed "); put(plural(fixed, "measured check", "measured checks"), true); put(" on visitors' screens with a reversible CSS change."); } if (poss) { put(" It also flagged "); put(plural(poss, "possible issue", "possible issues"), true); put(" at lower confidence, listed separately and not counted as problems."); } } function delta(now, before){ @@ -761,7 +773,8 @@ const DASH_JS = ` // the app's status: Healthy with no problems in the period; Critical when 100 or more were not fixed live, or // problems not fixed live reached 5% of page loads; otherwise Needs attention function verdict(S){ - var T0 = S.totals, n = T0.detections || 0, open = Math.max(0, n - (T0.interventions || 0)), loads = T0.sessions || 0; + // the measured checks count as problems too; a check fix still in place counts as fixed live + var T0 = S.totals, n = problemsOf(S), open = Math.max(0, n - (T0.interventions || 0) - checkFixed(S)), loads = T0.sessions || 0; var host = document.getElementById("verdict"); if (!host) return; var v = !n ? ["ok", "Healthy", "No problems found " + RANGE[S.range].charAt(0).toLowerCase() + RANGE[S.range].slice(1)] : open >= 100 || (loads && open / loads >= 0.05) ? ["bad", "Critical", plural(open, "problem", "problems") + " not fixed live"] @@ -773,12 +786,13 @@ const DASH_JS = ` function tiles(S){ var T0 = S.totals, B = S.breakdowns, host = document.getElementById("tiles"); host.textContent = ""; var ready = T0.model_ready || 0, failed = T0.model_load_failed || 0, modes = map(B.mode), acting = (modes.guard || 0) + (modes.heal || 0) > 0; - var outcomes = map(B.action_outcome), applied = outcomes.applied || T0.interventions || 0, undone = outcomes.undone || 0; - host.appendChild(stat("Problems found", fmt(T0.detections) + (P.demo ? "+" : ""), P.demo ? "detected in production" : delta(T0.detections || 0, (S.previous || {}).detections || 0))); + var outcomes = map(B.action_outcome), applied = outcomes.applied || T0.interventions || 0, undone = outcomes.undone || 0, prev = S.previous || {}, checks = T0.checks || 0, fixed = checkFixed(S); + var d = delta(problemsOf(S), (prev.detections || 0) + (prev.checks || 0)); + host.appendChild(stat("Problems found", fmt(problemsOf(S)) + (S.demo ? "+" : ""), S.demo ? "detected in production" : checks ? [d, plural(checks, "measured check", "measured checks")].filter(Boolean).join(" \\u00b7 ") : d)); var seenPages = count(S, "page_route"); - host.appendChild(stat("Pages affected", fmt(count(S, "detect_route")), seenPages ? "of " + plural(seenPages, "page", "pages") + " that sent data" : "")); - if (P.demo) host.appendChild(stat("Fixed live", fmt(T0.interventions), "hotfixed on users' screens, no redeploy")); - else host.appendChild(stat("Stopped by GenClass", fmt(T0.interventions), acting ? (applied ? Math.round(100 * undone / applied) + "% undone by users" : "nothing needed stopping") : "observe mode reports only")); + host.appendChild(stat("Pages affected", fmt(problemPages(S)), seenPages ? "of " + plural(seenPages, "page", "pages") + " that sent data" : "")); + if (S.demo) host.appendChild(stat("Fixed live", fmt(T0.interventions), "hotfixed on users' screens, no redeploy")); + else host.appendChild(stat("Stopped by GenClass", fmt((T0.interventions || 0) + fixed), acting ? (applied ? Math.round(100 * undone / applied) + "% undone by users" + (fixed ? ", " + plural(fixed, "check fix", "check fixes") : "") : fixed ? plural(fixed, "measured check", "measured checks") + " fixed live" : "nothing needed stopping") : "observe mode reports only")); host.appendChild(stat("Model loaded", share(ready, ready + failed), ready ? "average " + (T0.modelLoadAvgMs >= 1000 ? (T0.modelLoadAvgMs / 1000).toFixed(1) + " s" : fmt(T0.modelLoadAvgMs) + " ms") + ", " + plural(failed, "failure", "failures") : "no model loads yet")); } // ---------- problems over time @@ -787,22 +801,28 @@ const DASH_JS = ` host.textContent = ""; leg.textContent = ""; tbl.textContent = ""; var hourly = S.range === "24h"; document.getElementById("unit").textContent = hourly ? "hour" : "day"; + // the measured checks are one more slot, from the series (never folded into a model type or Other) + var chk = {}; (S.series || []).forEach(function(b){ chk[b.bucket] = b.checks || 0; }); var rows = (S.typeSeries || []).map(function(b){ var o = { bucket: b.bucket, total: 0, parts: {} }; Object.keys(b.counts).forEach(function(k){ var s = slotOf(k); o.parts[s] = (o.parts[s] || 0) + b.counts[k]; o.total += b.counts[k]; }); + if (chk[b.bucket]) { o.parts.checks = chk[b.bucket]; o.total += chk[b.bucket]; } return o; }); - var order = SLOTS.concat(["other"]).filter(function(s){ return rows.some(function(r){ return r.parts[s]; }); }); + var order = SLOTS.concat(["other", "checks"]).filter(function(s){ return rows.some(function(r){ return r.parts[s]; }); }); var total = rows.reduce(function(s, r){ return s + r.total; }, 0); document.getElementById("trendn").textContent = plural(total, "problem", "problems"); var poss = {}, ptotal = 0; (S.series || []).forEach(function(b){ poss[b.bucket] = b.possible || 0; ptotal += b.possible || 0; }); if (total || ptotal) trendTable(tbl, rows, order, hourly, poss, ptotal); if (!total) { host.appendChild(el("p", "quiet", "No problems found in this period.")); return; } - if (order.length > 1) order.forEach(function(s){ var x = el("span"); x.appendChild(key(colorOf(s))); x.appendChild(document.createTextNode(s === "other" ? "Other" : TYPES[s].name)); leg.appendChild(x); }); + if (order.length > 1 || order[0] === "checks") order.forEach(function(s){ var x = el("span"); x.appendChild(key(colorOf(s))); x.appendChild(document.createTextNode(slotName(s))); leg.appendChild(x); }); var W = 720, H = 210, PL = 30, PB = 24, PT = 6, n = rows.length, plotW = W - PL, plotH = H - PB - PT; var max = niceMax(Math.max.apply(null, rows.map(function(r){ return r.total; }))); var g = svg("svg", { viewBox: "0 0 " + W + " " + H, role: "img", "aria-label": "Problems found per " + (hourly ? "hour" : "day") + " by type" }); + // the measured checks' hatch (the legend's --chk, as an SVG pattern) + var defs = svg("defs"), pat = svg("pattern", { id: "chkp", width: 4, height: 4, patternUnits: "userSpaceOnUse", patternTransform: "rotate(45)" }); + pat.appendChild(svg("rect", { width: 4, height: 4, fill: "var(--panel)" })); pat.appendChild(svg("rect", { width: 2, height: 4, fill: "var(--ink2)" })); defs.appendChild(pat); g.appendChild(defs); [0, max / 2, max].forEach(function(v){ var y = PT + plotH - plotH * v / max; g.appendChild(svg("line", { x1: PL, x2: W, y1: y, y2: y, stroke: v ? "var(--grid)" : "var(--rule2)", "stroke-width": 1 })); @@ -814,7 +834,7 @@ const DASH_JS = ` var x = PL + i * step + (step - bw) / 2, y = PT + plotH, drawn = order.filter(function(s){ return r.parts[s]; }); drawn.forEach(function(s, j){ var h = plotH * r.parts[s] / max; y -= h; - g.appendChild(svg("rect", { x: x, y: y, width: bw, height: Math.max(1, h), rx: j === drawn.length - 1 ? 3 : 0, fill: colorOf(s), stroke: "var(--paper)", "stroke-width": drawn.length > 1 ? 2 : 0 })); + g.appendChild(svg("rect", { x: x, y: y, width: bw, height: Math.max(1, h), rx: j === drawn.length - 1 ? 3 : 0, fill: s === "checks" ? "url(#chkp)" : colorOf(s), stroke: "var(--paper)", "stroke-width": drawn.length > 1 ? 2 : 0, "data-slot": s })); }); if (i === 0 || i === n - 1 || i === Math.floor(n / 2)) { var lt = svg("text", { x: PL + i * step + step / 2, y: H - 6, "text-anchor": i === 0 ? "start" : i === n - 1 ? "end" : "middle" }); lt.textContent = label(r.bucket, hourly); g.appendChild(lt); @@ -822,7 +842,7 @@ const DASH_JS = ` var hit = svg("rect", { x: PL + i * step, y: PT, width: step, height: plotH, fill: "transparent", tabindex: 0 }); function show(){ tip.textContent = ""; tip.appendChild(el("div", "t", label(r.bucket, hourly) + ": " + plural(r.total, "problem", "problems"))); - order.forEach(function(s){ if (!r.parts[s]) return; var row = el("div", "r"); var a = el("span"); a.appendChild(key(colorOf(s))); a.appendChild(document.createTextNode(s === "other" ? "Other" : TYPES[s].name)); row.appendChild(a); row.appendChild(el("b", null, fmt(r.parts[s]))); tip.appendChild(row); }); + order.forEach(function(s){ if (!r.parts[s]) return; var row = el("div", "r"); var a = el("span"); a.appendChild(key(colorOf(s))); a.appendChild(document.createTextNode(slotName(s))); row.appendChild(a); row.appendChild(el("b", null, fmt(r.parts[s]))); tip.appendChild(row); }); tip.classList.remove("hidden"); var box = host.getBoundingClientRect(), left = (PL + i * step + step / 2) / W * box.width; tip.style.left = Math.min(Math.max(0, left - 80), box.width - 170) + "px"; tip.style.top = "0px"; @@ -834,7 +854,7 @@ const DASH_JS = ` host.appendChild(g); host.appendChild(tip); } function trendTable(tbl, rows, order, hourly, poss, ptotal){ - var tb = el("table", "dt"), hr = el("tr"); hr.appendChild(el("th", null, hourly ? "Hour" : "Day")); order.forEach(function(s){ hr.appendChild(el("th", null, s === "other" ? "Other" : TYPES[s].name)); }); hr.appendChild(el("th", null, "Total")); + var tb = el("table", "dt"), hr = el("tr"); hr.appendChild(el("th", null, hourly ? "Hour" : "Day")); order.forEach(function(s){ hr.appendChild(el("th", null, slotName(s))); }); hr.appendChild(el("th", null, "Total")); if (ptotal) hr.appendChild(el("th", null, "Possible (not in total)")); var th = el("thead"); th.appendChild(hr); tb.appendChild(th); var body = el("tbody"); rows.forEach(function(r){ var tr = el("tr"); tr.appendChild(el("td", null, label(r.bucket, hourly))); order.forEach(function(s){ tr.appendChild(el("td", null, fmt(r.parts[s] || 0))); }); tr.appendChild(el("td", null, fmt(r.total))); if (ptotal) tr.appendChild(el("td", "poss-n", fmt(poss[r.bucket] || 0))); body.appendChild(tr); }); @@ -880,11 +900,11 @@ const DASH_JS = ` } // ---------- one card per kind of problem - function spark(S, d){ - var vals = (S.typeSeries || []).map(function(b){ return b.counts[d] || 0; }), n = vals.length, max = Math.max.apply(null, vals) || 1; + function spark(S, d, check){ + var vals = ((check ? S.checkSeries : S.typeSeries) || []).map(function(b){ return b.counts[d] || 0; }), n = vals.length, max = Math.max.apply(null, vals) || 1; var g = svg("svg", { viewBox: "0 0 " + (n * 10) + " 24", preserveAspectRatio: "none", "class": "spark", "aria-hidden": "true" }); g.appendChild(svg("line", { x1: 0, x2: n * 10, y1: 23.5, y2: 23.5, stroke: "var(--rule)", "stroke-width": 1 })); - vals.forEach(function(v, i){ if (!v) return; var h = Math.max(2, 22 * v / max); g.appendChild(svg("rect", { x: i * 10 + 3, y: 23 - h, width: 4, height: h, rx: 1, fill: colorOf(slotOf(d)) })); }); + vals.forEach(function(v, i){ if (!v) return; var h = Math.max(2, 22 * v / max); g.appendChild(svg("rect", { x: i * 10 + 3, y: 23 - h, width: 4, height: h, rx: 1, fill: check ? "var(--ink2)" : colorOf(slotOf(d)) })); }); return g; } // where a kind of problem happened: the type x page counts, else the pages of the recent rows @@ -915,22 +935,22 @@ const DASH_JS = ` // one prompt for every kind of problem in the period, and the checks on the dev page: the loop for each (reproduce, // fix, prove), from one paste function fixAllPrompt(S){ - var list = (S.breakdowns.detect_diagnosis || []).filter(function(x){ return x.key !== "expected"; }); + var list = problemList(S), checks = list.some(function(x){ return x.check; }); var out = ["Fix the problems GenClass found in this web app (" + P.name + ") and prove each fix, with the GenClass MCP server (" + MCP + "; the project's MCP page in the GenClass dashboard says how to add it).", "", "In production, " + lower(RANGE[range]) + ":"]; list.forEach(function(x){ - var t = T(x.key), w = whereOf(S, x.key); - out.push("- " + t.name + ", " + plural(x.count, "time", "times") + (w.length ? ", on " + w.slice(0, 3).map(function(v){ return v.route; }).join(", ") : "") + "." + (t.fix ? " Usual fix: " + t.fix : "")); + var t = infoOf(x), w = whereFor(S, x); + out.push("- " + t.name + (x.check ? " (measured check " + x.key + ")" : "") + ", " + plural(x.count, "time", "times") + (w.length ? ", on " + w.slice(0, 3).map(function(v){ return v.route; }).join(", ") : "") + "." + (t.fix ? " Usual fix: " + t.fix : "")); }); - out.push("- And every check genclass_findings lists from the dev page (unreadable text, a scroll lock left on, a page that stops taking clicks).", "", + out.push("- And every " + (checks ? "other " : "") + "check genclass_findings lists from the dev page (unreadable text, a scroll lock left on, a page that stops taking clicks).", "", "1. genclass_status: the dev page must be connected; if it is not, tell me what to open.", - "2. For each problem: genclass_findings, then genclass_explain; reproduce it with genclass_explore or genclass_replay. " + WAIT, + "2. For each problem: genclass_findings, then genclass_explain; reproduce it with genclass_explore or genclass_replay. " + (checks ? "A measured check is a measurement, not a recorded order: run genclass_scan on its page instead, which lists it with the element, its selector and a suggested CSS change. " : "") + WAIT, "3. Fix the cause in the code, small and idiomatic; keep the regression test genclass_explain generates.", - "4. Replay the recorded order again: it must say not reproduced.", + "4. Replay the recorded order again: it must say not reproduced." + (checks ? " For a measured check, genclass_scan on its page must no longer list it." : ""), "5. Sum up in one line per problem: what it was, the proof, the fix."); return out.join("\\n"); } function fixAll(S){ - var any = (S.breakdowns.detect_diagnosis || []).some(function(x){ return x.key !== "expected"; }); + var any = problemList(S).length > 0; ["fixall", "fixall2"].forEach(function(id){ var b = document.getElementById(id); if (!b) return; b.classList.toggle("hidden", !any); b.setAttribute("data-copy", any ? fixAllPrompt(S) : ""); }); } function promptBtn(k, n, where, cls){ @@ -938,53 +958,146 @@ const DASH_JS = ` b.setAttribute("data-copy", fixPrompt(k, n, where)); b.setAttribute("aria-label", "Copy a prompt to fix " + lower(T(k).name)); return b; } + // ---------- measured visual checks (runtime 0.3.0-beta.4, docs/runtime/VISUAL.md): deterministic measurements with + // fixed thresholds, never a model finding. Their own kind on every view: a "Measured check" badge, a hatched key and + // their own slot in Fig. 1, their own prompt (genclass_scan measures the page again). Counts only reach the + // dashboard: never a selector, element text, colour or URL. + var CHECKS = { + "image-broken": { name: "Broken images", what: "An image failed to load or decode, so visitors saw an empty box or the browser's broken-image icon.", fix: "Fix the image's URL or the file it points to, and give images that matter a fallback." }, + "image-css-broken": { name: "Broken background images", what: "A background image set in CSS answered with an error, so its element showed without it (Chromium reports these).", fix: "Fix the url() in the stylesheet or the file it points to." }, + "media-failed": { name: "Video or audio that failed", what: "A video or audio element failed with a network or decoding error, or the browser could not play its format.", fix: "Check the media URL and offer a format every browser plays." }, + "stylesheet-failed": { name: "Stylesheets that failed", what: "A stylesheet did not load (an error, or a page served instead of CSS), so part of the page was unstyled.", fix: "Fix the stylesheet's URL or what the server answers for it (status and content type)." }, + "font-failed": { name: "Fonts that failed", what: "A web font failed to load, so its text fell back to another font.", fix: "Fix the font file's URL, format or CORS headers, and keep a close fallback in font-family." }, + "icon-font-failed": { name: "Missing icons", what: "An icon font failed to load, so icons showed as empty boxes or as their names.", fix: "Fix the icon font's URL or CORS headers, or draw the icons as inline SVG." }, + "csp-blocked": { name: "Blocked by the CSP", what: "The page's Content-Security-Policy blocked a stylesheet, a font or an image the page uses.", fix: "Allow that source in the policy (style-src, font-src, img-src), or serve the file from an allowed origin." }, + "text-invisible": { name: "Invisible text", what: "Text drawn in almost its background's colour (contrast under 1.5:1) in the visitor's own colour scheme, so nobody could read it.", fix: "Set the text colour and the background together on the same selector, the dark-mode variant included. In guard mode with visual fixes on, GenClass applies a readable colour on the visitor's screen.", live: true }, + "text-template-leak": { name: "Template text on the page", what: "Unrendered template or code text reached the page, such as {{ name }}, [object Object], undefined or NaN.", fix: "Find the value that was missing or unformatted when it was rendered, and render a fallback instead." }, + "text-placeholder": { name: "Placeholder content", what: "Lorem ipsum, or an image from a placeholder service, was still on the page.", fix: "Replace the placeholder text or image with the real content." }, + "text-encoding": { name: "Garbled characters", what: "Text with mojibake (UTF-8 read as another encoding), replacement characters, or HTML entities shown as text.", fix: "Serve and read the text as UTF-8 end to end, and escape it once, not twice." }, + "css-var-undefined": { name: "Undefined CSS variables", what: "A rule uses a custom property (var(--name)) that nothing defines, with no fallback, so the declaration does nothing.", fix: "Define the variable, fix its name, or give var() a fallback." }, + "image-distorted": { name: "Stretched images", what: "An image was shown at an aspect ratio far from its own, so it looked stretched or squashed.", fix: "Set object-fit: cover (or contain) on it, or size it to its own aspect ratio." }, + "page-overflow-x": { name: "Page scrolls sideways", what: "The page was wider than the screen, so it scrolled sideways, usually on phones.", fix: "Give the element that sticks out max-width: 100%, or let it wrap; avoid fixed widths wider than the screen." }, + "viewport-meta-missing": { name: "No viewport tag", what: "On a phone the page rendered at desktop width because it has no viewport meta tag, so everything was tiny.", fix: "Add a viewport meta tag with width=device-width, initial-scale=1 to the page's head." }, + "layout-shift": { name: "Layout shifts", what: "Content jumped while visitors were reading it, often an image without a size or a late banner.", fix: "Reserve space for images, ads and embeds (width and height, or aspect-ratio), and insert late content below what the visitor is looking at." }, + "page-blank": { name: "Blank page", what: "The page's main area showed no text and no media for seconds after it loaded.", fix: "Find why the content did not render (a failed request, an error while rendering) and show an error state instead of nothing." }, + "click-blocked": { name: "Clicks on an invisible layer", what: "Clicks landed on an invisible or transparent layer over the content, so the controls under it never got them.", fix: "Remove the leftover overlay, or give it pointer-events: none while it is not shown." }, + "pointer-events-leak": { name: "Page stopped taking clicks", what: "pointer-events: none was left on the page's html or body with nothing open, so nothing on the page responded to clicks.", fix: "Clear the click block in the same code path that sets it, failure and cancel paths included (a finally block). In guard mode with visual fixes on, GenClass removes the leaked declaration on the visitor's screen.", live: true }, + "scroll-lock-stuck": { name: "Scroll locked", what: "overflow: hidden (or a fixed body) was left on the page with no dialog or menu open, so the page stopped scrolling.", fix: "Remove the lock in the same code path that closes the dialog, menu or notice, error and cancel paths included. In guard mode with visual fixes on, GenClass removes the leaked lock on the visitor's screen.", live: true }, + "control-unreachable": { name: "Controls out of reach", what: "A button or link stayed under another element (a sticky footer, a cookie bar) even at the end of the page, so it could not be clicked.", fix: "Leave room for the fixed element (padding at the bottom of the page) or move the control out from under it." }, + "focus-obscured": { name: "Focused element hidden", what: "An element reached with the keyboard, or a link's target, ended up hidden behind a sticky header or another layer.", fix: "Add scroll-padding-top (or scroll-margin) for the sticky header, or keep the layer out of the way." }, + "dialog-covered": { name: "Dialogs under other layers", what: "A dialog opened behind another element, so parts of it, or its buttons, could not be seen or clicked.", fix: "Put the dialog above everything else: the top layer (a dialog element opened with showModal()), or a z-index above the other layer's stacking context." }, + "request-stalled": { name: "Requests that never finished", what: "A request stayed pending while a control waited on it, so visitors were stuck with a spinner or a disabled button.", fix: "Give the request a timeout (AbortSignal.timeout) and show an error with a retry when it runs out." }, + "ad-heavy": { name: "Ads covering content", what: "Ads took more of the screen than the Coalition for Better Ads standards allow, or one covered the content as an overlay.", fix: "Reserve each ad's space in the layout and keep in-content ads out of fixed overlays. GenClass reports these and leaves the ads alone." }, + "text-warped": { name: "Warped text", what: "Text on screen that could not be read as text: squeezed or stretched, overlapping other text, or its letters spread apart.", fix: "Remove the transform, letter-spacing or fixed height that distorts it, and let the text wrap." } + }; + // what to do on the dev page so the check measures it again (the rest: load the page and use it as visitors do) + var CHECK_DO = { + "scroll-lock-stuck": "Open and close every dialog, menu and notice on the page, error and cancel paths included, then try to scroll.", + "pointer-events-leak": "Run every action that blocks clicks while it works, the failing path too (a bad input, the network off), then click something.", + "click-blocked": "Open and close the overlays on the page, then click the controls under where they were.", + "request-stalled": "Repeat the action with the request held up (genclass_explore with slow_ms), and watch what the control does while it waits.", + "dialog-covered": "Open each dialog on the page.", + "focus-obscured": "Tab through the page and follow its in-page links.", + "control-unreachable": "Scroll to the end of the page on a phone-sized window.", + "text-invisible": "Check the page in light and dark mode, and the hover and focus states of what it names.", + "page-overflow-x": "Load the page in a phone-sized window.", + "viewport-meta-missing": "Load the page in a phone-sized window." + }; + var CHK = "var(--chk)"; + function CK(k){ return CHECKS[k] || { name: k ? k.charAt(0).toUpperCase() + k.slice(1).replace(/-/g, " ") : "Other check", what: "", fix: "" }; } + function checkList(S){ return (S.breakdowns.check_kind || []).filter(function(x){ return x.count > 0; }); } + function checkWhere(S, k){ return (issues(S.breakdowns, "check_issue")[k] || []).slice().sort(function(a, b){ return b.count - a.count; }); } + // check fixes still in place when their page loads ended (applied minus undone), at most one per finding + function checkFixed(S){ var o = map(S.breakdowns.check_fix); return Math.min(S.totals.checks || 0, Math.max(0, (o.applied || 0) - (o.undone || 0))); } + // every kind of problem in the period, the model's and the measured checks', the most frequent first + function problemList(S){ + var out = (S.breakdowns.detect_diagnosis || []).filter(function(x){ return x.key !== "expected"; }).map(function(x){ return { key: x.key, count: x.count, check: false }; }); + checkList(S).forEach(function(x){ out.push({ key: x.key, count: x.count, check: true }); }); + return out.sort(function(a, b){ return b.count - a.count; }); + } + function infoOf(x){ return x.check ? CK(x.key) : T(x.key); } + function whereFor(S, x){ return x.check ? checkWhere(S, x.key) : whereOf(S, x.key); } + function keyOf(x){ return key(x.check ? CHK : colorOf(slotOf(x.key))); } + function problemsOf(S){ return (S.totals.detections || 0) + (S.totals.checks || 0); } + // the pages with a problem of either kind: the model's (every one, past the top 25 too) and the checks' pages + function problemPages(S){ + var seen = {}, n = 0; + (S.breakdowns.detect_route || []).forEach(function(x){ if (!seen[x.key]) { seen[x.key] = 1; n++; } }); + var more = Math.max(0, count(S, "detect_route") - n); + (S.breakdowns.check_issue || []).forEach(function(x){ var r = x.key.slice(x.key.indexOf("\\t") + 1); if (!seen[r]) { seen[r] = 1; n++; } }); + return n + more; + } + function badge(){ var b = el("span", "bdg", "Measured check"); b.title = "A measurement with a fixed threshold, not a model finding"; return b; } + // what a coding agent gets from Copy prompt for one measured check: what was measured, where, the usual fix, and the + // MCP steps that measure the page again + function checkPrompt(k, n, where){ + var t = CK(k), pages = where.slice(0, 4).map(function(w){ return w.route + " (" + fmt(w.count) + ")"; }).join(", "), top = where.length ? where[0].route : "the page"; + var out = ["Fix a problem GenClass measured in production in this web app (" + P.name + "): " + lower(t.name) + " (the measured check " + k + "), " + plural(n, "time", "times") + " " + lower(RANGE[range]) + (pages ? ", on " + pages : "") + ".", ""]; + if (t.what) out.push("What it measured: " + t.what); + if (t.fix) out.push("How it is usually fixed: " + t.fix); + out.push("This is a measured check (a measurement against a fixed threshold, not a model finding): it says what was wrong on the page; the cause is in the code or the CSS.", + "", "Use the GenClass MCP server (" + MCP + "; the project's MCP page in the GenClass dashboard says how to add it):", + "1. genclass_scan on " + top + " (the dev server's URL, or the deployed page): it runs the same checks and lists this one with the element, its selector, what it measured and a suggested CSS change. With the dev page connected, genclass_findings lists it too, as kind check. " + (CHECK_DO[k] || "Load the page and use it as visitors do."), + " " + WAIT, + "2. Find the code or CSS behind it and fix it at the cause (the suggested CSS is a starting point, not a patch).", + "3. Run genclass_scan on the page again, after the same steps: it must not list " + k + " any more.", + "4. Tell me what you changed and what genclass_scan listed before and after."); + return out.join("\\n"); + } + function checkBtn(k, n, where, cls){ + var b = el("button", cls || "btn sm ghost", "Copy prompt"); b.type = "button"; + b.setAttribute("data-copy", checkPrompt(k, n, where)); b.setAttribute("aria-label", "Copy a prompt to fix " + lower(CK(k).name)); + return b; + } + function problemBtn(x, where, cls){ return x.check ? checkBtn(x.key, x.count, where, cls) : promptBtn(x.key, x.count, where, cls); } function cards(S){ - var B = S.breakdowns, host = document.getElementById("issues"); host.textContent = ""; - var list = (B.detect_diagnosis || []).filter(function(x){ return x.key !== "expected"; }), total = sum(list); + var host = document.getElementById("issues"); host.textContent = ""; + var list = problemList(S), total = sum(list); document.getElementById("issuesempty").classList.toggle("hidden", list.length > 0); list.forEach(function(x){ - var t = T(x.key), c = el("article", "find"); - var top = el("div", "top"), h = el("h3"); h.appendChild(key(colorOf(slotOf(x.key)))); h.appendChild(document.createTextNode(t.name)); top.appendChild(h); + var t = infoOf(x), c = el("article", "find" + (x.check ? " chk" : "")); + var top = el("div", "top"), h = el("h3"); h.appendChild(keyOf(x)); h.appendChild(document.createTextNode(t.name)); if (x.check) h.appendChild(badge()); top.appendChild(h); var nw = el("div", "nw"); nw.appendChild(el("span", "n", fmt(x.count))); nw.appendChild(el("span", "share", total ? Math.round(100 * x.count / total) + "% of problems" : "")); top.appendChild(nw); c.appendChild(top); - c.appendChild(spark(S, x.key)); + c.appendChild(spark(S, x.key, x.check)); if (t.what) c.appendChild(el("p", null, t.what)); - var where = whereOf(S, x.key); + var where = whereFor(S, x); if (where.length) { var ul = el("ul", "where"); where.slice(0, 4).forEach(function(w){ var li = el("li"); li.appendChild(el("code", null, w.route)); li.appendChild(el("span", null, fmt(w.count))); ul.appendChild(li); }); c.appendChild(ul); } if (t.fix) { var f = el("p", "fix"); f.appendChild(el("b", null, "Fix: ")); f.appendChild(document.createTextNode(t.fix)); c.appendChild(f); } - var acts = el("div", "acts"); acts.appendChild(promptBtn(x.key, x.count, where)); c.appendChild(acts); + var acts = el("div", "acts"); acts.appendChild(problemBtn(x, where)); c.appendChild(acts); host.appendChild(c); }); } // Project overview: the kinds of problems that need attention first (Studio's advisor cards), each with its prompt function advisor(S){ var host = document.getElementById("advisor"); if (!host) return; host.textContent = ""; - var list = (S.breakdowns.detect_diagnosis || []).filter(function(x){ return x.key !== "expected"; }); + var list = problemList(S); var lead = document.getElementById("advn"); if (lead) lead.textContent = list.length ? plural(list.length, "kind of problem needs", "kinds of problems need") + " attention" : "GenClass found no problems"; if (!list.length) { var none = el("div", "none"); none.appendChild(el("p", null, "Nothing broke " + lower(RANGE[range]) + ". GenClass only reports situations that stand out, so a quiet stretch is normal.")); host.appendChild(none); return; } list.slice(0, 3).forEach(function(x){ - var t = T(x.key), a = el("article"), top = el("div", "top"), h = el("h3"); - h.appendChild(key(colorOf(slotOf(x.key)))); h.appendChild(document.createTextNode(t.name)); top.appendChild(h); + var t = infoOf(x), a = el("article"), top = el("div", "top"), h = el("h3"); + h.appendChild(keyOf(x)); h.appendChild(document.createTextNode(t.name)); if (x.check) h.appendChild(badge()); top.appendChild(h); top.appendChild(el("span", "bdg" + (x.count >= 10 ? " destructive" : " warning"), plural(x.count, "time", "times"))); a.appendChild(top); - var where = whereOf(S, x.key); + var where = whereFor(S, x); a.appendChild(el("p", null, (where.length ? "On " + where.slice(0, 2).map(function(w){ return w.route; }).join(", ") + ". " : "") + (t.what || ""))); var acts = el("div", "acts"), v = el("a", "btn sm ghost", "View"); v.href = "#found"; v.setAttribute("data-goto", "found"); - acts.appendChild(promptBtn(x.key, x.count, where)); acts.appendChild(v); a.appendChild(acts); + acts.appendChild(problemBtn(x, where)); acts.appendChild(v); a.appendChild(acts); host.appendChild(a); }); } // Project overview: four usage cards (Studio's request charts), one bar per hour or day function usage(S){ var host = document.getElementById("usage"); if (!host) return; host.textContent = ""; - var hourly = S.range === "24h", rows = S.series || [], T0 = S.totals, prev = S.previous || {}; + // Problems found: the model's detections and the measured checks, per bucket and in total + var hourly = S.range === "24h", rows = (S.series || []).map(function(b){ var o = {}; for (var k in b) o[k] = b[k]; o.problems = (b.detections || 0) + (b.checks || 0); return o; }), T0 = S.totals, prev = S.previous || {}; var lead = document.getElementById("ulead"); if (lead) lead.textContent = fmt(T0.sessions); var ix = (T0.ix_sessions || 0) > 0 || rows.some(function(b){ return b.ix_sessions; }); // the number is the period's total (exact, from the totals); the bars are the series, one per hour or day - [["Page loads", "sessions", "var(--accent)", prev.sessions, T0.sessions], ["Problems found", "detections", "var(--t2)", prev.detections, T0.detections], + [["Page loads", "sessions", "var(--accent)", prev.sessions, T0.sessions], ["Problems found", "problems", "var(--t2)", typeof prev.detections === "number" ? prev.detections + (prev.checks || 0) : prev.detections, problemsOf(S)], ["Stepped in", "acted", "var(--accent)", prev.acted, T0.interventions], ix ? ["Visits", "ix_sessions", "var(--accent)", prev.ix_sessions, T0.ix_sessions] : ["Possible issues", "possible", "var(--ink3)", null, T0.possible]].forEach(function(c){ var total = typeof c[4] === "number" ? c[4] : rows.reduce(function(s, b){ return s + (b[c[1]] || 0); }, 0), card = el("div", "ucard"); @@ -1030,6 +1143,8 @@ const DASH_JS = ` function at(route){ return routes[route] = routes[route] || { route: route, total: 0, parts: {}, poss: 0 }; } Object.keys(byIssue).forEach(function(d){ byIssue[d].forEach(function(w){ var r = at(w.route); var s = slotOf(d); r.parts[s] = (r.parts[s] || 0) + w.count; r.total += w.count; }); }); if (!Object.keys(routes).length) (B.detect_route || []).forEach(function(x){ at(x.key).total += x.count; }); + var byCheck = issues(B, "check_issue"); + Object.keys(byCheck).forEach(function(k){ byCheck[k].forEach(function(w){ var r = at(w.route); r.parts.checks = (r.parts.checks || 0) + w.count; r.total += w.count; }); }); if (Object.keys(byPoss).length) Object.keys(byPoss).forEach(function(d){ byPoss[d].forEach(function(w){ at(w.route).poss += w.count; }); }); else (B.possible_route || []).forEach(function(x){ at(x.key).poss += x.count; }); var list = Object.keys(routes).map(function(k){ return routes[k]; }); @@ -1042,9 +1157,9 @@ const DASH_JS = ` hd.appendChild(hr); tb.appendChild(hd); var body = el("tbody"); list.slice(0, 8).forEach(function(r){ var tr = el("tr"); var rc = el("td", "route", r.route); rc.title = r.route; tr.appendChild(rc); tr.appendChild(el("td", "num", fmt(r.total))); - var td = el("td", "wide"), mix = el("div", "mix"), parts = SLOTS.concat(["other"]).filter(function(s){ return r.parts[s]; }); + var td = el("td", "wide"), mix = el("div", "mix"), parts = SLOTS.concat(["other", "checks"]).filter(function(s){ return r.parts[s]; }); if (!parts.length && r.total) { var i0 = el("i"); i0.style.width = (100 * r.total / max) + "%"; i0.style.background = "var(--t0)"; i0.title = "Not broken down by type: " + fmt(r.total); mix.appendChild(i0); } - parts.forEach(function(s){ var i = el("i"); i.style.width = (100 * r.parts[s] / max) + "%"; i.style.background = colorOf(s); i.title = (s === "other" ? "Other" : TYPES[s].name) + ": " + fmt(r.parts[s]); mix.appendChild(i); }); + parts.forEach(function(s){ var i = el("i"); i.style.width = (100 * r.parts[s] / max) + "%"; i.style.background = colorOf(s); i.title = slotName(s) + ": " + fmt(r.parts[s]); mix.appendChild(i); }); td.appendChild(mix); tr.appendChild(td); if (anyPoss) tr.appendChild(el("td", "num poss-n", r.poss ? fmt(r.poss) : "-")); body.appendChild(tr); @@ -1055,11 +1170,15 @@ const DASH_JS = ` if (typeof withPage === "number" && T0.detections > withPage) notes.push(plural(T0.detections - withPage, "problem", "problems") + " arrived without a page (the page comes with the decision record, which was in another batch or not sent); the totals count them."); var possWithPage = T0.possibleWithPage; if (anyPoss && typeof possWithPage === "number" && (T0.possible || 0) > possWithPage) notes.push(plural(T0.possible - possWithPage, "possible issue", "possible issues") + " arrived without a page."); + if (list.some(function(r){ return r.parts.checks; })) notes.push("Hatched: measured checks, not model findings."); + if (typeof T0.checksWithPage === "number" && (T0.checks || 0) > T0.checksWithPage) notes.push(plural(T0.checks - T0.checksWithPage, "measured check", "measured checks") + " arrived without a page (a page load sends the counts of its first 20 pages)."); if (notes.length) host.appendChild(el("figcaption", null, notes.join(" "))); } function acted(S){ var B = S.breakdowns, T0 = S.totals, host = document.getElementById("acted"); host.textContent = ""; - var modes = map(B.mode), acting = (modes.guard || 0) + (modes.heal || 0) > 0, list = B.intervention || []; + var modes = map(B.mode), acting = (modes.guard || 0) + (modes.heal || 0) > 0, list = (B.intervention || []).slice(), cf = map(B.check_fix); + // the measured checks' runtime fixes (one reversible CSS change each), counted apart from the model's actions + if (cf.applied) { list.push({ key: "check_fix", count: cf.applied }); list.sort(function(a, b){ return b.count - a.count; }); } if (!list.length) { host.appendChild(el("p", "quiet", acting ? "GenClass did not need to step in during this period." : "This app runs in observe mode, so GenClass only reports. Switch to guard mode when you trust what it reports, and what it stops will show here.")); return; @@ -1068,13 +1187,15 @@ const DASH_JS = ` hr.appendChild(el("th", null, "Action")); hr.appendChild(el("th", "num", "Times")); hr.appendChild(el("th", null, "")); hd.appendChild(hr); tb.appendChild(hd); var body = el("tbody"); list.slice(0, 8).forEach(function(x){ - var tr = el("tr"); tr.appendChild(el("td", null, ACTIONS[x.key] || x.key)); tr.appendChild(el("td", "num", fmt(x.count))); + var tr = el("tr"), name = el("td", null, ACTIONS[x.key] || x.key); if (x.key === "check_fix") name.appendChild(badge()); tr.appendChild(name); tr.appendChild(el("td", "num", fmt(x.count))); var td = el("td", "wide"), mix = el("div", "mix"), i = el("i"); i.style.width = (100 * x.count / max) + "%"; i.style.background = "var(--accent-f)"; mix.appendChild(i); td.appendChild(mix); tr.appendChild(td); body.appendChild(tr); }); tb.appendChild(body); host.appendChild(tb); - var o = map(B.action_outcome), applied = o.applied || T0.interventions || 0; - host.appendChild(el("figcaption", null, plural(o.undone || 0, "change", "changes") + " undone by users" + (applied ? " (" + Math.round(100 * (o.undone || 0) / applied) + "%)" : "") + (o.failed ? ", " + plural(o.failed, "action", "actions") + " failed" : "") + ".")); + var o = map(B.action_outcome), applied = o.applied || T0.interventions || 0, notes = []; + if (applied || !cf.applied) notes.push(plural(o.undone || 0, "change", "changes") + " undone by users" + (applied ? " (" + Math.round(100 * (o.undone || 0) / applied) + "%)" : "") + (o.failed ? ", " + plural(o.failed, "action", "actions") + " failed" : "") + "."); + if (cf.applied) notes.push(plural(cf.undone || 0, "check fix", "check fixes") + " undone: by the developer, by the page changing the element, or because the fix no longer fitted it."); + host.appendChild(el("figcaption", null, notes.join(" "))); } function row(dl, label0, value, sub, flag){ dl.appendChild(el("dt", null, label0)); var dd = el("dd"), v = el("div", "v", value); @@ -1102,6 +1223,13 @@ const DASH_JS = ` errs.slice(0, 4).forEach(function(x){ var li = el("li", null, fmt(x.count) + " \\u00D7 " + x.key); li.title = x.key; ul.appendChild(li); }); dd.appendChild(ul); } + // the measured checks' coverage (runtime 0.3.0-beta.4): the counts above are read against it + var vs = map(B.visual_setting), cl = T0.check_loads || 0, tiers = map(B.check_tier); + if (cl || vs.on || vs.local) { + var cs = cl ? [plural(cl, "page load", "page loads") + " ran them", tiers.light ? fmt(tiers.light) + " on the light tier (low-end devices: the near-free checks only)" : "", T0.visual_fix ? fmt(T0.visual_fix) + " with runtime fixes on" : "", T0.check_ms ? "about " + fmt(T0.check_ms / cl) + " ms of check time each" : ""].filter(Boolean).join(", ") + "." + : "They are off on public hosts unless the app turns them on: visual: true (on the script tag, data-visual=on), and visual: { on: true, fix: true } in guard mode (data-visual=fix) for the runtime fixes."; + row(dl, "Measured checks", T0.sessions ? share(cl, T0.sessions) + " of page loads" : fmt(cl), cs, cl ? null : ["bad", "not running"]); + } var rv = B.runtime || []; if (rv.length) row(dl, "Runtime versions", rv.slice(0, 3).map(function(x){ return x.key; }).join(", "), rv.length > 1 ? "more than one version is live" : ""); host.appendChild(dl); @@ -1125,7 +1253,8 @@ const DASH_JS = ` var B = S.breakdowns, host = document.getElementById("adv"); host.textContent = ""; [["Triggers", B.trigger], ["Diagnoses (all decisions)", B.diagnosis], ["Top routes (decision records)", B.route], ["Top pages (page loads)", B.page_route], ["Protected functions", B.fn], ["Model load states", B.model_state], ["Decision latency (records)", B.latency_bucket], ["Fail-opens (reason:trigger)", B.fail_open], ["Modes", B.mode], ["Model versions", B.model_version], - ["Hosts seen", B.host], ["Possible issues (lower confidence)", B.possible_diagnosis], ["Action limits", B.action_limits]].forEach(function(p){ + ["Hosts seen", B.host], ["Possible issues (lower confidence)", B.possible_diagnosis], ["Action limits", B.action_limits], + ["Measured checks (by check)", B.check_kind], ["Measured checks setting (page loads)", B.visual_setting]].forEach(function(p){ var c = el("div"); c.appendChild(el("h2", null, p[0])); if (!p[1] || !p[1].length) { c.appendChild(el("p", "empty", "None in this period.")); host.appendChild(c); return; } var max = Math.max.apply(null, p[1].map(function(x){ return x.count; })) || 1, ul = el("ul", "bars"); @@ -1605,8 +1734,9 @@ const DASH_JS = ` var T0 = S.totals, B = S.breakdowns, out = [], sumEl = document.getElementById("summary"); out.push("# " + P.name + " (GenClass dashboard)", "", RANGE[S.range] + ", to " + String(S.generated).slice(0, 16).replace("T", " ") + " UTC. " + (sumEl ? sumEl.textContent : ""), ""); out.push("| Metric | Value |", "| --- | ---: |"); - [["Page loads", T0.sessions], ["Decisions", T0.decisions], ["Problems found", T0.detections], ["Possible issues (lower confidence, not problems)", T0.possible], - ["Pages with a problem", count(S, "detect_route")], ["Times GenClass stepped in", T0.interventions], ["Visits", T0.ix_sessions], ["Page views", T0.ix_views]].forEach(function(r){ + var cf = map(B.check_fix); + [["Page loads", T0.sessions], ["Decisions", T0.decisions], ["Problems found", problemsOf(S)], ["Of them measured checks (not model findings)", T0.checks], ["Possible issues (lower confidence, not problems)", T0.possible], + ["Pages with a problem", problemPages(S)], ["Times GenClass stepped in", T0.interventions], ["Measured checks fixed on visitors' screens", cf.applied], ["Visits", T0.ix_sessions], ["Page views", T0.ix_views]].forEach(function(r){ if (typeof r[1] === "number") out.push("| " + r[0] + " | " + fmt(r[1]) + " |"); }); var list = (B.detect_diagnosis || []).filter(function(x){ return x.key !== "expected"; }); @@ -1619,7 +1749,17 @@ const DASH_JS = ` if (w.length) out.push("Pages: " + w.map(function(p){ return "\`" + p.route + "\` (" + fmt(p.count) + ")"; }).join(", "), ""); if (t.fix) out.push("Fix: " + t.fix, ""); }); - var acts = B.intervention || []; + var cl = checkList(S); + if (cl.length) out.push("## Measured checks by kind", "", "Measurements with fixed thresholds, not model findings: genclass_scan on the page lists each one with its element and a suggested CSS change.", ""); + cl.forEach(function(x){ + var t = CK(x.key), w = checkWhere(S, x.key).slice(0, 4); + out.push("### " + t.name + " (" + x.key + "): " + fmt(x.count), ""); + if (t.what) out.push(t.what, ""); + if (w.length) out.push("Pages: " + w.map(function(p){ return "\`" + p.route + "\` (" + fmt(p.count) + ")"; }).join(", "), ""); + if (t.fix) out.push("Fix: " + t.fix, ""); + }); + var acts = (B.intervention || []).slice(); + if (cf.applied) acts.push({ key: "check_fix", count: cf.applied }); if (acts.length) { out.push("## What GenClass did", "", "| Action | Times |", "| --- | ---: |"); acts.slice(0, 8).forEach(function(x){ out.push("| " + (ACTIONS[x.key] || x.key) + " | " + fmt(x.count) + " |"); }); @@ -1780,6 +1920,8 @@ const DASH_JS = ` function render(S){ shownS = S; + // the recorded demo's kinds get the chart's slots while its estimate is shown (src/demo.ts), else the model's five + SLOTS = S.demo && P.demo ? P.demo.slots : BASE_SLOTS; var live = !!S.lastEvent, st = document.getElementById("status"), stt = document.getElementById("statust"), hs = document.getElementById("hstatus"); if (st) st.classList.toggle("on", live); if (stt) stt.textContent = live ? "Last event " + ago(S.lastEvent) : "Waiting for the first event"; @@ -1799,16 +1941,15 @@ const DASH_JS = ` verdict(S); summary(S); tiles(S); usage(S); advisor(S); fixAll(S); trend(S); possibleTrend(S); cards(S); possibleCards(S); pages(S); acted(S); health(S); stages(S); agentLog(S); loop(S); recent(S); visits(S); advanced(S); var B2 = S.breakdowns, T2 = S.totals; function put(id, t){ var n = document.getElementById(id); if (n) n.textContent = t; } - put("c-trend", fmt(T2.detections)); - put("c-found", String((B2.detect_diagnosis || []).filter(function(x){ return x.key !== "expected"; }).length)); - put("c-pages", String(count(S, "detect_route"))); - put("c-did", fmt(T2.interventions)); + put("c-trend", fmt(problemsOf(S))); + put("c-found", String(problemList(S).length)); + put("c-pages", String(problemPages(S))); + put("c-did", fmt((T2.interventions || 0) + (map(B2.check_fix).applied || 0))); } if (P.demo) { Object.keys(P.demo.types).forEach(function(k){ var t = P.demo.types[k]; TYPES[k] = { name: t.name, c: t.c, what: t.what, fix: t.fix }; STEPS[k] = function(r){ return t.step.replace("{page}", r); }; }); Object.keys(P.demo.actions).forEach(function(k){ ACTIONS[k] = P.demo.actions[k]; }); - SLOTS = P.demo.slots; } document.getElementById("pname").textContent = P.name; // Views: each rail or product-menu link opens one view (the parts marked data-v with it; a part may name several); @@ -2437,7 +2578,7 @@ versions ignore it. ${LOCAL_HOSTS} Your deployed site sends by default.

    -
    Fig. 1. Problems GenClass found, per day, by type. Hover a bar for its breakdown.
    +
    Fig. 1. Problems GenClass found, per day, by type; hatched: measured checks (fixed thresholds, not the model). Hover a bar for its breakdown.