diff --git a/CHANGELOG.md b/CHANGELOG.md index 37728150..0a4ff7c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ grouped by area, with links to the docs for the details. - **Sessions.** A session that recorded weight edits gets `stats.weightEdits` (edits, distinct tensors, weights changed) and the `weights:edited` tag, and is listed even when it holds nothing else. `coding.sh show`, the viewer and MCP `session_get` print one line per edit, and `--where 'weights.changed>1000'` filters by size. Stored sessions are folded again. [Docs](https://coding.sh/docs/api) - **OpenTelemetry.** A span or log record named `weights.edit` with `coding_sh.weights.*` attributes is read as a weight edit, and the `otlp` sink sends one `weights.edit` span per edit (so does `[serve.forward]`, for edits posted to `/v1/ingest`). The `invoke_agent` span carries `coding_sh.stats.weight_edits` and `coding_sh.stats.weights_changed`. Metadata mode keeps only the counts and norms. [Docs](https://coding.sh/docs/api) - **Evals.** `coding.sh exp run --weights ` records which checkpoint a run's task ran (the SHA-256 of the weights file, which you compute). `exp compare`, `exp show`, `exp list`, the viewer and the MCP tools show the stamp, and a comparison warns when both sides ran the same weights or only one recorded them. [Compare an edited model with its base](https://github.com/jackulau/coding.sh/blob/main/docs/guides/evals.md#compare-an-edited-model-with-its-base) +- **SDK.** `coding_sh.weights_checkpoint()` and `weightsCheckpoint()` record the file a script saved its edited weights to: one `notification` event with `payload.kind: "weights.checkpoint"`, the file's SHA-256 (the value `exp run --weights` takes) and optionally the base file's. Both SDKs' tests check one fixture. [Docs](https://coding.sh/docs/api) +- **Sessions.** A session that recorded a weights checkpoint lists the file hashes in `stats.weightEdits.checkpoints` and gets the `weights:checkpoint` tag. `coding.sh show`, the viewer and MCP `session_get` print `checkpoint 58899f90572c ← 0496e5111097`, and `--where 'weights.checkpoint:58899f90572c*'` finds the session. Metadata capture keeps both hashes. Stored sessions are folded again. [Docs](https://coding.sh/docs/api) +- **OpenTelemetry.** A span or log record named `weights.checkpoint` with `coding_sh.weights.sha256` is read as a checkpoint. The `otlp` sink and `[serve.forward]` send one `weights.checkpoint` span per event. Metadata mode keeps the two file hashes and drops the note. [Docs](https://coding.sh/docs/api) +- **Evals.** `exp show` and `exp compare` name the session whose checkpoint recorded an experiment's `--weights` hash, with its edits. `--json` and the MCP tools carry the ids as `weights.sessions`. `coding.sh show ` and MCP `session_get` list the experiments that ran its checkpoints. [Docs](https://coding.sh/docs/experiments#weights) +- **Viewer.** A weights stamp on the experiment and compare pages links the session that recorded that file as a checkpoint, and the session's header links each experiment that ran it. `/api/compare`, `/api/experiments/{id}` and `/api/sessions/{id}` carry the same ids as the CLI's `--json`. [Docs](https://coding.sh/docs/experiments#weights) - **Evals.** `coding.sh exp compare A B --given ` compares only the cases where that grader passes in both runs, and prints how many it kept and dropped. With a `terminated` grader this separates "finished more answers" from "got more of the finished answers right". [Compare an edited model with its base](https://github.com/jackulau/coding.sh/blob/main/docs/guides/evals.md#compare-an-edited-model-with-its-base) - **MCP.** In 2026-07-28 requests, list, read and discover results carry `ttlMs` and `cacheScope`: the tool, prompt and template lists are cached for an hour, `server/discover` for five minutes, and sessions are not cached. [MCP server](https://coding.sh/docs/mcp#protocol) diff --git a/apps/site/src/pages/docs/experiments.astro b/apps/site/src/pages/docs/experiments.astro index 9a3d9175..71744287 100644 --- a/apps/site/src/pages/docs/experiments.astro +++ b/apps/site/src/pages/docs/experiments.astro @@ -94,6 +94,41 @@ coding.sh exp plateau --series triage-loop`; const metricsCmd = `coding.sh exp metrics --arm base='node bench.mjs --dir a' --arm cand='node bench.mjs --dir b'`; +// The weights outputs are real too: a six-question toy task whose base model is cut off on every +// other answer, with the edits recorded by the Python SDK, under a throwaway HOME and +// CODING_SH_HOME, 2026-11-12. +const weightsPy = `import coding_sh + +with coding_sh.trace("stop-looping edit"): + for name, base, edited in patches: + coding_sh.weights_edit(name, base_sha256=sha256_tensor(base), edited_sha256=sha256_tensor(edited), + **coding_sh.weights_delta(base, edited)) + save(model, "edited.safetensors") + coding_sh.weights_checkpoint(sha256_file("edited.safetensors"), + base_sha256=sha256_file("base.safetensors")) +coding_sh.flush()`; + +const weightsCmd = `coding.sh exp run -d math --task base.json -g terminated.json -g correct.json --name base \\ + --weights 6a78961da782b2795fa5bb6f0968f79c75799fbf1919aba8798c965001caf221 +coding.sh exp run -d math --task edited.json -g terminated.json -g correct.json --name edited \\ + --weights 6e4c46150bc5c8ef6f1223e3880c5415795233187be390daf290a1ee6cd72e5b +coding.sh exp compare base edited`; + +const weightsOut = `A base exp_01M793B934C7G6QEAP1DFBA810 pass 50.0% weights 6a78961da782 +B edited exp_01M793B998T36PYVGCE527YKMT pass 100.0% weights 6e4c46150bc5 + edits 74a74605-6002-4ae3-b53c-8aa065ba6ede 2 edits, 2 tensors, 3 weights changed +…`; + +const weightsShowCmd = `coding.sh sessions --where 'weights.checkpoint:6e4c46150bc5*' +coding.sh show 74a74605-6002-4ae3-b53c-8aa065ba6ede`; + +const weightsShowOut = ` Weight edits 2 edits, 2 tensors, 3 weights changed, checkpoint 6e4c46150bc5 + Experiments 6e4c46150bc5 ran in edited (exp_01M793B998T36PYVGCE527YKMT) + + +0ms weights layers.0.weight [2×2] · 1 of 4 weights · L2 0.5 · max 0.5 · 2927a02adf33 → ecfbe5382ed5 + +0ms weights layers.1.weight [1×3] · 2 of 3 weights · L2 1.414 · max 1 · d0a1c13caf23 → dca6036fedd3 + +0ms weights checkpoint 6e4c46150bc5 ← 6a78961da782`; + const replayCmd = `coding.sh exp replay latest#-1 --vary model=claude-haiku-4-5 --dry-run coding.sh exp replay latest#-1 --vary model=claude-sonnet-4-5,claude-haiku-4-5 --n 5`; --- @@ -309,6 +344,32 @@ coding.sh exp replay latest#-1 --vary model=claude-sonnet-4-5,claude-haiku-4-5 - +

Compare an edited model with its base

+

+ When the change is an edit to a model's weights, record it from the script that makes it. weights_edit() records each + tensor the script changes, and weights_checkpoint() the file it saves, by SHA-256. You compute the hashes; coding.sh + hashes nothing. +

+ +

+ patches, sha256_tensor, sha256_file and save stand for your own code. The + TypeScript SDK has weightsEdit() and weightsCheckpoint(), and an OpenTelemetry span named + weights.checkpoint records the same event. Then give each run the hash of the weights file its task ran: +

+ + +

+ The edits line names the session that saved B's file, with what it changed. The comparison warns when both sides ran the + same weights, or only one recorded them. The session names the experiments that ran its file: +

+ + +

+ The MCP tools carry the same session ids and experiments. The viewer links the session from the weights stamp on the experiment and + compare pages, and links each experiment from the session's header. See + Weights checkpoint for where each field appears. +

+

Replay one model call

exp replay sends a call that the proxy logged again, with the model, the system diff --git a/docs/ADAPTERS.md b/docs/ADAPTERS.md index e6ed7ce9..46b58012 100644 --- a/docs/ADAPTERS.md +++ b/docs/ADAPTERS.md @@ -265,6 +265,9 @@ Apps in production don't use an adapter. They send data over the network to `cod - A span or a log record named `weights.edit` with `coding_sh.weights.*` attributes is stored as the weight edit event the SDKs send (`notification`, `payload.kind: "weights.edit"`); the attributes are listed in [reference/api.md](reference/api.md). +- One named `weights.checkpoint` with `coding_sh.weights.sha256` is stored as the checkpoint event + (`payload.kind: "weights.checkpoint"`). Without a hash of 64 hex characters it is not read as a + checkpoint. - Hooks, transcript backfill and `coding.sh run` stay local. Serve does not take spool lines. - The ingest key picks the project, and session ids are stored as `/`. - SDK sessions that never send `session.end` have no `endedAt` yet; see the open item in diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index 5a8c1016..b5ecc1f2 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -265,11 +265,12 @@ max_buffer_spans = 20000 ``` - OTLP posted to `/v1/traces` keeps its traceId, spanId and parent ids. Native `/v1/ingest` batches - are sent as `invoke_agent`, `execute_tool`, `chat` and `weights.edit` spans, the same mapping as the `otlp` sink. + are sent as `invoke_agent`, `execute_tool`, `chat`, `weights.edit` and `weights.checkpoint` spans, the same mapping as the `otlp` sink. Every resource gets `coding_sh.project` set to the ingest key's project. - `metadata` mode keeps an allowlist: gen_ai operation, provider, model, response id and finish reasons, tool name, call id and type, agent and conversation ids, token usage, `service.*`, - `telemetry.*`, `coding_sh.*` (except the session title, the repo root, and a weight edit's tensor name, shape, hashes and note), and numbers and booleans. + `telemetry.*`, `coding_sh.*` (except the session title, the repo root, a weight edit's tensor name, shape, hashes and note, and a weights checkpoint's note), and numbers and booleans. + A `weights.checkpoint` span keeps `coding_sh.weights.sha256` and `coding_sh.weights.base_sha256`: they join the session to an experiment. Messages, prompts, tool arguments and results, `input.value`, `output.value`, span event attributes other than `exception.type`, and status messages are dropped. `full` mode sends every attribute after `capture.redact`. - Spans are sent in batches of up to 1,000, about once a second. A network error, 408, 429 or 5xx diff --git a/docs/concepts.md b/docs/concepts.md index 0130a040..88cbd0c8 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -75,6 +75,27 @@ session that recorded one has the `weights:edited` tag and counts its edits in ` or any OpenTelemetry SDK as a span named `weights.edit`. The fields are in the [API reference](https://coding.sh/docs/api). +### Weights checkpoint + +The file a script saved its edited weights to, recorded as one event: the SHA-256 of the file, and +of the file the edits started from. An edit's hashes cover one tensor, and +`coding.sh exp run --weights` records the hash of a whole file. The checkpoint is what links the +two. A session that recorded one has the `weights:checkpoint` tag and lists the hashes in +`stats.weightEdits.checkpoints`. + +- In the viewer: a `Saved weights` row on the session page, a `ran in ` link in its header + for each experiment that ran the file, and a link to the session next to an experiment's + weights stamp. +- In the CLI: `coding.sh sessions --where 'weights.checkpoint:58899f90572c*'` finds the session + that saved a file, and `coding.sh show ` prints `checkpoint 58899f90572c ← 0496e5111097` + and the experiments that ran that file. +- In evals: `coding.sh exp show` and `exp compare` name the session under the experiment's + `weights` line, with its edits. See + [Compare an edited model with its base](https://coding.sh/docs/experiments#weights). +- Recorded by the Python SDK's `coding_sh.weights_checkpoint()`, the TypeScript SDK's + `weightsCheckpoint()`, or any OpenTelemetry SDK as a span named `weights.checkpoint`. You compute + the hash; coding.sh hashes nothing. + ### Cost What a session's model calls cost, from their token counts and each model's price. A `~` marks an diff --git a/docs/guides/evals.md b/docs/guides/evals.md index be5aa727..ce6244fa 100644 --- a/docs/guides/evals.md +++ b/docs/guides/evals.md @@ -788,9 +788,40 @@ count such as tokens used has to become a pass or fail before it is a grader. {"kind": "json-path", "name": "correct", "path": "$.answer"} ``` +Record what the editing script did, so the comparison can name it. The script records each tensor +it changes with `weights_edit()`, and the file it saves with `weights_checkpoint()`: the SHA-256 +of the saved file, and of the file it started from. Put them in one `trace()`, so they are one +session. `CODING_SH_URL` points the SDK at your `coding.sh serve` +([SDK reference](https://coding.sh/docs/sdk)). + +```python +import hashlib +import coding_sh + +def sha256_file(path): + with open(path, "rb") as f: + return hashlib.sha256(f.read()).hexdigest() + +with coding_sh.trace("stop-looping edit"): + for name, base, edited in patches: + coding_sh.weights_edit(name, base_sha256=sha256_tensor(base), edited_sha256=sha256_tensor(edited), + **coding_sh.weights_delta(base, edited)) + save(model, "edited.safetensors") + coding_sh.weights_checkpoint(sha256_file("edited.safetensors"), + base_sha256=sha256_file("base.safetensors")) +coding_sh.flush() +``` + +`patches`, `sha256_tensor` and `save` stand for your own code: the tensors you changed with their +values before and after, a hash of one tensor, and the call that writes the file. +`weights_delta()` reads small arrays in pure Python; for real tensors pass `shape`, `changed`, `l2` +and `max_abs` from your own framework. The TypeScript SDK has the same two calls, `weightsEdit()` +and `weightsCheckpoint({ sha256, baseSha256 })`. + Run each model, then compare. `--weights` records which checkpoint each task ran: the SHA-256 of its weights file, 64 hex characters. You compute it, for example with -`shasum -a 256 model.safetensors`; coding.sh stores the value and hashes nothing. +`shasum -a 256 model.safetensors`; coding.sh stores the value and hashes nothing. Give the edited +run the hash the script recorded as its checkpoint. ```sh coding.sh exp run -d math --task base.json -g terminated.json -g correct.json --name base \ @@ -811,6 +842,26 @@ the same stamp. `exp list` adds a `WEIGHTS` column when a listed run has one, an rows carry `weights.sha256`. `exp run` without `--weights` stores no stamp, and a comparison of two such runs prints nothing about weights. +When a session recorded a side's stamp as a checkpoint, `exp compare` and `exp show` name it on the +next line, with the edits that session recorded: + +``` +A base exp_… pass 66.7% weights 0496e5111097 +B edited exp_… pass 100.0% weights 58899f90572c + edits 2 edits, 2 tensors, 66,560 weights changed +``` + +- `coding.sh show ` prints each edit, and an `Experiments` row with the runs that used + the file. +- `--json` carries the session ids as `weights.sessions`: on `comparison.a` and `comparison.b` + for `exp compare`, on each row of `exp list`, and in a top-level `weights` object + (`sha256`, `sessions`) for `exp show`. The MCP tools `experiments_list` and `experiment_compare` + carry the same field. +- At most 5 sessions are named for a hash, newest first. +- The viewer links the session from the stamp on the experiment and compare pages, and links + each experiment from the session's header as `ran in `. +- A stamp no session recorded prints as before, and so does a store that cannot answer the lookup. + The first compare prints a paired delta for `terminated` and one for `correct`, each with its bootstrap interval. If the edited model only stopped looping, `terminated` is up and `correct` is up with it. `--given terminated` keeps the cases where `terminated` passes in both runs, and diff --git a/docs/reference/api.md b/docs/reference/api.md index a4cf1b16..d7b9546e 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -518,7 +518,9 @@ curl -sS "$CSH/api/sessions?where=agent:codex%20modle:x" -H "Authorization: Bear ### `GET /api/sessions/{id}` -One session, the same shape as a list row without `childCount`. Percent-encode the id. +One session, the same shape as a list row without `childCount`. Percent-encode the id. A session +whose weights checkpoint an experiment ran also has `checkpointExperiments`: a list of +`{sha256, experiments: [{id, name}]}`. A key scoped to a project does not get it. ```sh curl -sS "$CSH/api/sessions/team%2Fchat-1" -H "Authorization: Bearer $READ_KEY" @@ -718,24 +720,37 @@ Three `notification` kinds carry the events behind these stats: | `fork` | `toolUseId` (the parent's Agent call) and `inheritedTokens` | | `hook.context` | `hookEvent`, `hookName`, `hookSource`, `command`, `chars`, `tokens` and `text` (clipped). Metadata mode keeps only `hookEvent`, `hookSource`, `chars` and `tokens` | -One more `notification` kind records weight edits. The Python SDK's `coding_sh.weights_edit()` and the TypeScript SDK's `weightsEdit()` send it, and any client can post it to `/v1/ingest`. Redaction runs on `tensor` and `note` like on any payload. +Two more `notification` kinds record weight edits and the file they were saved to. The Python SDK's `coding_sh.weights_edit()` and `coding_sh.weights_checkpoint()` and the TypeScript SDK's `weightsEdit()` and `weightsCheckpoint()` send them, and any client can post them to `/v1/ingest`. Redaction runs on `tensor` and `note` like on any payload. | `payload.kind` | Fields | |---|---| | `weights.edit` | `tensor` (string, the tensor's name), `shape` (array of non-negative integers, at most 16), `changed` (integer, the weights that differ, at most the product of `shape`), `l2` (number >= 0, the L2 norm of edited minus base), `maxAbs` (number >= 0, the largest absolute value of that change), `baseSha256` and `editedSha256` (64 lowercase hex characters, the tensor's hash before and after) and `note` (string, optional) | +| `weights.checkpoint` | `sha256` (64 hex characters, the SHA-256 of the saved weights file: the value you give to `exp run --weights`), `baseSha256` (optional, the SHA-256 of the file the edits started from) and `note` (string, optional) | | Field | Meaning | |---|---| -| `stats.weightEdits` (`events`, `tensors`, `weightsChanged`) | Weight edits the session recorded: how many, how many distinct tensor names, and their `changed` counts added up (a weight two edits changed counts twice). Absent when the session recorded none. `tensors` is absent when no edit kept its name | +| `stats.weightEdits` (`events`, `tensors`, `weightsChanged`, `checkpoints`) | Weight edits the session recorded: how many, how many distinct tensor names, and their `changed` counts added up (a weight two edits changed counts twice). `checkpoints` lists the distinct `sha256` of its `weights.checkpoint` events in the order they arrived, at most 16. Absent when the session recorded no edit and no checkpoint. `tensors` is absent when no edit kept its name, and `checkpoints` when there is none. A session with a checkpoint and no edit has `events: 0` and `weightsChanged: 0` | - A session with a weight edit gets the `weights:edited` tag. `coding.sh sessions --flag weights:edited` lists them, and `--where 'weights.changed>1000'` filters by size. -- `coding.sh show`, the viewer and MCP `session_get` print one line per edit: the tensor and its shape, the weights changed out of the total, both norms and the first 8 characters of each hash. +- A session with a checkpoint gets the `weights:checkpoint` tag. `--where 'weights.checkpoint:58899f90572c*'` finds the sessions that recorded a file by its hash or the start of it. A session that holds a checkpoint and nothing else is still listed. +- A checkpoint is read only when `sha256` is 64 hex characters. It is compared in lowercase, whatever case it arrived in. A `baseSha256` that is not a hash is left out and the checkpoint still counts. +- `coding.sh show`, the viewer and MCP `session_get` print one line per edit: the tensor and its shape, the weights changed out of the total, both norms and the first 12 characters of each hash. - A shape of more than 16 dimensions is not shown, and the total is left out when the count does not fit in the shape. An edit with no readable field still gets a line. -- Metadata mode: the SDK leaves out `note`. A server with `capture.mode = "metadata"` also drops `tensor`, `shape` and both hashes, and keeps `changed`, `l2` and `maxAbs`. +- A checkpoint gets one line in the same three places: `checkpoint 58899f90572c ← 0496e5111097`, the first 12 characters of the file's hash and of the base file's. With no base it reads `checkpoint 58899f90572c`. +- The session's `Weight edits` row names its checkpoints after the counts: `2 edits, 2 tensors, 66,560 weights changed, checkpoint 58899f90572c`. +- `coding.sh show` adds one `Experiments` row for each checkpoint an experiment ran with `exp run --weights`: `58899f90572c ran in edited (exp_…)`. It reads the newest 200 experiments. +- With `--json` those rows are `checkpointExperiments`, a list of `{sha256, experiments: [{id, name}]}`, absent when no experiment ran one. MCP `session_get` carries the same field. +- `GET /api/sessions/{id}` carries `checkpointExperiments` too, except for a project-scoped read key. The viewer's session header shows each experiment as a `ran in ` link. +- `exp show` and `exp compare` name the session from the other side: [Compare an edited model with its base](https://coding.sh/docs/experiments#weights). +- Metadata mode: the SDK leaves out `note`. A server with `capture.mode = "metadata"` also drops an edit's `tensor`, `shape` and both hashes, and keeps `changed`, `l2` and `maxAbs`. +- In metadata mode a checkpoint keeps `sha256` and `baseSha256`, the values that join it to an experiment, and loses `note`. - OpenTelemetry in: a span or log record named `weights.edit`, sent to `/v1/traces` or `/v1/logs`, is read as the same event. Its fields are the attributes below. A malformed one is left out and the edit still counts. - OpenTelemetry out: the `otlp` sink sends one `weights.edit` span per edit with the same attributes, as a child of the session's `invoke_agent` span. That span carries `coding_sh.stats.weight_edits` and `coding_sh.stats.weights_changed`. - `[serve.forward]` sends the same span for an edit posted to `/v1/ingest`, and passes on a `weights.edit` span posted to `/v1/traces` as it came. An edit that arrived as a log record is not forwarded. - In `metadata` mode the span keeps `coding_sh.weights.changed`, `.l2` and `.max_abs` only. +- A span or log record named `weights.checkpoint` is read as a checkpoint from `coding_sh.weights.sha256`, `.base_sha256` and `.note`. +- A checkpoint goes out the same way, as one `weights.checkpoint` span with those three attributes. In `metadata` mode it keeps the two hashes and drops `.note`. +- A session with a checkpoint and no edit has neither `coding_sh.stats` attribute. | Attribute | Payload field | |---|---| @@ -750,10 +765,12 @@ One more `notification` kind records weight edits. The Python SDK's `coding_sh.w - The caller computes the hashes and norms. Each SDK checks the values, drops an invalid edit with one warning, and never raises into the app. In Python, numpy scalars and 0-d tensors (what torch's `.norm()` returns) count as numbers. - `weights_edit()` (Python) and `weightsEdit()` (TypeScript) return true only when the event was queued. A `note` that is not a string is dropped with a warning, and the edit is still recorded. +- `weights_checkpoint(sha256, base_sha256=..., note=...)` (Python) and `weightsCheckpoint({ sha256, baseSha256, note })` (TypeScript) follow the same rules. A `sha256` or a `baseSha256` that is not 64 hex characters drops the event with one warning. +- Both hashes are sent in lowercase. coding.sh hashes nothing: you compute the file's hash, for example with `hashlib.sha256` or `shasum -a 256`. - `coding_sh.weights_delta()` computes `shape`, `changed`, `l2` and `max_abs` for two small array-likes in pure Python. It is a helper you call on purpose, so it raises `ValueError` on bad input. - The TypeScript SDK's `weightsDelta()` does the same for nested arrays and typed arrays, and throws a `TypeError` or a `RangeError`. - The server stores the payload without validating these fields, after redaction. -- A call outside `trace()` and `with_session()` (`withSession()` in TypeScript) makes a session of its own, so put a run of edits inside one of them. +- A call outside `trace()` and `with_session()` (`withSession()` in TypeScript) makes a session of its own, so put a run of edits and its checkpoint inside one of them. A missing session, or one outside a scoped key's project, gets 404: @@ -1502,7 +1519,7 @@ These fields are present only when they apply: | `samples`, `summary.caseVerdicts` | `exp run --samples adaptive` | `"samples": "adaptive"`, and verdicts `passed`, `failed`, `tieBreaks`. The `passRate` interval is then biased, so use the verdicts | | `regression` | `--stop-on-regression` stopped the run (status `cancelled`) | `{caseId, baseline}` | | `metadata.checkpoint` | A partial record written while the run was going (status `cancelled`) | `done`, `total`, `at` | -| `metadata.weights` | `exp run --weights ` | `sha256`: the SHA-256 of the checkpoint the task ran, 64 hex characters as given to the flag, stored in lowercase. `/api/compare` carries it as `a.weights` and `b.weights`, with `weightsWarning` when both are the same or only one side has one | +| `metadata.weights` | `exp run --weights ` | `sha256`: the SHA-256 of the checkpoint the task ran, 64 hex characters as given to the flag, stored in lowercase. `/api/compare` carries it as `a.weights` and `b.weights`, with `weightsWarning` when both are the same or only one side has one. `exp show --json`, `exp compare --json`, `exp list --json`, the MCP tools `experiments_list` and `experiment_compare`, `/api/compare` and `/api/experiments/{id}` add `weights.sessions`, the ids of the sessions whose `weights.checkpoint` event recorded that hash (at most 5, newest first). It is looked up when printed, never stored, and absent when no session recorded it | | `metadata.selection` | A run of part of the dataset | `from` (`{kind: "failing"\|"errors", experiment}`), `cases`, `sample`, `seed`, `firstN`, `selected`, `of` | | `metadata.seq` | `exp run --seq K` | `k`, `feedback`, `history` | | `summary.seq` | `exp run --seq K` | `k`, `feedback`, `history`, `curve` (`[{k, rate, lo, hi}]`, Seq@1..K), `egs`, `lgs`, `unchangedRate`, `costToSolve`; with 2+ trials also `passAtK` (`[{k, rate}]`), `front` (`[{protocol, k, cost, success, overall}]`) and `costUnit` (`usd` or `attempts`) | @@ -1514,6 +1531,10 @@ One experiment with its case runs (up to 5,000; `runsTruncated` says when more e 4,000 characters are cut and marked `outputTruncated`. Ids come from `coding.sh exp list`. Unknown ids, and any id for a scoped key, get 404 `{"error":"experiment not found","status":404}`. +When a session recorded the experiment's weights file as a checkpoint, a top-level `weights` object +holds `sha256` and `sessions`, as `exp show --json` prints it. The stamp itself is in +`metadata.weights`. + ```sh curl -sS "$CSH/api/experiments/$EXP_A" -H "Authorization: Bearer $READ_KEY" ``` @@ -1576,6 +1597,8 @@ These fields are present only when they apply: | `selectionWarning` | One side ran only on the other's failures (`exp run --failing`). The verdict is then `inconclusive` | | `seqK` | Both sides ran `--seq` with the same K and feedback: `k`, `feedback`, `n`, `a`, `b`, `diff` | | `seqNote` | Only one side ran `--seq`, or their K or feedback differ | +| `a.weights`, `b.weights` | The side ran with `exp run --weights`: `sha256`, and `sessions` when a session recorded that file as a checkpoint | +| `weightsWarning` | Both sides ran the same weights, or only one side records which it ran | A missing or malformed id gets 400 `{"error":"a and b must be experiment ids","status":400}`; an unknown one gets 404 `{"error":"experiment not found","status":404}`. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index d8626af0..395e40bf 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -508,7 +508,7 @@ coding.sh sessions --agent codex --since 7d ``` - --where: key:value clauses, AND by default; OR between clauses; ( ) to group; - to negate; a,b for either value; \* as a wildcard in text; > >= < <= and a..b on numbers. Bare words search prompts, replies and tool calls (every word in one event). -- Session keys: agent, model, provider, repo, launch, branch, dir, title, id, run, parent, flag, skill, hook, hook.tokens, weights.edits, weights.changed, since, until, duration, turns, tools, tool.errors, llm.errors, files, events, tests.passed, tests.failed, tokens, tokens.in, tokens.out, cost, cache.breaks, cache.lost, cache.cost, unapproved, has, is. Event keys (the session has a matching event; event keys in one group match the same event): kind, tool, ok, mcp, error, event.duration. Also sort:-cost and limit:N. Full list with examples: https://coding.sh/docs/query. +- Session keys: agent, model, provider, repo, launch, branch, dir, title, id, run, parent, flag, skill, hook, hook.tokens, weights.edits, weights.changed, weights.checkpoint, since, until, duration, turns, tools, tool.errors, llm.errors, files, events, tests.passed, tests.failed, tokens, tokens.in, tokens.out, cost, cache.breaks, cache.lost, cache.cost, unapproved, has, is. Event keys (the session has a matching event; event keys in one group match the same event): kind, tool, ok, mcp, error, event.duration. Also sort:-cost and limit:N. Full list with examples: https://coding.sh/docs/query. - --facets counts each key over the query without that key's own clauses, so every value of a picked key still shows. --limit sets the values per key (default 10). - With --json each session is a digest: its stats without stats.phases, and each derail without the event it started at (eventId, index). show <id> --json has both, and --fields stats.phases or --fields stats.derails prints them here. - With --json, an answer given before every capture was indexed carries "pending": {"files": N, "indexing": true\|false} (files still to ingest; indexing: an ingest is still running). The field is absent once all are indexed. @@ -1515,7 +1515,7 @@ coding.sh exp run [spec] [options] | `--replay ` | | Serve this recorded experiment's model calls again, up to each case-trial's first changed call; later calls run live. Adds the recorded config as a control arm (A) unless --no-control. | | `--replay-until ` | | With --replay: serve only calls 1..N from the recording; call N+1 onward runs live. | | `--series ` | | Tag the experiment as the next round of an iteration loop and print whether the loop should stop (see exp plateau) | -| `--weights ` | | Record the checkpoint the task ran: the SHA-256 of the weights file, 64 hex characters, which you compute (coding.sh hashes nothing). exp compare shows each side's and warns when both ran the same. With --arm: one per arm, as ARM=<SHA256>. Not with --watch or a matrix spec. Repeatable. | +| `--weights ` | | Record the checkpoint the task ran: the SHA-256 of the weights file, 64 hex characters, which you compute (coding.sh hashes nothing). exp compare shows each side's and warns when both ran the same; with exp show it names the session that recorded the hash as a weights.checkpoint event. With --arm: one per arm, as ARM=<SHA256>. Not with --watch or a matrix spec. Repeatable. | | `--watch` | | Rerun on every change to the spec, the task's script, grader files or --watch-path; a grader-only change regrades the stored outputs. Enter reruns, q quits. | | `--watch-path ` | | With --watch: more files whose change reruns the task (repeatable; also the spec's "watch" field) | | `--session` | `true` | In an experiment session (exp session new), add the run to its timeline and print the delta against the previous run; --no-session skips it. | diff --git a/docs/reference/query.md b/docs/reference/query.md index e1691acb..61fc6990 100644 --- a/docs/reference/query.md +++ b/docs/reference/query.md @@ -66,6 +66,7 @@ test each hit. | `hook.tokens` | session | Tokens hooks added to the context (characters / 4) | `hook.tokens>500` | | `weights.edits` | session | Weight edits the session recorded | `weights.edits>0` | | `weights.changed` | session | Weights those edits changed, added up | `weights.changed>1000` | +| `weights.checkpoint` | session | SHA-256 of a weights file the session recorded as a checkpoint, in lowercase: the whole hash, or the start of it with `*` | `weights.checkpoint:58899f90572c*` | | `since` | session | Started at or after: `7d`, `12h`, `today`, `yesterday` or a date | `since:7d` | | `until` | session | Started before (exclusive): `7d`, `12h`, `today` or a date | `until:2026-11-01` | | `duration` | session | Wall time of an ended session | `duration>10m` | @@ -223,6 +224,7 @@ tags out. | `model:switch` | Changed model mid-session. | | `model:fallback` | Recorded a fallback to another model, such as Claude Code's `--fallback-model`. | | `weights:edited` | Recorded a weight edit (`stats.weightEdits`). | +| `weights:checkpoint` | Recorded the file its weights were saved to (`stats.weightEdits.checkpoints`). | | `cache:low` | Wrote a prompt cache of at least 4,096 tokens over three or more turns, and read back less than it wrote. | | `fold:error` | Could not be folded into stats. `coding.sh ingest` tries again. | diff --git a/docs/reference/sdk.md b/docs/reference/sdk.md index 45d65e32..9219a687 100644 --- a/docs/reference/sdk.md +++ b/docs/reference/sdk.md @@ -114,8 +114,10 @@ That is AI SDK 7. On AI SDK 6 and older, pass `experimental_telemetry: { isEnabl | [`TraceHandle`](#tracehandle) | interface | | [`SpanOptions`](#spanoptions) | interface | | [`weightsEdit`](#weightseditedit) | function | +| [`weightsCheckpoint`](#weightscheckpointcheckpoint) | function | | [`weightsDelta`](#weightsdeltabase-edited) | function | | [`WeightsEdit`](#weightsedit) | interface | +| [`WeightsCheckpoint`](#weightscheckpoint) | interface | | [`WeightsDelta`](#weightsdelta) | interface | | [`WeightsArray`](#weightsarray) | type | | [`aiSdkTracer`](#aisdktraceropts) | function | @@ -249,6 +251,24 @@ function weightsEdit(edit: WeightsEdit): boolean |---|---|---| | `edit` | [`WeightsEdit`](#weightsedit) | required | +### `weightsCheckpoint(checkpoint)` + +Records one saved weights file as a `notification` event with `payload.kind` "weights.checkpoint". The Python SDK's `weights_checkpoint()` sends the same event. + +`sha256` is the hash of the whole file, as your script computed it: the value `coding.sh exp run --weights` takes, so the session that recorded the edits and the experiment that ran the file name each other. `baseSha256` is the hash of the file the edits started from. + +Returns true only when the event was queued. Returns false when a value is invalid, with one warning per process. Returns false without a warning when the SDK is off or shut down or the queue is full. Never throws. + +A `note` that is not a string is dropped with one warning; the checkpoint is still recorded. Inside `trace()` or `span()` the checkpoint joins that trace. Outside them it gets a session of its own, or a turn of the `withSession()` session, so record it inside the `trace()` that recorded the edits. + +```ts +function weightsCheckpoint(checkpoint: WeightsCheckpoint): boolean +``` + +| Parameter | Type | Default | +|---|---|---| +| `checkpoint` | [`WeightsCheckpoint`](#weightscheckpoint) | required | + ### `weightsDelta(base, edited)` Computes `{ shape, changed, l2, maxAbs }` for two arrays of the same shape, ready to spread into `weightsEdit()`. The Python SDK's `weights_delta()` gives the same numbers, with `max_abs` for `maxAbs`. @@ -390,6 +410,24 @@ interface WeightsEdit { | `editedSha256` | `string` | sha256 of the tensor after the edit: 64 hex characters, sent in lowercase. | | `note` | `string` | Free text, kept only in full capture mode. | +### `WeightsCheckpoint` + +One saved weights file, as `weightsCheckpoint()` records it. + +```ts +interface WeightsCheckpoint { + sha256: string; + baseSha256?: string; + note?: string; +} +``` + +| Field | Type | Description | +|---|---|---| +| `sha256` | `string` | sha256 of the whole weights file, the value `coding.sh exp run --weights` takes: 64 hex characters, sent in lowercase. | +| `baseSha256` | `string` | sha256 of the file the edits started from: 64 hex characters, sent in lowercase. | +| `note` | `string` | Free text, kept only in full capture mode. | + ### `WeightsDelta` What `weightsDelta()` returns. It spreads into `weightsEdit()`. @@ -524,6 +562,7 @@ Everything in `coding_sh.__all__`: | [`coding_sh.shutdown`](#coding_shshutdown) | function | | [`coding_sh.stats`](#coding_shstats) | function | | [`coding_sh.weights_edit`](#coding_shweights_edittensor-shape-changed-l2-max_abs-base_sha256-edited_sha256-) | function | +| [`coding_sh.weights_checkpoint`](#coding_shweights_checkpointsha256-) | function | | [`coding_sh.weights_delta`](#coding_shweights_deltabase-edited) | function | ### `coding_sh.__version__` @@ -704,6 +743,31 @@ def weights_edit( | `edited_sha256` | `str` | required | | `note` | `Optional[str]` | `None` | +### `coding_sh.weights_checkpoint(sha256, ...)` + +Record one saved weights file as a `notification` event with `payload.kind` "weights.checkpoint". + +sha256: the 64-hex hash of the whole file, as your script computed it. It is the value `coding.sh exp run --weights` takes, so the session that recorded the edits and the experiment that ran the file name each other. base_sha256: the hash of the file the edits started from. note: free text, kept only in full capture mode. + +Returns True only when the event was queued. Returns False when a value is invalid, with one warning per process. Returns False without a warning when the SDK is off or shut down or the queue is full. Never raises. + +A note that is not a string is dropped with one warning; the checkpoint is still recorded. Outside trace() and with_session() the checkpoint gets a session of its own, so record it inside the one that recorded the edits. + +```python +def weights_checkpoint( + sha256: str, + *, + base_sha256: Optional[str] = None, + note: Optional[str] = None, +) -> bool +``` + +| Parameter | Type | Default | +|---|---|---| +| `sha256` | `str` | required | +| `base_sha256` | `Optional[str]` | `None` | +| `note` | `Optional[str]` | `None` | + ### `coding_sh.weights_delta(base, edited)` {shape, changed, l2, max_abs} for two array-likes of the same shape, in pure Python. @@ -799,6 +863,7 @@ A constant: `"gen_ai.conversation.id"`. - TypeScript times are in milliseconds (`timeoutMs`, `flushIntervalMs`). Python timeouts are in seconds; only `flush_interval_ms` is in milliseconds. - In TypeScript, `trace()`, `span()` and `withSession()` take the function to run. In Python, they are context managers and decorators. - `aiSdkTracer()` and `observeFetch()` are TypeScript only. `coding_sh.otel` is Python only. -- `weightsEdit()` takes one object in TypeScript; `weights_edit()` takes arguments in Python. Both record the same event. `weightsDelta()` returns `maxAbs` where `weights_delta()` returns `max_abs`, and throws a `TypeError` or `RangeError` where Python raises `ValueError`. +- `weightsEdit()` and `weightsCheckpoint()` take one object in TypeScript. `weights_edit()` and `weights_checkpoint()` take arguments in Python, and `weights_checkpoint()` takes `base_sha256` and `note` by keyword only. Each pair records the same event. +- `weightsDelta()` returns `maxAbs` where `weights_delta()` returns `max_abs`, and throws a `TypeError` or `RangeError` where Python raises `ValueError`. - A weight count or dimension must be an integer below 2^53 in TypeScript; Python takes any integer. Python also takes numpy scalars as numbers, and `weights_delta()` reads anything with `.tolist()`; in TypeScript `weightsDelta()` reads arrays and typed arrays. - A long tensor name or note is cut. TypeScript cuts at 64 KiB of UTF-8 and sets `payload.truncated`; Python cuts at 65,536 characters and appends `…[truncated]`. diff --git a/docs/reference/stores.md b/docs/reference/stores.md index eed5e72d..393c94f0 100644 --- a/docs/reference/stores.md +++ b/docs/reference/stores.md @@ -187,6 +187,7 @@ What lands in `otel_traces`, with `ServiceName = 'coding.sh'`: | `execute_tool ` | tool call | `gen_ai.operation.name`, `gen_ai.tool.name`, `gen_ai.tool.call.id`, `gen_ai.agent.name`; `StatusCode` is `Error` when the call failed | | `chat ` | model call | `gen_ai.operation.name`, `gen_ai.request.model`, `gen_ai.provider.name`, `gen_ai.usage.*`, `coding_sh.cost_usd` | | `weights.edit` | weight edit | under `coding_sh.weights.`: `changed`, `l2` and `max_abs`; `mode = "full"` adds `tensor`, `shape`, `base_sha256`, `edited_sha256` and `note` | +| `weights.checkpoint` | weights checkpoint | under `coding_sh.weights.`: `sha256` and `base_sha256`, the hashes of the saved file and of its base; `mode = "full"` adds `note` | `gen_ai.usage.*` is `input_tokens`, `output_tokens`, `cache_read.input_tokens` and `cache_write.input_tokens`. `input_tokens` is the whole prompt, cache included; the uncached part @@ -197,7 +198,7 @@ trace, and `Duration` is in nanoseconds. - Map columns store every attribute as a string: cast with `toFloat64OrZero` and `toUInt64OrZero`. `coding_sh.session.flags` is a JSON array string. - Cost is on the `invoke_agent` span (the session) and on each `chat` span. Sum one, never both. -- A session that edited weights also has `weight_edits` and `weights_changed` under `coding_sh.stats.`, and one `weights.edit` span per edit. +- A session that edited weights also has `weight_edits` and `weights_changed` under `coding_sh.stats.`, and one `weights.edit` span per edit. Each checkpoint it recorded is one `weights.checkpoint` span. - `mode = "full"` adds prompts, code and tool output as span attributes. - The `invoke_agent` span is sent when the session ends. `coding.sh serve` and `coding.sh collector` also send it while a session is open: after 10 quiet seconds, and every 60 seconds while busy. diff --git a/packages/coding.sh/scripts/sdk-reference.ts b/packages/coding.sh/scripts/sdk-reference.ts index 067a30ff..6128e99b 100644 --- a/packages/coding.sh/scripts/sdk-reference.ts +++ b/packages/coding.sh/scripts/sdk-reference.ts @@ -665,9 +665,13 @@ function renderTsExport(e: TsExport, anchors: Map, options: Map< return out; } +/** The parameters a caller passes. A bare `*` is not one: it only makes the ones after it keyword-only, and the declaration shows it. */ +export const pyNamed = (a: PyApi): PyParam[] => a.params.filter((p) => p.name !== "*"); + function pyHeading(prefix: string, a: PyApi): string { if (a.kind !== "function") return `\`${prefix}${a.name}\``; - const args = [...a.params.filter((p) => p.dflt === undefined).map((p) => p.name), ...(a.params.some((p) => p.dflt !== undefined) ? ["..."] : [])]; + const params = pyNamed(a); + const args = [...params.filter((p) => p.dflt === undefined).map((p) => p.name), ...(params.some((p) => p.dflt !== undefined) ? ["..."] : [])]; return `\`${prefix}${a.name}(${args.join(", ")})\``; } @@ -689,7 +693,7 @@ function renderPy(prefix: string, a: PyApi, options?: Map): } } else { out.push("| Parameter | Type | Default |", "|---|---|---|"); - for (const p of a.params) out.push(`| \`${p.name}\` | ${p.type ? cell(code(p.type)) : ""} | ${p.dflt !== undefined ? cell(code(p.dflt)) : "required"} |`); + for (const p of pyNamed(a)) out.push(`| \`${p.name}\` | ${p.type ? cell(code(p.type)) : ""} | ${p.dflt !== undefined ? cell(code(p.dflt)) : "required"} |`); } out.push(""); } @@ -789,7 +793,8 @@ export function renderSdkReference(): string { "- TypeScript times are in milliseconds (`timeoutMs`, `flushIntervalMs`). Python timeouts are in seconds; only `flush_interval_ms` is in milliseconds.", "- In TypeScript, `trace()`, `span()` and `withSession()` take the function to run. In Python, they are context managers and decorators.", "- `aiSdkTracer()` and `observeFetch()` are TypeScript only. `coding_sh.otel` is Python only.", - "- `weightsEdit()` takes one object in TypeScript; `weights_edit()` takes arguments in Python. Both record the same event. `weightsDelta()` returns `maxAbs` where `weights_delta()` returns `max_abs`, and throws a `TypeError` or `RangeError` where Python raises `ValueError`.", + "- `weightsEdit()` and `weightsCheckpoint()` take one object in TypeScript. `weights_edit()` and `weights_checkpoint()` take arguments in Python, and `weights_checkpoint()` takes `base_sha256` and `note` by keyword only. Each pair records the same event.", + "- `weightsDelta()` returns `maxAbs` where `weights_delta()` returns `max_abs`, and throws a `TypeError` or `RangeError` where Python raises `ValueError`.", "- A weight count or dimension must be an integer below 2^53 in TypeScript; Python takes any integer. Python also takes numpy scalars as numbers, and `weights_delta()` reads anything with `.tolist()`; in TypeScript `weightsDelta()` reads arrays and typed arrays.", "- A long tensor name or note is cut. TypeScript cuts at 64 KiB of UTF-8 and sets `payload.truncated`; Python cuts at 65,536 characters and appends `…[truncated]`.", "", diff --git a/packages/coding.sh/src/cli/commands/exp.ts b/packages/coding.sh/src/cli/commands/exp.ts index 22dea01b..7aa5be5b 100644 --- a/packages/coding.sh/src/cli/commands/exp.ts +++ b/packages/coding.sh/src/cli/commands/exp.ts @@ -44,6 +44,7 @@ import { sortableId } from "../../core/ids.ts"; import { jobConfig, taskFingerprint } from "../../jobs/config.ts"; import type { Job, JobKind } from "../../jobs/store.ts"; import { positiveInt, UsageError } from "../args.ts"; +import { checkpointSessions, withSessions } from "../../experiments/checkpoints.ts"; import { parseWeights, shortWeights, stampedWith, weightsOf, withWeights } from "../../experiments/weights.ts"; import { finishJob, infraStopNote, openJob, resumeField, resumeNote, retryFlag, statusCommand, trialTimeout } from "../jobs.ts"; import { gateFlags, jsonArg, list, num, shellQuote, str, withContext } from "../common.ts"; @@ -1152,7 +1153,11 @@ export async function run(c: CommandContext): Promise { const hold = pws.length && cmp.gate?.verdict === "ship" ? judgeHold(replication ?? replicatePairwise([pws[0]!.summary], [pws[0]!.meta])) : undefined; if (hold && cmp.gate) Object.assign(cmp.gate, { decision: "abstain", verdict: "hold", judgeHold: hold }); const calField = (x: (typeof pws)[number]) => (x.calibration ? { ...x.calibration.report, abstained: x.calibration.abstained } : undefined); + // The sessions that recorded either side's weights file as a checkpoint (a weights.checkpoint event). + const checkpoints = await checkpointSessions(ctx.index, [cmp.a.weights?.sha256, cmp.b.weights?.sha256]); if (out.json) { + withSessions(cmp.a, checkpoints); + withSessions(cmp.b, checkpoints); out.data({ comparison: cmp, ...(pw ? { pairwise: pw.summary } : {}), @@ -1162,7 +1167,7 @@ export async function run(c: CommandContext): Promise { }); } else { const ds = await store.getDataset(b!.dataset.id, b!.dataset.version).catch(() => undefined); - out.print(renderComparison(cmp, { color: out.color, a: a!, b: b!, matrix: c.flags.matrix === true, cases: ds ? caseLabels(ds.cases) : undefined })); + out.print(renderComparison(cmp, { color: out.color, a: a!, b: b!, matrix: c.flags.matrix === true, cases: ds ? caseLabels(ds.cases) : undefined, checkpoints })); for (const x of pws) { out.print(`\n${renderPairwise(x.summary, { color: out.color })}`); if (x.calibration) { @@ -1305,10 +1310,18 @@ export async function run(c: CommandContext): Promise { } if (!e) throw new CliError(`Can't find experiment ${JSON.stringify(ref)}`, { reason: "not_found", next: ["List experiments:", "coding.sh exp list"] }); const verifiers = await verifierAudits(store, e); - if (out.json) out.data({ experiment: e, ...(verifiers.size ? { verifiers: [...verifiers.values()] } : {}) }); - else { + // The sessions that recorded the experiment's weights file as a checkpoint (a weights.checkpoint event). + const checkpoints = await checkpointSessions(ctx.index, [weightsOf(e)]); + if (out.json) { + const doc: Record = { experiment: e }; + // Next to the experiment, so the stored record prints as it is: its stamp with the ids of those sessions. + const { weights } = withSessions(stampedWith({}, weightsOf(e)), checkpoints); + if (weights?.sessions) doc.weights = weights; + if (verifiers.size) doc.verifiers = [...verifiers.values()]; + out.data(doc); + } else { const ds = await store.getDataset(e.dataset.id, e.dataset.version).catch(() => undefined); - out.print(renderExperiment(e, { color: out.color, verifiers, cases: ds ? caseLabels(ds.cases) : undefined })); + out.print(renderExperiment(e, { color: out.color, verifiers, cases: ds ? caseLabels(ds.cases) : undefined, checkpoints })); } return 0; } @@ -1316,8 +1329,9 @@ export async function run(c: CommandContext): Promise { const limit = positiveInt(c.flags.limit, "limit") ?? 20; const all = await store.listExperiments({ limit }); if (out.json) { - // Each row carries the checkpoint its run recorded (exp run --weights); no key when it recorded none. - out.data({ experiments: all.map((e) => stampedWith({ id: e.id, name: e.name, status: e.status ?? "completed", ...(isCheckpoint(e) ? { partial: true } : {}), dataset: e.dataset, trials: e.trials, runs: e.runs.length, createdAt: new Date(e.createdAt).toISOString(), summary: e.summary }, weightsOf(e))) }); + // Each row carries the checkpoint its run recorded (exp run --weights), with the sessions that recorded that file; no key when it recorded none. + const checkpoints = await checkpointSessions(ctx.index, all.map((e) => weightsOf(e))); + out.data({ experiments: all.map((e) => withSessions(stampedWith({ id: e.id, name: e.name, status: e.status ?? "completed", ...(isCheckpoint(e) ? { partial: true } : {}), dataset: e.dataset, trials: e.trials, runs: e.runs.length, createdAt: new Date(e.createdAt).toISOString(), summary: e.summary }, weightsOf(e)), checkpoints)) }); return 0; } if (!all.length) { diff --git a/packages/coding.sh/src/cli/commands/show.ts b/packages/coding.sh/src/cli/commands/show.ts index 9643b785..8e892889 100644 --- a/packages/coding.sh/src/cli/commands/show.ts +++ b/packages/coding.sh/src/cli/commands/show.ts @@ -11,8 +11,9 @@ import { costNote } from "../../core/pricing.ts"; import { failureFlags } from "../../core/sessions.ts"; import { SKILL_EVENT_KINDS, SKILL_TAG_PREFIX, skillUse, type SkillUseReport } from "../../core/skills-use.ts"; import { pageTree, subagentsOf, type TreeNode } from "../../core/tree.ts"; -import { WEIGHTS_EDIT_KIND, weightEditsText, weightsEditText } from "../../core/weights.ts"; +import { checkpointRunsText, weightsEventText, weightsSummaryText } from "../../core/weights.ts"; import type { AgentEvent, Session } from "../../core/types.ts"; +import type { CheckpointRun } from "../../experiments/checkpoints.ts"; import type { DurableStats } from "../../schema/index.ts"; import { positiveInt } from "../args.ts"; import { CallRefError, loadCall, resolveCall } from "../../proxy/calls.ts"; @@ -147,11 +148,14 @@ function describe(e: AgentEvent, starts: Map): string | unde case "session.end": case "turn.end": case "context.compact": - case "notification": + case "notification": { if (p.kind === "harness.loaded") return `harness ${oneLine(clean(harnessText(p)), 90)}`; - if (p.kind === WEIGHTS_EDIT_KIND) return `weights ${oneLine(clean(weightsEditText(p)), 140)}`; + // A weight edit or a checkpoint: one line from core/weights.ts. + const weights = weightsEventText(p); + if (weights !== undefined) return `weights ${oneLine(clean(weights), 140)}`; if (p.kind === "hook.context") return `hook ${str(p.hookEvent) ?? ""} ${str(p.hookSource) ?? "unknown"} ~${tokens(typeof p.tokens === "number" ? p.tokens : undefined)} tok ${oneLine(clean(str(p.text) ?? ""), 70)}`; return `${`${e.kind} `.padEnd(12)}${oneLine(clean(str(p.text) ?? str(p.message) ?? str(p.reason) ?? ""), 80)}`; + } default: return `${`${e.kind} `.padEnd(12)}${oneLine(clean(str(p.name) ?? ""), 80)}`; } @@ -334,6 +338,18 @@ export interface SnapshotChanges { files: FileDiff[]; } +/** + * Each checkpoint of the session that an experiment ran, in the order the session recorded them, with those + * experiments (newest first). Empty for a session with no checkpoint, which reads no experiment. + */ +async function ranIn(index: IndexStore, s: Session, env: NodeJS.ProcessEnv): Promise { + const hashes = s.stats.weightEdits?.checkpoints; + if (!hashes?.length) return []; + // Loaded here, so showing a session with no checkpoint loads no experiment code. + const [{ FileEvalStore }, { checkpointRuns }] = await Promise.all([import("../../datasets/store.ts"), import("../../experiments/checkpoints.ts")]); + return checkpointRuns(new FileEvalStore({ env, index }), hashes); +} + /** * What the working tree gained between the session's start state and its end * snapshot. Hooks only emit file.change for edit tools, so a file written by @@ -415,6 +431,8 @@ export async function run(c: CommandContext): Promise { if (changes?.startKind === "snapshot") for (const f of changes.files) changed.add(f.path); const filesChanged = Math.max(s.stats.filesChanged, changed.size); const skills = await sessionSkills(ctx.index, s); + // The experiments that ran each weights file the session recorded as a checkpoint (exp run --weights with the same hash). + const runs = await ranIn(ctx.index, s, c.env); if (out.json) { // A phased session answers with its phases, as the text does; --all or --limit lists the events. const listed = phased ? [] : ((all ? full : events) ?? []); @@ -432,6 +450,7 @@ export async function run(c: CommandContext): Promise { // The count alone does not say where the rest are. if (subagents.next !== undefined) Object.assign(doc, { subagentsTruncated: true, subagentsRest: `coding.sh tree ${s.id} --json` }); doc.skills = skills?.used ?? []; + if (runs.length) doc.checkpointExperiments = runs; Object.assign(doc, pendingField(backlog)); out.data(fields ? pickFields(doc, fields) : doc); return 0; @@ -477,7 +496,8 @@ export async function run(c: CommandContext): Promise { if (red) rows.push(red); const hooks = hookContextRow(s); if (hooks) rows.push(hooks); - if (s.stats.weightEdits) rows.push(["Weight edits", weightEditsText(s.stats.weightEdits)]); + if (s.stats.weightEdits) rows.push(["Weight edits", weightsSummaryText(s.stats.weightEdits)]); + for (const x of runs) rows.push(["Experiments", oneLine(clean(checkpointRunsText(x.sha256, x.experiments)), 140)]); if (skills?.used.length) rows.push(["Skills", skillsText(skills)]); if (derails.length) rows.push(["Derails", derails.slice(0, 4).map((d) => `${d.flag} at +${duration(Math.max(0, d.ts - s.startedAt))}`).join(", ") + (derails.length > 4 ? ` +${derails.length - 4} more` : "")]); for (const [k, v] of rows) out.print(` ${k.padEnd(14)}${v}`); diff --git a/packages/coding.sh/src/cli/registry.ts b/packages/coding.sh/src/cli/registry.ts index 7684832c..cb95f180 100644 --- a/packages/coding.sh/src/cli/registry.ts +++ b/packages/coding.sh/src/cli/registry.ts @@ -1122,7 +1122,7 @@ export const COMMANDS: CommandSpec[] = [ name: "weights", type: "list", value: "", - desc: "Record the checkpoint the task ran: the SHA-256 of the weights file, 64 hex characters, which you compute (coding.sh hashes nothing). exp compare shows each side's and warns when both ran the same. With --arm: one per arm, as ARM=. Not with --watch or a matrix spec", + desc: "Record the checkpoint the task ran: the SHA-256 of the weights file, 64 hex characters, which you compute (coding.sh hashes nothing). exp compare shows each side's and warns when both ran the same; with exp show it names the session that recorded the hash as a weights.checkpoint event. With --arm: one per arm, as ARM=. Not with --watch or a matrix spec", }, { name: "watch", type: "boolean", desc: "Rerun on every change to the spec, the task's script, grader files or --watch-path; a grader-only change regrades the stored outputs. Enter reruns, q quits" }, { name: "watch-path", type: "list", value: "", desc: "With --watch: more files whose change reruns the task (repeatable; also the spec's \"watch\" field)" }, diff --git a/packages/coding.sh/src/core/redact.ts b/packages/coding.sh/src/core/redact.ts index 9e5e73f7..e5d027c3 100644 --- a/packages/coding.sh/src/core/redact.ts +++ b/packages/coding.sh/src/core/redact.ts @@ -3,6 +3,7 @@ // name next to the value, so ordinary code, paths and sha hashes pass through. import { statSync } from "node:fs"; import type { AgentEvent, Session } from "../schema/index.ts"; +import { weightsCheckpointOf } from "./weights.ts"; export interface RedactRule { kind: string; @@ -794,7 +795,8 @@ const METADATA_STRING_KEYS = new Set([ /** * Metadata-only view of an event, for remote sinks and capture.mode="metadata": * keeps kind, ids, timings, tool names, ok flags, counts and usage; drops text, - * tool input/output, paths, cwd, repo root and blob refs. + * tool input/output, paths, cwd, repo root and blob refs. A weights checkpoint + * keeps its two file hashes. */ export function toMetadata(event: AgentEvent): AgentEvent { const payload: Record = {}; @@ -804,6 +806,13 @@ export function toMetadata(event: AgentEvent): AgentEvent { // The tools a model call offered, by name (the proxy), like `tool` on tool events. else if (k === "toolNames" && Array.isArray(v)) payload[k] = v.filter((n): n is string => typeof n === "string" && n.length <= 200).slice(0, 128); } + // A weights checkpoint keeps the hashes of its two files: they are the key that joins the session + // to the experiments that ran the file (which store the same value in any mode). The note is dropped. + const checkpoint = event.kind === "notification" ? weightsCheckpointOf(event.payload ?? {}) : undefined; + if (checkpoint) { + payload.sha256 = checkpoint.sha256; + if (checkpoint.baseSha256) payload.baseSha256 = checkpoint.baseSha256; + } // Redaction counts (kind → number, never values): the secret-exposure signal survives metadata mode. const counts = redactionCounts(event.payload?.redactions); if (counts) payload.redactions = counts; diff --git a/packages/coding.sh/src/core/sessions.ts b/packages/coding.sh/src/core/sessions.ts index de946f8d..1c328020 100644 --- a/packages/coding.sh/src/core/sessions.ts +++ b/packages/coding.sh/src/core/sessions.ts @@ -21,7 +21,7 @@ import { foldWaste, wasteState, wasteStatsOf, type WasteState } from "./waste.ts import { foldModels, mainModelOf, MODEL_FALLBACK, MODEL_SWITCH, modelFlags, modelState, modelSwitchOf, modelsOf, REFUSAL, type ModelState } from "./models.ts"; import { exposesSecret, foldRedactions, redactionsOf, redactionState, SECRET_EXPOSED, type RedactionState } from "./redaction-stats.ts"; import { foldUnapproved, unapprovedOf, unapprovedState, type UnapprovedState } from "./unapproved.ts"; -import { foldWeightEdits, weightEditFlags, weightEditsOf, weightEditState, WEIGHTS_EDITED, type WeightEditState } from "./weights.ts"; +import { foldWeightEdits, weightEditFlags, weightEditsOf, weightEditState, WEIGHTS_CHECKPOINT, WEIGHTS_EDITED, type WeightEditState } from "./weights.ts"; export { REOPEN_SLACK_MS }; @@ -58,6 +58,8 @@ export const FLAGS = { cacheLow: "cache:low", /** A tag, not a failure: the session recorded a weight edit (core/weights.ts, Session.stats.weightEdits). */ weightsEdited: WEIGHTS_EDITED, + /** A tag, not a failure: the session recorded the hash of a weights file it saved (core/weights.ts, Session.stats.weightEdits.checkpoints). */ + weightsCheckpoint: WEIGHTS_CHECKPOINT, /** A tag, not a failure: coding.sh could not fold the session's events into its stats (Session.stats.foldError, core/ingest.ts recomputeSession). */ foldError: "fold:error", /** Derails (core/derail.ts): where the session went off the rails. */ @@ -458,7 +460,7 @@ function turnCount(t: { live: number; backfill: number; liveIdless: number; back * new derivation. A new or retyped sub-state changes the stamp by itself; a * new optional field or a change of meaning still needs the bump. */ -export const SESSION_FOLD_VERSION = 20; // 3: TamperState tracks shell calls, own lines and source edits since the last green run. 4: skip markers inside string literals are not skips. 5: proxy calls (sansProxy, otherCalls). 6: hook context (stats.hookContext, hook: tags). 7: stats.launch. 8: premature-completion reads what the agent said as it stopped (PrematureState.said, reported). 9: models and proxyModels (stats.models, stats.modelSwitch, model:switch, refusal; model = the main thread's most used). 10: text-tool-call (FlagState.textToolCall). 11: cache:low tag (core/cache-breaks.ts cacheUnderused). 12: model switches per scope (recorded fallbacks, subagents; ModelSwitch count, switchedCalls, extraUsd; model:fallback). 13: idle cache breaks split by where the wait was (CacheBreak.idle, CacheBreakStats.byIdle). 14: unapproved-action (FlagState.unapproved, stats.unapprovedActions). 15: stats.inheritedTokens (a fork subagent's inherited context). 16: tool failures classified (core/failures.ts): toolErrors counts real ones, stats.benignToolErrors and stats.interruptedToolCalls the rest, repeated-tool-failure and phase toolErrors skip them; a REPL cell's bash()/subprocess commands count as verification (core/cells.ts). 17: a session.start an older ingest stamped with a harness marker names no parent (isSpawnMarker), so a ghost harness parent goes; a gateway record with no cost bills nothing (billedCallOf), so the call's cost stays in usage.estimatedCostUsd. 18: stats.memoryUse, the memory and instruction files a session's events name (core/memory-use.ts, SessionState.memory). 19: a session with transcript model calls counts them, not the copies its OTLP logs carry (sansOtel, transcriptCalls). 20: weight edits (stats.weightEdits, the weights:edited tag, SessionState.weightEdits). +export const SESSION_FOLD_VERSION = 21; // 3: TamperState tracks shell calls, own lines and source edits since the last green run. 4: skip markers inside string literals are not skips. 5: proxy calls (sansProxy, otherCalls). 6: hook context (stats.hookContext, hook: tags). 7: stats.launch. 8: premature-completion reads what the agent said as it stopped (PrematureState.said, reported). 9: models and proxyModels (stats.models, stats.modelSwitch, model:switch, refusal; model = the main thread's most used). 10: text-tool-call (FlagState.textToolCall). 11: cache:low tag (core/cache-breaks.ts cacheUnderused). 12: model switches per scope (recorded fallbacks, subagents; ModelSwitch count, switchedCalls, extraUsd; model:fallback). 13: idle cache breaks split by where the wait was (CacheBreak.idle, CacheBreakStats.byIdle). 14: unapproved-action (FlagState.unapproved, stats.unapprovedActions). 15: stats.inheritedTokens (a fork subagent's inherited context). 16: tool failures classified (core/failures.ts): toolErrors counts real ones, stats.benignToolErrors and stats.interruptedToolCalls the rest, repeated-tool-failure and phase toolErrors skip them; a REPL cell's bash()/subprocess commands count as verification (core/cells.ts). 17: a session.start an older ingest stamped with a harness marker names no parent (isSpawnMarker), so a ghost harness parent goes; a gateway record with no cost bills nothing (billedCallOf), so the call's cost stays in usage.estimatedCostUsd. 18: stats.memoryUse, the memory and instruction files a session's events name (core/memory-use.ts, SessionState.memory). 19: a session with transcript model calls counts them, not the copies its OTLP logs carry (sansOtel, transcriptCalls). 20: weight edits (stats.weightEdits, the weights:edited tag, SessionState.weightEdits). 21: weight checkpoints (stats.weightEdits.checkpoints, the weights:checkpoint tag), and stats.weightEdits on a session with a checkpoint and no edit. /** * A session's events folded so far: everything buildSession needs from them, @@ -1194,7 +1196,7 @@ function flagsOf(s: FlagState, derails: readonly Derail[], sorted?: readonly Age * A session with nothing to look at: no turn, no tool call and no model call * (a hook that fired with no agent work behind it, such as a wrapper process * or a subagent that never ran). A changed file, a start snapshot (what - * `coding.sh run` takes) or a recorded weight edit is something to look at. `coding.sh sessions` hides + * `coding.sh run` takes) or a recorded weight edit or checkpoint is something to look at. `coding.sh sessions` hides * the empty ones unless --empty. Applied when listing, not when folding. */ export function isEmptySession(s: Pick & Partial>): boolean { diff --git a/packages/coding.sh/src/core/weights.ts b/packages/coding.sh/src/core/weights.ts index 3cfe122f..28329216 100644 --- a/packages/coding.sh/src/core/weights.ts +++ b/packages/coding.sh/src/core/weights.ts @@ -2,13 +2,26 @@ // weights_edit helpers, or any client of /v1/ingest), folded into // Session.stats.weightEdits and the `weights:edited` session tag, and read // into one line of text for the CLI, the MCP server and the viewer's rows. +// +// A checkpoint: one `notification` with `payload.kind: "weights.checkpoint"` and +// the SHA-256 of the weights file the session saved. An edit hashes one tensor; +// `exp run --weights` records the hash of the whole file. The checkpoint event +// carries that file hash, so a session of edits and the experiments that ran its +// result name each other (Session.stats.weightEdits.checkpoints, the +// `weights:checkpoint` tag, `--where weights.checkpoint:`). import type { AgentEvent, WeightEditStats } from "../schema/index.ts"; +import { isSha256Hex } from "./ids.ts"; export const WEIGHTS_EDIT_KIND = "weights.edit"; /** A tag, not a failure: the session recorded at least one weight edit. */ export const WEIGHTS_EDITED = "weights:edited"; +export const WEIGHTS_CHECKPOINT_KIND = "weights.checkpoint"; +/** A tag, not a failure: the session recorded the hash of a weights file it saved. */ +export const WEIGHTS_CHECKPOINT = "weights:checkpoint"; /** Distinct tensor names kept per session; `tensors` stops counting there. */ const TENSORS_MAX = 10_000; +/** Distinct checkpoint hashes kept per session, in arrival order; later ones are not kept. */ +export const CHECKPOINTS_MAX = 16; /** Dimensions a shape may have (what the SDKs accept); a longer one is not read as a shape. */ const SHAPE_MAX = 16; @@ -17,9 +30,11 @@ export interface WeightEditState { events: number; weightsChanged: number; tensors: Set; + /** Distinct file hashes of the session's weights.checkpoint events, in arrival order (CHECKPOINTS_MAX). */ + checkpoints: string[]; } -export const weightEditState = (): WeightEditState => ({ events: 0, weightsChanged: 0, tensors: new Set() }); +export const weightEditState = (): WeightEditState => ({ events: 0, weightsChanged: 0, tensors: new Set(), checkpoints: [] }); /** True for a weights.edit event. */ export const isWeightsEdit = (e: AgentEvent): boolean => e.kind === "notification" && e.payload.kind === WEIGHTS_EDIT_KIND; @@ -28,6 +43,15 @@ const count = (v: unknown): number | undefined => (typeof v === "number" && Numb const norm = (v: unknown): number | undefined => (typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : undefined); const text = (v: unknown): string | undefined => (typeof v === "string" && v ? v : undefined); +/** A SHA-256 as stored: trimmed and lowercased; undefined when it is not 64 hex characters. */ +export function parseWeights(v: unknown): string | undefined { + const s = typeof v === "string" ? v.trim().toLowerCase() : undefined; + return isSha256Hex(s) ? s : undefined; +} + +/** The first 12 hex characters: how every weights hash (a tensor's, a file's) is written in a line of text. */ +export const shortWeights = (sha256: string): string => sha256.slice(0, 12); + /** The fields of a weights.edit payload that are present and well-formed. Metadata capture keeps only the numbers. */ export interface WeightsEdit { tensor?: string; @@ -80,7 +104,59 @@ export function weightsEditAttrs(p: Record): Record): WeightsCheckpoint | undefined { + const sha256 = p.kind === WEIGHTS_CHECKPOINT_KIND ? parseWeights(p.sha256) : undefined; + if (sha256 === undefined) return undefined; + const out: WeightsCheckpoint = { sha256 }; + const base = parseWeights(p.baseSha256); + if (base !== undefined) out.baseSha256 = base; + const note = text(p.note); + if (note !== undefined) out.note = note; + return out; +} + +/** True for a weights.checkpoint event with a well-formed `sha256`. */ +export const isWeightsCheckpoint = (e: AgentEvent): boolean => e.kind === "notification" && weightsCheckpointOf(e.payload) !== undefined; + +/** True for an event that gets a `weights` line: a weight edit, or a checkpoint. */ +export const isWeightsEvent = (e: AgentEvent): boolean => isWeightsEdit(e) || isWeightsCheckpoint(e); + +/** + * OTLP: a span or a log record named `weights.checkpoint` carries the checkpoint in these + * attributes. The base hash and the note share their names with a weight edit's. + */ +export const WEIGHTS_CHECKPOINT_ATTRS: Record = { + sha256: "coding_sh.weights.sha256", + baseSha256: WEIGHTS_EDIT_ATTRS.baseSha256, + note: WEIGHTS_EDIT_ATTRS.note, +}; + +/** A weights.checkpoint payload as OTLP attributes; empty when it is not a checkpoint. */ +export function weightsCheckpointAttrs(p: Record): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(weightsCheckpointOf(p) ?? {})) out[WEIGHTS_CHECKPOINT_ATTRS[k as keyof WeightsCheckpoint]] = v; + return out; +} + export function foldWeightEdits(s: WeightEditState, e: AgentEvent): void { + if (e.kind !== "notification") return; + const c = weightsCheckpointOf(e.payload); + if (c && s.checkpoints.length < CHECKPOINTS_MAX && !s.checkpoints.includes(c.sha256)) s.checkpoints.push(c.sha256); if (!isWeightsEdit(e)) return; const w = weightsEditOf(e.payload); s.events++; @@ -88,15 +164,24 @@ export function foldWeightEdits(s: WeightEditState, e: AgentEvent): void { if (w.tensor && s.tensors.size < TENSORS_MAX) s.tensors.add(w.tensor); } -/** Session.stats.weightEdits, or undefined when the session recorded no weight edit. */ +/** + * Session.stats.weightEdits, or undefined when the session recorded no weight edit and no + * checkpoint. A session with a checkpoint and no edit has `events: 0`. + */ export function weightEditsOf(s: WeightEditState | undefined): WeightEditStats | undefined { - if (!s?.events) return undefined; + if (!s || (!s.events && !s.checkpoints.length)) return undefined; const out: WeightEditStats = { events: s.events, weightsChanged: s.weightsChanged }; if (s.tensors.size) out.tensors = s.tensors.size; + if (s.checkpoints.length) out.checkpoints = [...s.checkpoints]; return out; } -export const weightEditFlags = (s: WeightEditState | undefined): string[] => (s?.events ? [WEIGHTS_EDITED] : []); +export function weightEditFlags(s: WeightEditState | undefined): string[] { + const out: string[] = []; + if (s?.events) out.push(WEIGHTS_EDITED); + if (s?.checkpoints.length) out.push(WEIGHTS_CHECKPOINT); + return out; +} const int = (n: number): string => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ","); /** Four significant digits; a number that rounds past the largest double is written as it is. */ @@ -104,11 +189,10 @@ const real = (n: number): string => { const r = Number(n.toPrecision(4)); return String(Number.isFinite(r) ? r : n); }; -const short = (sha: string): string => sha.slice(0, 8); /** * One line for a weights.edit payload, most useful part first: - * `layers.3.mlp.down_proj.weight [4096×11008] · 1,204 of 45,088,768 weights · L2 0.0183 · max 0.004 · 9f2c1a7e → 4b1d0c33`. + * `layers.3.mlp.down_proj.weight [4096×11008] · 1,204 of 45,088,768 weights · L2 0.0183 · max 0.004 · 9f2c1a7e03b5 → 4b1d0c33a9e2`. * A part whose field is missing is left out, so an event captured in metadata mode reads * `1,204 weights · L2 0.0183 · max 0.004`. */ @@ -125,14 +209,53 @@ export function weightsEditText(p: Record): string { } if (w.l2 !== undefined) parts.push(`L2 ${real(w.l2)}`); if (w.maxAbs !== undefined) parts.push(`max ${real(w.maxAbs)}`); - if (w.baseSha256 && w.editedSha256) parts.push(`${short(w.baseSha256)} → ${short(w.editedSha256)}`); + if (w.baseSha256 && w.editedSha256) parts.push(`${shortWeights(w.baseSha256)} → ${shortWeights(w.editedSha256)}`); return parts.join(" · "); } -/** The session line: `2 edits, 2 tensors, 66,560 weights changed`. */ +/** + * One line for a weights.checkpoint payload: `checkpoint 58899f90572c ← 0496e5111097`, the + * saved file's hash and the base file's, or `checkpoint 58899f90572c` when no base was + * recorded. Empty when the payload is not a checkpoint. + */ +export function weightsCheckpointText(p: Record): string { + const c = weightsCheckpointOf(p); + if (!c) return ""; + return `checkpoint ${shortWeights(c.sha256)}${c.baseSha256 ? ` ← ${shortWeights(c.baseSha256)}` : ""}`; +} + +/** + * The `weights` line of an event's payload, for `coding.sh show`, MCP session_get and the + * viewer's rows: a weight edit's (empty when no field is readable) or a checkpoint's. + * Undefined when the payload is neither. + */ +export function weightsEventText(p: Record): string | undefined { + if (p.kind === WEIGHTS_EDIT_KIND) return weightsEditText(p); + return weightsCheckpointOf(p) ? weightsCheckpointText(p) : undefined; +} + +/** The edits of a session: `2 edits, 2 tensors, 66,560 weights changed`, or `no edits` for a session that only recorded a checkpoint. */ export function weightEditsText(w: WeightEditStats): string { + if (!w.events) return "no edits"; const parts = [`${int(w.events)} ${w.events === 1 ? "edit" : "edits"}`]; if (w.tensors) parts.push(`${int(w.tensors)} ${w.tensors === 1 ? "tensor" : "tensors"}`); parts.push(`${int(w.weightsChanged)} ${w.weightsChanged === 1 ? "weight" : "weights"} changed`); return parts.join(", "); } + +/** + * The session's summary row: its edits, then the checkpoints it recorded. + * `2 edits, 2 tensors, 66,560 weights changed, checkpoint 58899f90572c`; a session with a + * checkpoint and no edit reads `checkpoint 58899f90572c`. + */ +export function weightsSummaryText(w: WeightEditStats): string { + const parts = w.events ? [weightEditsText(w)] : []; + const cps = w.checkpoints ?? []; + if (cps.length) parts.push(`${cps.length === 1 ? "checkpoint" : "checkpoints"} ${cps.map(shortWeights).join(", ")}`); + return parts.join(", "); +} + +/** `58899f90572c ran in edited (exp_…), edited-again (exp_…)`: the experiments stamped with one checkpoint's hash. */ +export function checkpointRunsText(sha256: string, experiments: readonly { id: string; name: string }[]): string { + return `${shortWeights(sha256)} ran in ${experiments.map((e) => `${e.name} (${e.id})`).join(", ")}`; +} diff --git a/packages/coding.sh/src/experiments/checkpoints.ts b/packages/coding.sh/src/experiments/checkpoints.ts new file mode 100644 index 00000000..c104ee66 --- /dev/null +++ b/packages/coding.sh/src/experiments/checkpoints.ts @@ -0,0 +1,131 @@ +// The join between sessions and experiments on a weights file's SHA-256. +// A session records the hash of the file it saved in a `weights.checkpoint` +// event (Session.stats.weightEdits.checkpoints); an experiment records the hash +// of the file its task ran in its weights stamp (`exp run --weights`). The same +// hash on both sides says which session's edits an experiment evaluated, and +// which experiments ran what a session produced. +// +// Both lookups are extras on top of what a command prints anyway: when the +// store can't answer (a store plugin without the key, no experiments), they +// return nothing and the caller prints nothing more. +import type { IndexStore } from "../core/types.ts"; +import { weightEditsText } from "../core/weights.ts"; +import type { ExperimentResult } from "../datasets/types.ts"; +import { findSessions } from "../query/run.ts"; +import { parseWhere } from "../query/where.ts"; +import type { WeightEditStats } from "../schema/index.ts"; +import { parseWeights, weightsOf, type WeightsStamp } from "./weights.ts"; + +/** Sessions named per weights hash, newest first. */ +export const CHECKPOINT_SESSIONS_MAX = 5; +/** Sessions read per query. */ +const SCAN_MAX = 1000; +/** + * Hashes asked for in one query. A `--where` query holds 2,000 characters at most (parseWhere), and + * `weights.checkpoint:` with 25 hashes and their commas is 1,643: a list of experiments with more + * distinct stamps than that is looked up in several queries. + */ +const HASHES_PER_QUERY = 25; +/** Experiments read to find the ones that ran a session's checkpoints, newest first. */ +const EXPERIMENTS_MAX = 200; + +/** A session whose `weights.checkpoint` event recorded a hash, with the edits it recorded. */ +export interface CheckpointSession { + id: string; + edits: WeightEditStats; +} + +/** Weights hash → the sessions that recorded it as a checkpoint. A hash no session recorded has no entry. */ +export type CheckpointSessions = ReadonlyMap; + +/** + * The sessions that recorded each of these hashes as a checkpoint: newest first, at most + * CHECKPOINT_SESSIONS_MAX per hash. Values that are not a SHA-256 are skipped. Empty when + * none matches or the session index can't answer the query. + */ +export async function checkpointSessions(index: IndexStore, hashes: Iterable): Promise { + const want = new Set(); + for (const h of hashes) { + const sha = parseWeights(h); + if (sha) want.add(sha); + } + const out = new Map(); + if (!want.size) return out; + const all = [...want]; + try { + for (let i = 0; i < all.length; i += HASHES_PER_QUERY) { + // A hash is in one query only, so each hash's sessions arrive in that query's order: newest first. + const asked = new Set(all.slice(i, i + HASHES_PER_QUERY)); + const { sessions } = await findSessions(index, parseWhere(`weights.checkpoint:${[...asked].join(",")}`), { limit: SCAN_MAX }); + for (const s of sessions) { + const edits = s.stats.weightEdits; + for (const sha of edits?.checkpoints ?? []) { + if (!asked.has(sha)) continue; + const list = out.get(sha) ?? []; + if (list.length < CHECKPOINT_SESSIONS_MAX) list.push({ id: s.id, edits: edits! }); + out.set(sha, list); + } + } + } + } catch { + // An older store plugin has no SQL for the key, or the index is not there: nothing to add. + return new Map(); + } + return out; +} + +/** + * One line per session that recorded this hash, for under the line that shows the stamp: + * ` edits 2 edits, 2 tensors, 66,560 weights changed`. + */ +export function checkpointLines(sha256: string | undefined, by: CheckpointSessions | undefined): string[] { + return ((sha256 !== undefined && by?.get(sha256)) || []).map((s) => ` edits ${s.id} ${weightEditsText(s.edits)}`); +} + +/** Sets `sessions` (the ids) on a stamp whose hash some session recorded; leaves any other stamp as it is. */ +export function withSessions(o: T, by: CheckpointSessions): T { + const found = o.weights ? by.get(o.weights.sha256) : undefined; + if (o.weights && found?.length) o.weights = { ...o.weights, sessions: found.map((s) => s.id) }; + return o; +} + +/** An experiment that ran a checkpoint. */ +export interface CheckpointExperiment { + id: string; + name: string; +} + +/** + * The experiments stamped with each of these hashes, newest first, among the newest + * EXPERIMENTS_MAX. A hash no experiment ran has no entry. Empty when the experiments can't be read. + */ +export async function checkpointExperiments(store: { listExperiments(filter: { limit: number }): Promise }, hashes: readonly string[] | undefined): Promise> { + const out = new Map(); + if (!hashes?.length) return out; + try { + for (const e of await store.listExperiments({ limit: EXPERIMENTS_MAX })) { + const sha = weightsOf(e); + if (sha === undefined || !hashes.includes(sha)) continue; + out.set(sha, [...(out.get(sha) ?? []), { id: e.id, name: e.name }]); + } + } catch { + return new Map(); + } + return out; +} + +/** A checkpoint of a session, with the experiments that ran it. */ +export interface CheckpointRun { + sha256: string; + experiments: CheckpointExperiment[]; +} + +/** + * Each checkpoint of a session that an experiment ran, in the order the session recorded them, with those + * experiments (newest first). `hashes` is Session.stats.weightEdits.checkpoints. What `coding.sh show` + * and MCP session_get print as `checkpointExperiments`. + */ +export async function checkpointRuns(store: Parameters[0], hashes: readonly string[] | undefined): Promise { + const by = await checkpointExperiments(store, hashes); + return (hashes ?? []).flatMap((sha256) => (by.has(sha256) ? [{ sha256, experiments: by.get(sha256)! }] : [])); +} diff --git a/packages/coding.sh/src/experiments/render.ts b/packages/coding.sh/src/experiments/render.ts index af66d090..70b6da1b 100644 --- a/packages/coding.sh/src/experiments/render.ts +++ b/packages/coding.sh/src/experiments/render.ts @@ -22,6 +22,7 @@ import type { ExperimentBlock, ExpSession, SessionBlock } from "./session.ts"; import { stabilityLine } from "./stability.ts"; import type { SweepResult } from "./sweep.ts"; import type { Turnover } from "./turnover.ts"; +import { checkpointLines, type CheckpointSessions } from "./checkpoints.ts"; import { shortWeights, weightsOf } from "./weights.ts"; import { fixed, ms, paint, pct, pValue, signedPct, table, truncate, type RenderOpts } from "./text.ts"; import { formatBestSet } from "../stats/mcb.ts"; @@ -100,7 +101,7 @@ export function selectionText(s: CaseSelection): string { return `${s.selected} of ${s.of}${parts.length ? `: ${parts.join(", ")}` : ""}`; } -export function renderExperiment(r: ExperimentResult, opts: RenderOpts & { verifiers?: ReadonlyMap; cases?: ReadonlyMap }): string { +export function renderExperiment(r: ExperimentResult, opts: RenderOpts & { verifiers?: ReadonlyMap; cases?: ReadonlyMap; checkpoints?: CheckpointSessions }): string { const color = opts.color; const limit = opts.limit ?? 20; const s = r.summary; @@ -116,6 +117,8 @@ export function renderExperiment(r: ExperimentResult, opts: RenderOpts & { verif // The checkpoint the task ran, as given to exp run --weights: whole, so it can be checked against the file's hash. const weights = weightsOf(r); if (weights) out.push(`weights ${weights}`); + // The sessions that recorded that file as a checkpoint, each with the edits it recorded. + out.push(...checkpointLines(weights, opts.checkpoints)); if (r.status === "cancelled" && s.n === 0) { out.push(paint("cancelled before any case run finished: nothing was graded", "yellow", color)); return out.join("\n"); @@ -248,7 +251,7 @@ export function renderExperiment(r: ExperimentResult, opts: RenderOpts & { verif return out.join("\n"); } -export function renderComparison(c: Comparison, opts: RenderOpts & { a?: ExperimentResult; b?: ExperimentResult; matrix?: boolean; cases?: ReadonlyMap }): string { +export function renderComparison(c: Comparison, opts: RenderOpts & { a?: ExperimentResult; b?: ExperimentResult; matrix?: boolean; cases?: ReadonlyMap; checkpoints?: CheckpointSessions }): string { const color = opts.color; const limit = opts.limit ?? 20; const out: string[] = []; @@ -257,15 +260,17 @@ export function renderComparison(c: Comparison, opts: RenderOpts & { a?: Experim // Which checkpoint each side ran (exp run --weights), once either side says: the first 12 hex characters, or that it was not recorded. const stamped = c.a.weights !== undefined || c.b.weights !== undefined; const weights = (x: Comparison["a"]) => (stamped ? ` weights ${x.weights ? shortWeights(x.weights.sha256) : "not recorded"}` : ""); + // Under each side: the sessions that recorded its weights file as a checkpoint, with the edits they recorded. + const edits = (x: Comparison["a"]) => checkpointLines(x.weights?.sha256, opts.checkpoints); if (c.given && !c.given.kept) { const g = c.given; - out.push(`${paint("A", "bold", color)} ${c.a.name} ${paint(c.a.id, "dim", color)}${weights(c.a)}`, `${paint("B", "bold", color)} ${c.b.name} ${paint(c.b.id, "dim", color)}${weights(c.b)}`); + out.push(`${paint("A", "bold", color)} ${c.a.name} ${paint(c.a.id, "dim", color)}${weights(c.a)}`, ...edits(c.a), `${paint("B", "bold", color)} ${c.b.name} ${paint(c.b.id, "dim", color)}${weights(c.b)}`, ...edits(c.b)); if (c.weightsWarning) out.push(paint(`! ${c.weightsWarning}`, "yellow", color)); out.push(paint(`! --given ${g.grader}: no case passes ${g.grader} in both A and B (${g.dropped} dropped), so there is nothing to compare`, "yellow", color)); return out.join("\n"); } - out.push(`${paint("A", "bold", color)} ${c.a.name} ${paint(c.a.id, "dim", color)} pass ${pct(c.a.passRate)}${partial(opts.a)}${weights(c.a)}`); - out.push(`${paint("B", "bold", color)} ${c.b.name} ${paint(c.b.id, "dim", color)} pass ${pct(c.b.passRate)}${partial(opts.b)}${weights(c.b)}`); + out.push(`${paint("A", "bold", color)} ${c.a.name} ${paint(c.a.id, "dim", color)} pass ${pct(c.a.passRate)}${partial(opts.a)}${weights(c.a)}`, ...edits(c.a)); + out.push(`${paint("B", "bold", color)} ${c.b.name} ${paint(c.b.id, "dim", color)} pass ${pct(c.b.passRate)}${partial(opts.b)}${weights(c.b)}`, ...edits(c.b)); if (c.weightsWarning) out.push(paint(`! ${c.weightsWarning}`, "yellow", color)); if (c.given) out.push(paint(`! ${c.given.note}`, "yellow", color)); if (c.datasetWarning) out.push(paint(`! ${c.datasetWarning}`, "yellow", color)); diff --git a/packages/coding.sh/src/experiments/weights.ts b/packages/coding.sh/src/experiments/weights.ts index f6d34fee..143cafe0 100644 --- a/packages/coding.sh/src/experiments/weights.ts +++ b/packages/coding.sh/src/experiments/weights.ts @@ -3,19 +3,21 @@ // nothing itself; it stores the value in `metadata.weights` and shows it where // two experiments are compared, so "base vs edited" names both checkpoints and // a comparison of one checkpoint with itself is called out. -import { isSha256Hex } from "../core/ids.ts"; +import { parseWeights, shortWeights } from "../core/weights.ts"; import type { ExperimentResult } from "../datasets/types.ts"; -/** `ExperimentResult.metadata.weights`. */ +// One parser and one 12-character form for every weights hash: core/weights.ts has both. +export { parseWeights, shortWeights }; + +/** `ExperimentResult.metadata.weights`, and the stamp as commands and tools print it. */ export interface WeightsStamp { /** SHA-256 of the checkpoint file: 64 lowercase hex characters. */ sha256: string; -} - -/** A `--weights` value as stored: trimmed and lowercased; undefined when it is not 64 hex characters. */ -export function parseWeights(v: unknown): string | undefined { - const s = typeof v === "string" ? v.trim().toLowerCase() : undefined; - return isSha256Hex(s) ? s : undefined; + /** + * Ids of the sessions whose `weights.checkpoint` event recorded this hash (experiments/checkpoints.ts), + * newest first. Looked up when the stamp is printed, never stored; absent when no session recorded it. + */ + sessions?: string[]; } /** The stamp an experiment carries, or undefined (never stamped, or a value that is not a SHA-256). */ @@ -24,9 +26,6 @@ export function weightsOf(e: Pick | undefined): st return w && typeof w === "object" ? parseWeights((w as { sha256?: unknown }).sha256) : undefined; } -/** The first 12 hex characters, for a line of text. */ -export const shortWeights = (sha256: string): string => sha256.slice(0, 12); - /** Metadata with the stamp set; `metadata` itself when there is no stamp. */ export function withWeights(metadata: Record | undefined, sha256: string | undefined): Record | undefined { return sha256 === undefined ? metadata : { ...(metadata ?? {}), weights: { sha256 } satisfies WeightsStamp }; diff --git a/packages/coding.sh/src/mcp/data.ts b/packages/coding.sh/src/mcp/data.ts index 743d5a58..6e4ce679 100644 --- a/packages/coding.sh/src/mcp/data.ts +++ b/packages/coding.sh/src/mcp/data.ts @@ -9,7 +9,7 @@ import { CACHE_BREAK_FIX, CACHE_IDLE_FIX } from "../core/cache-breaks.ts"; import { cacheBaseline, CACHE_BASELINE_NOTE } from "../core/cache-baselines.ts"; import { mergeCallCopies } from "../core/call-copies.ts"; import { failureFlags, percentile } from "../core/sessions.ts"; -import { WEIGHTS_EDIT_KIND, weightsEditText } from "../core/weights.ts"; +import { weightsEventText } from "../core/weights.ts"; import { sessionMatcher } from "../query/run.ts"; import type { Node, Query } from "../query/where.ts"; @@ -331,11 +331,13 @@ export function timeline(events: AgentEvent[], startedAt: number): TimelineEntry case "context.compact": push("compact", `context compacted (${str(p.trigger) ?? "auto"})`); break; - case "notification": - // An edit with no readable field still gets its line: push() leaves out an empty one. - if (p.kind === WEIGHTS_EDIT_KIND) push("weights", clip(weightsEditText(p), 240) || "no details recorded"); + case "notification": { + // A weight edit or a checkpoint. An edit with no readable field still gets its line: push() leaves out an empty one. + const weights = weightsEventText(p); + if (weights !== undefined) push("weights", clip(weights, 240) || "no details recorded"); else push("notification", clip(str(p.message) ?? str(p.text), 160)); break; + } default: break; } diff --git a/packages/coding.sh/src/mcp/digest.ts b/packages/coding.sh/src/mcp/digest.ts index 9c325781..8c51b060 100644 --- a/packages/coding.sh/src/mcp/digest.ts +++ b/packages/coding.sh/src/mcp/digest.ts @@ -3,7 +3,7 @@ import { costNote } from "../core/pricing.ts"; import { SKILL_EVENT_KINDS, SKILL_TAG_PREFIX, skillUse } from "../core/skills-use.ts"; import type { IndexStore, Session } from "../core/types.ts"; -import { weightEditsText } from "../core/weights.ts"; +import { weightsSummaryText } from "../core/weights.ts"; import { clip, countNodes, fmtMs, isRunning, iso, lastActivity, sessionSummary, splitTurns, subagentTree, timeline, turnIndex, type SubagentNode, type TimelineEntry, type TurnInfo } from "./data.ts"; type Obj = Record; @@ -113,7 +113,7 @@ export async function sessionMarkdown(index: IndexStore, s: Session, maxEntries if (s.stats.waste) row("Re-fetches", `${s.stats.waste.duplicateFetches} of ${s.stats.waste.readOnlyCalls} read-only calls repeated with the same result (~${s.stats.waste.duplicateFetchTokens} tokens)`); const cb = s.stats.cacheBreaks; if (cb) row("Cache breaks", `${cb.count}, ${cb.lostTokens} prompt tokens billed again${cb.extraUsd !== undefined ? `, $${cb.extraUsd.toFixed(4)} extra` : ""} (${Object.entries(cb.byCause).map(([k, c]) => `${k} ${c!.count}${k === "idle" && cb.byIdle ? ` (${Object.entries(cb.byIdle).map(([w, i]) => `${w} ${i!.count}`).join(", ")})` : ""}`).join(", ")})`); - if (s.stats.weightEdits) row("Weight edits", weightEditsText(s.stats.weightEdits)); + if (s.stats.weightEdits) row("Weight edits", weightsSummaryText(s.stats.weightEdits)); for (const d of s.stats.derails ?? []) row("Derail", `${d.flag} at +${fmtMs(d.ts - s.startedAt)} (event ${d.index + 1}): ${d.detail}`); if (s.parent) row("Parent", `\`${s.parent.sessionId}\``); if (turns.length > 1) { diff --git a/packages/coding.sh/src/mcp/tools.ts b/packages/coding.sh/src/mcp/tools.ts index 58977774..130a7b24 100644 --- a/packages/coding.sh/src/mcp/tools.ts +++ b/packages/coding.sh/src/mcp/tools.ts @@ -426,7 +426,15 @@ function toolDefs(): ToolDef[] { const ctx = await c.store(); const s = await resolveSession(ctx, strArg(a, "sessionId")!); try { - return await sessionDigest(ctx.index, s, { turn: typeof a.turn === "number" ? a.turn : undefined, limit: numArg(a, "limit", 40), offset: cursorArg(a), from: a.tail === true ? "tail" : "head", now: c.now() }); + const digest = await sessionDigest(ctx.index, s, { turn: typeof a.turn === "number" ? a.turn : undefined, limit: numArg(a, "limit", 40), offset: cursorArg(a), from: a.tail === true ? "tail" : "head", now: c.now() }); + // The experiments that ran each weights file the session recorded as a checkpoint, as `coding.sh show --json` has them. + const hashes = s.stats.weightEdits?.checkpoints; + if (hashes?.length) { + const [{ FileEvalStore }, { checkpointRuns }] = await Promise.all([import("../datasets/store.ts"), import("../experiments/checkpoints.ts")]); + const runs = await checkpointRuns(new FileEvalStore({ env: c.env, index: ctx.index }), hashes); + if (runs.length) digest.checkpointExperiments = runs; + } + return digest; } catch (e) { if (e instanceof RangeError) throw new ToolError(e.message); throw e; @@ -989,7 +997,7 @@ function toolDefs(): ToolDef[] { annotations: READ, async run(a, c) { const ctx = await c.store(); - const [{ FileEvalStore }, { weightsOf, stampedWith }] = await Promise.all([import("../datasets/store.ts"), import("../experiments/weights.ts")]); + const [{ FileEvalStore }, { weightsOf, stampedWith }, { checkpointSessions, withSessions }] = await Promise.all([import("../datasets/store.ts"), import("../experiments/weights.ts"), import("../experiments/checkpoints.ts")]); const store = new FileEvalStore({ env: c.env, index: ctx.index }); const offset = cursorArg(a); const limit = numArg(a, "limit", 20); @@ -997,9 +1005,11 @@ function toolDefs(): ToolDef[] { const name = strArg(a, "name"); const all = await store.listExperiments({ limit: offset + limit + 1, ...(dataset ? { datasetId: dataset } : {}), ...(name ? { name } : {}) }); const page = all.slice(offset, offset + limit); + const checkpoints = await checkpointSessions(ctx.index, page.map((e) => weightsOf(e))); return { - // Each row carries the checkpoint its run recorded (exp run --weights); no key when it recorded none. - experiments: page.map((e) => stampedWith({ + // Each row carries the checkpoint its run recorded (exp run --weights), with the ids of the sessions that + // recorded that file (weights.sessions); no key when it recorded none. + experiments: page.map((e) => withSessions(stampedWith({ id: e.id, name: e.name, dataset: e.dataset.id, @@ -1014,7 +1024,7 @@ function toolDefs(): ToolDef[] { passRateLo: round(e.summary.passRate.lo), passRateHi: round(e.summary.passRate.hi), createdAt: iso(e.createdAt), - }, weightsOf(e))), + }, weightsOf(e)), checkpoints)), ...(all.length > offset + limit ? { nextCursor: String(offset + limit) } : {}), }; }, @@ -1061,7 +1071,7 @@ function toolDefs(): ToolDef[] { annotations: READ, async run(a, c) { const ctx = await c.store(); - const [{ FileEvalStore }, { compareExperiments, IncompleteExperimentError }] = await Promise.all([import("../datasets/store.ts"), import("../experiments/compare.ts")]); + const [{ FileEvalStore }, { compareExperiments, IncompleteExperimentError }, { checkpointSessions, withSessions }] = await Promise.all([import("../datasets/store.ts"), import("../experiments/compare.ts"), import("../experiments/checkpoints.ts")]); const store = new FileEvalStore({ env: c.env, index: ctx.index }); const ids = [strArg(a, "baseline")!, strArg(a, "candidate")!]; const [ea, eb] = await Promise.all(ids.map((id) => store.getExperiment(id).catch(() => undefined))); @@ -1085,9 +1095,11 @@ function toolDefs(): ToolDef[] { if (cmp.weightsWarning) warnings.unshift(cmp.weightsWarning); const gate = cmp.gate; const t = cmp.turnover; + // Each side's stamp names the sessions that recorded that weights file as a checkpoint (weights.sessions). + const checkpoints = await checkpointSessions(ctx.index, [cmp.a.weights?.sha256, cmp.b.weights?.sha256]); return { - baseline: { ...cmp.a, passRate: round(cmp.a.passRate) }, - candidate: { ...cmp.b, passRate: round(cmp.b.passRate) }, + baseline: withSessions({ ...cmp.a, passRate: round(cmp.a.passRate) }, checkpoints), + candidate: withSessions({ ...cmp.b, passRate: round(cmp.b.passRate) }, checkpoints), verdict: cmp.verdict, diff: { mean: round(cmp.diff.mean), lo: round(cmp.diff.lo), hi: round(cmp.diff.hi) }, paired: cmp.paired, diff --git a/packages/coding.sh/src/otel/map.ts b/packages/coding.sh/src/otel/map.ts index 7c5ab3ab..db2de479 100644 --- a/packages/coding.sh/src/otel/map.ts +++ b/packages/coding.sh/src/otel/map.ts @@ -35,7 +35,7 @@ import { SCHEMA_VERSION } from "../schema/index.ts"; import { hashParts, sha256Hex } from "../core/ids.ts"; import { noteBlobHits, type RedactionTally, type Redactor, redactBlobValue, redactionTally } from "../core/redact.ts"; import { geoValue, tierValue } from "../core/pricing.ts"; -import { WEIGHTS_EDIT_ATTRS, WEIGHTS_EDIT_KIND, weightsEditOf } from "../core/weights.ts"; +import { WEIGHTS_CHECKPOINT_ATTRS, WEIGHTS_CHECKPOINT_KIND, WEIGHTS_EDIT_ATTRS, WEIGHTS_EDIT_KIND, weightsCheckpointOf, weightsEditOf } from "../core/weights.ts"; import { anthropicPriceHints } from "../adapters/_shared/normalize.ts"; import { type DurableTag, durableOf, durableSessionId } from "./durable.ts"; import { @@ -819,6 +819,22 @@ function weightsEditDraft(d: Draft, a: Attrs): Draft { return d; } +/** + * The payload of a span or log record named `weights.checkpoint`: `kind` plus the hash of the + * saved weights file, the base file's and the note (WEIGHTS_CHECKPOINT_ATTRS), the event the SDKs' + * weights_checkpoint helpers send. Without a `coding_sh.weights.sha256` of 64 hex characters the + * payload is `kind` alone, which nothing reads as a checkpoint. + */ +function weightsCheckpointDraft(d: Draft, a: Attrs): Draft { + const K = WEIGHTS_CHECKPOINT_ATTRS; + d.payload.kind = WEIGHTS_CHECKPOINT_KIND; + Object.assign(d.payload, weightsCheckpointOf({ kind: WEIGHTS_CHECKPOINT_KIND, sha256: str(a, K.sha256), baseSha256: str(a, K.baseSha256), note: str(a, K.note) })); + return d; +} + +/** The draft a span or log record of this name fills: a weight edit's or a checkpoint's. Undefined for any other name. */ +const weightsDraft = (name: string | undefined): ((d: Draft, a: Attrs) => Draft) | undefined => (name === WEIGHTS_EDIT_KIND ? weightsEditDraft : name === WEIGHTS_CHECKPOINT_KIND ? weightsCheckpointDraft : undefined); + async function mapSpan(c: SpanCtx, ctx: MapContext, store: Blobs): Promise { const { span, traceId, spanId } = c; const a = span.attributes; @@ -864,8 +880,8 @@ async function mapSpan(c: SpanCtx, ctx: MapContext, store: Blobs): Promise new Draft(store, max, raw); - // A span named weights.edit is a weight edit, whatever else its attributes say. - const weightsEdit = span.name === WEIGHTS_EDIT_KIND; + // A span named weights.edit or weights.checkpoint is that event, whatever else its attributes say. + const weights = weightsDraft(span.name); const agentName = str(a, "gen_ai.agent.name", "agent.name") ?? (/^invoke_agent\s+(.+)$/.exec(span.name)?.[1] || undefined); // ---- model call @@ -942,7 +958,7 @@ async function mapSpan(c: SpanCtx, ctx: MapContext, store: Blobs): Promise 0 ? { durationMs } : {}); + } else if (weights) { + emit(0, "notification", startMs, weights(draft(), a), durationMs > 0 ? { durationMs } : {}); } else if (klass === "llm" || klass === "embedding") { await llm(0); } else if (klass === "tool") { @@ -1361,8 +1377,8 @@ async function mapLog(rec: OtelLogRecord, resource: Attrs, scope: OtelScope, ctx .set("plugin", str(a, "plugin.name", "skill.plugin_id")) .set("event", name); emit(0, "notification", ts, d); - } else if (suffix === WEIGHTS_EDIT_KIND) { - emit(0, "notification", ts, weightsEditDraft(draft(), a)); + } else if (weightsDraft(suffix)) { + emit(0, "notification", ts, weightsDraft(suffix)!(draft(), a)); } else if (suffix === "agent_response" && content(str(a, "response"))) { // Codex: a final answer of the main agent or of a spawned subagent (otel.log_agent_responses). const d = draft().set("responseLength", num(a, "response_length")).set("truncated", bool(a, "response_truncated") || undefined); diff --git a/packages/coding.sh/src/query/compile.ts b/packages/coding.sh/src/query/compile.ts index 5c3e60ea..d142eb56 100644 --- a/packages/coding.sh/src/query/compile.ts +++ b/packages/coding.sh/src/query/compile.ts @@ -160,7 +160,7 @@ export function orderBy(sort: { key: string; desc: boolean } | undefined): strin */ export function facetSource(key: string): { value: string; from: string; prefix?: string; facetKey?: string } { const def = KEYS.get(key)!; - if (def.multi) return { value: def.multi.prefix ? `substr(j.value, ${def.multi.prefix.length + 1})` : "j.value", from: `sessions s, json_each(s.flags) j`, prefix: def.multi.prefix }; + if (def.multi) return { value: def.multi.prefix ? `substr(j.value, ${def.multi.prefix.length + 1})` : "j.value", from: `sessions s, json_each(${def.multi.from}) j`, prefix: def.multi.prefix }; if (def.level === "event") return { value: "f.value", from: "session_facets f JOIN sessions s ON s.id = f.session_id", facetKey: key }; return { value: def.facet?.sql ?? def.sql[0]!, from: "sessions s" }; } diff --git a/packages/coding.sh/src/query/where.ts b/packages/coding.sh/src/query/where.ts index a20822ba..6184e513 100644 --- a/packages/coding.sh/src/query/where.ts +++ b/packages/coding.sh/src/query/where.ts @@ -121,7 +121,7 @@ export interface KeyDef { example: string; /** * SQL expressions over `s` (sessions) or `e` (events); a clause matches when any - * of them does. Multi keys (flags) use `j.value` from `multi.from` instead. + * of them does. Multi keys (flags, weights.checkpoint) use `j.value` from `multi.from` instead. */ sql: string[]; /** Columns that are never NULL (no COALESCE needed under negation). */ @@ -133,7 +133,7 @@ export interface KeyDef { * guard per `sql` expression, so each branch of the OR can start from its own index. */ present?: string | string[]; - /** A list-valued field: one EXISTS over json_each, values compared after `prefix`. */ + /** A list-valued field: one EXISTS over json_each of `from` (a JSON array, or NULL for none), values compared after `prefix`. */ multi?: { from: string; prefix: string }; /** The values the scan fallback compares (one per `sql` expression, or every list item for multi). */ get(row: Session & AgentEvent): unknown[]; @@ -203,6 +203,16 @@ const KEY_LIST: KeyDef[] = [ { name: "hook.tokens", level: "session", type: "number", unit: "count", doc: "Tokens hooks added to the context (characters / 4)", example: "hook.tokens>500", sql: [stat("hookContext.tokens")], get: (s) => [num(s.stats?.hookContext?.tokens)] }, { name: "weights.edits", level: "session", type: "number", unit: "count", doc: "Weight edits the session recorded", example: "weights.edits>0", sql: [stat("weightEdits.events")], get: (s) => [num(s.stats?.weightEdits?.events)] }, { name: "weights.changed", level: "session", type: "number", unit: "count", doc: "Weights those edits changed, added up", example: "weights.changed>1000", sql: [stat("weightEdits.weightsChanged")], get: (s) => [num(s.stats?.weightEdits?.weightsChanged)] }, + { + name: "weights.checkpoint", + level: "session", + type: "text", + doc: "SHA-256 of a weights file the session recorded as a checkpoint (the value exp run --weights takes): 64 lowercase hex characters, or a prefix with *", + example: "weights.checkpoint:58899f90572c*", + sql: ["j.value"], + multi: { from: stat("weightEdits.checkpoints"), prefix: "" }, + get: (s) => s.stats?.weightEdits?.checkpoints ?? [], + }, { name: "since", level: "session", type: "time", bound: "since", doc: "Started at or after: 7d, 12h, today, yesterday or a date. A date without a zone is local time: 2026-11-09 is local midnight", example: "since:7d", sql: ["s.started_at"], notNull: true, get: (s) => [s.startedAt] }, { name: "until", level: "session", type: "time", bound: "until", doc: "Started before (exclusive): 7d, 12h, today or a date in local time. until:2026-11-01 leaves that day out", example: "until:2026-11-01", sql: ["s.started_at"], notNull: true, get: (s) => [s.startedAt] }, { name: "duration", level: "session", type: "number", unit: "ms", doc: "Wall time of an ended session (30s, 10m, 2h)", example: "duration>10m", sql: ["(s.ended_at - s.started_at)"], get: (s) => [s.endedAt !== undefined ? s.endedAt - s.startedAt : undefined] }, diff --git a/packages/coding.sh/src/refine/taxonomy.ts b/packages/coding.sh/src/refine/taxonomy.ts index 708f5e08..e5f53aad 100644 --- a/packages/coding.sh/src/refine/taxonomy.ts +++ b/packages/coding.sh/src/refine/taxonomy.ts @@ -208,6 +208,8 @@ export const FLAG_DIAGNOSIS: Readonly`, `settings`, …). Unset when no hook added any. */ hookContext?: HookContextStats; - /** Weight edits the session recorded; it also tags the session `weights:edited`. Unset when it recorded none. */ + /** Weight edits and checkpoints the session recorded; they also tag the session `weights:edited` and `weights:checkpoint`. Unset when it recorded neither. */ weightEdits?: WeightEditStats; /** * Memory and instruction files (AGENTS.md, rules, skills, Claude auto-memory) the diff --git a/packages/coding.sh/src/schema/trace.schema.json b/packages/coding.sh/src/schema/trace.schema.json index 70a3c8ac..a6fd1e19 100644 --- a/packages/coding.sh/src/schema/trace.schema.json +++ b/packages/coding.sh/src/schema/trace.schema.json @@ -931,7 +931,7 @@ "$ref": "#/$defs/HookContextStats" }, "weightEdits": { - "description": "Weight edits the session recorded; it also tags the session `weights:edited`. Unset when it recorded none.", + "description": "Weight edits and checkpoints the session recorded; they also tag the session `weights:edited` and `weights:checkpoint`. Unset when it recorded neither.", "$ref": "#/$defs/WeightEditStats" }, "memoryUse": { @@ -1308,11 +1308,11 @@ "additionalProperties": false }, "WeightEditStats": { - "description": "Session.stats.weightEdits: weight edits the session recorded (`notification`\nevents with `payload.kind: \"weights.edit\"`).", + "description": "Session.stats.weightEdits: weight edits the session recorded (`notification`\nevents with `payload.kind: \"weights.edit\"`), and the checkpoints it recorded\n(`payload.kind: \"weights.checkpoint\"`).", "type": "object", "properties": { "events": { - "description": "Edits recorded.", + "description": "Edits recorded. 0 for a session that recorded a checkpoint and no edit.", "type": "number" }, "tensors": { @@ -1322,6 +1322,13 @@ "weightsChanged": { "description": "The edits' `changed` counts added up: a weight two edits changed counts twice.", "type": "number" + }, + "checkpoints": { + "description": "SHA-256 of each weights file the session recorded as a checkpoint: distinct, in\narrival order, at most 16. The value `coding.sh exp run --weights` takes. Unset when\nthe session recorded no checkpoint.", + "type": "array", + "items": { + "type": "string" + } } }, "required": [ diff --git a/packages/coding.sh/src/server/forward.ts b/packages/coding.sh/src/server/forward.ts index 3f8bf73b..3330e9d2 100644 --- a/packages/coding.sh/src/server/forward.ts +++ b/packages/coding.sh/src/server/forward.ts @@ -5,7 +5,7 @@ // // - Spans received over OTLP keep their own trace and span ids (they line up // with the app's other spans in the backend). Native /v1/ingest batches are -// converted with toOtlpRequest (invoke_agent / execute_tool / chat / weights.edit spans). +// converted with toOtlpRequest (invoke_agent / execute_tool / chat / weights.edit / weights.checkpoint spans). // - Every resource gets `coding_sh.project`. // - Metadata mode is an allowlist (see keepInMetadata); full mode keeps every // attribute and runs serve's redactor over every string. @@ -17,7 +17,7 @@ import type { Redactor } from "../core/redact.ts"; import type { OtelValue, ResourceSpans } from "../otel/model.ts"; import type { OpenSession } from "../core/types.ts"; import { otlpTracesUrl, toOtlpOpenSessionRequest, toOtlpRequest } from "../sinks/otlp.ts"; -import { WEIGHTS_EDIT_CONTENT_ATTRS } from "../core/weights.ts"; +import { parseWeights, WEIGHTS_CHECKPOINT_ATTRS, WEIGHTS_CHECKPOINT_KIND, WEIGHTS_EDIT_CONTENT_ATTRS } from "../core/weights.ts"; import { DEFAULT_TIMEOUT_MS, parseRetryAfter, redactUrl } from "../sinks/post.ts"; import type { ForwardConfig } from "./config.ts"; import type { Logger } from "./log.ts"; @@ -108,8 +108,14 @@ const DROP_EXACT = new Set(["coding_sh.session.title", "coding_sh.repo.root", .. const isScalarNonString = (v: AnyValue): boolean => "boolValue" in v || "intValue" in v || "doubleValue" in v; -/** Whether metadata mode forwards this attribute: an allowlisted key, or any boolean or number. */ -export function keepInMetadata(key: string, value: AnyValue): boolean { +/** + * Whether metadata mode forwards this attribute: an allowlisted key, or any boolean or number. + * `span` is the name of the span it is on: a `weights.checkpoint` span keeps the SHA-256 of its + * base file, which a weight edit's span drops (there it is one tensor's hash). The two file + * hashes join a session to the experiments that ran the file. Its note is dropped. + */ +export function keepInMetadata(key: string, value: AnyValue, span?: string): boolean { + if (span === WEIGHTS_CHECKPOINT_KIND && key === WEIGHTS_CHECKPOINT_ATTRS.baseSha256) return "stringValue" in value && parseWeights(value.stringValue) !== undefined; if (DROP_EXACT.has(key)) return false; if (KEEP_EXACT.has(key) || KEEP_PREFIX.some((p) => key.startsWith(p))) return true; return isScalarNonString(value); @@ -135,11 +141,11 @@ function deepRedact(v: AnyValue, r: Redactor): AnyValue { return v; } -/** Applies the forward mode to a list of attributes (a new list). */ -export function scrubAttrs(kvs: KeyValue[] | undefined, mode: "full" | "metadata", r: Redactor): KeyValue[] { +/** Applies the forward mode to a list of attributes (a new list). `span`: the name of the span they are on. */ +export function scrubAttrs(kvs: KeyValue[] | undefined, mode: "full" | "metadata", r: Redactor, span?: string): KeyValue[] { const out: KeyValue[] = []; for (const kv of kvs ?? []) { - if (mode === "metadata" && !keepInMetadata(kv.key, kv.value)) continue; + if (mode === "metadata" && !keepInMetadata(kv.key, kv.value, span)) continue; out.push({ key: kv.key, value: deepRedact(kv.value, r) }); } return out; @@ -148,7 +154,7 @@ export function scrubAttrs(kvs: KeyValue[] | undefined, mode: "full" | "metadata /** Applies the forward mode to one span (a new span). */ export function scrubSpan(s: JsonSpan, mode: "full" | "metadata", r: Redactor): JsonSpan { const meta = mode === "metadata"; - const out: JsonSpan = { ...s, name: r.string(s.name), attributes: scrubAttrs(s.attributes, mode, r) }; + const out: JsonSpan = { ...s, name: r.string(s.name), attributes: scrubAttrs(s.attributes, mode, r, s.name) }; if (s.events) { out.events = s.events.map((e) => ({ timeUnixNano: e.timeUnixNano, diff --git a/packages/coding.sh/src/sinks/otlp.ts b/packages/coding.sh/src/sinks/otlp.ts index 933b40dc..dec9e471 100644 --- a/packages/coding.sh/src/sinks/otlp.ts +++ b/packages/coding.sh/src/sinks/otlp.ts @@ -8,6 +8,7 @@ // tool.start/tool.end → span "execute_tool {tool}" (child, emitted with tool.end) // assistant.message → span "chat {model}" (child, carries token usage) // weights.edit → span "weights.edit" (child, one per edit: coding_sh.weights.*) +// weights.checkpoint → span "weights.checkpoint" (child, one per event: coding_sh.weights.sha256, .base_sha256, .note) // Token usage follows the GenAI gen_ai.usage.* attributes: input_tokens counts the // whole prompt (uncached + cache reads + cache writes), and the cache parts go to // cache_read.input_tokens and cache_write.input_tokens (renamed from @@ -20,7 +21,7 @@ import type { AgentEvent, Session, Usage } from "../schema/index.ts"; import type { BlobStore, OpenSession, Sink, SinkAck, SinkConfig } from "../core/types.ts"; import { hashParts } from "../core/ids.ts"; import { sessionToMetadata, toMetadata } from "../core/redact.ts"; -import { isWeightsEdit, WEIGHTS_EDIT_KIND, weightsEditAttrs } from "../core/weights.ts"; +import { isWeightsCheckpoint, isWeightsEdit, WEIGHTS_CHECKPOINT_KIND, WEIGHTS_EDIT_KIND, weightsCheckpointAttrs, weightsEditAttrs } from "../core/weights.ts"; import { postJson } from "./post.ts"; export const SEMCONV_SCHEMA_URL = "https://opentelemetry.io/schemas/1.37.0"; @@ -216,17 +217,19 @@ export function toOtlpSpans(batch: AgentEvent[], sessions: Session[], mode: "ful : {}), }), }); - } else if (isWeightsEdit(e)) { - // Metadata mode has already dropped the tensor's name, its shape, both hashes and the note (toMetadata above). + } else if (isWeightsEdit(e) || isWeightsCheckpoint(e)) { + // Metadata mode has already dropped an edit's tensor name, shape, both hashes and note, and a + // checkpoint's note; a checkpoint keeps the hashes of its two files (toMetadata above). + const edit = isWeightsEdit(e); spans.push({ traceId, spanId: hashParts("otel-span", "weights", e.id).slice(0, 16), parentSpanId, - name: WEIGHTS_EDIT_KIND, + name: edit ? WEIGHTS_EDIT_KIND : WEIGHTS_CHECKPOINT_KIND, kind: SPAN_KIND_INTERNAL, startTimeUnixNano: nanos(e.ts), endTimeUnixNano: nanos(e.ts + (e.durationMs ?? 0)), - attributes: attrs({ "gen_ai.agent.name": e.agent, "coding_sh.event.id": e.id, ...weightsEditAttrs(e.payload) }), + attributes: attrs({ "gen_ai.agent.name": e.agent, "coding_sh.event.id": e.id, ...(edit ? weightsEditAttrs(e.payload) : weightsCheckpointAttrs(e.payload)) }), }); } else if (e.kind === "session.end") { const s = sessionsById.get(e.sessionId); @@ -264,8 +267,9 @@ function sessionSpan(s: Session, endMs: number, full: boolean, traceId: string): "coding_sh.parent.session.id": s.parent?.sessionId, "coding_sh.stats.tests_passed": s.stats.testsPassed, "coding_sh.stats.tests_failed": s.stats.testsFailed, - "coding_sh.stats.weight_edits": s.stats.weightEdits?.events, - "coding_sh.stats.weights_changed": s.stats.weightEdits?.weightsChanged, + // Absent on a session that recorded a checkpoint and no edit. + "coding_sh.stats.weight_edits": s.stats.weightEdits?.events || undefined, + "coding_sh.stats.weights_changed": s.stats.weightEdits?.events ? s.stats.weightEdits.weightsChanged : undefined, "vcs.ref.head.revision": s.repo?.head || undefined, ...(full ? { "coding_sh.session.title": s.title, "coding_sh.repo.root": s.repo?.root } : {}), }), diff --git a/packages/coding.sh/src/viewer/model.ts b/packages/coding.sh/src/viewer/model.ts index c92563cb..fe2c4c59 100644 --- a/packages/coding.sh/src/viewer/model.ts +++ b/packages/coding.sh/src/viewer/model.ts @@ -10,10 +10,10 @@ import { failureState, foldFailure } from "../core/failures.ts"; import type { IndexStore } from "../core/types.ts"; import type { AgentEvent, Session } from "../schema/index.ts"; -import { isWeightsEdit, weightsEditText } from "../core/weights.ts"; +import { isWeightsEvent, weightsEventText } from "../core/weights.ts"; /** Bump when the model's shape or its rules change: cached models are rebuilt. */ -export const MODEL_VERSION = 2; // 2: a weight edit has its own code, so a fold never hides it. +export const MODEL_VERSION = 3; // 2: a weight edit has its own code, so a fold never hides it. 3: a weights checkpoint has that code too. const KINDS = ["other", "session.start", "session.end", "user.message", "assistant.message", "llm.call", "tool.start", "tool.end", "context.compact", "turn.end", "subagent.start", "subagent.end", "file.change", "test.result", "notification", "span", "error", "weights.edit"] as const; const CODE = new Map(KINDS.map((k, i) => [k, i])); @@ -128,7 +128,7 @@ export function modelBuilder(key = ""): { add(batch: readonly AgentEvent[]): voi grown.set(codes); codes = grown; } - const code = isWeightsEdit(e) ? C["weights.edit"] : (CODE.get(e.kind) ?? C.other); + const code = isWeightsEvent(e) ? C["weights.edit"] : (CODE.get(e.kind) ?? C.other); codes[i] = code; const toolFailed = foldFailure(fs, e) !== undefined; // A turn opens at the main agent's prompt, or at a turn id not seen before (an agent @@ -311,8 +311,10 @@ export function rowLabel(e: AgentEvent): string { return cut(text(p.message), 200); case "span": return text(p.name); - case "notification": - return isWeightsEdit(e) ? cut(weightsEditText(p), 200) || "weights.edit" : e.kind; + case "notification": { + const weights = weightsEventText(p); + return weights !== undefined ? cut(weights, 200) || "weights.edit" : e.kind; + } default: return e.kind; } diff --git a/packages/coding.sh/src/viewer/server.ts b/packages/coding.sh/src/viewer/server.ts index 91e462c0..7f35089e 100644 --- a/packages/coding.sh/src/viewer/server.ts +++ b/packages/coding.sh/src/viewer/server.ts @@ -23,7 +23,9 @@ import { phaseLabel, sessionOutline, storedOutline } from "../core/outline.ts"; import { loadReplay, type Baseline, type Replay } from "../core/replay.ts"; import { buildSessionTree, worstFirst, worstNode, type TreeNode } from "../core/tree.ts"; import type { ExperimentResult } from "../datasets/types.ts"; +import { checkpointRuns, checkpointSessions, withSessions } from "../experiments/checkpoints.ts"; import { compareExperiments } from "../experiments/compare.ts"; +import { stampedWith, weightsOf } from "../experiments/weights.ts"; import { facetCounts, findEvents, findSessions, searchWhere } from "../query/run.ts"; import type { AgentEvent } from "../schema/index.ts"; import { searchableText, searchTerms, snippet } from "../store/common.ts"; @@ -668,7 +670,14 @@ async function routeApi(req: IncomingMessage, res: ServerResponse, url: URL, ctx if (sub !== "fork") get(); if (id === undefined || (id === "" && sub === undefined)) return sendJson(res, 200, await listSessions(ctx.index, q, projectParam(ctx, q))); const sessionId = checkId(id, "session id"); - if (sub === undefined) return sendJson(res, 200, await visibleSession(ctx, sessionId)); + if (sub === undefined) { + const s = await visibleSession(ctx, sessionId); + // The experiments that ran each weights file the session recorded as a checkpoint, as `coding.sh show --json` + // has them. Eval assets belong to no project: a project-scoped key sees none. + const hashes = s.stats.weightEdits?.checkpoints; + const runs = hashes?.length && ctx.project === undefined && ctx.experiments ? await checkpointRuns(ctx.experiments, hashes) : []; + return sendJson(res, 200, runs.length ? { ...s, checkpointExperiments: runs } : s); + } if (sub === "events") { await visibleSession(ctx, sessionId); const limit = intParam(q, "limit", 5000, 1, 20_000); @@ -975,7 +984,9 @@ async function routeApi(req: IncomingMessage, res: ServerResponse, url: URL, ctx if (!EXPERIMENT_ID_RE.test(id) || id.includes("..")) throw new HttpError(400, "invalid experiment id"); const exp = scoped || !ctx.experiments ? undefined : await ctx.experiments.getExperiment(id); if (!exp) throw new HttpError(404, "experiment not found"); - return sendJson(res, 200, experimentDetail(exp)); + // Next to the stored record, as `exp show --json`: the stamp with the sessions that recorded that weights file as a checkpoint. + const { weights } = withSessions(stampedWith({}, weightsOf(exp)), await checkpointSessions(ctx.index, [weightsOf(exp)])); + return sendJson(res, 200, weights?.sessions ? { ...experimentDetail(exp), weights } : experimentDetail(exp)); } case "workflows": { @@ -1025,7 +1036,12 @@ async function routeApi(req: IncomingMessage, res: ServerResponse, url: URL, ctx if (!a || !b) throw new HttpError(404, "experiment not found"); // A cancelled side still compares (its cases are real), flagged as partial. const partial = [a, b].filter((x) => x.status === "cancelled").map((x) => x.id); - return sendJson(res, 200, { ...compareExperiments(a, b, { iters: 4000, force: true }), partial }); + const cmp = compareExperiments(a, b, { iters: 4000, force: true }); + // Each side's stamp names the sessions that recorded that weights file as a checkpoint (weights.sessions). + const checkpoints = await checkpointSessions(ctx.index, [cmp.a.weights?.sha256, cmp.b.weights?.sha256]); + withSessions(cmp.a, checkpoints); + withSessions(cmp.b, checkpoints); + return sendJson(res, 200, { ...cmp, partial }); } case "labels": { diff --git a/packages/coding.sh/src/viewer/ui/index.html b/packages/coding.sh/src/viewer/ui/index.html index b6e9995d..707d98ab 100644 --- a/packages/coding.sh/src/viewer/ui/index.html +++ b/packages/coding.sh/src/viewer/ui/index.html @@ -296,6 +296,8 @@ border-radius: 5px; font-family: var(--font-mono); font-size: 11px; white-space: nowrap; color: var(--text-2); box-shadow: inset 0 0 0 1px var(--line); } +/* The session behind a weights stamp, and the experiment that ran a session's checkpoint. */ +.badge a, .weights a { text-decoration: underline; text-decoration-color: var(--line-strong); text-underline-offset: 3px; } .badge.pass { color: var(--pass-ink); background: var(--pass-wash); box-shadow: none; } .badge.fail { color: var(--fail-ink); background: var(--fail-wash); box-shadow: none; } .badge.warn { color: var(--warn-ink); background: var(--warn-wash); box-shadow: none; } @@ -3840,9 +3842,20 @@ } if (norm(p.l2)) parts.push("L2 " + real(p.l2)); if (norm(p.maxAbs)) parts.push("max " + real(p.maxAbs)); - if (text(p.baseSha256) && text(p.editedSha256)) parts.push(p.baseSha256.slice(0, 8) + " → " + p.editedSha256.slice(0, 8)); + if (text(p.baseSha256) && text(p.editedSha256)) parts.push(shortWeights(p.baseSha256) + " → " + shortWeights(p.editedSha256)); return parts.join(" · "); } +/** Every weights hash is written in 12 hex characters (shortWeights in core/weights.ts). */ +function shortWeights(sha) { return String(sha).slice(0, 12); } +// One line for a weights.checkpoint payload, "" when it is not one (no kind, or a sha256 that is not +// 64 hex characters); the same text as weightsCheckpointText in core/weights.ts. +function weightsCheckpointText(p) { + const sha = (v) => { const s = typeof v === "string" ? v.trim().toLowerCase() : ""; return /^[0-9a-f]{64}$/.test(s) ? s : ""; }; + const file = p.kind === "weights.checkpoint" ? sha(p.sha256) : ""; + if (!file) return ""; + const base = sha(p.baseSha256); + return "checkpoint " + shortWeights(file) + (base ? " ← " + shortWeights(base) : ""); +} function eventView(ev) { const p = ev.payload || {}; switch (ev.kind) { @@ -3881,6 +3894,9 @@ const t = weightsEditText(p); return { name: "Edited weights", arg: clip(t, 200), label: "Edited weights" + (t ? " · " + clip(t, 90) : ""), tone: "text", kindText: "weights.edit" }; } + // The hash of the weights file the session saved: what exp run --weights takes. + const checkpoint = weightsCheckpointText(p); + if (checkpoint) return { name: "Saved weights", arg: checkpoint, label: "Saved weights · " + checkpoint, tone: "text", kindText: "weights.checkpoint" }; // Claude Code loading a nested CLAUDE.md or a skill mid-session. if (p.kind !== "harness.loaded") return { name: KIND_LABEL[ev.kind], arg: "", label: KIND_LABEL[ev.kind], tone: "muted", kindText: ev.kind }; const names = (v) => (Array.isArray(v) ? v.filter((x) => typeof x === "string") : []); @@ -5098,7 +5114,11 @@ : [st.toolCalls ? h("span", { class: "badge" }, plural(st.toolCalls, "tool call")) : null, st.toolErrors ? h("span", { class: "badge fail" }, plural(st.toolErrors, "tool error")) : null], st.filesChanged ? h("span", { class: "badge" }, plural(st.filesChanged, "file") + " changed") : null, - st.weightEdits ? h("span", { class: "badge", title: fmtNum(st.weightEdits.weightsChanged) + " weights changed" + (st.weightEdits.tensors ? " in " + plural(st.weightEdits.tensors, "tensor") : "") }, plural(st.weightEdits.events, "weight edit")) : null, + st.weightEdits && st.weightEdits.events ? h("span", { class: "badge", title: fmtNum(st.weightEdits.weightsChanged) + " weights changed" + (st.weightEdits.tensors ? " in " + plural(st.weightEdits.tensors, "tensor") : "") }, plural(st.weightEdits.events, "weight edit")) : null, + st.weightEdits && st.weightEdits.checkpoints && st.weightEdits.checkpoints.length ? h("span", { class: "badge", title: "SHA-256 of the weights file this session saved (exp run --weights takes it): " + st.weightEdits.checkpoints.join(", ") }, st.weightEdits.checkpoints.length === 1 ? "checkpoint " + shortWeights(st.weightEdits.checkpoints[0]) : plural(st.weightEdits.checkpoints.length, "checkpoint")) : null, + // The experiments that ran a file this session saved (checkpointExperiments, as coding.sh show --json). + (Array.isArray(s.checkpointExperiments) ? s.checkpointExperiments : []).map((c) => (c.experiments || []).slice(0, 3).map((e) => + h("span", { class: "badge", title: "This experiment ran the weights file " + c.sha256 + " (exp run --weights)" }, "ran in ", h("a", { href: expHref(e.id), title: e.id }, e.name || e.id)))), st.turns && !api_ ? h("span", { class: "badge" }, plural(st.turns, "turn")) : null, kids.length ? h("span", { class: "badge" }, plural(kids.length, "subagent")) : null, s.parent && s.parent.type ? h("span", { class: "badge" }, "subagent · " + s.parent.type) : null, @@ -6225,7 +6245,7 @@ h("div", { class: "head-row" }, h("h1", { title: d.id }, d.name || d.id), h("div", { class: "badges" }, d.status === "cancelled" ? h("span", { class: "badge warn" }, "cancelled · partial") : null, copyBtn(d.id, "Copy experiment id", "experiment id", "Copy id"))), // Facts wrap between items, never inside one; the task command gets its own line. - h("p", { class: "meta fact-line" }, ...[datasetText(d), (d.trials || 1) > 1 ? plural(d.trials, "trial") + " per case" : "", fmtDate(d.createdAt || 0)].filter(Boolean).map((t) => h("span", null, t)), weightsFact(d.metadata && d.metadata.weights, "")), + h("p", { class: "meta fact-line" }, ...[datasetText(d), (d.trials || 1) > 1 ? plural(d.trials, "trial") + " per case" : "", fmtDate(d.createdAt || 0)].filter(Boolean).map((t) => h("span", null, t)), weightsFact(d.weights || (d.metadata && d.metadata.weights), "")), d.task ? h("p", { class: "meta mono truncate task-line", title: d.task.kind + " " + d.task.ref }, // Each argument wraps as a whole, so a phone never splits "agents-md-v2" at its hyphen. ...(d.task.kind + " " + d.task.ref).split(" ").flatMap((t, i) => [i ? " " : null, h("span", { class: "tok" }, t)])) : null), @@ -6353,7 +6373,10 @@ function weightsFact(w, none) { const sha = w && typeof w.sha256 === "string" ? w.sha256 : ""; if (!sha && !none) return null; - return h("span", { class: "meta weights", title: sha ? "SHA-256 of the checkpoint this run recorded with --weights: " + sha : "This run was started without --weights" }, sha ? "weights " + sha.slice(0, 12) : none); + // The sessions that recorded this file as a checkpoint (weights.sessions): where the edits behind it are. + const from = sha && Array.isArray(w.sessions) ? w.sessions.filter((id) => typeof id === "string" && id) : []; + return h("span", { class: "meta weights", title: sha ? "SHA-256 of the checkpoint this run recorded with --weights: " + sha : "This run was started without --weights" }, sha ? "weights " + shortWeights(sha) : none, + from.map((id, i) => [i ? ", " : " · edits ", h("a", { href: sessionHref(id), title: "The session that recorded this weights file as a checkpoint: " + id }, short(id, 8))])); } /** One decimal, so both sides add up to the difference shown in tenths of a point. */ function pct1(x) { return ((x || 0) * 100).toFixed(1) + "%"; } diff --git a/packages/coding.sh/test/cli/weights-show.test.ts b/packages/coding.sh/test/cli/weights-show.test.ts index 58724308..7c691c70 100644 --- a/packages/coding.sh/test/cli/weights-show.test.ts +++ b/packages/coding.sh/test/cli/weights-show.test.ts @@ -1,5 +1,7 @@ -// Weight edits in the CLI: `sessions` lists a session that holds nothing but an edit and finds -// it by tag and by count, and `show` prints the Weight edits row and one line per edit. +// Weight edits and checkpoints in the CLI: `sessions` lists a session that holds nothing but an +// edit or a checkpoint and finds it by tag, by count and by the checkpoint's hash, and `show` +// prints the Weight edits row, one line per edit and checkpoint, and the experiments that ran a +// checkpoint. import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; @@ -9,13 +11,22 @@ import { after, before, describe, it } from "node:test"; import { fileURLToPath } from "node:url"; import { openContext } from "../../src/core/context.ts"; import { buildSession } from "../../src/core/sessions.ts"; +import { createDataset } from "../../src/datasets/dataset.ts"; +import { FileEvalStore } from "../../src/datasets/store.ts"; +import type { Grader } from "../../src/datasets/types.ts"; +import { runExperiment } from "../../src/experiments/run.ts"; import type { AgentEvent } from "../../src/schema/index.ts"; const CLI = fileURLToPath(new URL("../../src/cli/main.ts", import.meta.url)); const A = "a".repeat(64); const B = "b".repeat(64); +// Hashes of whole weights files, as a script would compute them: the edited file and the one it started from. +const FILE = `c0ffee${"c".repeat(58)}`; +const FILE_BASE = "d".repeat(64); +const LATER = `beef${"e".repeat(60)}`; const EDITS = "sess-edits"; const PLAIN = "sess-plain"; +const SAVED = "sess-saved"; describe("weight edits in the CLI", () => { let root = ""; @@ -60,8 +71,26 @@ describe("weight edits in the CLI", () => { { ...e, id: "w-0", ts: t0, kind: "session.start", payload: { name: "weights.edit" } }, { ...e, id: "w-1", ts: t0 + 100, kind: "notification", payload: { kind: "weights.edit", tensor: "model.layers.15.mlp.down_proj.weight", shape: [4096, 11008], changed: 65536, l2: 0.0183, maxAbs: 0.004, baseSha256: A, editedSha256: B } }, { ...e, id: "w-2", ts: t0 + 200, kind: "notification", payload: { kind: "weights.edit", changed: 1024, l2: 0.002, maxAbs: 0.001 } }, + { ...e, id: "w-2c", ts: t0 + 250, kind: "notification", payload: { kind: "weights.checkpoint", sha256: FILE, baseSha256: FILE_BASE, note: "after layer 15" } }, { ...e, id: "w-3", ts: t0 + 300, kind: "session.end", payload: {} }, ]); + // A script that only saves: two checkpoints (one hash in upper case, as some tools print it) and no edit. + const c = { v: 0 as const, sessionId: SAVED, agent: "weights-run" }; + await put([ + { ...c, id: "c-0", ts: t0, kind: "session.start", payload: { name: "weights.checkpoint" } }, + { ...c, id: "c-1", ts: t0 + 100, kind: "notification", payload: { kind: "weights.checkpoint", sha256: LATER.toUpperCase() } }, + { ...c, id: "c-2", ts: t0 + 200, kind: "notification", payload: { kind: "weights.checkpoint", sha256: FILE_BASE } }, + { ...c, id: "c-3", ts: t0 + 300, kind: "notification", payload: { kind: "weights.checkpoint", sha256: "not a hash" } }, + { ...c, id: "c-4", ts: t0 + 400, kind: "session.end", payload: {} }, + ]); + // Two experiments ran the file EDITS saved, one ran the file both sessions name, none ran LATER. + const ds = createDataset({ name: "wt", createdAt: 1, cases: [{ id: "m1", input: 1, expected: 1 }] }); + const exact: Grader = { id: "exact", version: "1", kind: "custom", grade: async (x) => ({ score: 1, pass: x.output === x.expected }) }; + const evals = new FileEvalStore({ env, index: ctx.index }); + for (const [name, sha256] of [["edited", FILE], ["base", FILE_BASE], ["edited-again", FILE]] as const) { + await evals.putExperiment(await runExperiment({ name, dataset: ds, task: (n) => n, graders: [exact], metadata: { weights: { sha256 } } })); + } + await evals.putExperiment(await runExperiment({ name: "unstamped", dataset: ds, task: (n) => n, graders: [exact] })); const p = { v: 0 as const, sessionId: PLAIN, agent: "codex" }; await put([ { ...p, id: "p-0", ts: t0, kind: "session.start", payload: {} }, @@ -73,24 +102,67 @@ describe("weight edits in the CLI", () => { after(() => rmSync(root, { recursive: true, force: true })); it("sessions lists a session that holds only weight edits, and finds it by tag and by count", () => { - assert.deepEqual(ids(), [EDITS, PLAIN].sort(), "not hidden as an empty session"); + assert.deepEqual(ids(), [EDITS, PLAIN, SAVED].sort(), "not hidden as an empty session"); assert.deepEqual(ids("--flag", "weights:edited"), [EDITS]); assert.deepEqual(ids("--where", "flag:weights:edited"), [EDITS]); assert.deepEqual(ids("--where", "weights.edits>0"), [EDITS]); assert.deepEqual(ids("--where", "weights.changed>=66560"), [EDITS]); assert.deepEqual(ids("--where", "weights.changed>66560"), []); - assert.deepEqual(ids("--where", "-flag:weights:edited"), [PLAIN]); + assert.deepEqual(ids("--where", "-flag:weights:edited"), [PLAIN, SAVED].sort()); + }); + + it("sessions finds a session by the hash of a checkpoint it recorded, whole or by prefix", () => { + assert.deepEqual(ids("--flag", "weights:checkpoint"), [EDITS, SAVED].sort()); + assert.deepEqual(ids("--where", `weights.checkpoint:${FILE}`), [EDITS]); + assert.deepEqual(ids("--where", "weights.checkpoint:c0ffee*"), [EDITS]); + assert.deepEqual(ids("--where", "weights.checkpoint:C0FFEE*"), [EDITS], "a prefix matches in any case"); + assert.deepEqual(ids("--where", `weights.checkpoint:${LATER}`), [SAVED], "stored in lower case"); + assert.deepEqual(ids("--where", `weights.checkpoint:${FILE_BASE}`), [SAVED], "the hash a checkpoint started from is not a checkpoint"); + assert.deepEqual(ids("--where", `weights.checkpoint:${FILE},${LATER}`), [EDITS, SAVED].sort()); + assert.deepEqual(ids("--where", "weights.checkpoint:c0ffee* weights.edits>0"), [EDITS]); + assert.deepEqual(ids("--where", "-weights.checkpoint:*"), [PLAIN]); + assert.deepEqual(ids("--where", `weights.checkpoint:${"9".repeat(64)}`), []); + }); + + it("a session that holds only checkpoints is listed, tagged and shown with no edit count", () => { + const j = JSON.parse(cli("show", SAVED, "--json")) as { session: { flags: string[]; stats: { weightEdits?: unknown } }; checkpointExperiments?: unknown }; + assert.deepEqual(j.session.stats.weightEdits, { events: 0, weightsChanged: 0, checkpoints: [LATER, FILE_BASE] }, "in the order recorded; a value that is not a hash is not a checkpoint"); + assert.ok(j.session.flags.includes("weights:checkpoint")); + assert.equal(j.session.flags.includes("weights:edited"), false); + assert.deepEqual(ids("--where", "weights.edits:0 flag:weights:checkpoint"), [SAVED]); + const text = cli("show", SAVED); + assert.match(text, /^\s*Weight edits\s+checkpoints beefeeeeeeee, dddddddddddd$/m); + assert.match(text, /weights\s+checkpoint beefeeeeeeee$/m); + assert.match(text, /weights\s+checkpoint dddddddddddd$/m); + assert.doesNotMatch(text, /Flags/, "the tag is not a failure"); + }); + + it("show names the experiments that ran each checkpoint, newest first, and nothing for one no experiment ran", () => { + const text = cli("show", EDITS); + assert.match(text, /^\s*Experiments\s+c0ffeecccccc ran in edited-again \(exp_\w+\), edited \(exp_\w+\)$/m); + const saved = cli("show", SAVED); + assert.match(saved, /^\s*Experiments\s+dddddddddddd ran in base \(exp_\w+\)$/m); + assert.equal(saved.match(/^\s*Experiments/gm)?.length, 1, "no row for the checkpoint no experiment ran"); + const j = JSON.parse(cli("show", EDITS, "--json")) as { checkpointExperiments?: { sha256: string; experiments: { id: string; name: string }[] }[] }; + assert.deepEqual(j.checkpointExperiments?.map((x) => [x.sha256, x.experiments.map((e) => e.name)]), [[FILE, ["edited-again", "edited"]]]); + assert.ok(j.checkpointExperiments![0]!.experiments.every((e) => /^exp_\w+$/.test(e.id))); + for (const id of [PLAIN]) { + assert.doesNotMatch(cli("show", id), /^\s*Experiments/m); + assert.equal(Object.hasOwn(JSON.parse(cli("show", id, "--json")), "checkpointExperiments"), false); + } }); - it("show prints the Weight edits row and one line per edit", () => { + it("show prints the Weight edits row and one line per edit and checkpoint", () => { const text = cli("show", EDITS); - assert.match(text, /Weight edits\s+2 edits, 1 tensor, 66,560 weights changed/); - assert.match(text, /weights\s+model\.layers\.15\.mlp\.down_proj\.weight \[4096×11008\] · 65,536 of 45,088,768 weights · L2 0\.0183 · max 0\.004 · aaaaaaaa → bbbbbbbb/); + assert.match(text, /^\s*Weight edits\s+2 edits, 1 tensor, 66,560 weights changed, checkpoint c0ffeecccccc$/m); + assert.match(text, /weights\s+checkpoint c0ffeecccccc ← dddddddddddd$/m); + assert.doesNotMatch(text, /after layer 15/, "the line is the two hashes; the note stays in the event"); + assert.match(text, /weights\s+model\.layers\.15\.mlp\.down_proj\.weight \[4096×11008\] · 65,536 of 45,088,768 weights · L2 0\.0183 · max 0\.004 · aaaaaaaaaaaa → bbbbbbbbbbbb/); assert.match(text, /weights\s+1,024 weights · L2 0\.002 · max 0\.001/, "an edit captured without its name still says what changed"); assert.doesNotMatch(text, /^\s*\+\S+\s+notification\s*$/m, "no blank notification line"); const j = JSON.parse(cli("show", EDITS, "--json")) as { session: { flags: string[]; stats: { weightEdits?: unknown } } }; - assert.deepEqual(j.session.stats.weightEdits, { events: 2, tensors: 1, weightsChanged: 66560 }); - assert.ok(j.session.flags.includes("weights:edited")); + assert.deepEqual(j.session.stats.weightEdits, { events: 2, tensors: 1, weightsChanged: 66560, checkpoints: [FILE] }); + assert.ok(j.session.flags.includes("weights:edited") && j.session.flags.includes("weights:checkpoint")); }); it("a session with no weight edit has no row", () => { diff --git a/packages/coding.sh/test/core/reparse.test.ts b/packages/coding.sh/test/core/reparse.test.ts index 753bef70..775b5c5e 100644 --- a/packages/coding.sh/test/core/reparse.test.ts +++ b/packages/coding.sh/test/core/reparse.test.ts @@ -329,6 +329,24 @@ describe("reparse", () => { await t.index.close(); }); + it("a session stored before checkpoints were folded gains the file hashes and the tag on the refold", async () => { + const t = await setup(); + const file = `58899f90${"c".repeat(56)}`; + await t.index.replaceSessionEvents!("sdk", [], [ + { id: "c0", v: 0, sessionId: "sdk", ts: T0, agent: "sdk", kind: "session.start", payload: {} }, + { id: "c1", v: 0, sessionId: "sdk", ts: T0 + 1, agent: "sdk", kind: "notification", payload: { kind: "weights.checkpoint", sha256: file.toUpperCase(), baseSha256: "d".repeat(64) } }, + ]); + // What fold version 20 stored for it: a checkpoint was a notification like any other. + await t.index.upsertSession({ id: "sdk", agent: "sdk", startedAt: T0, stats: { events: 2, turns: 0, toolCalls: 0, toolErrors: 0, filesChanged: 0 }, flags: [] }); + await t.index.setCursor(REFOLD_CURSOR, 20 * 1e9); + assert.equal(await t.due(), true); + assert.equal((await t.reparse()).refolded, 1); + const s = (await t.index.getSession("sdk"))!; + assert.deepEqual(s.stats.weightEdits, { events: 0, weightsChanged: 0, checkpoints: [file] }); + assert.deepEqual(s.flags, ["weights:checkpoint"]); + await t.index.close(); + }); + it("one re-parse at a time", async () => { const t = await setup(); const lock = await acquireLock(paths(t.env).reparseLock, { waitMs: 0 }); diff --git a/packages/coding.sh/test/core/weights.test.ts b/packages/coding.sh/test/core/weights.test.ts index b41c6be7..3cd73333 100644 --- a/packages/coding.sh/test/core/weights.test.ts +++ b/packages/coding.sh/test/core/weights.test.ts @@ -3,7 +3,7 @@ import { describe, it } from "node:test"; import type { AgentEvent, EventKind } from "../../src/schema/index.ts"; import { toMetadata } from "../../src/core/redact.ts"; import { buildSession, failureFlags, isEmptySession } from "../../src/core/sessions.ts"; -import { weightEditsText, weightsEditText } from "../../src/core/weights.ts"; +import { CHECKPOINTS_MAX, weightEditsText, weightsCheckpointOf, weightsCheckpointText, weightsEditText, weightsEventText, weightsSummaryText } from "../../src/core/weights.ts"; const A = "a".repeat(64); const B = "b".repeat(64); @@ -50,11 +50,105 @@ describe("weight edits in a session", () => { }); }); +// The SHA-256 of a weights file, as `shasum -a 256 model.safetensors` prints it: what `exp run --weights` takes. +const FILE = `58899f90572c${"0".repeat(52)}`; +const BASE = `0496e5111097${"1".repeat(52)}`; +const checkpoint = (o: Record = {}): AgentEvent => ev("notification", { kind: "weights.checkpoint", sha256: FILE, baseSha256: BASE, ...o }); + +describe("a weights checkpoint in a session", () => { + it("is kept next to the edits, with its own tag, which is not a failure", () => { + const s = buildSession([ev("session.start"), edit(), edit({ tensor: "layers.4.mlp.down_proj.weight", changed: 96 }), checkpoint(), ev("session.end")]); + assert.deepEqual(s.stats.weightEdits, { events: 2, tensors: 2, weightsChanged: 1300, checkpoints: [FILE] }); + assert.deepEqual(s.flags.filter((f) => f.startsWith("weights:")), ["weights:edited", "weights:checkpoint"]); + assert.deepEqual(failureFlags(s.flags), []); + }); + + it("edits alone leave `checkpoints` unset", () => { + const s = buildSession([ev("session.start"), edit(), ev("session.end")]); + assert.equal(Object.hasOwn(s.stats.weightEdits!, "checkpoints"), false); + assert.equal(s.flags.includes("weights:checkpoint"), false); + }); + + it("a session with a checkpoint and no edit has the stat with zero edits, is not tagged as edited and is not hidden as empty", () => { + const s = buildSession([ev("session.start", { name: "weights.checkpoint" }), checkpoint(), ev("session.end")]); + assert.deepEqual(s.stats.weightEdits, { events: 0, weightsChanged: 0, checkpoints: [FILE] }); + assert.deepEqual(s.flags, ["weights:checkpoint"]); + assert.equal(s.stats.turns, 0, "the test's premise: nothing else makes this session count"); + assert.equal(isEmptySession(s), false); + }); + + it("keeps distinct hashes in arrival order, lowercased, and stops at the cap", () => { + const h = (i: number) => i.toString(16).padStart(64, "0"); + const two = buildSession([ev("session.start"), checkpoint({ sha256: BASE }), checkpoint({ sha256: FILE.toUpperCase() }), checkpoint({ sha256: ` ${BASE} ` }), checkpoint()]); + assert.deepEqual(two.stats.weightEdits!.checkpoints, [BASE, FILE]); + const many = buildSession([ev("session.start"), ...Array.from({ length: CHECKPOINTS_MAX + 4 }, (_, i) => checkpoint({ sha256: h(i) }))]); + assert.deepEqual(many.stats.weightEdits!.checkpoints, Array.from({ length: CHECKPOINTS_MAX }, (_, i) => h(i))); + assert.equal(CHECKPOINTS_MAX, 16); + }); + + it("an event whose sha256 is not 64 hex characters is not read as a checkpoint", () => { + for (const sha256 of [undefined, "", "58899f90572c", `${FILE}0`, `g${FILE.slice(1)}`, 7, null, [FILE]]) { + const e = checkpoint({ sha256 }); + assert.equal(weightsCheckpointOf(e.payload), undefined, JSON.stringify(sha256)); + assert.equal(weightsEventText(e.payload), undefined); + const s = buildSession([ev("session.start"), e, ev("session.end")]); + assert.equal(Object.hasOwn(s.stats, "weightEdits"), false, JSON.stringify(sha256)); + assert.deepEqual(s.flags, []); + } + // The hash under another kind is not a checkpoint either. + assert.equal(weightsCheckpointOf({ kind: "weights.edit", sha256: FILE }), undefined); + assert.equal(weightsCheckpointOf({ sha256: FILE }), undefined); + }); + + it("a base hash that is not a SHA-256 is left out, and the checkpoint still counts", () => { + assert.deepEqual(weightsCheckpointOf(checkpoint({ baseSha256: "0496e5111097" }).payload), { sha256: FILE }); + assert.deepEqual(weightsCheckpointOf(checkpoint({ baseSha256: BASE.toUpperCase(), note: "after the rank-1 patch" }).payload), { sha256: FILE, baseSha256: BASE, note: "after the rank-1 patch" }); + assert.deepEqual(weightsCheckpointOf(checkpoint({ note: 7 }).payload), { sha256: FILE, baseSha256: BASE }); + }); + + it("metadata capture keeps both file hashes and drops the note; an edit still keeps only its numbers", () => { + const full = checkpoint({ note: "after the rank-1 patch on layer 3" }); + assert.deepEqual(toMetadata(full).payload, { kind: "weights.checkpoint", sha256: FILE, baseSha256: BASE }); + assert.deepEqual(toMetadata(checkpoint({ sha256: FILE.toUpperCase(), baseSha256: undefined })).payload, { kind: "weights.checkpoint", sha256: FILE }); + // Not a checkpoint: nothing but the kind passes. + assert.deepEqual(toMetadata(checkpoint({ sha256: "not-a-hash" })).payload, { kind: "weights.checkpoint" }); + // The same field names anywhere else are still dropped. + assert.deepEqual(toMetadata(ev("notification", { kind: "other", sha256: FILE, baseSha256: BASE })).payload, { kind: "other" }); + assert.deepEqual(toMetadata(ev("tool.end", { tool: "Bash", sha256: FILE })).payload, { tool: "Bash" }); + assert.deepEqual(toMetadata(edit({ note: "x" })).payload, { kind: "weights.edit", changed: 1204, l2: 0.0183, maxAbs: 0.004 }); + + // Full and metadata capture fold to the same checkpoints, so the join works in either mode. + const events = [ev("session.start"), edit(), full, ev("session.end")]; + const a = buildSession(events); + const b = buildSession(events.map(toMetadata)); + assert.deepEqual(a.stats.weightEdits, { events: 1, tensors: 1, weightsChanged: 1204, checkpoints: [FILE] }); + assert.deepEqual(b.stats.weightEdits, { events: 1, weightsChanged: 1204, checkpoints: [FILE] }); + assert.equal(weightsEventText(toMetadata(full).payload), weightsEventText(full.payload)); + }); +}); + +describe("weightsCheckpointText", () => { + it("writes the saved file's hash and the base file's, 12 characters each", () => { + assert.equal(weightsCheckpointText(checkpoint().payload), "checkpoint 58899f90572c ← 0496e5111097"); + assert.equal(weightsCheckpointText(checkpoint({ baseSha256: undefined }).payload), "checkpoint 58899f90572c"); + assert.equal(weightsCheckpointText(checkpoint({ note: "not shown" }).payload), "checkpoint 58899f90572c ← 0496e5111097"); + assert.equal(weightsCheckpointText({ kind: "weights.checkpoint" }), ""); + }); + + it("weightsEventText is the line of either event, and undefined for anything else", () => { + assert.equal(weightsEventText(checkpoint().payload), "checkpoint 58899f90572c ← 0496e5111097"); + assert.equal(weightsEventText(edit().payload), weightsEditText(edit().payload)); + assert.equal(weightsEventText({ kind: "weights.edit" }), "", "an edit with no readable field is still an edit"); + assert.equal(weightsEventText({ kind: "hook.context", text: "x" }), undefined); + assert.equal(weightsEventText({}), undefined); + }); +}); + describe("weightsEditText", () => { it("names the tensor, its shape, the share of weights changed, both norms and both hashes", () => { assert.equal( weightsEditText(edit().payload), - "layers.3.mlp.down_proj.weight [4096×11008] · 1,204 of 45,088,768 weights · L2 0.0183 · max 0.004 · aaaaaaaa → bbbbbbbb", + "layers.3.mlp.down_proj.weight [4096×11008] · 1,204 of 45,088,768 weights · L2 0.0183 · max 0.004 · aaaaaaaaaaaa → bbbbbbbbbbbb", ); }); @@ -86,3 +180,16 @@ describe("weightEditsText", () => { assert.equal(weightEditsText({ events: 1, weightsChanged: 1 }), "1 edit, 1 weight changed"); }); }); + +describe("weightsSummaryText", () => { + it("names the checkpoints after the edits", () => { + assert.equal(weightsSummaryText({ events: 2, tensors: 2, weightsChanged: 66560 }), "2 edits, 2 tensors, 66,560 weights changed"); + assert.equal(weightsSummaryText({ events: 2, tensors: 2, weightsChanged: 66560, checkpoints: [FILE] }), "2 edits, 2 tensors, 66,560 weights changed, checkpoint 58899f90572c"); + assert.equal(weightsSummaryText({ events: 1, weightsChanged: 1, checkpoints: [BASE, FILE] }), "1 edit, 1 weight changed, checkpoints 0496e5111097, 58899f90572c"); + }); + + it("a session with a checkpoint and no edit reads as the checkpoint alone", () => { + assert.equal(weightsSummaryText({ events: 0, weightsChanged: 0, checkpoints: [FILE] }), "checkpoint 58899f90572c"); + assert.equal(weightEditsText({ events: 0, weightsChanged: 0, checkpoints: [FILE] }), "no edits"); + }); +}); diff --git a/packages/coding.sh/test/docs/sdk-reference.test.ts b/packages/coding.sh/test/docs/sdk-reference.test.ts index 448a12eb..8d8af9f6 100644 --- a/packages/coding.sh/test/docs/sdk-reference.test.ts +++ b/packages/coding.sh/test/docs/sdk-reference.test.ts @@ -4,7 +4,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { describe, it } from "node:test"; -import { pyOtel, pyPublic, renderSdkReference, SDK_REFERENCE_PATH, tsExports } from "../../scripts/sdk-reference.ts"; +import { pyNamed, pyOtel, pyPublic, renderSdkReference, SDK_REFERENCE_PATH, tsExports } from "../../scripts/sdk-reference.ts"; const repo = new URL("../../../../", import.meta.url); @@ -38,7 +38,7 @@ describe("docs/reference/sdk.md", () => { for (const p of e.params) assert.ok(doc.includes(`\`${p.name}\``), `${e.name} parameter ${p.name}`); for (const f of e.fields) assert.ok(doc.includes(`| \`${f.name}\` |`), `${e.name}.${f.name}`); } - for (const a of [...pyPublic(), ...pyOtel()]) for (const p of a.params) assert.ok(doc.includes(`\`${p.name}\``), `${a.name} parameter ${p.name}`); + for (const a of [...pyPublic(), ...pyOtel()]) for (const p of pyNamed(a)) assert.ok(doc.includes(`\`${p.name}\``), `${a.name} parameter ${p.name}`); for (const v of ["CODING_SH_URL", "CODING_SH_INGEST_KEY", "CODING_SH_SERVICE", "CODING_SH_CAPTURE_MODE", "CODING_SH_DISABLED", "CODING_SH_DEBUG"]) { assert.ok(doc.includes(`| \`${v}\` | yes | yes |`), `${v} read by both SDKs`); } diff --git a/packages/coding.sh/test/experiments/checkpoint-join.test.ts b/packages/coding.sh/test/experiments/checkpoint-join.test.ts new file mode 100644 index 00000000..d4714cc0 --- /dev/null +++ b/packages/coding.sh/test/experiments/checkpoint-join.test.ts @@ -0,0 +1,190 @@ +// The join between a session's weights.checkpoint event and an experiment's weights stamp: for a +// stamped experiment, exp show and both sides of exp compare name the sessions that recorded that +// file's hash, each with the edits it recorded, and --json carries their ids next to the stamp. +// The hashes below are of labels, not of files: nothing here hashes a file. +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { after, before, describe, it } from "node:test"; +import { buildSession } from "../../src/core/sessions.ts"; +import type { IndexStore } from "../../src/core/types.ts"; +import type { ExperimentResult } from "../../src/datasets/types.ts"; +import { CHECKPOINT_SESSIONS_MAX, checkpointExperiments, checkpointLines, checkpointSessions, withSessions } from "../../src/experiments/checkpoints.ts"; +import type { AgentEvent } from "../../src/schema/index.ts"; +import { MemoryIndexStore } from "../../src/store/memory.ts"; +import { tmpEnv, type TmpEnv } from "../adapters/_helpers.ts"; +import { ok, seed, T0 } from "../query/_seed.ts"; + +const sha = (label: string) => createHash("sha256").update(label).digest("hex"); +const BASE = sha("base file"); +const EDITED = sha("edited file"); +const OTHER = sha("a file no session saved"); +const TENSOR_A = sha("tensor before"); +const TENSOR_B = sha("tensor after"); + +/** A session as an SDK run leaves it: `edits` weight edits on distinct tensors, then one checkpoint per hash. */ +function run(id: string, at: number, edits: number, checkpoints: { sha256: string; baseSha256?: string }[]): AgentEvent[] { + const e = { v: 0 as const, sessionId: id, agent: "weights-run" }; + const evs: AgentEvent[] = [{ ...e, id: `${id}-0`, ts: at, kind: "session.start", payload: { name: "edit" } }]; + for (let i = 0; i < edits; i++) { + evs.push({ ...e, id: `${id}-e${i}`, ts: at + 1 + i, kind: "notification", payload: { kind: "weights.edit", tensor: `model.layers.${i}.weight`, shape: [256, 260], changed: 33_280, l2: 0.5, maxAbs: 0.01, baseSha256: TENSOR_A, editedSha256: TENSOR_B } }); + } + checkpoints.forEach((c, i) => evs.push({ ...e, id: `${id}-c${i}`, ts: at + 100 + i, kind: "notification", payload: { kind: "weights.checkpoint", ...c } })); + evs.push({ ...e, id: `${id}-z`, ts: at + 200, kind: "session.end", payload: {} }); + return evs; +} + +describe("checkpointSessions", () => { + const put = async (index: IndexStore, evs: AgentEvent[]) => { + await index.putEvents(evs); + await index.upsertSession(buildSession(evs)); + }; + + it("groups sessions by the hash they recorded, newest first, with each session's edits", async () => { + const index = new MemoryIndexStore(); + await put(index, run("old", T0, 2, [{ sha256: EDITED, baseSha256: BASE }])); + await put(index, run("new", T0 + 60_000, 1, [{ sha256: EDITED }, { sha256: BASE }])); + await put(index, run("none", T0 + 120_000, 3, [])); + const by = await checkpointSessions(index, [EDITED, BASE.toUpperCase(), OTHER, undefined, "not a hash"]); + assert.deepEqual([...by.keys()].sort(), [BASE, EDITED].sort(), "a hash no session recorded has no entry"); + assert.deepEqual(by.get(EDITED)!.map((s) => [s.id, s.edits.events]), [["new", 1], ["old", 2]]); + assert.deepEqual(by.get(BASE)!.map((s) => s.id), ["new"], "the hash a checkpoint started from does not make it a session of that file"); + assert.deepEqual(checkpointLines(EDITED, by), [" edits new 1 edit, 1 tensor, 33,280 weights changed", " edits old 2 edits, 2 tensors, 66,560 weights changed"]); + assert.deepEqual(checkpointLines(OTHER, by), []); + assert.deepEqual(checkpointLines(undefined, by), []); + assert.deepEqual(checkpointLines(EDITED, undefined), []); + }); + + it(`names at most ${CHECKPOINT_SESSIONS_MAX} sessions per hash`, async () => { + const index = new MemoryIndexStore(); + for (let i = 0; i < CHECKPOINT_SESSIONS_MAX + 2; i++) await put(index, run(`s${i}`, T0 + i * 1000, 0, [{ sha256: EDITED }])); + const by = await checkpointSessions(index, [EDITED]); + assert.deepEqual(by.get(EDITED)!.map((s) => s.id), ["s6", "s5", "s4", "s3", "s2"]); + assert.deepEqual(checkpointLines(EDITED, by)[0], " edits s6 no edits", "a session that saved without recording an edit says so"); + }); + + it("finds the session among more hashes than one query holds", async () => { + // exp list and experiments_list ask for every stamp on the page. A --where query holds 2,000 + // characters, about 30 hashes: one query for 120 would be refused, and the join would say nothing. + const index = new MemoryIndexStore(); + const many = Array.from({ length: 120 }, (_, i) => sha(`file ${i}`)); + await put(index, run("first", T0, 1, [{ sha256: many[0]! }])); + await put(index, run("last", T0 + 1000, 2, [{ sha256: many[119]! }, { sha256: many[60]! }])); + const by = await checkpointSessions(index, many); + assert.deepEqual([...by].map(([k, v]) => [many.indexOf(k), v.map((s) => s.id)]).sort((a, b) => Number(a[0]) - Number(b[0])), [[0, ["first"]], [60, ["last"]], [119, ["last"]]]); + }); + + it("is empty, with no error, when the index can't answer or nothing was asked", async () => { + // What a store plugin from before the key does: its query path throws on a key it has no SQL for. + const broken = new MemoryIndexStore(); + await put(broken, run("one", T0, 1, [{ sha256: EDITED }])); + broken.listSessions = async () => Promise.reject(new Error("no such key")); + assert.equal((await checkpointSessions(broken, [EDITED])).size, 0); + let asked = 0; + const counting = new MemoryIndexStore(); + counting.listSessions = async () => (asked++, []); + assert.equal((await checkpointSessions(counting, [undefined, "abc"])).size, 0); + assert.equal(asked, 0, "no query for no hash"); + }); + + it("withSessions sets the ids on a stamp some session recorded, and leaves any other object alone", async () => { + const index = new MemoryIndexStore(); + await put(index, run("one", T0, 1, [{ sha256: EDITED }])); + const by = await checkpointSessions(index, [EDITED, BASE]); + const row = (id: string, sha256?: string): { id: string; weights?: { sha256: string } } => (sha256 === undefined ? { id } : { id, weights: { sha256 } }); + assert.deepEqual(withSessions(row("a", EDITED), by), { id: "a", weights: { sha256: EDITED, sessions: ["one"] } }); + assert.deepEqual(withSessions(row("b", BASE), by), { id: "b", weights: { sha256: BASE } }); + assert.deepEqual(withSessions(row("c"), by), { id: "c" }); + }); + + it("checkpointExperiments lists the experiments stamped with each hash, and nothing when they can't be read", async () => { + const exp = (id: string, name: string, sha256?: string): ExperimentResult => { + const e = { id, name } as ExperimentResult; + if (sha256) e.metadata = { weights: { sha256 } }; + return e; + }; + const store = { listExperiments: async () => [exp("exp_3", "again", EDITED), exp("exp_2", "plain"), exp("exp_1", "first", EDITED), exp("exp_0", "base", BASE)] }; + const by = await checkpointExperiments(store, [EDITED, OTHER]); + assert.deepEqual([...by], [[EDITED, [{ id: "exp_3", name: "again" }, { id: "exp_1", name: "first" }]]]); + assert.equal((await checkpointExperiments({ listExperiments: async () => Promise.reject(new Error("no store")) }, [EDITED])).size, 0); + assert.equal((await checkpointExperiments(store, undefined)).size, 0); + }); +}); + +describe("exp show, exp compare and exp list name the sessions that recorded a stamp", () => { + let env: TmpEnv; + let proj = ""; + const task = (model: string) => JSON.stringify({ kind: "command", command: process.execPath, args: [join(proj, "gen.mjs")], env: { MODEL: model } }); + const G = '{"kind":"json-path","name":"correct","path":"$.answer"}'; + const runArgs = (model: string, name: string, ...more: string[]) => ["exp", "run", "-d", "wt", "--task", task(model), "-g", G, "--name", name, ...more]; + + before(async () => { + env = tmpEnv(); + proj = join(env.root, "proj"); + mkdirSync(proj, { recursive: true }); + mkdirSync(env.dataHome, { recursive: true }); + // MODEL=base answers m1 and m2; edited answers all four. + writeFileSync( + join(proj, "gen.mjs"), + `let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{const n=Number(JSON.parse(s).slice(1));const right=process.env.MODEL==="edited"||n<=2;process.stdout.write(JSON.stringify({answer:right?"ok":"wrong"}))});\n`, + ); + const cases = [1, 2, 3, 4].flatMap((n) => ["--case", JSON.stringify({ id: `m${n}`, input: `m${n}`, expected: { answer: "ok" } })]); + ok(["dataset", "add", "wt", ...cases]); + // Two sessions saved the edited file (the newer one recorded no edit); nobody recorded the base file. + const sessions = [run("sess-edit", T0, 2, [{ sha256: EDITED, baseSha256: BASE }]), run("sess-resave", T0 + 3600e3, 0, [{ sha256: EDITED }]), run("sess-plain", T0, 1, [])]; + await seed(env.dataHome, { events: sessions.flat(), sessions: sessions.map((evs) => buildSession(evs)) }); + ok(runArgs("base", "base", "--weights", BASE)); + ok(runArgs("edited", "edited", "--weights", EDITED)); + ok(runArgs("edited", "other", "--weights", OTHER)); + ok(runArgs("base", "plain")); + }); + after(() => env.restore()); + + const EDIT_LINES = [" edits sess-resave no edits", " edits sess-edit 2 edits, 2 tensors, 66,560 weights changed"]; + + it("exp show prints one edits line per session under the stamp, and --json carries the ids next to it", () => { + const lines = ok(["exp", "show", "edited"]).stdout.split("\n"); + const at = lines.indexOf(`weights ${EDITED}`); + assert.ok(at > 0, lines.join("\n")); + assert.deepEqual(lines.slice(at + 1, at + 3), EDIT_LINES); + const j = JSON.parse(ok(["exp", "show", "edited", "--json"]).stdout) as { experiment: ExperimentResult; weights?: unknown }; + assert.deepEqual(j.weights, { sha256: EDITED, sessions: ["sess-resave", "sess-edit"] }); + assert.deepEqual(j.experiment.metadata?.weights, { sha256: EDITED }, "the stored record prints as it is"); + }); + + it("a stamp no session recorded, and an experiment with no stamp, print nothing extra", () => { + for (const name of ["base", "other", "plain"]) { + assert.doesNotMatch(ok(["exp", "show", name]).stdout, /^ {2}edits /m, name); + assert.equal(Object.hasOwn(JSON.parse(ok(["exp", "show", name, "--json"]).stdout), "weights"), false, name); + } + }); + + it("exp compare names them under the side that ran that file, in text and --json", () => { + const lines = ok(["exp", "compare", "base", "edited"]).stdout.split("\n"); + assert.match(lines[0]!, /^A base exp_\w+ {2}pass 50\.0% {2}weights \w{12}$/); + assert.match(lines[1]!, /^B edited exp_\w+ {2}pass 100\.0% {2}weights \w{12}$/); + assert.deepEqual(lines.slice(2, 4), EDIT_LINES); + const flipped = ok(["exp", "compare", "edited", "base"]).stdout.split("\n"); + assert.deepEqual(flipped.slice(1, 3), EDIT_LINES, "under A when A ran the edited file"); + assert.match(flipped[3]!, /^B base /); + const c = JSON.parse(ok(["exp", "compare", "base", "edited", "--json"]).stdout).comparison; + assert.deepEqual([c.a.weights, c.b.weights], [{ sha256: BASE }, { sha256: EDITED, sessions: ["sess-resave", "sess-edit"] }]); + const twins = ok(["exp", "compare", "edited", "edited"]).stdout.split("\n"); + assert.deepEqual([twins.slice(1, 3), twins.slice(4, 6)], [EDIT_LINES, EDIT_LINES], "both sides, when both ran it"); + assert.doesNotMatch(ok(["exp", "compare", "base", "plain"]).stdout, /^ {2}edits /m); + }); + + it("exp list --json carries the ids on each stamped row", () => { + const rows = (JSON.parse(ok(["exp", "list", "--json"]).stdout) as { experiments: { name: string; weights?: unknown }[] }).experiments; + const row = (name: string) => rows.find((r) => r.name === name)!; + assert.deepEqual(row("edited").weights, { sha256: EDITED, sessions: ["sess-resave", "sess-edit"] }); + assert.deepEqual(row("base").weights, { sha256: BASE }); + assert.equal(Object.hasOwn(row("plain"), "weights"), false); + }); + + it("the session, in turn, names the experiment that ran its checkpoint", () => { + assert.match(ok(["show", "sess-edit"]).stdout, new RegExp(`^\\s*Experiments\\s+${EDITED.slice(0, 12)} ran in edited \\(exp_\\w+\\)$`, "m")); + assert.doesNotMatch(ok(["show", "sess-plain"]).stdout, /^\s*Experiments/m); + }); +}); diff --git a/packages/coding.sh/test/mcp/weights.test.ts b/packages/coding.sh/test/mcp/weights.test.ts index cbfc4cc6..e72ef023 100644 --- a/packages/coding.sh/test/mcp/weights.test.ts +++ b/packages/coding.sh/test/mcp/weights.test.ts @@ -1,9 +1,15 @@ -// Weight edits over MCP: sessions_list finds the session by its tag and carries the stat, -// session_get's timeline has one line per edit, and the session resource has the row. +// Weight edits and checkpoints over MCP: sessions_list finds the session by its tag and by a +// checkpoint's hash and carries the stat, session_get's timeline has one line per edit and +// checkpoint, the session resource has the row, and the experiment tools name the session +// that recorded the weights file an experiment ran. import assert from "node:assert/strict"; import { after, before, describe, it } from "node:test"; import { openContext } from "../../src/core/context.ts"; import { buildSession } from "../../src/core/sessions.ts"; +import { createDataset } from "../../src/datasets/dataset.ts"; +import { FileEvalStore } from "../../src/datasets/store.ts"; +import type { Grader } from "../../src/datasets/types.ts"; +import { runExperiment } from "../../src/experiments/run.ts"; import type { AgentEvent } from "../../src/schema/index.ts"; import { RawClient, cleanup, makeWorld, seed, type World } from "./helpers.ts"; @@ -11,6 +17,11 @@ const SID = "sess-weights"; const A = "a".repeat(64); const B = "b".repeat(64); const C = "c".repeat(64); +// Hashes of whole weights files: the one the session saved and the one it started from. +const FILE = `c0ffee${"1".repeat(58)}`; +const FILE_BASE = `ba5e${"2".repeat(60)}`; +const STATS = { events: 3, tensors: 2, weightsChanged: 66560, checkpoints: [FILE] }; +const ids: Record = {}; let w: World; let c: RawClient; @@ -27,10 +38,20 @@ before(async () => { { ...base, id: "w-1", ts: t0 + 100, kind: "notification", payload: edit("model.layers.15.mlp.down_proj.weight", 65536, A, B) }, { ...base, id: "w-2", ts: t0 + 200, kind: "notification", payload: edit("model.layers.16.mlp.down_proj.weight", 1024, B, C) }, { ...base, id: "w-2b", ts: t0 + 250, kind: "notification", payload: { kind: "weights.edit" } }, + { ...base, id: "w-2c", ts: t0 + 275, kind: "notification", payload: { kind: "weights.checkpoint", sha256: FILE, baseSha256: FILE_BASE, note: "saved after layer 16" } }, { ...base, id: "w-3", ts: t0 + 300, kind: "session.end", payload: {} }, ]; await ctx.index.putEvents(evs); await ctx.index.upsertSession(buildSession(evs)); + // One experiment ran the file the session saved, one ran the file it started from, one recorded none. + const ds = createDataset({ name: "wt", createdAt: 1, cases: [1, 2].map((n) => ({ id: `m${n}`, input: n, expected: n })) }); + const exact: Grader = { id: "exact", version: "1", kind: "custom", grade: async (x) => ({ score: 1, pass: x.output === x.expected }) }; + const evals = new FileEvalStore({ env: w.env, index: ctx.index }); + for (const [name, sha256] of [["wt-base", FILE_BASE], ["wt-edited", FILE], ["wt-plain", undefined]] as const) { + const r = await runExperiment({ name, dataset: ds, task: (n) => n, graders: [exact], metadata: sha256 ? { weights: { sha256 } } : undefined }); + await evals.putExperiment(r); + ids[name] = r.id; + } await ctx.close(); c = new RawClient(w.env, w.repo); await c.initialize(); @@ -46,8 +67,14 @@ describe("weight edits over MCP", () => { const r = await c.call("sessions_list", { flag: "weights:edited" }); const sessions = r.structuredContent!.sessions as { id: string; flags?: string[]; weightEdits?: unknown }[]; assert.deepEqual(sessions.map((s) => s.id), [SID]); - assert.deepEqual(sessions[0]!.weightEdits, { events: 3, tensors: 2, weightsChanged: 66560 }); - assert.ok(sessions[0]!.flags!.includes("weights:edited")); + assert.deepEqual(sessions[0]!.weightEdits, STATS); + assert.ok(sessions[0]!.flags!.includes("weights:edited") && sessions[0]!.flags!.includes("weights:checkpoint")); + for (const q of [`weights.checkpoint:${FILE}`, "weights.checkpoint:c0ffee*", "flag:weights:checkpoint"]) { + const hit = await c.call("sessions_list", { where: q }); + assert.deepEqual((hit.structuredContent!.sessions as { id: string }[]).map((s) => s.id), [SID], q); + } + const miss = await c.call("sessions_list", { where: `weights.checkpoint:${FILE_BASE}` }); + assert.deepEqual(miss.structuredContent!.sessions, [], "the hash a checkpoint started from is not a checkpoint"); const where = await c.call("sessions_list", { where: "weights.changed>1000" }); assert.deepEqual((where.structuredContent!.sessions as { id: string }[]).map((s) => s.id), [SID]); const all = await c.call("sessions_list", {}); @@ -55,26 +82,54 @@ describe("weight edits over MCP", () => { assert.ok(other.length > 0 && other.every((s) => !Object.hasOwn(s, "weightEdits")), "sessions without edits carry no weightEdits key"); }); - it("session_get has one timeline line per edit", async () => { + it("session_get has one timeline line per edit and checkpoint", async () => { const r = await c.call("session_get", { sessionId: SID }); const d = r.structuredContent as { session: { weightEdits?: unknown }; timeline: { kind: string; text: string }[]; timelineTotal: number }; - assert.deepEqual(d.session.weightEdits, { events: 3, tensors: 2, weightsChanged: 66560 }); + assert.deepEqual(d.session.weightEdits, STATS); assert.deepEqual( d.timeline.map((t) => [t.kind, t.text]), [ - ["weights", "model.layers.15.mlp.down_proj.weight [4096×11008] · 65,536 of 45,088,768 weights · L2 0.0183 · max 0.004 · aaaaaaaa → bbbbbbbb"], - ["weights", "model.layers.16.mlp.down_proj.weight [4096×11008] · 1,024 of 45,088,768 weights · L2 0.0183 · max 0.004 · bbbbbbbb → cccccccc"], + ["weights", "model.layers.15.mlp.down_proj.weight [4096×11008] · 65,536 of 45,088,768 weights · L2 0.0183 · max 0.004 · aaaaaaaaaaaa → bbbbbbbbbbbb"], + ["weights", "model.layers.16.mlp.down_proj.weight [4096×11008] · 1,024 of 45,088,768 weights · L2 0.0183 · max 0.004 · bbbbbbbbbbbb → cccccccccccc"], ["weights", "no details recorded"], + ["weights", "checkpoint c0ffee111111 ← ba5e22222222"], ], ); - assert.equal(d.timelineTotal, 3, "an edit with no readable field keeps its line"); + assert.equal(d.timelineTotal, 4, "an edit with no readable field keeps its line"); + }); + + it("session_get names the experiments that ran the session's checkpoint, as coding.sh show --json does", async () => { + const r = await c.call("session_get", { sessionId: SID }); + const d = r.structuredContent as { checkpointExperiments?: unknown }; + // The file the session saved, not the one it started from: wt-base ran that one. + assert.deepEqual(d.checkpointExperiments, [{ sha256: FILE, experiments: [{ id: ids["wt-edited"], name: "wt-edited" }] }]); + const plain = await c.call("session_get", { sessionId: "sess-a" }); + assert.equal(Object.hasOwn(plain.structuredContent!, "checkpointExperiments"), false, "absent on a session with no checkpoint"); }); it("the session resource has the weight edits row and both lines", async () => { const r = await c.request("resources/read", { uri: `codingsh://session/${SID}` }); const text = (r.result!.contents as { text: string }[])[0]!.text; - assert.match(text, /Weight edits.*3 edits, 2 tensors, 66,560 weights changed/); + assert.match(text, /Weight edits.*3 edits, 2 tensors, 66,560 weights changed, checkpoint c0ffee111111/); + assert.match(text, /checkpoint c0ffee111111 ← ba5e22222222/); + assert.doesNotMatch(text, /saved after layer 16/); assert.match(text, /model\.layers\.15\.mlp\.down_proj\.weight \[4096×11008\]/); assert.match(text, /model\.layers\.16\.mlp\.down_proj\.weight \[4096×11008\]/); }); + + it("experiments_list and experiment_compare name the session that recorded the weights file a run used", async () => { + const list = await c.call("experiments_list", {}); + const rows = list.structuredContent!.experiments as { id: string; weights?: unknown }[]; + const row = (name: string) => rows.find((r) => r.id === ids[name])!; + assert.deepEqual(row("wt-edited").weights, { sha256: FILE, sessions: [SID] }); + assert.deepEqual(row("wt-base").weights, { sha256: FILE_BASE }, "no session recorded this file: the stamp alone"); + assert.equal(Object.hasOwn(row("wt-plain"), "weights"), false); + + const cmp = await c.call("experiment_compare", { baseline: ids["wt-base"], candidate: ids["wt-edited"] }); + const d = cmp.structuredContent as { baseline: { weights?: unknown }; candidate: { weights?: unknown } }; + assert.deepEqual(d.baseline.weights, { sha256: FILE_BASE }); + assert.deepEqual(d.candidate.weights, { sha256: FILE, sessions: [SID] }); + const plain = await c.call("experiment_compare", { baseline: ids["wt-plain"], candidate: ids["wt-edited"] }); + assert.equal(Object.hasOwn((plain.structuredContent as { baseline: object }).baseline, "weights"), false); + }); }); diff --git a/packages/coding.sh/test/otel/weights.test.ts b/packages/coding.sh/test/otel/weights.test.ts index c3eb1d10..739d9b6e 100644 --- a/packages/coding.sh/test/otel/weights.test.ts +++ b/packages/coding.sh/test/otel/weights.test.ts @@ -1,11 +1,11 @@ -// A weight edit over OTLP: a span or a log record named `weights.edit` with coding_sh.weights.* -// attributes becomes the notification the SDKs' weights_edit helpers send, so the session gets -// the same stat, tag and line whichever way the edit arrived. +// A weight edit and a weights checkpoint over OTLP: a span or a log record named `weights.edit` +// or `weights.checkpoint` with coding_sh.weights.* attributes becomes the notification the SDKs' +// helpers send, so the session gets the same stat, tag and line whichever way it arrived. import assert from "node:assert/strict"; import { describe, it } from "node:test"; import type { AgentEvent } from "../../src/schema/index.ts"; import { buildSession } from "../../src/core/sessions.ts"; -import { isWeightsEdit, weightsEditText } from "../../src/core/weights.ts"; +import { isWeightsCheckpoint, isWeightsEdit, weightsCheckpointText, weightsEditText } from "../../src/core/weights.ts"; import { decodeLogsJson, decodeTraceJson } from "../../src/otel/json.ts"; import { type MapContext, mapLogs, mapTraces, TraceSessionMap } from "../../src/otel/map.ts"; import { iAttr, otlpLogs, otlpSpan, otlpTraces, sAttr } from "./_encode.ts"; @@ -28,7 +28,7 @@ const FULL: J[] = [ sAttr("coding_sh.weights.note", "rank-1 edit"), ]; const PAYLOAD = { kind: "weights.edit", tensor: "layers.3.mlp.down_proj.weight", shape: [4096, 11008], changed: 1204, l2: 0.0183, maxAbs: 0.004, baseSha256: A, editedSha256: B, note: "rank-1 edit" }; -const LINE = "layers.3.mlp.down_proj.weight [4096×11008] · 1,204 of 45,088,768 weights · L2 0.0183 · max 0.004 · aaaaaaaa → bbbbbbbb"; +const LINE = "layers.3.mlp.down_proj.weight [4096×11008] · 1,204 of 45,088,768 weights · L2 0.0183 · max 0.004 · aaaaaaaaaaaa → bbbbbbbbbbbb"; const ctx = (): MapContext => { const blobs = new MemBlobs(); @@ -88,3 +88,66 @@ describe("a weights.edit log record", () => { assert.deepEqual(events.map((e) => e.payload), [{ kind: "weights.edit", changed: 12 }]); }); }); + +const FILE = `58899f90572c${"0".repeat(52)}`; +const FILE_BASE = `0496e5111097${"f".repeat(52)}`; +const SAVED: J[] = [sAttr("coding_sh.weights.sha256", FILE.toUpperCase()), sAttr("coding_sh.weights.base_sha256", FILE_BASE), sAttr("coding_sh.weights.note", "after layer 3")]; +const SAVED_PAYLOAD = { kind: "weights.checkpoint", sha256: FILE, baseSha256: FILE_BASE, note: "after layer 3" }; +const checkpoints = (events: AgentEvent[]) => events.filter(isWeightsCheckpoint); + +describe("a weights.checkpoint span", () => { + it("under a parent span becomes the checkpoint notification, with the hash in lower case", async () => { + const events = await spans( + otlpSpan(TRACE, "1111111111111111", undefined, "edit run", [], 0, 5000), + otlpSpan(TRACE, "2222222222222222", "1111111111111111", "weights.edit", FULL, 1000, 1200), + otlpSpan(TRACE, "3333333333333333", "1111111111111111", "weights.checkpoint", SAVED, 2000, 2000), + ); + const own = events.filter((e) => e.spanId === "3333333333333333"); + assert.deepEqual(own.map((e) => e.kind), ["notification"]); + assert.deepEqual(own[0]!.payload, SAVED_PAYLOAD); + assert.equal(weightsCheckpointText(own[0]!.payload), "checkpoint 58899f90572c ← 0496e5111097"); + const s = buildSession(events); + assert.deepEqual(s.stats.weightEdits, { events: 1, tensors: 1, weightsChanged: 1204, checkpoints: [FILE] }); + assert.ok(s.flags.includes("weights:edited") && s.flags.includes("weights:checkpoint")); + }); + + it("as the root span of its trace still records the checkpoint", async () => { + const events = await spans(otlpSpan(TRACE, "1111111111111111", undefined, "weights.checkpoint", SAVED, 0, 0)); + assert.deepEqual(checkpoints(events).map((e) => e.payload), [SAVED_PAYLOAD]); + assert.ok(events.some((e) => e.kind === "session.start"), "the root span still opens the session"); + assert.deepEqual(buildSession(events).stats.weightEdits, { events: 0, weightsChanged: 0, checkpoints: [FILE] }); + }); + + it("keeps the hash alone when there is no base, and leaves out a base that is not a SHA-256", async () => { + const one = async (attrs: J[]) => (await spans(otlpSpan(TRACE, "1111111111111111", undefined, "run", [], 0, 9), otlpSpan(TRACE, "3333333333333333", "1111111111111111", "weights.checkpoint", attrs, 1, 2))).find((e) => e.spanId === "3333333333333333")!.payload; + assert.deepEqual(await one([sAttr("coding_sh.weights.sha256", FILE)]), { kind: "weights.checkpoint", sha256: FILE }); + assert.deepEqual(await one([sAttr("coding_sh.weights.sha256", FILE), sAttr("coding_sh.weights.base_sha256", "abc")]), { kind: "weights.checkpoint", sha256: FILE }); + }); + + it("without a well-formed sha256 is not read as a checkpoint", async () => { + for (const attrs of [[], [sAttr("coding_sh.weights.sha256", "abc")], [sAttr("coding_sh.weights.base_sha256", FILE_BASE)], [iAttr("coding_sh.weights.sha256", 7)]]) { + const events = await spans(otlpSpan(TRACE, "1111111111111111", undefined, "run", [], 0, 9), otlpSpan(TRACE, "3333333333333333", "1111111111111111", "weights.checkpoint", attrs, 1, 2)); + assert.deepEqual(checkpoints(events), [], JSON.stringify(attrs)); + assert.equal(Object.hasOwn(buildSession(events).stats, "weightEdits"), false); + } + }); + + it("a span with another name is not a checkpoint, whatever its attributes", async () => { + const events = await spans(otlpSpan(TRACE, "1111111111111111", undefined, "run", [], 0, 9), otlpSpan(TRACE, "4444444444444444", "1111111111111111", "weights.save", SAVED, 1, 2)); + assert.deepEqual(checkpoints(events), []); + }); +}); + +describe("a weights.checkpoint log record", () => { + it("becomes the same notification", async () => { + const events = (await mapLogs(decodeLogsJson(otlpLogs("trainer", "weights.checkpoint", [sAttr("session.id", "run-7"), ...SAVED])), ctx())).events; + assert.deepEqual(events.map((e) => [e.kind, e.sessionId]), [["notification", "run-7"]]); + assert.deepEqual(events[0]!.payload, SAVED_PAYLOAD); + }); + + it("named by its event.name attribute too", async () => { + const req = otlpLogs("trainer", "saved the edited file", [sAttr("event.name", "weights.checkpoint"), sAttr("coding_sh.weights.sha256", FILE)]); + const events = (await mapLogs(decodeLogsJson(req), ctx())).events; + assert.deepEqual(events.map((e) => e.payload), [{ kind: "weights.checkpoint", sha256: FILE }]); + }); +}); diff --git a/packages/coding.sh/test/query/_where-fixture.ts b/packages/coding.sh/test/query/_where-fixture.ts index 932455dc..073c23fc 100644 --- a/packages/coding.sh/test/query/_where-fixture.ts +++ b/packages/coding.sh/test/query/_where-fixture.ts @@ -5,14 +5,18 @@ import type { AgentEvent, Session } from "../../src/schema/index.ts"; import { event, session, T0 } from "./_seed.ts"; const H = 3_600_000; +/** Weights files the fixture's sessions recorded as checkpoints: s2 saved CP_A then CP_B, s5 only CP_C (and no edit). */ +export const CP_A = `c0ffee${"a".repeat(58)}`; +export const CP_B = `beef${"b".repeat(60)}`; +export const CP_C = `c0ffee${"c".repeat(58)}`; export function fixture(): { sessions: Session[]; events: AgentEvent[] } { const repo = (root: string, branch?: string) => ({ root, head: "abc", ...(branch ? { branch } : {}) }); const sessions: Session[] = [ session("s1", { agent: "codex", model: "gpt-5.1-codex", repo: repo("/work/demo", "main"), cwd: "/work/demo", startedAt: T0, endedAt: T0 + 20 * 60_000, flags: ["no-verification", "skill:pdf"], title: "Fix the Build", stats: { events: 6, turns: 3, toolCalls: 4, toolErrors: 1, filesChanged: 2, testsFailed: 2, usage: { inputTokens: 900_000, outputTokens: 200_000, cacheReadTokens: 50_000, costUsd: 1.25 }, cacheBreaks: { count: 2, lostTokens: 30_000, extraUsd: 0.14, byCause: { idle: { count: 2, lostTokens: 30_000, extraUsd: 0.14 } } } } }), - session("s2", { agent: "claude-code", model: "claude-sonnet-4-5", repo: repo("/work/other", "feat/x"), cwd: "/work/other/sub", startedAt: T0 + H, endedAt: T0 + H + 5 * 60_000, flags: ["compaction", "unapproved-action"], title: "write docs", stats: { events: 4, turns: 25, toolCalls: 120, toolErrors: 0, filesChanged: 0, testsPassed: 10, usage: { inputTokens: 1000, outputTokens: 100, costUsd: 0.02 }, unapprovedActions: { count: 2, byKind: { "rm-rf": 1, paid: 1 }, first: { kind: "rm-rf", command: "rm -rf src", at: T0 + H + 60_000, afterError: true } } } }), + session("s2", { agent: "claude-code", model: "claude-sonnet-4-5", repo: repo("/work/other", "feat/x"), cwd: "/work/other/sub", startedAt: T0 + H, endedAt: T0 + H + 5 * 60_000, flags: ["compaction", "unapproved-action"], title: "write docs", stats: { events: 4, turns: 25, toolCalls: 120, toolErrors: 0, filesChanged: 0, testsPassed: 10, usage: { inputTokens: 1000, outputTokens: 100, costUsd: 0.02 }, unapprovedActions: { count: 2, byKind: { "rm-rf": 1, paid: 1 }, first: { kind: "rm-rf", command: "rm -rf src", at: T0 + H + 60_000, afterError: true } }, weightEdits: { events: 2, tensors: 2, weightsChanged: 66_560, checkpoints: [CP_A, CP_B] } } }), session("s3", { agent: "claude-code", model: "claude-opus-4-1", cwd: "C:\\Users\\me\\proj", startedAt: T0 + 2 * H, flags: [], title: "50% done_now", stats: { events: 3, turns: 1, toolCalls: 1, toolErrors: 0, filesChanged: 0 } }), session("s4", { agent: "codex", model: "gpt-5.1-codex", repo: repo("/work/demo", "main"), cwd: "/work/demo", startedAt: T0 + 3 * H, endedAt: T0 + 3 * H + 2 * H, parent: { sessionId: "s1", type: "explorer" }, flags: ["skill:docx", "gateway:vercel"], stats: { events: 2, turns: 1, toolCalls: 2, toolErrors: 2, filesChanged: 0, llmErrors: { count: 1, statusCode: 429 }, usage: { inputTokens: 10, outputTokens: 5 }, cacheBreaks: { count: 1, lostTokens: 5000, byCause: { other: { count: 1, lostTokens: 5000 } }, unpriced: 1 } } }), - session("s5", { agent: "gemini", runId: "run_42", cwd: "/work/*", title: "done*now", startedAt: T0 + 4 * H, endedAt: T0 + 4 * H + 30_000, flags: ["repeated-tool-failure"], stats: { events: 1, turns: 0, toolCalls: 0, toolErrors: 0, filesChanged: 0, usage: { inputTokens: 5, outputTokens: 5, costUsd: 0 } } }), + session("s5", { agent: "gemini", runId: "run_42", cwd: "/work/*", title: "done*now", startedAt: T0 + 4 * H, endedAt: T0 + 4 * H + 30_000, flags: ["repeated-tool-failure"], stats: { events: 1, turns: 0, toolCalls: 0, toolErrors: 0, filesChanged: 0, usage: { inputTokens: 5, outputTokens: 5, costUsd: 0 }, weightEdits: { events: 0, weightsChanged: 0, checkpoints: [CP_C] } } }), ]; const tool = (sid: string, n: number, kind: "tool.start" | "tool.end", payload: Record, over: Partial = {}) => event(sid, n, { kind, payload, ...over, agent: sessions.find((s) => s.id === sid)!.agent }); const events: AgentEvent[] = [ @@ -117,6 +121,19 @@ export const QUERIES = [ "unapproved>0", "unapproved>=2", "-unapproved>0", + "weights.edits>0", + "weights.edits:0", + "weights.changed>1000", + `weights.checkpoint:${CP_A}`, + `weights.checkpoint:${CP_B}`, + `weights.checkpoint:${CP_A.toUpperCase()}`, + "weights.checkpoint:c0ffee*", + "weights.checkpoint:C0FFEE*", + "weights.checkpoint:beef*", + "weights.checkpoint:*cc", + `weights.checkpoint:${CP_B},${CP_C}`, + "-weights.checkpoint:c0ffee*", + "weights.checkpoint:c0ffee* weights.edits>0", "files>=1", "events<3", "run:run_*", diff --git a/packages/coding.sh/test/server/forward.test.ts b/packages/coding.sh/test/server/forward.test.ts index 9c9b898e..7c210979 100644 --- a/packages/coding.sh/test/server/forward.test.ts +++ b/packages/coding.sh/test/server/forward.test.ts @@ -101,6 +101,9 @@ async function post(h: ServeHandle, path: string, key: string | undefined, body: const attr = (kvs: J[] | undefined, key: string): J => kvs?.find((kv: J) => kv.key === key)?.value; const TRACE = "0af7651916cd43dd8448eb211c80319c"; +// SHA-256 of a weights file and of the file it was edited from, as a weights.checkpoint event carries them. +const WEIGHTS_FILE = `58899f90572c${"0".repeat(52)}`; +const WEIGHTS_BASE = `0496e5111097${"f".repeat(52)}`; /** A chat span carrying every kind of content metadata mode must drop. */ function contentSpan(spanId: string, convo = "conv-1"): J { @@ -239,6 +242,7 @@ describe("serve forward: metadata allowlist and full-mode redaction", () => { ["coding_sh.weights.base_sha256", str, false], ["coding_sh.weights.edited_sha256", str, false], ["coding_sh.weights.note", str, false], + ["coding_sh.weights.sha256", { stringValue: WEIGHTS_FILE }, true], ["gen_ai.input.messages", str, false], ["gen_ai.output.messages", str, false], ["gen_ai.system_instructions", str, false], @@ -253,12 +257,93 @@ describe("serve forward: metadata allowlist and full-mode redaction", () => { ["app.note", str, false], ]; for (const [k, v, want] of cases) assert.equal(keepInMetadata(k, v as never), want, k); + // The SHA-256 of a checkpoint's base file passes on a weights.checkpoint span only, and only as a SHA-256. + const base = "coding_sh.weights.base_sha256"; + assert.equal(keepInMetadata(base, { stringValue: WEIGHTS_BASE }, "weights.checkpoint"), true); + assert.equal(keepInMetadata(base, { stringValue: WEIGHTS_BASE.toUpperCase() }, "weights.checkpoint"), true); + assert.equal(keepInMetadata(base, { stringValue: "layers.3.mlp.down_proj.weight" }, "weights.checkpoint"), false); + assert.equal(keepInMetadata(base, { intValue: "7" } as never, "weights.checkpoint"), false); + assert.equal(keepInMetadata(base, { stringValue: WEIGHTS_BASE }, "weights.edit"), false); + assert.equal(keepInMetadata(base, { stringValue: WEIGHTS_BASE }), false); + assert.equal(keepInMetadata("coding_sh.weights.note", str as never, "weights.checkpoint"), false); + assert.equal(keepInMetadata("coding_sh.weights.tensor", str as never, "weights.checkpoint"), false); // span helper: events keep exception.type only, status message cleared const span: JsonSpan = { traceId: TRACE, spanId: "b7ad6b7169203331", name: "x", kind: 1, startTimeUnixNano: "1", endTimeUnixNano: "2", attributes: [], status: { code: 2, message: "boom" } }; assert.deepEqual(scrubSpan(span, "metadata", createRedactor()).status, { code: 2 }); }); }); +describe("serve forward: a weight edit and a weights checkpoint in one metadata batch", () => { + const TENSOR_A = "a".repeat(64); + const TENSOR_B = "b".repeat(64); + const weights = (s: J) => Object.fromEntries((s.attributes as J[]).filter((kv) => kv.key.startsWith("coding_sh.weights.")).map((kv) => [kv.key, kv.value])); + const want = { + "weights.edit": { "coding_sh.weights.changed": { intValue: "1204" }, "coding_sh.weights.l2": { doubleValue: 0.0183 }, "coding_sh.weights.max_abs": { doubleValue: 0.004 } }, + "weights.checkpoint": { "coding_sh.weights.sha256": { stringValue: WEIGHTS_FILE }, "coding_sh.weights.base_sha256": { stringValue: WEIGHTS_BASE } }, + }; + const leaked = ["layers.3.mlp.down_proj.weight", TENSOR_A, TENSOR_B, "rank-1 edit", "after layer 3"]; + + it("from /v1/ingest: the checkpoint keeps its two file hashes, the edit its counts, and neither its note", async () => { + const c = await stub(); + const ik = generateKey("ingest"); + const index = new MemoryIndexStore(); + const { h } = await serve(tmpEnv({ CODING_SH_INGEST_KEYS: `demo:${ik}`, CODING_SH_FORWARD_URL: c.url, CODING_SH_CAPTURE_MODE: "metadata" }), { stores: { index, blobs: new MemoryBlobStore() } }); + const now = Date.now() - 5000; + const events = [ + { v: 0, id: "s", sessionId: "edit-1", ts: now, kind: "session.start", payload: {} }, + { v: 0, id: "w", sessionId: "edit-1", ts: now + 10, kind: "notification", payload: { kind: "weights.edit", tensor: "layers.3.mlp.down_proj.weight", shape: [4096, 11008], changed: 1204, l2: 0.0183, maxAbs: 0.004, baseSha256: TENSOR_A, editedSha256: TENSOR_B, note: "rank-1 edit" } }, + { v: 0, id: "c", sessionId: "edit-1", ts: now + 20, kind: "notification", payload: { kind: "weights.checkpoint", sha256: WEIGHTS_FILE, baseSha256: WEIGHTS_BASE, note: "after layer 3" } }, + { v: 0, id: "e", sessionId: "edit-1", ts: now + 30, kind: "session.end", payload: {} }, + ]; + const r = await post(h, "/v1/ingest", ik, { v: 0, service: "trainer", events }); + assert.equal(r.status, 200, JSON.stringify(r.json)); + const spans = (await c.spans(3)).filter((s) => String(s.name).startsWith("weights.")); + assert.deepEqual(Object.fromEntries(spans.map((s) => [s.name, weights(s)])), want); + const text = JSON.stringify(c.bodies); + for (const needle of leaked) assert.ok(!text.includes(needle), needle); + // The store kept the same: a checkpoint is still one the session can be found by. + const stored = (await index.listEvents((await index.listSessions({}))[0]!.id)).filter((e) => e.kind === "notification").map((e) => e.payload); + assert.deepEqual(stored, [ + { kind: "weights.edit", changed: 1204, l2: 0.0183, maxAbs: 0.004 }, + { kind: "weights.checkpoint", sha256: WEIGHTS_FILE, baseSha256: WEIGHTS_BASE }, + ]); + assert.deepEqual((await index.listSessions({}))[0]!.stats.weightEdits, { events: 1, weightsChanged: 1204, checkpoints: [WEIGHTS_FILE] }); + await h.shutdown(); + }); + + it("from /v1/traces: the same, on the app's own spans", async () => { + const c = await stub(); + const { h } = await serve(tmpEnv({ CODING_SH_FORWARD_URL: c.url, CODING_SH_CAPTURE_MODE: "metadata" })); + const body = otlpTraces("trainer", [ + otlpSpan(TRACE, "b7ad6b7169203331", undefined, "edit run", [], 0, 9), + otlpSpan( + TRACE, + "b7ad6b7169203332", + "b7ad6b7169203331", + "weights.edit", + [ + sAttr("coding_sh.weights.tensor", "layers.3.mlp.down_proj.weight"), + iAttr("coding_sh.weights.changed", 1204), + { key: "coding_sh.weights.l2", value: { doubleValue: 0.0183 } }, + { key: "coding_sh.weights.max_abs", value: { doubleValue: 0.004 } }, + sAttr("coding_sh.weights.base_sha256", TENSOR_A), + sAttr("coding_sh.weights.edited_sha256", TENSOR_B), + sAttr("coding_sh.weights.note", "rank-1 edit"), + ], + 1, + 2, + ), + otlpSpan(TRACE, "b7ad6b7169203333", "b7ad6b7169203331", "weights.checkpoint", [sAttr("coding_sh.weights.sha256", WEIGHTS_FILE), sAttr("coding_sh.weights.base_sha256", WEIGHTS_BASE), sAttr("coding_sh.weights.note", "after layer 3")], 3, 4), + ]); + assert.equal((await post(h, "/v1/traces", undefined, body)).status, 200); + const spans = (await c.spans(3)).filter((s) => String(s.name).startsWith("weights.")); + assert.deepEqual(Object.fromEntries(spans.map((s) => [s.name, weights(s)])), want); + const text = JSON.stringify(c.bodies); + for (const needle of leaked) assert.ok(!text.includes(needle), needle); + await h.shutdown(); + }); +}); + describe("serve forward: native ingest", () => { it("converts /v1/ingest batches to invoke_agent, execute_tool and chat spans", async () => { const c = await stub(); diff --git a/packages/coding.sh/test/server/serve.test.ts b/packages/coding.sh/test/server/serve.test.ts index 5a64bec8..51e6fcf1 100644 --- a/packages/coding.sh/test/server/serve.test.ts +++ b/packages/coding.sh/test/server/serve.test.ts @@ -19,7 +19,7 @@ import type { IndexStore, Sink } from "../../src/core/types.ts"; import { otlpSpan, otlpTraces, sAttr, secretRequests } from "../otel/_encode.ts"; import { FileEvalStore } from "../../src/datasets/store.ts"; import { openIndexStore } from "../../src/store/factory.ts"; -import { experimentResult } from "../viewer/fakes.ts"; +import { experimentResult, session } from "../viewer/fakes.ts"; const dirs: string[] = []; const handles: ServeHandle[] = []; @@ -254,16 +254,29 @@ for (const kind of ["memory", "sqlite"] as const) { const evals = new FileEvalStore({ env, index: writer }); assert.equal(evals.kind, "index"); await evals.putExperiment(experimentResult("exp-prod-smoke")); + // An experiment stamped with a weights file, and a session of project `demo` that recorded the same file as a checkpoint. + const sha = "ab12".repeat(16); + await evals.putExperiment(experimentResult("exp-edited", { name: "edited", metadata: { weights: { sha256: sha } } })); + const stats = { events: 1, turns: 0, toolCalls: 0, toolErrors: 0, filesChanged: 0, weightEdits: { events: 1, weightsChanged: 4, checkpoints: [sha] } }; + await writer.upsertSession(session("demo/ck-1", { project: "demo", stats, flags: ["weights:edited", "weights:checkpoint"] })); if (kind !== "memory") await writer.close(); const list = await call(h, "/api/experiments", { key: rkAll }); assert.equal(list.status, 200, list.text); - assert.deepEqual(list.json.items.map((e: { id: string; name: string; runCount: number }) => [e.id, e.name, e.runCount]), [["exp-prod-smoke", "prod-smoke", 2]]); + assert.deepEqual(list.json.items.map((e: { id: string; name: string; runCount: number }) => [e.id, e.name, e.runCount]).sort(), [["exp-edited", "edited", 2], ["exp-prod-smoke", "prod-smoke", 2]]); const one = await call(h, "/api/experiments/exp-prod-smoke", { key: rkAll }); assert.equal(one.json.runs.length, 2); assert.equal((await call(h, "/api/experiments")).status, 401); assert.deepEqual((await call(h, "/api/experiments", { key: rkDemo })).json, { items: [] }); assert.equal((await call(h, "/api/experiments/exp-prod-smoke", { key: rkDemo })).status, 404); + + // The join on the weights file: an all-projects key reads it both ways; a project key reads its session without the experiments. + const ck = `/api/sessions/${encodeURIComponent("demo/ck-1")}`; + assert.deepEqual((await call(h, "/api/experiments/exp-edited", { key: rkAll })).json.weights, { sha256: sha, sessions: ["demo/ck-1"] }); + assert.deepEqual((await call(h, ck, { key: rkAll })).json.checkpointExperiments, [{ sha256: sha, experiments: [{ id: "exp-edited", name: "edited" }] }]); + const scoped = await call(h, ck, { key: rkDemo }); + assert.equal(scoped.status, 200, scoped.text); + assert.equal("checkpointExperiments" in scoped.json, false); await h.shutdown(); }); diff --git a/packages/coding.sh/test/sinks/otlp.test.ts b/packages/coding.sh/test/sinks/otlp.test.ts index a3b9b0dc..0df62c60 100644 --- a/packages/coding.sh/test/sinks/otlp.test.ts +++ b/packages/coding.sh/test/sinks/otlp.test.ts @@ -4,6 +4,7 @@ // leaves the sink comes back as the same Usage. import assert from "node:assert/strict"; import { describe, it } from "node:test"; +import { buildSession } from "../../src/core/sessions.ts"; import type { AgentEvent, Session, Usage } from "../../src/schema/index.ts"; import { type KeyValue, OtlpSink, toOtlpRequest, toOtlpSpans } from "../../src/sinks/otlp.ts"; import { decodeTraceJson } from "../../src/otel/json.ts"; @@ -205,6 +206,54 @@ describe("otlp sink: weight edits", () => { assert.equal(Object.hasOwn(none, "coding_sh.stats.weight_edits") || Object.hasOwn(none, "coding_sh.stats.weights_changed"), false); }); + const FILE = `58899f90572c${"0".repeat(52)}`; + const FILE_BASE = `0496e5111097${"f".repeat(52)}`; + const saved = { kind: "weights.checkpoint", sha256: FILE, baseSha256: FILE_BASE, note: "after layer 3" }; + + it("sends one weights.checkpoint span per checkpoint, with both file hashes and the note", () => { + const spans = toOtlpSpans([edit("w1"), edit("c1", saved), edit("c2", { kind: "weights.checkpoint", sha256: FILE_BASE })], [], "full"); + assert.deepEqual(spans.map((s) => s.name), ["weights.edit", "weights.checkpoint", "weights.checkpoint"]); + assert.equal(new Set(spans.map((s) => s.spanId)).size, 3); + assert.equal(spans[1]!.parentSpanId, spans[0]!.parentSpanId, "under the session span, like an edit"); + assert.deepEqual(weights(attrsOf(spans[1]!.attributes)), { "coding_sh.weights.sha256": FILE, "coding_sh.weights.base_sha256": FILE_BASE, "coding_sh.weights.note": "after layer 3" }); + assert.deepEqual(weights(attrsOf(spans[2]!.attributes)), { "coding_sh.weights.sha256": FILE_BASE }); + }); + + it("metadata mode keeps a checkpoint's two file hashes and drops its note", () => { + const spans = toOtlpSpans([edit("c1", saved)], [], "metadata"); + assert.deepEqual(spans.map((s) => s.name), ["weights.checkpoint"]); + assert.deepEqual(weights(attrsOf(spans[0]!.attributes)), { "coding_sh.weights.sha256": FILE, "coding_sh.weights.base_sha256": FILE_BASE }); + }); + + it("a checkpoint whose sha256 is not a SHA-256 is not exported", () => { + assert.deepEqual(toOtlpSpans([edit("c1", { kind: "weights.checkpoint", sha256: "abc", baseSha256: FILE_BASE }), edit("c2", { kind: "weights.checkpoint" })], [], "full"), []); + }); + + it("the session span of a session with a checkpoint and no edit carries no edit count", () => { + const end: AgentEvent = { ...edit("end"), kind: "session.end", payload: {} }; + const s = session(); + s.stats.weightEdits = { events: 0, weightsChanged: 0, checkpoints: [FILE] }; + s.flags = ["weights:checkpoint"]; + const a = attrsOf(toOtlpSpans([end], [s], "full")[0]!.attributes); + assert.equal(Object.hasOwn(a, "coding_sh.stats.weight_edits") || Object.hasOwn(a, "coding_sh.stats.weights_changed"), false); + }); + + it("the receiver reads an exported checkpoint back as the same payload, and the session folds to the same stat", async () => { + const cases: Record[] = [saved, { kind: "weights.checkpoint", sha256: FILE, baseSha256: FILE_BASE }, { kind: "weights.checkpoint", sha256: FILE }]; + for (const [i, p] of cases.entries()) { + const req = JSON.parse(JSON.stringify(toOtlpRequest([edit(`cp${i}`, p)], [], "full"))); + const { events } = await mapTraces(decodeTraceJson(req), ctx()); + const back = events.filter((e) => e.kind === "notification"); + assert.equal(back.length, 1, JSON.stringify(p)); + assert.deepEqual(back[0]!.payload, p); + assert.deepEqual(buildSession(events).stats.weightEdits, { events: 0, weightsChanged: 0, checkpoints: [FILE] }); + } + // Metadata mode: the note is gone, the join on the file's hash still works. + const req = JSON.parse(JSON.stringify(toOtlpRequest([edit("cpm", saved)], [], "metadata"))); + const { events } = await mapTraces(decodeTraceJson(req), ctx()); + assert.deepEqual(events.filter((e) => e.kind === "notification").map((e) => e.payload), [{ kind: "weights.checkpoint", sha256: FILE, baseSha256: FILE_BASE }]); + }); + it("the receiver reads an exported edit back as the same payload", async () => { const cases: Record[] = [ payload, diff --git a/packages/coding.sh/test/viewer/server.test.ts b/packages/coding.sh/test/viewer/server.test.ts index 05bbb49d..ed975cfa 100644 --- a/packages/coding.sh/test/viewer/server.test.ts +++ b/packages/coding.sh/test/viewer/server.test.ts @@ -640,6 +640,26 @@ describe("viewer server: experiment compare", () => { assert.deepEqual((await req(v, "/api/experiments/w-base")).json().metadata, { weights: { sha256: x } }); }); + test("a stamp names the sessions that recorded it as a checkpoint, and the session names the experiments that ran it", async () => { + const [x, y] = ["ef56".repeat(16), "0a9b".repeat(16)]; + await index.putExperiment(experimentResult("j-base", { name: "base", metadata: { weights: { sha256: x } } })); + await index.putExperiment(experimentResult("j-edit", { name: "edited", metadata: { weights: { sha256: y } } })); + const stats = { events: 3, turns: 1, toolCalls: 0, toolErrors: 0, filesChanged: 0 }; + await index.upsertSession(session("sess-edit", { stats: { ...stats, weightEdits: { events: 2, weightsChanged: 3, tensors: 2, checkpoints: [y] } }, flags: ["weights:edited", "weights:checkpoint"] })); + await index.upsertSession(session("sess-plain", { stats })); + + // Both ways of the same join that `exp compare --json`, `exp show --json` and `show --json` print. + const cmp = (await req(v, "/api/compare?a=j-base&b=j-edit")).json(); + assert.deepEqual([cmp.a.weights, cmp.b.weights], [{ sha256: x }, { sha256: y, sessions: ["sess-edit"] }]); + assert.deepEqual((await req(v, "/api/experiments/j-edit")).json().weights, { sha256: y, sessions: ["sess-edit"] }); + const base = (await req(v, "/api/experiments/j-base")).json(); + assert.equal("weights" in base, false, "no session recorded the base file: the stamp stays in metadata only"); + assert.deepEqual(base.metadata, { weights: { sha256: x } }); + + assert.deepEqual((await req(v, "/api/sessions/sess-edit")).json().checkpointExperiments, [{ sha256: y, experiments: [{ id: "j-edit", name: "edited" }] }]); + assert.equal("checkpointExperiments" in (await req(v, "/api/sessions/sess-plain")).json(), false); + }); + test("rejects bad ids and missing experiments", async () => { assert.equal((await req(v, "/api/compare?a=base")).status, 400); assert.equal((await req(v, "/api/compare?a=..%2Fx&b=base")).status, 400); diff --git a/packages/coding.sh/test/viewer/ui-browser.test.ts b/packages/coding.sh/test/viewer/ui-browser.test.ts index e795b55a..7be46697 100644 --- a/packages/coding.sh/test/viewer/ui-browser.test.ts +++ b/packages/coding.sh/test/viewer/ui-browser.test.ts @@ -764,10 +764,15 @@ describe("viewer UI: timeline, inspector and compare", { skip: chrome ? false : const payload = { kind: "weights.edit", tensor: "layers.3.mlp.down_proj.weight", shape: [4096, 11008], changed: 1204, l2: 0.0183, maxAbs: 0.004, baseSha256: "a".repeat(64), editedSha256: "b".repeat(64) }; const v = await page.eval(`eventView({ kind: "notification", payload: ${JSON.stringify(payload)} })`); assert.equal(v.name, "Edited weights"); - assert.equal(v.arg, "layers.3.mlp.down_proj.weight [4096×11008] · 1,204 of 45,088,768 weights · L2 0.0183 · max 0.004 · aaaaaaaa → bbbbbbbb"); + assert.equal(v.arg, "layers.3.mlp.down_proj.weight [4096×11008] · 1,204 of 45,088,768 weights · L2 0.0183 · max 0.004 · aaaaaaaaaaaa → bbbbbbbbbbbb"); assert.equal(v.kindText, "weights.edit"); const other = await page.eval(`eventView({ kind: "notification", payload: { kind: "hook.context", text: "x" } })`); assert.equal(other.name, "Notification"); + // The file the edits were saved to: its hash and the base file's, at 12 characters like every weights hash. + const saved = await page.eval(`eventView({ kind: "notification", payload: { kind: "weights.checkpoint", sha256: "58899F90572C" + "c".repeat(52), baseSha256: "0496e5111097" + "d".repeat(52), note: "after layer 3" } })`); + assert.deepEqual([saved.name, saved.arg, saved.kindText], ["Saved weights", "checkpoint 58899f90572c ← 0496e5111097", "weights.checkpoint"]); + const notOne = await page.eval(`eventView({ kind: "notification", payload: { kind: "weights.checkpoint", sha256: "abc" } })`); + assert.equal(notOne.name, "Notification", "without a SHA-256 it is not a checkpoint"); } finally { await page.close(); } @@ -1255,6 +1260,78 @@ describe("viewer UI: failed requests and the top bar", { skip: chrome ? false : } }); +// A weights file's SHA-256 joins the session that saved it (a weights.checkpoint event) with the +// experiments that ran it (exp run --weights). The pages link both ways, as the CLI prints both. +describe("viewer UI: a checkpoint links its session and the experiments that ran it", { skip: chrome ? false : "no Chrome/Chromium found (set CODING_SH_TEST_CHROME)" }, () => { + const EDITS = "5d0c9a1e-3b7f-4c2a-8e6d-9f1a2b3c4d5e"; + const PLAIN = "plain-1"; + const SHA_BASE = "ab12".repeat(16); + const SHA_EDIT = "cd34".repeat(16); + const index = new FakeIndex(); + let v: Viewer; + let browser: Browser; + let page: Page; + const ran = "[...document.querySelectorAll('.detail-head .badges .badge')].filter((b) => b.textContent.startsWith('ran in ')).map((b) => [b.textContent, b.querySelector('a').getAttribute('href'), b.title])"; + + before(async () => { + const stats = { events: 1, turns: 1, toolCalls: 0, toolErrors: 0, filesChanged: 0 }; + await index.upsertSession(session(EDITS, { startedAt: T0, title: "Edit the layer", stats: { ...stats, weightEdits: { events: 2, weightsChanged: 3, tensors: 2, checkpoints: [SHA_EDIT] } }, flags: ["weights:edited", "weights:checkpoint"] })); + await index.upsertSession(session(PLAIN, { startedAt: T0 - 1000, title: "No weights here", stats })); + await index.putEvents([ev(EDITS, 1, { kind: "user.message", payload: { text: "edit the layer" } }), ev(PLAIN, 1, { kind: "user.message", payload: { text: "hello" } })]); + const exps = [ + experimentResult("exp-w-base", { name: "base", metadata: { weights: { sha256: SHA_BASE } } }), + experimentResult("exp-w-edit", { name: "edited", metadata: { weights: { sha256: SHA_EDIT } } }), + ]; + v = await startViewer({ + index, blobs: new FakeBlobs(), port: 0, updateStatus: async () => ({ disabled: "test" }) as any, + experiments: { listExperiments: async () => exps, getExperiment: async (id) => exps.find((e) => e.id === id) }, + }); + browser = await launchBrowser(chrome!); + }); + after(async () => { + await browser?.close(); + await v?.close(); + }); + + test("the stamp links the session that recorded it; that session's header links the experiment back", async () => { + page = await browser.newPage(); + try { + await page.viewport(1280, 900); + await page.goto(v.url + "#/experiments/exp-w-base/vs/exp-w-edit"); + const sides: [string, string | null, string | null][] = await page.waitFor( + "(() => { const w = [...document.querySelectorAll('.vs .side .weights')]; return w.length === 2 && w.map((e) => [e.textContent, e.querySelector('a')?.getAttribute('href') ?? null, e.querySelector('a')?.title ?? null]); })()", + "both stamps on the comparison", + ); + assert.deepEqual(sides[0], ["weights " + SHA_BASE.slice(0, 12), null, null], "no session recorded the base file: the stamp alone"); + assert.deepEqual(sides[1]!.slice(0, 2), [`weights ${SHA_EDIT.slice(0, 12)} · edits ${EDITS.slice(0, 8)}`, "#/s/" + EDITS]); + assert.ok(sides[1]![2]!.endsWith(EDITS), "the link's title carries the whole session id"); + + await page.eval(`location.hash = "#/experiments/exp-w-edit"`); + const fact: [string, string] = await page.waitFor("(() => { const a = document.querySelector('.exp-top .fact-line .weights a'); return a && [a.parentElement.textContent, a.getAttribute('href')]; })()", "the session link on the experiment page"); + assert.deepEqual(fact, [`weights ${SHA_EDIT.slice(0, 12)} · edits ${EDITS.slice(0, 8)}`, "#/s/" + EDITS]); + + // Follow the link: the session names the experiment that ran the file it saved. + await page.eval("document.querySelector('.exp-top .fact-line .weights a').click()"); + const badges: [string, string, string][] = await page.waitFor(`(() => { const b = ${ran}; return b.length ? b : null; })()`, "the experiment on the session header"); + assert.deepEqual(badges.map((b) => b.slice(0, 2)), [["ran in edited", "#/experiments/exp-w-edit"]]); + assert.ok(badges[0]![2].includes(SHA_EDIT), "the badge's title names the weights file"); + + await page.eval(`location.hash = "#/s/${PLAIN}"`); + await page.waitFor("/No weights here/.test(document.querySelector('.detail-head')?.textContent ?? '')", "a session with no checkpoint"); + assert.deepEqual(await page.eval(ran), []); + + // The usual way in: a click on the session's row in the list. + await page.eval(`location.hash = "#/"`); + await page.waitFor("document.querySelector('[role=listbox] [role=option]')", "the sessions list"); + await page.eval("[...document.querySelectorAll('[role=listbox] [role=option]')].find((r) => r.querySelector('.row-title').textContent === 'Edit the layer').click()"); + const fromList: [string, string, string][] = await page.waitFor(`(() => { const b = ${ran}; return b.length ? b : null; })()`, "the experiment on the header, from the list"); + assert.deepEqual(fromList.map((b) => b.slice(0, 2)), [["ran in edited", "#/experiments/exp-w-edit"]]); + } finally { + await page.close(); + } + }); +}); + describe("session replay in a browser", { skip: chrome ? false : "no Chrome/Chromium found (set CODING_SH_TEST_CHROME)" }, () => { const RP = "7f3c1e2a-5b6d-4e8f-9a0b-1c2d3e4f5a6b"; const index = new FakeIndex(); diff --git a/packages/coding.sh/test/viewer/weights-row.test.ts b/packages/coding.sh/test/viewer/weights-row.test.ts index efc2a37c..1b65d621 100644 --- a/packages/coding.sh/test/viewer/weights-row.test.ts +++ b/packages/coding.sh/test/viewer/weights-row.test.ts @@ -1,11 +1,11 @@ -// A weight edit in the viewer: the page's own weightsEditText (run straight from index.html) gives -// the same line as core/weights.ts, the rows route labels the event with it, and a fold of routine -// events never hides one. +// A weight edit and a weights checkpoint in the viewer: the page's own weightsEditText and +// weightsCheckpointText (run straight from index.html) give the same lines as core/weights.ts, the +// rows route labels the event with them, and a fold of routine events never hides one. import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { runInNewContext } from "node:vm"; import { test } from "node:test"; -import { weightsEditText } from "../../src/core/weights.ts"; +import { weightsCheckpointText, weightsEditText } from "../../src/core/weights.ts"; import type { AgentEvent } from "../../src/schema/index.ts"; import { buildModel, FOLD_MIN, layoutRows, rowLabel } from "../../src/viewer/model.ts"; @@ -14,6 +14,7 @@ const from = html.indexOf("function weightsEditText("); const to = html.indexOf("function eventView(", from); assert.ok(from >= 0 && to > from, "the page defines weightsEditText just before eventView"); const page: (p: Record) => string = runInNewContext(`${html.slice(from, to)}\nweightsEditText`, { Number, String, Array }); +const pageCheckpoint: (p: Record) => string = runInNewContext(`${html.slice(from, to)}\nweightsCheckpointText`, { Number, String, Array }); const A = "a".repeat(64); const B = "b".repeat(64); @@ -42,7 +43,31 @@ test("the page and the CLI write the same line for a weight edit", () => { { tensor: "w", shape: Array.from({ length: 17 }, () => 1), changed: 1 }, ]; for (const p of payloads) assert.equal(page(p), weightsEditText(p), JSON.stringify(p)); - assert.equal(page(FULL), "layers.3.mlp.down_proj.weight [4096×11008] · 1,204 of 45,088,768 weights · L2 0.0183 · max 0.004 · aaaaaaaa → bbbbbbbb"); + assert.equal(page(FULL), "layers.3.mlp.down_proj.weight [4096×11008] · 1,204 of 45,088,768 weights · L2 0.0183 · max 0.004 · aaaaaaaaaaaa → bbbbbbbbbbbb"); +}); + +const SAVED = { kind: "weights.checkpoint", sha256: `58899f90572c${"0".repeat(52)}`, baseSha256: `0496e5111097${"f".repeat(52)}` }; + +test("the page and the CLI write the same line for a weights checkpoint", () => { + const payloads: Record[] = [ + SAVED, + { ...SAVED, note: "after the layer 15 edit" }, + { kind: "weights.checkpoint", sha256: SAVED.sha256 }, + { kind: "weights.checkpoint", sha256: SAVED.sha256.toUpperCase(), baseSha256: ` ${SAVED.baseSha256.toUpperCase()}\n` }, + { kind: "weights.checkpoint", sha256: SAVED.sha256, baseSha256: "abc" }, + { kind: "weights.checkpoint", sha256: SAVED.sha256, baseSha256: 7 }, + { kind: "weights.checkpoint", sha256: SAVED.sha256.slice(1) }, + { kind: "weights.checkpoint", sha256: `${SAVED.sha256.slice(2)}zz` }, + { kind: "weights.checkpoint", sha256: 7 }, + { kind: "weights.checkpoint" }, + { kind: "weights.edit", sha256: SAVED.sha256 }, + { sha256: SAVED.sha256 }, + {}, + ]; + for (const p of payloads) assert.equal(pageCheckpoint(p), weightsCheckpointText(p), JSON.stringify(p)); + assert.equal(pageCheckpoint(SAVED), "checkpoint 58899f90572c ← 0496e5111097"); + assert.equal(pageCheckpoint({ kind: "weights.checkpoint", sha256: SAVED.sha256 }), "checkpoint 58899f90572c"); + assert.equal(pageCheckpoint({ kind: "weights.checkpoint", sha256: "abc" }), "", "a value that is not a SHA-256 is not a checkpoint"); }); const base = { v: 0, sessionId: "w-1", agent: "sdk" } as const; @@ -54,6 +79,25 @@ test("the rows route labels a weight edit with that line", () => { assert.equal(rowLabel(ev(1, "notification", { kind: "hook.context", text: "x" })), "notification"); }); +test("the rows route labels a weights checkpoint with its line", () => { + assert.equal(rowLabel(ev(0, "notification", SAVED)), "checkpoint 58899f90572c ← 0496e5111097"); + assert.equal(rowLabel(ev(1, "notification", { kind: "weights.checkpoint", sha256: "abc" })), "notification", "not a checkpoint without a SHA-256"); +}); + +test("a fold of routine events never hides a weights checkpoint", () => { + const n = FOLD_MIN * 6; + const mid = n / 2; + const evs = [ev(0, "session.start"), ev(1, "user.message", { text: "save the weights" })]; + for (let i = 0; i < n; i++) evs.push(ev(2 + i, "notification", i === mid ? SAVED : { message: `tick ${i}` })); + const m = buildModel(evs); + const { slots } = layoutRows(m, m.turns[0]!, m.turns[0]!.start, 10_000); + const folds = slots.filter((s): s is { from: number; to: number } => "from" in s); + const at = 2 + mid; + assert.ok(folds.length >= 2, `the runs on both sides fold: ${JSON.stringify(slots)}`); + assert.ok(slots.some((s) => "i" in s && s.i === at), "the checkpoint is a row of its own"); + assert.ok(folds.every((f) => at < f.from || at > f.to)); +}); + test("a fold of routine events never hides a weight edit", () => { // One turn: a long run of plain notifications with one weight edit in the middle. const n = FOLD_MIN * 6; diff --git a/packages/sdk-python/src/coding_sh/__init__.py b/packages/sdk-python/src/coding_sh/__init__.py index 9d9b2845..e3eeaf36 100644 --- a/packages/sdk-python/src/coding_sh/__init__.py +++ b/packages/sdk-python/src/coding_sh/__init__.py @@ -20,7 +20,7 @@ from ._client import __version__ from ._providers import observe from ._trace import current_session_id, flush, init, shutdown, span, stats, trace, with_session -from ._weights import weights_delta, weights_edit +from ._weights import weights_checkpoint, weights_delta, weights_edit __all__ = [ "__version__", @@ -34,5 +34,6 @@ "shutdown", "stats", "weights_edit", + "weights_checkpoint", "weights_delta", ] diff --git a/packages/sdk-python/src/coding_sh/_weights.py b/packages/sdk-python/src/coding_sh/_weights.py index fa256d7b..3c329cd7 100644 --- a/packages/sdk-python/src/coding_sh/_weights.py +++ b/packages/sdk-python/src/coding_sh/_weights.py @@ -1,4 +1,5 @@ -"""Weight edits: one `notification` event per edit, `payload.kind` "weights.edit". +"""Weight edits: one `notification` event per edit, `payload.kind` "weights.edit", and one per +saved file, `payload.kind` "weights.checkpoint". coding_sh.weights_edit( "model.layers.3.mlp.down_proj.weight", @@ -9,12 +10,13 @@ base_sha256=sha_before, edited_sha256=sha_after, ) + coding_sh.weights_checkpoint(sha_of_edited_file, base_sha256=sha_of_base_file) The caller's script computes the hashes and norms; this module only checks them and sends them. It imports no numpy or torch; numpy and torch scalars are accepted as numbers. weights_edit() -never raises into the app: a bad value drops the event with a warning (once per process) and it -returns False. weights_delta() is a helper you call on purpose, so it raises ValueError on bad -input. The event lands in the current trace or session, or in a +and weights_checkpoint() never raise into the app: a bad value drops the event with a warning +(once per process) and they return False. weights_delta() is a helper you call on purpose, so +it raises ValueError on bad input. The event lands in the current trace or session, or in a session of its own when there is none. Payload fields are in docs/reference/api.md. """ @@ -29,6 +31,7 @@ from ._trace import Scope, _get, clip, emit, full_capture KIND = "weights.edit" +CHECKPOINT_KIND = "weights.checkpoint" _SHA256 = re.compile(r"^[0-9a-fA-F]{64}$") MAX_DIMS = 16 @@ -115,16 +118,32 @@ def _payload( "baseSha256": _sha(base_sha256, "base_sha256"), "editedSha256": _sha(edited_sha256, "edited_sha256"), } - if note is not None: - if not isinstance(note, str): - # The edit is still recorded; only the note is dropped. - _warn("weights_edit_note", "coding.sh: weights_edit ignored a note that is not a string") - elif full_capture() and note: - p["note"] = clip(note) + _put_note(p, note, "weights_edit_note", "weights_edit") + return p + + +def _put_note(p: Dict[str, Any], note: Any, key: str, fn: str) -> None: + """Sets `note` in full capture mode. One that is not a string is dropped with a warning; the + event is still recorded.""" + if note is None: + return + if not isinstance(note, str): + _warn(key, "coding.sh: %s ignored a note that is not a string" % fn) + elif full_capture() and note: + p["note"] = clip(note) + + +def _checkpoint_payload(sha256: Any, base_sha256: Any, note: Any) -> Dict[str, Any]: + p: Dict[str, Any] = {"kind": CHECKPOINT_KIND, "sha256": _sha(sha256, "sha256")} + if base_sha256 is not None: + p["baseSha256"] = _sha(base_sha256, "base_sha256") + _put_note(p, note, "weights_checkpoint_note", "weights_checkpoint") return p class _WeightsEdit(Scope): + """One weights notification: an edit, or a checkpoint.""" + bind_context = False def __init__(self, payload: Dict[str, Any]) -> None: @@ -133,7 +152,7 @@ def __init__(self, payload: Dict[str, Any]) -> None: self.queued = False def _root_name(self) -> str: - return KIND + return str(self.payload["kind"]) def _on_start(self) -> None: if self.node is not None: @@ -170,8 +189,13 @@ def weights_edit( except Exception as e: _warn("weights_edit", "coding.sh: weights_edit dropped an event: %s" % e) return False + return _record(p) + + +def _record(payload: Dict[str, Any]) -> bool: + """Queues one weights notification. True only when it was queued; never raises.""" try: - scope = _WeightsEdit(p) + scope = _WeightsEdit(payload) scope.start() queued = scope.queued scope.end() @@ -180,6 +204,31 @@ def weights_edit( return False +def weights_checkpoint(sha256: str, *, base_sha256: Optional[str] = None, note: Optional[str] = None) -> bool: + """Record one saved weights file as a `notification` event with `payload.kind` + "weights.checkpoint". + + sha256: the 64-hex hash of the whole file, as your script computed it. It is the value + `coding.sh exp run --weights` takes, so the session that recorded the edits and the + experiment that ran the file name each other. base_sha256: the hash of the file the edits + started from. note: free text, kept only in full capture mode. + + Returns True only when the event was queued. Returns False when a value is invalid, with + one warning per process. Returns False without a warning when the SDK is off or shut down + or the queue is full. Never raises. + + A note that is not a string is dropped with one warning; the checkpoint is still recorded. + Outside trace() and with_session() the checkpoint gets a session of its own, so record it + inside the one that recorded the edits. + """ + try: + p = _checkpoint_payload(sha256, base_sha256, note) + except Exception as e: + _warn("weights_checkpoint", "coding.sh: weights_checkpoint dropped an event: %s" % e) + return False + return _record(p) + + def _walk(x: Any, shape: List[int], depth: int, out: List[float], leaves: Set[int]) -> None: if hasattr(x, "tolist") and not isinstance(x, (list, tuple, range)): x = x.tolist() diff --git a/packages/sdk-python/tests/test_weights.py b/packages/sdk-python/tests/test_weights.py index b412ca14..fcec57b4 100644 --- a/packages/sdk-python/tests/test_weights.py +++ b/packages/sdk-python/tests/test_weights.py @@ -1,13 +1,15 @@ -"""weights_edit: one `notification` event with payload.kind "weights.edit" (docs/reference/api.md). +"""weights_edit: one `notification` event with payload.kind "weights.edit", and weights_checkpoint: +one with payload.kind "weights.checkpoint" (docs/reference/api.md). The unit tests read the batch the SDK posts to a fake ingest endpoint. The parity tests check those batches against packages/sdk/test/fixtures/weights-edit.json, which the TypeScript SDK's weights.test.ts checks too. The last test starts a real `coding.sh serve` on a temp -CODING_SH_HOME and reads the event back, redacted. +CODING_SH_HOME and reads the events back, redacted. """ from __future__ import annotations +import hashlib import json import logging import numbers @@ -16,6 +18,7 @@ import subprocess import time import unittest +import urllib.parse import urllib.request from typing import Any, Callable, Dict, List, Optional @@ -28,7 +31,10 @@ A = "a" * 64 B = "b" * 64 CLI = os.path.join(support.ROOT, "..", "coding.sh", "src", "cli", "main.ts") -# What both SDKs must send for the same edit. The TypeScript SDK's tests read the same file. +# SHA-256 of a whole weights file and of the file it was edited from, as a script would compute them. +FILE = "58899f90572c" + "0" * 52 +FILE_BASE = "0496e5111097" + "f" * 52 +# What both SDKs must send for the same edit and the same checkpoint. The TypeScript SDK's tests read the same file. FIXTURE = os.path.join(support.ROOT, "..", "sdk", "test", "fixtures", "weights-edit.json") NODE = shutil.which("node") # Built in pieces so push protection does not flag the test file. @@ -355,6 +361,162 @@ def test_weights_delta_feeds_weights_edit(self) -> None: self.assertTrue(coding_sh.weights_edit("w", base_sha256=A, edited_sha256=B, **d)) +def _checkpoint(**kw: Any) -> bool: + args: Dict[str, Any] = dict(sha256=FILE, base_sha256=FILE_BASE) + args.update(kw) + return coding_sh.weights_checkpoint(args.pop("sha256"), **args) + + +class WeightsCheckpointTest(unittest.TestCase): + """weights_checkpoint() follows weights_edit() case for case; weights.test.ts has the same list.""" + + def setUp(self) -> None: + # The warnings are once per process: each test starts with none given. + _weights._reset_warned() + self.server = IngestServer() + + def tearDown(self) -> None: + coding_sh.shutdown(1.0) + self.server.close() + + def _payloads(self) -> List[Dict[str, Any]]: + return [e["payload"] for e in by_kind(collect(self.server), "notification")] + + def test_one_notification_with_the_payload(self) -> None: + start_sdk(self.server) + self.assertIs(_checkpoint(note="after layer 3"), True) + events = collect(self.server) + (e,) = by_kind(events, "notification") + self.assertEqual(e["payload"], {"kind": "weights.checkpoint", "sha256": FILE, "baseSha256": FILE_BASE, "note": "after layer 3"}) + self.assertEqual(e["source"], "sdk") + (start,) = by_kind(events, "session.start") # a session of its own, named after the event + self.assertEqual(start["payload"]["name"], "weights.checkpoint") + + def test_the_hash_alone_is_enough_and_a_missing_base_is_left_out(self) -> None: + start_sdk(self.server) + self.assertIs(coding_sh.weights_checkpoint(FILE), True) + self.assertIs(_checkpoint(base_sha256=None), True) + self.assertEqual(self._payloads(), [{"kind": "weights.checkpoint", "sha256": FILE}] * 2) + + def test_base_and_note_are_keyword_only(self) -> None: + start_sdk(self.server) + with self.assertRaises(TypeError): + coding_sh.weights_checkpoint(FILE, FILE_BASE) # type: ignore[misc] + + def test_joins_the_trace_that_recorded_the_edits(self) -> None: + start_sdk(self.server) + with coding_sh.trace("edit-run", session_id="conv-1"): + _edit() + _edit(tensor="lm_head.weight") + _checkpoint() + events = collect(self.server) + notes = by_kind(events, "notification") + self.assertEqual( + [(n["sessionId"], n["payload"]["kind"]) for n in notes], + [("conv-1", "weights.edit"), ("conv-1", "weights.edit"), ("conv-1", "weights.checkpoint")], + ) + self.assertEqual(len(by_kind(events, "session.start")), 1) + + def test_metadata_mode_drops_the_note_only(self) -> None: + start_sdk(self.server, capture="metadata") + _checkpoint(note="free text") + self.assertEqual(self._payloads(), [{"kind": "weights.checkpoint", "sha256": FILE, "baseSha256": FILE_BASE}]) + + def test_invalid_values_are_dropped_not_raised(self) -> None: + start_sdk(self.server) + bad = [ + dict(sha256="abc"), + dict(sha256="g" * 64), + dict(sha256=FILE + "\n"), # a trailing newline is not 64 hex characters + dict(sha256="sha256:" + FILE), + dict(sha256=None), + dict(sha256=5), + dict(sha256=bytes.fromhex(FILE)), + dict(base_sha256="abc"), + dict(base_sha256=FILE[1:]), + dict(base_sha256=5), + dict(base_sha256=""), + ] + with self.assertLogs("coding_sh", level="WARNING") as cm: + for kw in bad: + self.assertIs(_checkpoint(**kw), False, kw) + self.assertEqual(len(cm.records), 1) + self.assertEqual(cm.records[0].getMessage(), "coding.sh: weights_checkpoint dropped an event: sha256 must be 64 hex characters") + coding_sh.flush(2.0) + self.assertEqual(by_kind(self.server.events(), "notification"), []) + + def test_uppercase_sha_is_lowercased(self) -> None: + start_sdk(self.server) + self.assertIs(_checkpoint(sha256=FILE.upper(), base_sha256=FILE_BASE.upper()), True) + (p,) = self._payloads() + self.assertEqual((p["sha256"], p["baseSha256"]), (FILE, FILE_BASE)) + + def test_returns_false_when_disabled(self) -> None: + coding_sh.init(disabled=True) + self.assertIs(_checkpoint(), False) + + def test_returns_false_without_a_url(self) -> None: + coding_sh.init(url="") + self.assertIs(_checkpoint(), False) + + def test_returns_false_after_shutdown(self) -> None: + start_sdk(self.server) + self.assertIs(_checkpoint(), True) + coding_sh.shutdown(2.0) + self.assertIs(_checkpoint(), False) + + def test_returns_false_when_the_queue_is_full(self) -> None: + start_sdk(self.server, max_queue=1, flush_interval_ms=60000) + with coding_sh.trace("full"): # session.start takes the one slot + self.assertIs(_checkpoint(), False) + + def test_warns_once_per_process_apart_from_weights_edit(self) -> None: + start_sdk(self.server) + with self.assertLogs("coding_sh", level="WARNING") as cm: + _checkpoint(sha256="abc") + _checkpoint(base_sha256="abc") + _checkpoint(sha256=5) + _edit(changed=-1) + self.assertEqual(len([r for r in cm.records if "weights_checkpoint dropped" in r.getMessage()]), 1) + self.assertEqual(len([r for r in cm.records if "weights_edit dropped" in r.getMessage()]), 1) + + def test_note_that_is_not_a_string_is_dropped_with_a_warning_an_empty_one_without(self) -> None: + start_sdk(self.server) + with self.assertLogs("coding_sh", level="WARNING") as cm: + self.assertIs(_checkpoint(note=5), True) + self.assertIs(_checkpoint(note={"text": "x"}), True) + self.assertIs(_checkpoint(note=""), True) + self.assertIs(_checkpoint(note=None), True) + self.assertEqual([r.getMessage() for r in cm.records], ["coding.sh: weights_checkpoint ignored a note that is not a string"]) + self.assertEqual(self._payloads(), [{"kind": "weights.checkpoint", "sha256": FILE, "baseSha256": FILE_BASE}] * 4) + + def test_inside_with_session(self) -> None: + start_sdk(self.server) + with coding_sh.with_session("run-7"): + _checkpoint() + self.assertEqual([n["sessionId"] for n in by_kind(collect(self.server), "notification")], ["run-7"]) + + def test_inside_a_span_links_to_it(self) -> None: + start_sdk(self.server) + with coding_sh.trace("run"): + with coding_sh.span("tool", "save"): + _checkpoint() + events = collect(self.server) + (note,) = by_kind(events, "notification") + (tool,) = by_kind(events, "tool.start") + self.assertEqual(note["parentId"], tool["id"]) + self.assertEqual(note["parentSpanId"], tool["spanId"]) + + def test_server_down_never_raises(self) -> None: + dead = IngestServer() + url = dead.url + dead.close() + coding_sh.init(url=url, key=support.KEY, flush_interval_ms=5) + self.assertIs(_checkpoint(), True) # queued; delivery fails in the background + coding_sh.flush(1.0) + coding_sh.shutdown(1.0) + + def _alone(edit: Callable[[], bool], sid: Optional[str]) -> bool: return edit() @@ -393,10 +555,12 @@ def _step_span(edit: Callable[[], bool], sid: Optional[str]) -> bool: class WeightsEditParityTest(unittest.TestCase): - """weights_edit() here and weightsEdit() in the TypeScript SDK send the same events. + """weights_edit() and weights_checkpoint() here send the same events as weightsEdit() and + weightsCheckpoint() in the TypeScript SDK. - The fixture holds, for one edit in each place it can be recorded, the events in the order - they are posted: each event's kind, its keys and its payload. A missing fixture is a failure. + The fixture holds, for one event in each place it can be recorded, the events in the order + they are posted: each event's kind, its keys and its payload. Every scenario runs once for + the edit and once for the checkpoint. A missing fixture is a failure. """ fx: Dict[str, Any] = {} @@ -422,11 +586,15 @@ def _edit(self) -> bool: note=e["note"], ) - def _posted(self, name: str) -> List[Dict[str, Any]]: + def _checkpoint(self) -> bool: + c = self.fx["checkpoint"] + return coding_sh.weights_checkpoint(c["sha256"], base_sha256=c["baseSha256"], note=c["note"]) + + def _posted(self, name: str, record: Callable[[], bool]) -> List[Dict[str, Any]]: server = IngestServer() try: start_sdk(server) - self.assertIs(SCENARIOS[name](self._edit, self.fx["scenarios"][name]["sessionId"]), True) + self.assertIs(SCENARIOS[name](record, self.fx["scenarios"][name]["sessionId"]), True) return collect(server) finally: coding_sh.shutdown(1.0) @@ -438,34 +606,45 @@ def test_every_scenario_in_the_fixture_runs_here(self) -> None: def test_the_same_events_as_the_typescript_sdk(self) -> None: volatile = set(self.fx["volatilePayloadKeys"]) ignored = set(self.fx["ignoredEventKeys"]) + # Each scenario once per helper: what is recorded, and the payload the fixture says it sends. + records = [("weights_edit", self._edit, self.fx["payload"]), ("weights_checkpoint", self._checkpoint, self.fx["checkpointPayload"])] for name, sc in self.fx["scenarios"].items(): - with self.subTest(name): - events = self._posted(name) - got = [ - { - "kind": e["kind"], - "keys": sorted(k for k in e if k not in ignored), - "payload": {k: "" if k in volatile else v for k, v in e["payload"].items()}, - } - for e in events - ] - want = [dict(e, payload=self.fx["payload"]) if e["kind"] == "notification" else e for e in sc["events"]] - self.assertEqual(got, want) - - # One trace, one session, and the edit hangs off the event the fixture names. - first = events[0] - (note,) = by_kind(events, "notification") - self.assertEqual({e["traceId"] for e in events}, {first["traceId"]}) - self.assertEqual({e["sessionId"] for e in events}, {sc["sessionId"] or first["sessionId"]}) - if sc["parent"] is None: - self.assertEqual(note["spanId"], first["spanId"]) - else: - (parent,) = by_kind(events, sc["parent"]) - self.assertEqual(note["parentId"], parent["id"]) - self.assertEqual(note["parentSpanId"], parent["spanId"]) - self.assertNotEqual(note["spanId"], parent["spanId"]) - if "turnId" in note: - self.assertEqual(note["turnId"], note["traceId"]) + for helper, record, payload in records: + with self.subTest(name, helper=helper): + events = self._posted(name, record) + got = [ + { + "kind": e["kind"], + "keys": sorted(k for k in e if k not in ignored), + "payload": {k: "" if k in volatile else v for k, v in e["payload"].items()}, + } + for e in events + ] + # "" in the fixture: a session the helper opens by itself is named after the event. + want = [ + dict(e, payload=payload) + if e["kind"] == "notification" + else dict(e, payload=dict(e["payload"], name=payload["kind"])) + if e.get("payload", {}).get("name") == "" + else e + for e in sc["events"] + ] + self.assertEqual(got, want) + + # One trace, one session, and the event hangs off the event the fixture names. + first = events[0] + (note,) = by_kind(events, "notification") + self.assertEqual({e["traceId"] for e in events}, {first["traceId"]}) + self.assertEqual({e["sessionId"] for e in events}, {sc["sessionId"] or first["sessionId"]}) + if sc["parent"] is None: + self.assertEqual(note["spanId"], first["spanId"]) + else: + (parent,) = by_kind(events, sc["parent"]) + self.assertEqual(note["parentId"], parent["id"]) + self.assertEqual(note["parentSpanId"], parent["spanId"]) + self.assertNotEqual(note["spanId"], parent["spanId"]) + if "turnId" in note: + self.assertEqual(note["turnId"], note["traceId"]) def test_the_same_delta_as_the_typescript_sdk(self) -> None: d = self.fx["delta"] @@ -487,16 +666,28 @@ def _node_ok() -> bool: @unittest.skipUnless(_node_ok(), "needs node >= 22.16 and the core package") class WeightsEditServerTest(unittest.TestCase): - """A real coding.sh serve on a temp CODING_SH_HOME: the event comes back with the secret redacted.""" + """A real coding.sh serve on a temp CODING_SH_HOME: the events come back with the secret redacted + and the hashes whole, and the session is found by the checkpoint's hash.""" def test_round_trip_through_serve(self) -> None: home = os.path.join(support._tmp_home, ".coding.sh-serve") ingest = "csh_ik_" + "w" * 43 read = "csh_rk_" + "r" * 43 + # Nothing the server reads or writes is the user's: every home it knows is under the temp home. + tmp = support._tmp_home env = dict( os.environ, - HOME=support._tmp_home, + HOME=tmp, + USERPROFILE=tmp, CODING_SH_HOME=home, + CLAUDE_CONFIG_DIR=os.path.join(tmp, "claude"), + CODEX_HOME=os.path.join(tmp, "codex"), + XDG_CONFIG_HOME=os.path.join(tmp, "xdg-config"), + XDG_DATA_HOME=os.path.join(tmp, "xdg-data"), + CURSOR_CONFIG_DIR=os.path.join(tmp, "cursor"), + OPENCODE_CONFIG_DIR=os.path.join(tmp, "opencode"), + OPENCODE_DB=os.path.join(tmp, "opencode.db"), + CODING_SH_NO_AUTO_INGEST="1", CODING_SH_INGEST_KEYS="demo:" + ingest, CODING_SH_READ_KEYS="demo:" + read, NODE_NO_WARNINGS="1", @@ -514,22 +705,41 @@ def test_round_trip_through_serve(self) -> None: if not url.startswith("http://"): self.fail("serve did not start: " + (proc.stderr.read() if proc.poll() is not None else url)) coding_sh.init(url=url, key=ingest, service="weights-test", flush_interval_ms=5) - self.assertTrue( - coding_sh.weights_edit( - "blocks.0." + FAKE_TOKEN, - shape=[2, 3], - changed=2, - l2=0.5, - max_abs=0.4, - base_sha256=A, - edited_sha256=B, - note="deploy with " + FAKE_TOKEN, + # Hashes of real bytes: the server's redactor must leave a SHA-256 alone, or the join on it breaks. + file = hashlib.sha256(b"edited weights file").hexdigest() + file_base = hashlib.sha256(b"base weights file").hexdigest() + with coding_sh.trace("edit-run"): + self.assertTrue( + coding_sh.weights_edit( + "blocks.0." + FAKE_TOKEN, + shape=[2, 3], + changed=2, + l2=0.5, + max_abs=0.4, + base_sha256=A, + edited_sha256=B, + note="deploy with " + FAKE_TOKEN, + ) ) - ) + self.assertTrue(coding_sh.weights_checkpoint(file.upper(), base_sha256=file_base, note="saved with " + FAKE_TOKEN)) self.assertTrue(coding_sh.flush(5.0)) self.assertEqual(coding_sh.stats()["failed"], 0) items = self._events(url, read) - (e,) = [i for i in items if i["kind"] == "notification"] + (saved,) = [i for i in items if i["payload"].get("kind") == "weights.checkpoint"] + kept = {k: v for k, v in saved["payload"].items() if k != "redactions"} + self.assertEqual( + kept, + {"kind": "weights.checkpoint", "sha256": file, "baseSha256": file_base, "note": "saved with [redacted:github-token]"}, + ) + self.assertNotIn(FAKE_TOKEN, json.dumps(saved)) + # The session the two events share is found by the file's hash, with the edit it recorded. + found = self._get(url + "/api/sessions?where=" + urllib.parse.quote("weights.checkpoint:" + file), read) + sessions = found["items"] if isinstance(found, dict) else found + self.assertEqual( + [s["stats"]["weightEdits"] for s in sessions], + [{"events": 1, "tensors": 1, "weightsChanged": 2, "checkpoints": [file]}], + ) + (e,) = [i for i in items if i["payload"].get("kind") == "weights.edit"] p = e["payload"] self.assertEqual(p["kind"], "weights.edit") self.assertEqual((p["shape"], p["changed"], p["l2"], p["maxAbs"]), ([2, 3], 2, 0.5, 0.4)) @@ -547,14 +757,17 @@ def test_round_trip_through_serve(self) -> None: proc.stdout.close() proc.stderr.close() + def _get(self, url: str, read: str) -> Any: + req = urllib.request.Request(url, headers={"Authorization": "Bearer " + read}) + with urllib.request.urlopen(req, timeout=5) as r: + return json.loads(r.read()) + def _events(self, url: str, read: str) -> list: deadline = time.monotonic() + 10 while True: - req = urllib.request.Request(url + "/api/events?limit=50", headers={"Authorization": "Bearer " + read}) - with urllib.request.urlopen(req, timeout=5) as r: - body = json.loads(r.read()) + body = self._get(url + "/api/events?limit=50", read) items = body["items"] if isinstance(body, dict) else body - if any(i.get("kind") == "notification" for i in items) or time.monotonic() > deadline: + if len([i for i in items if i.get("kind") == "notification"]) >= 2 or time.monotonic() > deadline: return items time.sleep(0.1) diff --git a/packages/sdk-python/tests/typing/trace_example.py b/packages/sdk-python/tests/typing/trace_example.py index 46c8cbfd..01434d90 100644 --- a/packages/sdk-python/tests/typing/trace_example.py +++ b/packages/sdk-python/tests/typing/trace_example.py @@ -35,4 +35,10 @@ async def double(x: int) -> int: return x * 2 +def record_weights(file_sha256: str, base_sha256: Optional[str]) -> bool: + edited: bool = coding_sh.weights_edit("w", shape=[2, 3], changed=1, l2=0.5, max_abs=0.5, base_sha256="a" * 64, edited_sha256="b" * 64) + saved: bool = coding_sh.weights_checkpoint(file_sha256, base_sha256=base_sha256, note="after the edit") + return edited and saved + + n: int = decorated(1) diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 5a2e0810..3beb23be 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -65,6 +65,7 @@ app.post("/chat", async (req, res) => { | `withSession(id, fn)` | Every trace and untraced call inside `fn` goes to session `id`. | | `aiSdkTracer({sessionId?})` | An OpenTelemetry `Tracer` for the Vercel AI SDK (see below). | | `weightsEdit({tensor, shape, changed, l2, maxAbs, baseSha256, editedSha256, note?})` | Records one weight edit as a `notification` event with `payload.kind: "weights.edit"`, the same event the Python SDK's `weights_edit()` sends. Returns `true` only when the event was queued. An invalid value drops the event with one warning per process. Inside `trace()` or `span()` it joins that trace; outside it gets a session of its own. | +| `weightsCheckpoint({sha256, baseSha256?, note?})` | Records one saved weights file as a `notification` event with `payload.kind: "weights.checkpoint"`, the same event the Python SDK's `weights_checkpoint()` sends. `sha256` is the hash of the whole file, the value `coding.sh exp run --weights` takes. Returns `true` only when the event was queued. An invalid value drops the event with one warning per process. Record it inside the `trace()` that recorded the edits. | | `weightsDelta(base, edited)` | Computes `{shape, changed, l2, maxAbs}` for two nested arrays or typed arrays of the same shape, ready to spread into `weightsEdit()`. Throws a `TypeError` or `RangeError` on bad input. | | `init(opts?)` | Optional. The SDK reads env on first use. Options: `url`, `key`, `service`, `capture: "full" \| "metadata"`, `flushIntervalMs` (1000), `maxBatch` (500), `maxQueue` (10000), `disabled`, `debug`, `fetch`. | | `flush(timeoutMs?)` / `shutdown(timeoutMs?)` | Send what is queued. `flush` resolves `true` when the queue is drained. `shutdown` also stops the SDK. | diff --git a/packages/sdk/src/context.ts b/packages/sdk/src/context.ts index 445f56f0..7a5d6562 100644 --- a/packages/sdk/src/context.ts +++ b/packages/sdk/src/context.ts @@ -10,7 +10,7 @@ // model call llm.call (ts = start, durationMs, usage, model) · assistant.message (ts = end) // tool span tool.start · tool.end // step span span { name, ok } -// notification notification { kind, … } (weightsEdit: one event, a child of the current span) +// notification notification { kind, … } (weightsEdit, weightsCheckpoint: one event, a child of the current span) // // A model call or tool span outside any trace is its own root (a one-call // session), unless withSession() names the session it belongs to. diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 435b1d81..ab8b00ea 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -15,8 +15,8 @@ export { observe } from "./observe.ts"; export { observeFetch } from "./fetch.ts"; export { trace, span, withSession } from "./context.ts"; export type { TraceOptions, TraceHandle, SpanOptions } from "./context.ts"; -export { weightsEdit, weightsDelta } from "./weights.ts"; -export type { WeightsEdit, WeightsDelta, WeightsArray } from "./weights.ts"; +export { weightsEdit, weightsCheckpoint, weightsDelta } from "./weights.ts"; +export type { WeightsEdit, WeightsCheckpoint, WeightsDelta, WeightsArray } from "./weights.ts"; export { aiSdkTracer } from "./ai-sdk.ts"; export type { AiSdkTracer, AiSdkTracerOptions } from "./ai-sdk.ts"; export type { TransportStats } from "./transport.ts"; diff --git a/packages/sdk/src/weights.ts b/packages/sdk/src/weights.ts index f836fe1f..47f8a9c1 100644 --- a/packages/sdk/src/weights.ts +++ b/packages/sdk/src/weights.ts @@ -1,4 +1,5 @@ -// Weight edits: one `notification` event per edit, `payload.kind` "weights.edit". +// Weight edits: one `notification` event per edit, `payload.kind` "weights.edit", +// and one per saved file, `payload.kind` "weights.checkpoint". // // weightsEdit({ // tensor: "model.layers.3.mlp.down_proj.weight", @@ -9,19 +10,22 @@ // baseSha256: shaBefore, // editedSha256: shaAfter, // }); +// weightsCheckpoint({ sha256: shaOfEditedFile, baseSha256: shaOfBaseFile }); // // The caller's script computes the hashes and norms; this module only checks -// them and sends them. weightsEdit() never throws into the app: a bad value -// drops the event with a warning (once per process) and it returns false. -// weightsDelta() is a helper you call on purpose, so it throws on bad input. -// The Python SDK's weights_edit() sends the same event (packages/sdk-python, -// _weights.py); test/fixtures/weights-edit.json holds what both must send. -// Payload fields are in docs/reference/api.md. +// them and sends them. weightsEdit() and weightsCheckpoint() never throw into +// the app: a bad value drops the event with a warning (once per process) and +// they return false. weightsDelta() is a helper you call on purpose, so it +// throws on bad input. The Python SDK's weights_edit() and weights_checkpoint() +// send the same events (packages/sdk-python, _weights.py); +// test/fixtures/weights-edit.json holds what both must send. Payload fields are +// in docs/reference/api.md. import { config, warnOnce } from "./client.ts"; import { recordNotification } from "./context.ts"; import { errorMessage, quiet, truncate } from "./util.ts"; const KIND = "weights.edit"; +const CHECKPOINT_KIND = "weights.checkpoint"; const SHA256 = /^[0-9a-fA-F]{64}$/; const MAX_DIMS = 16; @@ -45,6 +49,16 @@ export interface WeightsEdit { note?: string; } +/** One saved weights file, as `weightsCheckpoint()` records it. */ +export interface WeightsCheckpoint { + /** sha256 of the whole weights file, the value `coding.sh exp run --weights` takes: 64 hex characters, sent in lowercase. */ + sha256: string; + /** sha256 of the file the edits started from: 64 hex characters, sent in lowercase. */ + baseSha256?: string; + /** Free text, kept only in full capture mode. */ + note?: string; +} + /** What `weightsDelta()` returns. It spreads into `weightsEdit()`. */ export interface WeightsDelta { /** Dimensions of both arrays. */ @@ -104,11 +118,24 @@ function payloadOf(edit: WeightsEdit): Record { p.maxAbs = norm(maxAbs, "maxAbs"); p.baseSha256 = sha(baseSha256, "baseSha256"); p.editedSha256 = sha(editedSha256, "editedSha256"); - if (note !== undefined && note !== null) { - // The edit is still recorded; only the note is dropped. - if (typeof note !== "string") quiet(warnOnce)("weights-edit-note", "[coding.sh] weightsEdit ignored a note that is not a string"); - else if (note && config().capture === "full") put(p, "note", note); - } + putNote(p, note, "weights-edit-note", "weightsEdit"); + return p; +} + +/** Sets `note` in full capture mode. One that is not a string is dropped with a warning; the event is still recorded. */ +function putNote(p: Record, note: unknown, key: string, fn: string): void { + if (note === undefined || note === null) return; + if (typeof note !== "string") quiet(warnOnce)(key, `[coding.sh] ${fn} ignored a note that is not a string`); + else if (note && config().capture === "full") put(p, "note", note); +} + +/** The event payload for a checkpoint. Throws an Error that says which value is invalid. */ +function checkpointPayloadOf(checkpoint: WeightsCheckpoint): Record { + if (typeof checkpoint !== "object" || checkpoint === null) throw new Error("expected an object with sha256"); + const { sha256, baseSha256, note } = checkpoint as { [K in keyof WeightsCheckpoint]: unknown }; + const p: Record = { kind: CHECKPOINT_KIND, sha256: sha(sha256, "sha256") }; + if (baseSha256 !== undefined && baseSha256 !== null) p.baseSha256 = sha(baseSha256, "baseSha256"); + putNote(p, note, "weights-checkpoint-note", "weightsCheckpoint"); return p; } @@ -140,6 +167,39 @@ export function weightsEdit(edit: WeightsEdit): boolean { } } +/** + * Records one saved weights file as a `notification` event with `payload.kind` + * "weights.checkpoint". The Python SDK's `weights_checkpoint()` sends the same event. + * + * `sha256` is the hash of the whole file, as your script computed it: the value + * `coding.sh exp run --weights` takes, so the session that recorded the edits and + * the experiment that ran the file name each other. `baseSha256` is the hash of + * the file the edits started from. + * + * Returns true only when the event was queued. Returns false when a value is + * invalid, with one warning per process. Returns false without a warning when + * the SDK is off or shut down or the queue is full. Never throws. + * + * A `note` that is not a string is dropped with one warning; the checkpoint is + * still recorded. Inside `trace()` or `span()` the checkpoint joins that trace. + * Outside them it gets a session of its own, or a turn of the `withSession()` + * session, so record it inside the `trace()` that recorded the edits. + */ +export function weightsCheckpoint(checkpoint: WeightsCheckpoint): boolean { + try { + let payload: Record; + try { + payload = checkpointPayloadOf(checkpoint); + } catch (err) { + quiet(warnOnce)("weights-checkpoint", `[coding.sh] weightsCheckpoint dropped an event: ${errorMessage(err)}`); + return false; + } + return recordNotification(CHECKPOINT_KIND, payload); + } catch { + return false; + } +} + const isTyped = (x: unknown): x is ArrayBufferView & ArrayLike => ArrayBuffer.isView(x) && !(x instanceof DataView); const isArrayish = (x: unknown): x is ArrayLike => Array.isArray(x) || isTyped(x); diff --git a/packages/sdk/test/fixtures/weights-edit.json b/packages/sdk/test/fixtures/weights-edit.json index 1a324190..d3a52e37 100644 --- a/packages/sdk/test/fixtures/weights-edit.json +++ b/packages/sdk/test/fixtures/weights-edit.json @@ -1,5 +1,5 @@ { - "about": "What weights_edit() in the Python SDK and weightsEdit() in the TypeScript SDK must both send for the same edit. packages/sdk-python/tests/test_weights.py and packages/sdk/test/weights.test.ts both assert against this file. Recorded from the batches the Python SDK posts to /v1/ingest.", + "about": "What the Python SDK and the TypeScript SDK must both send for the same weight edit (weights_edit() and weightsEdit(): `edit` in, `payload` out) and for the same weights checkpoint (weights_checkpoint() and weightsCheckpoint(): `checkpoint` in, `checkpointPayload` out). packages/sdk-python/tests/test_weights.py and packages/sdk/test/weights.test.ts both assert against this file, running every scenario once for the edit and once for the checkpoint. In a scenario, \"\" stands for the kind of the event recorded: a session the helper opens by itself is named after it. Recorded from the batches the Python SDK posts to /v1/ingest.", "edit": { "tensor": "layers.3.mlp.down_proj.weight", "shape": [4, 8], @@ -21,32 +21,43 @@ "editedSha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "note": "rank-1 patch" }, + "checkpoint": { + "sha256": "58899F90572C0E1D5A7B3C2F4E6D8091A2B3C4D5E6F708192A3B4C5D6E7F8091", + "baseSha256": "0496e5111097ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766", + "note": "after the layer 3 patch" + }, + "checkpointPayload": { + "kind": "weights.checkpoint", + "sha256": "58899f90572c0e1d5a7b3c2f4e6d8091a2b3c4d5e6f708192a3b4c5d6e7f8091", + "baseSha256": "0496e5111097ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766", + "note": "after the layer 3 patch" + }, "volatilePayloadKeys": ["traceId", "durationMs", "toolUseId"], "ignoredEventKeys": ["source"], "scenarios": { "alone": { - "how": "One edit outside trace(), span() and withSession().", + "how": "One event outside trace(), span() and withSession().", "sessionId": null, "parent": null, "events": [ - {"kind": "session.start", "keys": ["id", "kind", "payload", "sessionId", "spanId", "traceId", "ts", "v"], "payload": {"name": "weights.edit", "traceId": ""}}, + {"kind": "session.start", "keys": ["id", "kind", "payload", "sessionId", "spanId", "traceId", "ts", "v"], "payload": {"name": "", "traceId": ""}}, {"kind": "notification", "keys": ["id", "kind", "payload", "sessionId", "spanId", "traceId", "ts", "v"]}, {"kind": "turn.end", "keys": ["durationMs", "id", "kind", "payload", "sessionId", "spanId", "traceId", "ts", "v"], "payload": {"ok": true, "durationMs": ""}}, {"kind": "session.end", "keys": ["durationMs", "id", "kind", "payload", "sessionId", "spanId", "traceId", "ts", "v"], "payload": {"reason": "completed"}} ] }, "withSession": { - "how": "One edit inside withSession(sessionId).", + "how": "One event inside withSession(sessionId).", "sessionId": "run-7", "parent": null, "events": [ - {"kind": "session.start", "keys": ["id", "kind", "payload", "sessionId", "spanId", "traceId", "ts", "turnId", "v"], "payload": {"name": "weights.edit", "traceId": ""}}, + {"kind": "session.start", "keys": ["id", "kind", "payload", "sessionId", "spanId", "traceId", "ts", "turnId", "v"], "payload": {"name": "", "traceId": ""}}, {"kind": "notification", "keys": ["id", "kind", "payload", "sessionId", "spanId", "traceId", "ts", "turnId", "v"]}, {"kind": "turn.end", "keys": ["durationMs", "id", "kind", "payload", "sessionId", "spanId", "traceId", "ts", "turnId", "v"], "payload": {"ok": true, "durationMs": ""}} ] }, "trace": { - "how": "One edit inside trace(\"run\").", + "how": "One event inside trace(\"run\").", "sessionId": null, "parent": "session.start", "events": [ @@ -57,7 +68,7 @@ ] }, "traceWithSessionId": { - "how": "One edit inside trace(\"run\") with sessionId.", + "how": "One event inside trace(\"run\") with sessionId.", "sessionId": "conv-1", "parent": "session.start", "events": [ @@ -67,7 +78,7 @@ ] }, "toolSpan": { - "how": "One edit inside span(\"tool\", \"patch\") inside trace(\"run\").", + "how": "One event inside span(\"tool\", \"patch\") inside trace(\"run\").", "sessionId": null, "parent": "tool.start", "events": [ @@ -80,7 +91,7 @@ ] }, "stepSpan": { - "how": "One edit inside span(\"step\", \"patch\") inside trace(\"run\").", + "how": "One event inside span(\"step\", \"patch\") inside trace(\"run\").", "sessionId": null, "parent": "span", "events": [ diff --git a/packages/sdk/test/weights.test.ts b/packages/sdk/test/weights.test.ts index 6a8a8b3c..3d774004 100644 --- a/packages/sdk/test/weights.test.ts +++ b/packages/sdk/test/weights.test.ts @@ -1,11 +1,13 @@ -// weightsEdit: one `notification` event with payload.kind "weights.edit" (docs/reference/api.md). +// weightsEdit: one `notification` event with payload.kind "weights.edit", and weightsCheckpoint: +// one with payload.kind "weights.checkpoint" (docs/reference/api.md). // // These mirror packages/sdk-python/tests/test_weights.py case for case. Most read the batch the // SDK posts to a fake ingest endpoint. The parity tests check those batches against // test/fixtures/weights-edit.json, which the Python SDK's tests check too. The last test starts a -// real `coding.sh serve` on a temp CODING_SH_HOME and reads the event back, redacted. +// real `coding.sh serve` on a temp CODING_SH_HOME and reads the events back, redacted. import assert from "node:assert/strict"; import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; import { once } from "node:events"; import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -13,7 +15,7 @@ import { dirname, join } from "node:path"; import { createInterface } from "node:readline"; import { after, afterEach, before, beforeEach, test } from "node:test"; import { fileURLToPath } from "node:url"; -import { flush, init, shutdown, span, stats, trace, weightsDelta, weightsEdit, withSession, type WeightsArray, type WeightsEdit } from "../src/index.ts"; +import { flush, init, shutdown, span, stats, trace, weightsCheckpoint, weightsDelta, weightsEdit, withSession, type WeightsArray, type WeightsCheckpoint, type WeightsEdit } from "../src/index.ts"; import { _reset } from "../src/client.ts"; import { MAX_TEXT_BYTES } from "../src/util.ts"; import { type FakeServe, startFakeServe } from "./helpers/fake-serve.ts"; @@ -41,13 +43,15 @@ interface Wire { interface Fixture { edit: WeightsEdit; payload: Record; + checkpoint: WeightsCheckpoint; + checkpointPayload: Record; volatilePayloadKeys: string[]; ignoredEventKeys: string[]; scenarios: Record }[] }>; delta: { base: WeightsArray; edited: WeightsArray; result: unknown }; } -// What both SDKs must send for the same edit. A missing fixture fails every test in this file. +// What both SDKs must send for the same edit and the same checkpoint. A missing fixture fails every test in this file. const fx = JSON.parse(readFileSync(join(here, "fixtures", "weights-edit.json"), "utf8")) as Fixture; let serve: FakeServe; @@ -312,6 +316,159 @@ test("server down never throws", async () => { }); }); +// ---------------------------------------------------------------- weightsCheckpoint + +// SHA-256 of a whole weights file and of the file it was edited from, as a script would compute them. +const FILE = `58899f90572c${"0".repeat(52)}`; +const FILE_BASE = `0496e5111097${"f".repeat(52)}`; +const checkpoint = (over: Record = {}): boolean => weightsCheckpoint({ sha256: FILE, baseSha256: FILE_BASE, ...over } as WeightsCheckpoint); + +test("checkpoint: one notification with the payload", async () => { + assert.equal(checkpoint({ note: "after layer 3" }), true); + const events = await posted(); + assert.deepEqual(one(byKind(events, "notification")).payload, { kind: "weights.checkpoint", sha256: FILE, baseSha256: FILE_BASE, note: "after layer 3" }); + assert.equal(one(byKind(events, "session.start")).payload.name, "weights.checkpoint", "a session of its own, named after the event"); + assert.equal(one(serve.events().filter((e) => e.kind === "notification" && e.sessionId === events[0]!.sessionId)).source, "sdk"); +}); + +test("checkpoint: the hash alone is enough, and a missing base is left out", async () => { + assert.equal(weightsCheckpoint({ sha256: FILE }), true); + for (const baseSha256 of [undefined, null]) assert.equal(checkpoint({ baseSha256 }), true); + for (const n of byKind(await posted(), "notification")) assert.deepEqual(n.payload, { kind: "weights.checkpoint", sha256: FILE }); +}); + +test("checkpoint: joins the trace that recorded the edits", async () => { + trace( + "edit-run", + () => { + edit(); + edit({ tensor: "lm_head.weight" }); + checkpoint(); + }, + { sessionId: "conv-1" }, + ); + const events = await posted(); + assert.deepEqual(byKind(events, "notification").map((n) => [n.sessionId, n.payload.kind]), [["conv-1", "weights.edit"], ["conv-1", "weights.edit"], ["conv-1", "weights.checkpoint"]]); + assert.equal(byKind(events, "session.start").length, 1); +}); + +test("checkpoint: metadata mode drops the note only", async () => { + start({ capture: "metadata" }); + checkpoint({ note: "free text" }); + assert.deepEqual(one(byKind(await posted(), "notification")).payload, { kind: "weights.checkpoint", sha256: FILE, baseSha256: FILE_BASE }); +}); + +test("checkpoint: invalid values are dropped, not thrown", async () => { + const bad: Record[] = [ + { sha256: "abc" }, + { sha256: "g".repeat(64) }, + { sha256: FILE + "\n" }, // a trailing newline is not 64 hex characters + { sha256: `sha256:${FILE}` }, + { sha256: undefined }, + { sha256: null }, + { sha256: 5 }, + { baseSha256: "abc" }, + { baseSha256: FILE.slice(1) }, + { baseSha256: 5 }, + { baseSha256: "" }, + ]; + const warns = await warnings(() => { + for (const over of bad) assert.equal(checkpoint(over), false, JSON.stringify(over)); + // A call from plain JavaScript can pass anything. + const untyped = weightsCheckpoint as (checkpoint: unknown) => boolean; + for (const arg of [undefined, null, FILE, 5, {}]) assert.equal(untyped(arg), false, String(arg)); + }); + assert.equal(warns.length, 1); + assert.match(warns[0]!, /^\[coding\.sh\] weightsCheckpoint dropped an event: sha256 must be 64 hex characters$/); + assert.deepEqual(byKind(await posted(), "notification"), []); +}); + +test("checkpoint: uppercase sha is lowercased", async () => { + assert.equal(checkpoint({ sha256: FILE.toUpperCase(), baseSha256: FILE_BASE.toUpperCase() }), true); + const p = one(byKind(await posted(), "notification")).payload; + assert.deepEqual([p.sha256, p.baseSha256], [FILE, FILE_BASE]); +}); + +test("checkpoint: returns false when disabled, without a URL, after shutdown and when the queue is full", async () => { + _reset(); + init({ url: serve.url, disabled: true }); + assert.deepEqual(await warnings(() => assert.equal(checkpoint(), false)), [], "a valid checkpoint is silent when disabled"); + const invalid = await warnings(() => assert.equal(checkpoint({ sha256: "abc" }), false)); + assert.equal(invalid.length, 1, "an invalid one still warns, as in Python"); + _reset(); + init({ url: "" }); + assert.equal(checkpoint(), false); + start(); + assert.equal(checkpoint(), true); + await shutdown(2000); + assert.deepEqual(await warnings(() => assert.equal(checkpoint(), false)), [], "no warning: nothing is wrong with the checkpoint"); + start({ maxQueue: 1, flushIntervalMs: 60_000 }); + // The trace's session.start takes the one slot. + await warnings(() => trace("full", () => assert.equal(checkpoint(), false))); + assert.equal(stats().queued, 1); +}); + +test("checkpoint: warns once per process, apart from weightsEdit's warning", async () => { + const warns = await warnings(() => { + checkpoint({ sha256: "abc" }); + checkpoint({ baseSha256: "abc" }); + init({ url: serve.url, flushIntervalMs: 5 }); // a second init() is the same process + checkpoint({ sha256: 5 }); + edit({ changed: -1 }); + }); + assert.equal(warns.filter((w) => w.includes("weightsCheckpoint dropped")).length, 1); + assert.equal(warns.filter((w) => w.includes("weightsEdit dropped")).length, 1); +}); + +test("checkpoint: a note that is not a string is dropped with a warning, an empty one without", async () => { + const warns = await warnings(() => { + assert.equal(checkpoint({ note: 5 }), true); + assert.equal(checkpoint({ note: { text: "x" } }), true); + for (const note of ["", null, undefined]) assert.equal(checkpoint({ note }), true); + }); + assert.equal(warns.length, 1); + assert.match(warns[0]!, /weightsCheckpoint ignored a note that is not a string/); + const notes = byKind(await posted(), "notification"); + assert.equal(notes.length, 5); + for (const n of notes) assert.deepEqual(n.payload, { kind: "weights.checkpoint", sha256: FILE, baseSha256: FILE_BASE }); +}); + +test("checkpoint: a long note is cut like every other SDK string", async () => { + assert.equal(checkpoint({ note: "x".repeat(MAX_TEXT_BYTES + 10) }), true); + const p = one(byKind(await posted(), "notification")).payload; + assert.equal((p.note as string).length, MAX_TEXT_BYTES); + assert.equal(p.truncated, true); +}); + +test("checkpoint: inside withSession, inside a span, and after an await", async () => { + withSession("run-7", () => void checkpoint()); + assert.deepEqual(byKind(await posted(), "notification").map((n) => n.sessionId), ["run-7"]); + start(); + trace("run", () => span("tool", "save", () => void checkpoint())); + let events = await posted(); + assert.equal(one(byKind(events, "notification")).parentId, one(byKind(events, "tool.start")).id); + start(); + await trace("run", async () => { + await new Promise((r) => setTimeout(r, 1)); + checkpoint(); + }); + events = await posted(); + assert.equal(one(byKind(events, "notification")).parentId, one(byKind(events, "session.start")).id); +}); + +test("checkpoint: server down never throws", async () => { + const dead = await startFakeServe(); + const url = dead.url; + await dead.close(); + _reset(); + init({ url, flushIntervalMs: 5 }); + await warnings(async () => { + assert.equal(checkpoint(), true, "queued; delivery fails in the background"); + assert.equal(await flush(200), false); + await shutdown(200); + }); +}); + // ---------------------------------------------------------------- weightsDelta test("weightsDelta", () => { @@ -375,27 +532,35 @@ test("weightsDelta feeds weightsEdit", async () => { // ---------------------------------------------------------------- parity with the Python SDK -type Run = (edit: () => void, sessionId: string | null) => void; +type Run = (record: () => void, sessionId: string | null) => void; // One entry per scenario in the fixture; test_weights.py has the same list. The functions return // nothing, so no span records an output the Python `with` block would not have. const SCENARIOS: Record = { - alone: (edit) => edit(), - withSession: (edit, sid) => withSession(sid!, edit), - trace: (edit) => trace("run", edit), - traceWithSessionId: (edit, sid) => trace("run", edit, { sessionId: sid! }), - toolSpan: (edit) => trace("run", () => span("tool", "patch", edit)), - stepSpan: (edit) => trace("run", () => span("step", "patch", edit)), + alone: (record) => record(), + withSession: (record, sid) => withSession(sid!, record), + trace: (record) => trace("run", record), + traceWithSessionId: (record, sid) => trace("run", record, { sessionId: sid! }), + toolSpan: (record) => trace("run", () => span("tool", "patch", record)), + stepSpan: (record) => trace("run", () => span("step", "patch", record)), }; +// Each scenario runs once per helper: what is recorded, and the payload the fixture says it sends. +const RECORDS: [string, () => boolean, Record][] = [ + ["weightsEdit", () => weightsEdit(fx.edit), fx.payload], + ["weightsCheckpoint", () => weightsCheckpoint(fx.checkpoint), fx.checkpointPayload], +]; +/** A fixture payload with "" filled in: a session the helper opens by itself is named after the event. */ +const named = (payload: Record | undefined, kind: unknown) => (payload?.name === "" ? { ...payload, name: kind } : payload); + test("every scenario in the fixture runs here", () => { assert.deepEqual(Object.keys(SCENARIOS).sort(), Object.keys(fx.scenarios).sort()); }); -for (const [name, sc] of Object.entries(fx.scenarios)) { - test(`the same events as the Python SDK: ${name}`, async () => { +for (const [name, sc] of Object.entries(fx.scenarios)) for (const [helper, record, payload] of RECORDS) { + test(`the same events as the Python SDK: ${helper} ${name}`, async () => { let queued = false; - SCENARIOS[name]!(() => void (queued = weightsEdit(fx.edit)), sc.sessionId); + SCENARIOS[name]!(() => void (queued = record()), sc.sessionId); assert.equal(queued, true); const events = await posted(); const got = events.map((e) => ({ @@ -403,9 +568,9 @@ for (const [name, sc] of Object.entries(fx.scenarios)) { keys: Object.keys(e).filter((k) => !fx.ignoredEventKeys.includes(k)).sort(), payload: Object.fromEntries(Object.entries(e.payload).map(([k, v]) => [k, fx.volatilePayloadKeys.includes(k) ? "" : v])), })); - assert.deepEqual(got, sc.events.map((e) => (e.kind === "notification" ? { ...e, payload: fx.payload } : e))); + assert.deepEqual(got, sc.events.map((e) => (e.kind === "notification" ? { ...e, payload } : e.payload ? { ...e, payload: named(e.payload, payload.kind) } : e))); - // One trace, one session, and the edit hangs off the event the fixture names. + // One trace, one session, and the event hangs off the event the fixture names. const first = events[0]!; const note = one(byKind(events, "notification")); assert.deepEqual([...new Set(events.map((e) => e.traceId))], [first.traceId]); @@ -431,7 +596,7 @@ const [major = 0, minor = 0] = process.versions.node.split(".").map(Number); // The same condition as WeightsEditServerTest in test_weights.py. const serveOk = existsSync(CLI) && (major > 22 || (major === 22 && minor >= 15)); -test("round trip through a real coding.sh serve: the event comes back with the secret redacted", { skip: serveOk ? false : "needs node >= 22.15 and the core package" }, async () => { +test("round trip through a real coding.sh serve: the events come back with the secret redacted and the hashes whole", { skip: serveOk ? false : "needs node >= 22.15 and the core package" }, async () => { const root = mkdtempSync(join(tmpdir(), "csh-sdk-weights-")); const ingest = "csh_ik_" + "w".repeat(43); const read = "csh_rk_" + "r".repeat(43); @@ -468,7 +633,13 @@ test("round trip through a real coding.sh serve: the event comes back with the s assert.match(url, /^http:\/\//, `serve did not start: ${stderr || url}`); _reset(); init({ url, key: ingest, service: "weights-test", flushIntervalMs: 5 }); - assert.equal(weightsEdit({ tensor: `blocks.0.${FAKE_TOKEN}`, shape: [2, 3], changed: 2, l2: 0.5, maxAbs: 0.4, baseSha256: A, editedSha256: B, note: `deploy with ${FAKE_TOKEN}` }), true); + // Hashes of real bytes: the server's redactor must leave a SHA-256 alone, or the join on it breaks. + const file = createHash("sha256").update("edited weights file").digest("hex"); + const fileBase = createHash("sha256").update("base weights file").digest("hex"); + trace("edit-run", () => { + assert.equal(weightsEdit({ tensor: `blocks.0.${FAKE_TOKEN}`, shape: [2, 3], changed: 2, l2: 0.5, maxAbs: 0.4, baseSha256: A, editedSha256: B, note: `deploy with ${FAKE_TOKEN}` }), true); + assert.equal(weightsCheckpoint({ sha256: file.toUpperCase(), baseSha256: fileBase, note: `saved with ${FAKE_TOKEN}` }), true); + }); assert.equal(await flush(5000), true); assert.equal(stats().failed, 0); @@ -479,10 +650,19 @@ test("round trip through a real coding.sh serve: the event comes back with the s assert.equal(res.status, 200); const body = (await res.json()) as { items?: typeof items } | typeof items; items = Array.isArray(body) ? body : (body.items ?? []); - if (items.some((i) => i.kind === "notification") || Date.now() > until) break; + if (items.filter((i) => i.kind === "notification").length >= 2 || Date.now() > until) break; await new Promise((r) => setTimeout(r, 100)); } - const e = one(items.filter((i) => i.kind === "notification")); + const saved = one(items.filter((i) => i.payload.kind === "weights.checkpoint")); + const { redactions: _counts, ...kept } = saved.payload; + assert.deepEqual(kept, { kind: "weights.checkpoint", sha256: file, baseSha256: fileBase, note: "saved with [redacted:github-token]" }); + assert.ok(!JSON.stringify(saved).includes(FAKE_TOKEN)); + // The session the two events share is found by the file's hash, with the edit it recorded. + const found = await fetch(`${url}/api/sessions?where=${encodeURIComponent(`weights.checkpoint:${file}`)}`, { headers: { authorization: `Bearer ${read}` } }); + assert.equal(found.status, 200); + const listed = (await found.json()) as { items?: { stats: { weightEdits?: unknown } }[] } | { stats: { weightEdits?: unknown } }[]; + assert.deepEqual((Array.isArray(listed) ? listed : (listed.items ?? [])).map((s) => s.stats.weightEdits), [{ events: 1, tensors: 1, weightsChanged: 2, checkpoints: [file] }]); + const e = one(items.filter((i) => i.payload.kind === "weights.edit")); const p = e.payload; assert.equal(p.kind, "weights.edit"); assert.deepEqual([p.shape, p.changed, p.l2, p.maxAbs], [[2, 3], 2, 0.5, 0.4]); diff --git a/packages/store-clickhouse/src/where.ts b/packages/store-clickhouse/src/where.ts index b38d7ddf..ba7681a4 100644 --- a/packages/store-clickhouse/src/where.ts +++ b/packages/store-clickhouse/src/where.ts @@ -54,7 +54,7 @@ const tokens = ["inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTok /** * SQL over the sessions row, or (event keys) over an events row; a clause - * matches when any expression does. Flags use the lambda argument `f`. + * matches when any expression does. List-valued keys (flags, weights.checkpoint) use the lambda argument `f`, an item of MULTI_FROM. */ export const KEY_SQL: Record = { agent: ["agent"], @@ -74,6 +74,7 @@ export const KEY_SQL: Record = { "hook.tokens": [stat("hookContext.tokens")], "weights.edits": [stat("weightEdits.events")], "weights.changed": [stat("weightEdits.weightsChanged")], + "weights.checkpoint": ["f"], since: ["started_at"], until: ["started_at"], duration: ["(ended_at - started_at)"], @@ -143,6 +144,13 @@ export function assertSupported(n: Node): void { } } +/** The array a list-valued key reads: the session's flags, unless the key is named here. */ +const MULTI_FROM: Record = { + // An empty array when the session recorded no checkpoint. + "weights.checkpoint": "JSONExtract(stats, 'weightEdits', 'checkpoints', 'Array(String)')", +}; +const multiFrom = (key: string): string => MULTI_FROM[key] ?? "flags"; + class Compiler { private readonly bind: Bind; private readonly events: string; @@ -168,7 +176,7 @@ class Compiler { /** A key clause; `negated` wraps nullable columns so NOT(NULL) reads as true, as in the scan. */ cmp(c: Cmp, negated: boolean): string { const def = KEYS.get(c.key)!; - if (def.multi) return `arrayExists(f -> ${c.values.map((v) => this.valueSql(def, c, v, "f")).join(" OR ")}, flags)`; + if (def.multi) return `arrayExists(f -> ${c.values.map((v) => this.valueSql(def, c, v, "f")).join(" OR ")}, ${multiFrom(c.key)})`; const parts = KEY_SQL[c.key]!.flatMap((expr) => c.values.map((v) => this.valueSql(def, c, v, expr))); const sql = parts.length === 1 ? parts[0]! : `(${parts.join(" OR ")})`; return negated && !def.notNull ? `ifNull(${sql}, 0)` : sql; @@ -251,7 +259,8 @@ export function facetRows(key: string, bind: Bind, sessions: string, events: str if (!def || !KEY_SQL[key]) throw unknown(key); if (def.multi) { const p = def.multi.prefix; - const value = p ? `arrayJoin(arrayMap(f -> substring(f, ${p.length + 1}), arrayFilter(f -> lower(f) LIKE ${bind("String", `${esc(asciiLower(p))}%`)}, flags)))` : "arrayJoin(flags)"; + const from = multiFrom(key); + const value = p ? `arrayJoin(arrayMap(f -> substring(f, ${p.length + 1}), arrayFilter(f -> lower(f) LIKE ${bind("String", `${esc(asciiLower(p))}%`)}, ${from})))` : `arrayJoin(${from})`; return `SELECT ${value} AS fv, id AS sid FROM ${sessions} FINAL ${where}`; } const value = FACET_SQL[key] ?? KEY_SQL[key]![0]!; diff --git a/packages/store-postgres/src/where.ts b/packages/store-postgres/src/where.ts index e5baab2e..3ff37567 100644 --- a/packages/store-postgres/src/where.ts +++ b/packages/store-postgres/src/where.ts @@ -41,11 +41,13 @@ const jnum = (col: string, path: string) => `(CASE WHEN json_typeof(${col}::json /** A JSON string at `path` of a JSON text column, else NULL. */ const jstr = (col: string, path: string) => `(CASE WHEN json_typeof(${col}::json #> '{${path}}') = 'string' THEN ${col}::json #>> '{${path}}' END)`; const stat = (path: string) => jnum("s.stats", path); +/** A JSON array at `path` of a JSON text column, else an empty one. */ +const jarr = (col: string, path: string) => `(CASE WHEN json_typeof(${col}::json #> '{${path}}') = 'array' THEN ${col}::json #> '{${path}}' ELSE '[]'::json END)`; /** A repo's name as coding.sh's repoName: the text after the last / or \ (empty after a trailing slash), "tmp" for the macOS temp dir. */ const base = (x: string) => `(CASE WHEN replace(${x}, chr(92), '/') ~ '/var/folders/.*/T$' THEN 'tmp' ELSE regexp_replace(replace(${x}, chr(92), '/'), '^.*/', '') END)`; const tokens = ["inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens"].map((k) => `coalesce(${stat(`usage,${k}`)}, 0)`).join(" + "); -/** SQL over `s` (sessions) or `e` (events) per key; a clause matches when any expression does. Flags use `f.v`. */ +/** SQL over `s` (sessions) or `e` (events) per key; a clause matches when any expression does. List-valued keys (flags, weights.checkpoint) use `f.v`, an item of MULTI_FROM. */ export const KEY_SQL: Record = { agent: ["s.agent"], model: ["s.model"], @@ -64,6 +66,7 @@ export const KEY_SQL: Record = { "hook.tokens": [stat("hookContext,tokens")], "weights.edits": [stat("weightEdits,events")], "weights.changed": [stat("weightEdits,weightsChanged")], + "weights.checkpoint": ["f.v"], since: ["s.started_at"], until: ["s.started_at"], duration: ["(s.ended_at - s.started_at)"], @@ -133,6 +136,12 @@ export function assertSupported(n: Node): void { } } +/** The JSON array a list-valued key reads, one row per item: the session's flags, unless the key is named here. */ +const MULTI_FROM: Record = { + "weights.checkpoint": jarr("s.stats", "weightEdits,checkpoints"), +}; +const multiFrom = (key: string): string => MULTI_FROM[key] ?? "s.flags::json"; + class Compiler { private readonly bind: Bind; private readonly events: string; @@ -159,7 +168,7 @@ class Compiler { const def = KEYS.get(c.key)!; if (def.multi) { const any = c.values.map((v) => this.valueSql(def, c, v, "f.v")).join(" OR "); - return `EXISTS (SELECT 1 FROM json_array_elements_text(s.flags::json) AS f(v) WHERE ${any})`; + return `EXISTS (SELECT 1 FROM json_array_elements_text(${multiFrom(c.key)}) AS f(v) WHERE ${any})`; } const parts = KEY_SQL[c.key]!.flatMap((expr) => c.values.map((v) => this.valueSql(def, c, v, expr))); const sql = parts.length === 1 ? parts[0]! : `(${parts.join(" OR ")})`; @@ -245,7 +254,7 @@ export function facetSource(key: string, bind: Bind, sessions: string, events: s const def = KEYS.get(key); if (!def || !KEY_SQL[key]) throw unknown(key); if (def.multi) { - const from = `${sessions} s CROSS JOIN LATERAL json_array_elements_text(s.flags::json) AS ff(v)`; + const from = `${sessions} s CROSS JOIN LATERAL json_array_elements_text(${multiFrom(key)}) AS ff(v)`; const p = def.multi.prefix; if (!p) return { value: "ff.v", from }; return { value: `substr(ff.v, ${p.length + 1})`, from, where: `${lowerSql("ff.v")} LIKE ${bind(`${esc(asciiLower(p))}%`)}` }; diff --git a/scripts/lint/allowlist.json b/scripts/lint/allowlist.json index 3749eab4..36cf14aa 100644 --- a/scripts/lint/allowlist.json +++ b/scripts/lint/allowlist.json @@ -1011,7 +1011,7 @@ { "rule": "conditional-empty-spread", "path": "packages/coding.sh/src/cli/commands/exp.ts", - "max": 67, + "max": 66, "reason": "baseline 2026-11-09: existing code, do not add more" }, {