From ab101a5b0a1b1be682bb45db4f54af2587474aff Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 24 Oct 2026 20:52:40 -0400 Subject: [PATCH 1/4] generate-parts: the app-specific parts of a workflow, generated and verified openadapt-flow generate-parts builds the four parts of a workflow that are specific to one application at setup time, and admits each only by a check that does not depend on the generator. Healthy runs still make no model calls; nothing here calls a model. - Read channel (generate/channel.py): candidates from an OpenAPI document, a FHIR CapabilityStatement or teach-time traffic. The channel test reads each demonstration's record, every page, and must read as unreadable through a fault proxy (no auth, 401, 500, truncated, not JSON, a failed next page). - Record contract (generate/contract.py): from the before/after difference across 2-3 varied demonstrations (bound fields, constants, volatile fields, match keys, frame), refined on a test copy by the fault gate until every fault class is killed or reported. - Duplicate rule (generate/identity.py): roles from the schema, clauses calibrated on the system's own records with a generic error model and per-comparison false-match rates; a person labels the pairs that block a clause; unknown goes to a person. - Input types (generate/inputs.py): labels, dates and the source's date order, code tables, admitted by a round trip; ambiguity halts. - A floor generated parts can't go below (generate/baseline.py); a plain- language review, a digest manifest and a person's approval bound to it (generate/build.py, generate/cli.py). - Verifiers: the FHIR read follows next links on the same origin and refuses a short read; the REST read follows records_next_path. - benchmark/generated_parts: the generated duplicate rule on #1516's stream (676 items, 223 duplicates): 54 missed, 0 false stops, against the hand-built check's 24 and 2. Co-Authored-By: Claude Opus 5.5 --- CAPABILITIES.md | 2 + llms-full.txt | 3 + .../flow/benchmark/generated_parts/README.md | 69 ++ .../benchmark/generated_parts/__init__.py | 0 .../benchmark/generated_parts/dup_backtest.py | 203 +++++ .../generated_parts/results/duplicates.json | 749 ++++++++++++++++++ packages/flow/docs/GENERATED_PARTS.md | 89 +++ .../02-check-and-admit/generate-parts.md | 0 packages/flow/openadapt_flow/__main__.py | 4 + packages/flow/openadapt_flow/deployment.py | 9 +- .../flow/openadapt_flow/generate/__init__.py | 0 .../flow/openadapt_flow/generate/baseline.py | 105 +++ .../flow/openadapt_flow/generate/build.py | 508 ++++++++++++ .../flow/openadapt_flow/generate/channel.py | 543 +++++++++++++ packages/flow/openadapt_flow/generate/cli.py | 240 ++++++ .../flow/openadapt_flow/generate/contract.py | 544 +++++++++++++ .../flow/openadapt_flow/generate/identity.py | 634 +++++++++++++++ .../flow/openadapt_flow/generate/inputs.py | 306 +++++++ .../openadapt_flow/runtime/effects/fhir.py | 71 +- .../openadapt_flow/runtime/effects/rest.py | 80 +- packages/flow/public-artifacts.json | 1 + packages/flow/tests/test_generate_channel.py | 388 +++++++++ packages/flow/tests/test_generate_contract.py | 256 ++++++ packages/flow/tests/test_generate_inputs.py | 103 +++ .../tests/test_generate_parts_pipeline.py | 133 ++++ 25 files changed, 5017 insertions(+), 23 deletions(-) create mode 100644 packages/flow/benchmark/generated_parts/README.md create mode 100644 packages/flow/benchmark/generated_parts/__init__.py create mode 100644 packages/flow/benchmark/generated_parts/dup_backtest.py create mode 100644 packages/flow/benchmark/generated_parts/results/duplicates.json create mode 100644 packages/flow/docs/GENERATED_PARTS.md create mode 100644 packages/flow/docs/commands/02-check-and-admit/generate-parts.md create mode 100644 packages/flow/openadapt_flow/generate/__init__.py create mode 100644 packages/flow/openadapt_flow/generate/baseline.py create mode 100644 packages/flow/openadapt_flow/generate/build.py create mode 100644 packages/flow/openadapt_flow/generate/channel.py create mode 100644 packages/flow/openadapt_flow/generate/cli.py create mode 100644 packages/flow/openadapt_flow/generate/contract.py create mode 100644 packages/flow/openadapt_flow/generate/identity.py create mode 100644 packages/flow/openadapt_flow/generate/inputs.py create mode 100644 packages/flow/tests/test_generate_channel.py create mode 100644 packages/flow/tests/test_generate_contract.py create mode 100644 packages/flow/tests/test_generate_inputs.py create mode 100644 packages/flow/tests/test_generate_parts_pipeline.py diff --git a/CAPABILITIES.md b/CAPABILITIES.md index e541d413f..7e7eb6f63 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -137,6 +137,7 @@ improvising or retrying when something is off. | Judge any executor's write, including an agent's or a script's, from the system of record | Released | Synthetic | `openadapt-flow verify-run`, [VERIFY_RUN.md](packages/flow/docs/VERIFY_RUN.md) | | Prove that nothing outside the declared change moved (frame conditions), and attribute concurrent edits by other users through the app's audit log or a modified-by field; the receipt signs which parts were checked | On main | Synthetic (MockMed, SQLite audit log) | `verify-run`, [VERIFICATION_COVERAGE.md](packages/flow/docs/VERIFICATION_COVERAGE.md) | | Admit a record check only after a fault suite written to a store copy shows it catches every fault it claims, and sign its coverage into the receipt | On main | Synthetic (in-memory REST store, SQLite copy) | `openadapt_flow/fault_gate.py`, `scaffold-verifier --fault-gate`, `verify-run` `admission:` | +| Generate the app-specific parts of a workflow (read channel, record contract, duplicate rule, input types) from two or three demonstrations, each admitted only by a check that doesn't depend on the generator, and approved by a person in plain words | On main | Synthetic (a fake REST and FHIR app); duplicates on #1516's stream: 54 of 223 missed against the hand-built 24, 0 false stops against 2 | `openadapt-flow generate-parts`, [GENERATED_PARTS.md](packages/flow/docs/GENERATED_PARTS.md) | | Check that the saved record matches its source document, not only the parameters: intake keeps each value's line, a second reading must agree before a guarded save, and a round trip compares the read-back with the document | On main | Synthetic (MockMed, text referral) | `openadapt_flow/provenance.py`, `intake`, `verify-run --provenance`, guarded save | | Guarded save: a safety layer any agent wraps around one write (check the record, hold for approval, read back) | On main ([#1258](https://github.com/OpenAdaptAI/OpenAdapt/pull/1258)) | Synthetic | `GUARDED_SAVE.md` | | Draft a record check from the demonstration's write steps (`scaffold-verifier`) | Released | Synthetic | `openadapt-flow scaffold-verifier` (a person fills in the draft) | @@ -302,6 +303,7 @@ A test fails when a command has no file or this list is out of date. - `bind-record-checks` - `certify` - `check` +- `generate-parts` - `jit` - `known-ids` - `lint` diff --git a/llms-full.txt b/llms-full.txt index 02ba1a00d..f2c6f7e35 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -534,6 +534,7 @@ improvising or retrying when something is off. | Judge any executor's write, including an agent's or a script's, from the system of record | Released | Synthetic | `openadapt-flow verify-run`, [VERIFY_RUN.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/VERIFY_RUN.md) | | Prove that nothing outside the declared change moved (frame conditions), and attribute concurrent edits by other users through the app's audit log or a modified-by field; the receipt signs which parts were checked | On main | Synthetic (MockMed, SQLite audit log) | `verify-run`, [VERIFICATION_COVERAGE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/VERIFICATION_COVERAGE.md) | | Admit a record check only after a fault suite written to a store copy shows it catches every fault it claims, and sign its coverage into the receipt | On main | Synthetic (in-memory REST store, SQLite copy) | `openadapt_flow/fault_gate.py`, `scaffold-verifier --fault-gate`, `verify-run` `admission:` | +| Generate the app-specific parts of a workflow (read channel, record contract, duplicate rule, input types) from two or three demonstrations, each admitted only by a check that doesn't depend on the generator, and approved by a person in plain words | On main | Synthetic (a fake REST and FHIR app); duplicates on #1516's stream: 54 of 223 missed against the hand-built 24, 0 false stops against 2 | `openadapt-flow generate-parts`, [GENERATED_PARTS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/GENERATED_PARTS.md) | | Check that the saved record matches its source document, not only the parameters: intake keeps each value's line, a second reading must agree before a guarded save, and a round trip compares the read-back with the document | On main | Synthetic (MockMed, text referral) | `openadapt_flow/provenance.py`, `intake`, `verify-run --provenance`, guarded save | | Guarded save: a safety layer any agent wraps around one write (check the record, hold for approval, read back) | On main ([#1258](https://github.com/OpenAdaptAI/OpenAdapt/pull/1258)) | Synthetic | `GUARDED_SAVE.md` | | Draft a record check from the demonstration's write steps (`scaffold-verifier`) | Released | Synthetic | `openadapt-flow scaffold-verifier` (a person fills in the draft) | @@ -563,6 +564,7 @@ improvising or retrying when something is off. | Learn one program with branches from two or more recordings, or say which recordings are missing | Released | Synthetic | `openadapt-flow induce`, `disambiguate`; [CAPABILITIES.md (flow)](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/CAPABILITIES.md) | | Learn a guarded branch from several halts fixed the same way | Released | Synthetic | `openadapt-flow teach-branch`, `learning/branch_teach.py` | | Run a program over a worklist | Released | Real app | `openadapt-flow for-each` | +| Learn a worklist loop from recordings that search a record, open it, work it and go back to the list, even when the last row skips the return or rows differ (a field some rows lack, a write some rows get behind a screen check a person accepts). Each pass selects the row's own record by its values, verifies it, and checks that row's write. Certified and governed runs refuse it until qualification covers it | On main ([#1435](https://github.com/OpenAdaptAI/OpenAdapt/pull/1435)) | Synthetic (held-out rows run through the interpreter against an in-memory app) | `openadapt-flow induce`; `Step.target_param` | | Typed business decisions inside a run | Released | Synthetic | `business_decision` states, `openadapt-flow business-decisions` | | Watch many real runs in the background and mine candidate branches (shadow mode) | On main ([#1212](https://github.com/OpenAdaptAI/OpenAdapt/pull/1212)) | Synthetic | [SHADOW_MINING.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/capture/docs/SHADOW_MINING.md), [LEARN_FROM_MANY_RUNS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/LEARN_FROM_MANY_RUNS.md) | | Fill a workflow's inputs from a referral PDF, fax image or email | Released | Synthetic | `openadapt-flow intake`, [INTAKE.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/INTAKE.md) | @@ -698,6 +700,7 @@ A test fails when a command has no file or this list is out of date. - `bind-record-checks` - `certify` - `check` +- `generate-parts` - `jit` - `known-ids` - `lint` diff --git a/packages/flow/benchmark/generated_parts/README.md b/packages/flow/benchmark/generated_parts/README.md new file mode 100644 index 000000000..0d012afdf --- /dev/null +++ b/packages/flow/benchmark/generated_parts/README.md @@ -0,0 +1,69 @@ +# Generated parts against hand-built parts + +Measures `openadapt-flow generate-parts` +([`docs/GENERATED_PARTS.md`](../../docs/GENERATED_PARTS.md)) against the parts +written by hand for the OpenEMR referral workflow. + +## Duplicates: the generated rule against #1516's record check + +`dup_backtest.py` runs both checks on #1516's own streams +(`benchmark/dup_check/measure.py` at `09d0298dc`, read from a checkout of that +branch). The long stream has 676 items: 440 new patients and 223 true +duplicates, 176 of them planted variants of the existing charts. The +generated rule is calibrated only on the 30 charts that exist before the +stream starts, with the generic error model in +`openadapt_flow/generate/identity.py`. It never sees the stream or its labels. +No app, no model calls; generation takes under half a second. + +```bash +python -m generated_parts.dup_backtest --dup-check +``` + +| Long stream, 676 items | generated rule | #1516 record check (hand-built) | +|---|---:|---:| +| missed duplicates, of 223 | 54 | 24 | +| false stops, of 440 new patients | 0 | 2 | +| false stops per 100 items | 0.0 | 0.3 | + +Generation is worse at catching duplicates. The 54 misses: + +- 24 are the two kinds #1516 also misses (a new name, new contact details + and the same or a mistyped date of birth: nothing else to match on); +- 12 `pl_married_moved` (a married name and new contact details). The rule + would need "same first name and same date of birth", and this system's own + records hold six pairs of separate charts that share both, so calibration + refuses it. #1516's hand-written rule accepts that risk; +- 12 `pl_both_typos_same_contact` (a typo in both names); +- 6 `r2_prior_name_moved_dob_typo` (a prior name, new contact details and a + mistyped day). + +The rule's own calibration estimates a 21% miss rate on made-up duplicates +of this system's records, and lists the error kinds it misses most (a changed +last name, or swapped names, together with new contact details). That text +goes to the person who approves it. The hand-built rule has a nickname table; +the generated one has none. + +The rule's false-stop budget (estimated false stops per new record) changes +calibration, but not these results: at 0.005 and at 0.05 the stream gives 54 +misses and 0 false stops. + +### How the rule got here + +Three versions were run on this stream. The second and third were changed +after seeing the first's result, so this stream is no longer a clean held-out +test for the generator: + +| Version | Change | Missed | False stops | +|---|---|---:|---:| +| 1 | one error at a time | 78 | 0 | +| 2 | each error with contact details kept and changed; swapped names, an initial, a second surname, a prior name | 42 | 12 | +| 3 | admit a clause only if its estimated false stops per record, from per-comparison rates on this system's records, fit the budget | 54 | 0 | + +Version 1 picked one clause that relied on the email: every made-up +duplicate kept its email. Version 2's loosest clauses ("same first name, last +name one letter apart") never fired on 30 records but did on 440 new +patients; version 3 estimates that risk from each comparison's rate instead +of waiting to see it. + +The committed result is `results/duplicates.json`, including each rule in +words. diff --git a/packages/flow/benchmark/generated_parts/__init__.py b/packages/flow/benchmark/generated_parts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/flow/benchmark/generated_parts/dup_backtest.py b/packages/flow/benchmark/generated_parts/dup_backtest.py new file mode 100644 index 000000000..b927e20ee --- /dev/null +++ b/packages/flow/benchmark/generated_parts/dup_backtest.py @@ -0,0 +1,203 @@ +"""Back-test: the generated duplicate rule against #1516's record check. + + python -m generated_parts.dup_backtest --dup-check + +Uses #1516's own streams (``benchmark/dup_check/measure.py``): the soak's 55 +items and the 500-item seed-7 stream, each with the planted true duplicates, +676 items with 223 true duplicates on the long stream. The generated rule is +calibrated only on the charts that exist before the stream starts (the +system's own data) with the generic error model in +``openadapt_flow/generate/identity.py``; it never sees the stream, the +planted kinds or their labels. Both checks then meet the same items in the +same order, with each earlier item's right outcome added to the table. + +No app, no model calls. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent + + +_DUMP = r""" +import dataclasses, json, sys +sys.path = [p for p in sys.path if "openadapt-evals" not in p] +from benchmark.dup_check import measure +long_items = int(sys.argv[1]) +out = {} +for name, (existing, items) in ( + ("soak-seed7-55", measure.from_ledger(measure.SOAK / "workload.jsonl")), + (f"seed7-{long_items}", measure.from_generator(7, long_items, 10)), +): + extra, extra_items = measure.extra_existing(existing) + profile = existing + extra + stream = measure.plant(existing, items + extra_items, 7) + hand = measure.measure(profile, stream) + out[name] = { + "profile": profile, + "items": [dataclasses.asdict(i) for i in stream], + "duplicate_kinds": sorted(measure.DUPLICATE_KINDS), + "hand_rows": hand["rows"], + } +json.dump(out, sys.stdout) +""" + + +def dump_streams(checkout: Path, long_items: int) -> dict[str, Any]: + """#1516's streams and its record check's decisions, from its own checkout.""" + + import os + import subprocess + + flow = checkout / "packages" / "flow" + env = { + **os.environ, + "PYTHONPATH": os.pathsep.join( + [str(flow), str(checkout / "packages" / "types")] + ), + } + out = subprocess.run( + [sys.executable, "-c", _DUMP, str(long_items)], + capture_output=True, + text=True, + env=env, + cwd=str(flow), + check=True, + ).stdout + return json.loads(out) + + +def _iso(value: str) -> str: + import re + + m = re.fullmatch(r"(\d{2})/(\d{2})/(\d{4})", value or "") + return f"{m.group(3)}-{m.group(1)}-{m.group(2)}" if m else (value or "") + + +def run(stream: dict[str, Any], *, seed: int, budget: float = 0.005) -> dict[str, Any]: + from openadapt_flow.generate.identity import generate_rule + + profile = [dict(p, DOB=_iso(p.get("DOB", ""))) for p in stream["profile"]] + started = time.perf_counter() + # The table a new record meets grows to the stream's size: the known + # operational volume, not any outcome. + expected = len(profile) + sum( + 1 for i in stream["items"] if i["intended"] is not None + ) + rule = generate_rule( + profile, seed=seed, expected_records=expected, false_stop_budget=budget + ) + gen_s = time.perf_counter() - started + table: list[dict] = [dict(p) for p in profile] + kinds = set(stream["duplicate_kinds"]) + hand = {r["item"]: r for r in stream["hand_rows"]} + rows = [] + for item in stream["items"]: + table += [dict(p, DOB=_iso(p.get("DOB", ""))) for p in item["before"]] + params = { + k: (v.strip() if isinstance(v, str) else v) + for k, v in item["params"].items() + } + params["DOB"] = _iso(params.get("DOB", "")) + decision, _why = rule.decide(params, table) + rows.append( + { + "item": item["item_id"], + "exception": item["exception"], + "right": "new chart" + if item["intended"] is not None + else ( + "no chart: duplicate" if item["exception"] in kinds else "no chart" + ), + "generated": decision, + "record": hand[item["item_id"]]["record"], + } + ) + if item["intended"] is not None: + table.append( + dict(item["intended"], DOB=_iso(item["intended"].get("DOB", ""))) + ) + return {"rule": rule, "rows": rows, "generation_s": round(gen_s, 3)} + + +def summarize(rows: list[dict], key: str, stops: Any) -> dict: + new = [r for r in rows if r["right"] == "new chart"] + dups = [r for r in rows if r["right"] == "no chart: duplicate"] + false = [r for r in new if stops(r)] + missed = [r for r in dups if not stops(r)] + + def count(rs: list[dict]) -> dict: + out: dict[str, int] = {} + for r in rs: + out[r["exception"]] = out.get(r["exception"], 0) + 1 + return dict(sorted(out.items())) + + return { + "items": len(rows), + "new_chart_items": len(new), + "true_duplicates": len(dups), + "false_stops": len(false), + "false_stops_per_100_items": round(100 * len(false) / max(1, len(rows)), 1), + "missed_duplicates": len(missed), + "missed_by_exception": count(missed), + "false_stops_by_exception": count(false), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--dup-check", type=Path, required=True) + parser.add_argument("--long-items", type=int, default=500) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument( + "--budgets", + default="0.005,0.01,0.02,0.05", + help="estimated false stops per new record the rule may spend (a sweep)", + ) + parser.add_argument("--out", type=Path, default=HERE / "results") + args = parser.parse_args() + streams = dump_streams(args.dup_check, args.long_items) + args.out.mkdir(parents=True, exist_ok=True) + report: dict[str, Any] = {} + budgets = [float(b) for b in args.budgets.split(",")] + for name, stream in streams.items(): + for budget in budgets: + result = run(stream, seed=args.seed, budget=budget) + rows = result["rows"] + report[f"{name}@{budget}"] = { + "budget": budget, + "profile_records": len(stream["profile"]), + "generation_s": result["generation_s"], + "rule": result["rule"].as_dict(), + "rule_text": result["rule"].describe(), + "generated": summarize( + rows, "generated", lambda r: r["generated"] != "none" + ), + "hand_built_record_check": summarize( + rows, "record", lambda r: r["record"] != "none" + ), + } + g = report[f"{name}@{budget}"]["generated"] + h = report[f"{name}@{budget}"]["hand_built_record_check"] + print( + f"{name} budget={budget}: generated missed {g['missed_duplicates']}/" + f"{g['true_duplicates']}, false stops {g['false_stops']} " + f"({g['false_stops_per_100_items']}/100); hand-built missed " + f"{h['missed_duplicates']}, false stops {h['false_stops']} " + f"({h['false_stops_per_100_items']}/100)" + ) + (args.out / "duplicates.json").write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/generated_parts/results/duplicates.json b/packages/flow/benchmark/generated_parts/results/duplicates.json new file mode 100644 index 000000000..8a27daa25 --- /dev/null +++ b/packages/flow/benchmark/generated_parts/results/duplicates.json @@ -0,0 +1,749 @@ +{ + "seed7-500@0.005": { + "budget": 0.005, + "generated": { + "false_stops": 0, + "false_stops_by_exception": {}, + "false_stops_per_100_items": 0.0, + "items": 676, + "missed_by_exception": { + "limit_married_moved_dob_typo": 12, + "limit_new_name_moved_same_dob": 12, + "pl_both_typos_same_contact": 12, + "pl_married_moved": 12, + "r2_prior_name_moved_dob_typo": 6 + }, + "missed_duplicates": 54, + "new_chart_items": 440, + "true_duplicates": 223 + }, + "generation_s": 0.456, + "hand_built_record_check": { + "false_stops": 2, + "false_stops_by_exception": { + "shared_dob": 2 + }, + "false_stops_per_100_items": 0.3, + "items": 676, + "missed_by_exception": { + "limit_married_moved_dob_typo": 12, + "limit_new_name_moved_same_dob": 12 + }, + "missed_duplicates": 24, + "new_chart_items": 440, + "true_duplicates": 223 + }, + "profile_records": 30, + "rule": { + "calibration": { + "calibration_duplicates": 526, + "distinct_queries": 870, + "false_match_rate": 0.0, + "miss_by_error": { + "contact_changed+contact_changed": 0.0, + "day_month_swapped+contact_changed": 0.0, + "day_month_swapped+contact_kept": 0.0, + "dob_digit+contact_changed": 0.0, + "dob_digit+contact_kept": 0.0, + "first_initial+contact_changed": 0.0, + "first_initial+contact_kept": 0.0, + "first_typo+contact_changed": 0.9, + "first_typo+contact_kept": 0.0, + "last_changed+contact_changed": 1.0, + "last_changed+contact_kept": 0.0, + "last_hyphenated+contact_changed": 0.0, + "last_hyphenated+contact_kept": 0.0, + "last_second_surname+contact_changed": 0.0, + "last_second_surname+contact_kept": 0.0, + "last_typo+contact_changed": 0.8, + "last_typo+contact_kept": 0.0, + "names_swapped+contact_changed": 1.0, + "names_swapped+contact_kept": 0.0 + }, + "miss_rate": 0.211 + }, + "close": [ + [ + [ + "email", + "eq" + ], + [ + "phone", + "eq" + ], + [ + "street", + "eq" + ] + ], + [ + [ + "first", + "initial" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + [ + [ + "first", + "eq" + ], + [ + "last", + "share" + ], + [ + "dob", + "swap" + ] + ], + [ + [ + "first", + "eq" + ], + [ + "last", + "share" + ], + [ + "dob", + "near" + ] + ], + [ + [ + "first", + "edit1" + ], + [ + "last", + "eq" + ], + [ + "dob", + "swap" + ] + ] + ], + "estimated_false_stops_per_record": 0.00497, + "exact": [ + [ + "first", + "eq" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + "notes": [], + "roles": { + "dob": "DOB", + "email": "email", + "first": "fname", + "last": "lname", + "phone": "phone_home", + "prior_last": "prior_lname", + "street": "street" + }, + "schema_version": "openadapt.duplicate-rule/v1" + }, + "rule_text": [ + "Stop: the record exists when first the same and last the same and dob the same.", + "Stop for a person when email the same and phone the same and street the same.", + "Stop for a person when first matching by initial and last the same and dob the same.", + "Stop for a person when first the same and last sharing a name part and dob with day and month swapped.", + "Stop for a person when first the same and last sharing a name part and dob one part off.", + "Stop for a person when first one letter apart and last the same and dob with day and month swapped.", + "Stop for a person when the new record lacks a name or a date of birth." + ] + }, + "seed7-500@0.05": { + "budget": 0.05, + "generated": { + "false_stops": 0, + "false_stops_by_exception": {}, + "false_stops_per_100_items": 0.0, + "items": 676, + "missed_by_exception": { + "limit_married_moved_dob_typo": 12, + "limit_new_name_moved_same_dob": 12, + "pl_both_typos_same_contact": 12, + "pl_married_moved": 12, + "r2_prior_name_moved_dob_typo": 6 + }, + "missed_duplicates": 54, + "new_chart_items": 440, + "true_duplicates": 223 + }, + "generation_s": 0.423, + "hand_built_record_check": { + "false_stops": 2, + "false_stops_by_exception": { + "shared_dob": 2 + }, + "false_stops_per_100_items": 0.3, + "items": 676, + "missed_by_exception": { + "limit_married_moved_dob_typo": 12, + "limit_new_name_moved_same_dob": 12 + }, + "missed_duplicates": 24, + "new_chart_items": 440, + "true_duplicates": 223 + }, + "profile_records": 30, + "rule": { + "calibration": { + "calibration_duplicates": 526, + "distinct_queries": 870, + "false_match_rate": 0.0, + "miss_by_error": { + "contact_changed+contact_changed": 0.0, + "day_month_swapped+contact_changed": 0.0, + "day_month_swapped+contact_kept": 0.0, + "dob_digit+contact_changed": 0.0, + "dob_digit+contact_kept": 0.0, + "first_initial+contact_changed": 0.0, + "first_initial+contact_kept": 0.0, + "first_typo+contact_changed": 0.0, + "first_typo+contact_kept": 0.0, + "last_changed+contact_changed": 1.0, + "last_changed+contact_kept": 0.0, + "last_hyphenated+contact_changed": 0.0, + "last_hyphenated+contact_kept": 0.0, + "last_second_surname+contact_changed": 0.0, + "last_second_surname+contact_kept": 0.0, + "last_typo+contact_changed": 0.8, + "last_typo+contact_kept": 0.0, + "names_swapped+contact_changed": 0.0, + "names_swapped+contact_kept": 0.0 + }, + "miss_rate": 0.1027 + }, + "close": [ + [ + [ + "email", + "eq" + ], + [ + "phone", + "eq" + ], + [ + "street", + "eq" + ] + ], + [ + [ + "first", + "initial" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + [ + [ + "first", + "eq" + ], + [ + "last", + "share" + ], + [ + "dob", + "swap" + ] + ], + [ + [ + "first", + "eq" + ], + [ + "last", + "share" + ], + [ + "dob", + "near" + ] + ], + [ + [ + "first", + "edit1" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + [ + [ + "dob", + "eq" + ], + [ + "names_swapped", + "eq" + ] + ] + ], + "estimated_false_stops_per_record": 0.04539, + "exact": [ + [ + "first", + "eq" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + "notes": [], + "roles": { + "dob": "DOB", + "email": "email", + "first": "fname", + "last": "lname", + "phone": "phone_home", + "prior_last": "prior_lname", + "street": "street" + }, + "schema_version": "openadapt.duplicate-rule/v1" + }, + "rule_text": [ + "Stop: the record exists when first the same and last the same and dob the same.", + "Stop for a person when email the same and phone the same and street the same.", + "Stop for a person when first matching by initial and last the same and dob the same.", + "Stop for a person when first the same and last sharing a name part and dob with day and month swapped.", + "Stop for a person when first the same and last sharing a name part and dob one part off.", + "Stop for a person when first one letter apart and last the same and dob the same.", + "Stop for a person when dob the same and first and last names swapped.", + "Stop for a person when the new record lacks a name or a date of birth." + ] + }, + "soak-seed7-55@0.005": { + "budget": 0.005, + "generated": { + "false_stops": 0, + "false_stops_by_exception": {}, + "false_stops_per_100_items": 0.0, + "items": 231, + "missed_by_exception": { + "limit_married_moved_dob_typo": 12, + "limit_new_name_moved_same_dob": 12, + "pl_both_typos_same_contact": 12, + "pl_married_moved": 12, + "r2_prior_name_moved_dob_typo": 6 + }, + "missed_duplicates": 54, + "new_chart_items": 45, + "true_duplicates": 184 + }, + "generation_s": 0.447, + "hand_built_record_check": { + "false_stops": 0, + "false_stops_by_exception": {}, + "false_stops_per_100_items": 0.0, + "items": 231, + "missed_by_exception": { + "limit_married_moved_dob_typo": 12, + "limit_new_name_moved_same_dob": 12 + }, + "missed_duplicates": 24, + "new_chart_items": 45, + "true_duplicates": 184 + }, + "profile_records": 30, + "rule": { + "calibration": { + "calibration_duplicates": 526, + "distinct_queries": 870, + "false_match_rate": 0.0, + "miss_by_error": { + "contact_changed+contact_changed": 0.0, + "day_month_swapped+contact_changed": 0.0, + "day_month_swapped+contact_kept": 0.0, + "dob_digit+contact_changed": 0.0, + "dob_digit+contact_kept": 0.0, + "first_initial+contact_changed": 0.0, + "first_initial+contact_kept": 0.0, + "first_typo+contact_changed": 0.0, + "first_typo+contact_kept": 0.0, + "last_changed+contact_changed": 1.0, + "last_changed+contact_kept": 0.0, + "last_hyphenated+contact_changed": 0.0, + "last_hyphenated+contact_kept": 0.0, + "last_second_surname+contact_changed": 0.0, + "last_second_surname+contact_kept": 0.0, + "last_typo+contact_changed": 0.733, + "last_typo+contact_kept": 0.0, + "names_swapped+contact_changed": 0.9, + "names_swapped+contact_kept": 0.0 + }, + "miss_rate": 0.1502 + }, + "close": [ + [ + [ + "email", + "eq" + ], + [ + "phone", + "eq" + ], + [ + "street", + "eq" + ] + ], + [ + [ + "first", + "initial" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + [ + [ + "first", + "eq" + ], + [ + "last", + "share" + ], + [ + "dob", + "swap" + ] + ], + [ + [ + "first", + "eq" + ], + [ + "last", + "share" + ], + [ + "dob", + "near" + ] + ], + [ + [ + "first", + "edit1" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + [ + [ + "dob", + "swap" + ], + [ + "names_swapped", + "eq" + ] + ], + [ + [ + "first", + "eq" + ], + [ + "last", + "edit1" + ], + [ + "dob", + "swap" + ] + ] + ], + "estimated_false_stops_per_record": 0.00358, + "exact": [ + [ + "first", + "eq" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + "notes": [], + "roles": { + "dob": "DOB", + "email": "email", + "first": "fname", + "last": "lname", + "phone": "phone_home", + "prior_last": "prior_lname", + "street": "street" + }, + "schema_version": "openadapt.duplicate-rule/v1" + }, + "rule_text": [ + "Stop: the record exists when first the same and last the same and dob the same.", + "Stop for a person when email the same and phone the same and street the same.", + "Stop for a person when first matching by initial and last the same and dob the same.", + "Stop for a person when first the same and last sharing a name part and dob with day and month swapped.", + "Stop for a person when first the same and last sharing a name part and dob one part off.", + "Stop for a person when first one letter apart and last the same and dob the same.", + "Stop for a person when dob with day and month swapped and first and last names swapped.", + "Stop for a person when first the same and last one letter apart and dob with day and month swapped.", + "Stop for a person when the new record lacks a name or a date of birth." + ] + }, + "soak-seed7-55@0.05": { + "budget": 0.05, + "generated": { + "false_stops": 0, + "false_stops_by_exception": {}, + "false_stops_per_100_items": 0.0, + "items": 231, + "missed_by_exception": { + "limit_married_moved_dob_typo": 12, + "limit_new_name_moved_same_dob": 12, + "pl_both_typos_same_contact": 12, + "pl_married_moved": 12, + "r2_prior_name_moved_dob_typo": 6 + }, + "missed_duplicates": 54, + "new_chart_items": 45, + "true_duplicates": 184 + }, + "generation_s": 0.441, + "hand_built_record_check": { + "false_stops": 0, + "false_stops_by_exception": {}, + "false_stops_per_100_items": 0.0, + "items": 231, + "missed_by_exception": { + "limit_married_moved_dob_typo": 12, + "limit_new_name_moved_same_dob": 12 + }, + "missed_duplicates": 24, + "new_chart_items": 45, + "true_duplicates": 184 + }, + "profile_records": 30, + "rule": { + "calibration": { + "calibration_duplicates": 526, + "distinct_queries": 870, + "false_match_rate": 0.0, + "miss_by_error": { + "contact_changed+contact_changed": 0.0, + "day_month_swapped+contact_changed": 0.0, + "day_month_swapped+contact_kept": 0.0, + "dob_digit+contact_changed": 0.0, + "dob_digit+contact_kept": 0.0, + "first_initial+contact_changed": 0.0, + "first_initial+contact_kept": 0.0, + "first_typo+contact_changed": 0.0, + "first_typo+contact_kept": 0.0, + "last_changed+contact_changed": 0.967, + "last_changed+contact_kept": 0.0, + "last_hyphenated+contact_changed": 0.0, + "last_hyphenated+contact_kept": 0.0, + "last_second_surname+contact_changed": 0.0, + "last_second_surname+contact_kept": 0.0, + "last_typo+contact_changed": 0.0, + "last_typo+contact_kept": 0.0, + "names_swapped+contact_changed": 0.0, + "names_swapped+contact_kept": 0.0 + }, + "miss_rate": 0.0551 + }, + "close": [ + [ + [ + "email", + "eq" + ], + [ + "phone", + "eq" + ], + [ + "street", + "eq" + ] + ], + [ + [ + "first", + "initial" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + [ + [ + "first", + "eq" + ], + [ + "last", + "share" + ], + [ + "dob", + "swap" + ] + ], + [ + [ + "first", + "eq" + ], + [ + "last", + "share" + ], + [ + "dob", + "near" + ] + ], + [ + [ + "first", + "edit1" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + [ + [ + "dob", + "eq" + ], + [ + "names_swapped", + "eq" + ] + ], + [ + [ + "first", + "eq" + ], + [ + "last", + "edit1" + ], + [ + "dob", + "eq" + ] + ] + ], + "estimated_false_stops_per_record": 0.01387, + "exact": [ + [ + "first", + "eq" + ], + [ + "last", + "eq" + ], + [ + "dob", + "eq" + ] + ], + "notes": [], + "roles": { + "dob": "DOB", + "email": "email", + "first": "fname", + "last": "lname", + "phone": "phone_home", + "prior_last": "prior_lname", + "street": "street" + }, + "schema_version": "openadapt.duplicate-rule/v1" + }, + "rule_text": [ + "Stop: the record exists when first the same and last the same and dob the same.", + "Stop for a person when email the same and phone the same and street the same.", + "Stop for a person when first matching by initial and last the same and dob the same.", + "Stop for a person when first the same and last sharing a name part and dob with day and month swapped.", + "Stop for a person when first the same and last sharing a name part and dob one part off.", + "Stop for a person when first one letter apart and last the same and dob the same.", + "Stop for a person when dob the same and first and last names swapped.", + "Stop for a person when first the same and last one letter apart and dob the same.", + "Stop for a person when the new record lacks a name or a date of birth." + ] + } +} diff --git a/packages/flow/docs/GENERATED_PARTS.md b/packages/flow/docs/GENERATED_PARTS.md new file mode 100644 index 000000000..e734af281 --- /dev/null +++ b/packages/flow/docs/GENERATED_PARTS.md @@ -0,0 +1,89 @@ +# Generated application parts + +`openadapt-flow generate-parts` builds the parts of a +workflow that are specific to one application at setup time, and admits each +one only after a check that does not depend on the generator. Healthy runs +still make no model calls: everything here happens once, before a person +approves the workflow. + +## What is generated, and what never is + +| Part | Generated from | Admitted by | +|---|---|---| +| Read channel: how to read the system of record | the network traffic captured while teaching (`network.jsonl`), an OpenAPI document, or a FHIR CapabilityStatement | the channel test: reads back exactly the records each demonstration created or changed, across every page, and reads as unreadable (never as empty) when auth, paging or a response is broken | +| Record contract: what a save must do and must not do | the before/after difference in the system of record across two or three varied demonstrations | the fault gate (`fault_gate.py`): wrong record, wrong value, unrelated change, missing write, partial write, duplicate, delayed write, with coverage per class | +| Identity and duplicate rule: when a new record is already there | the record schema plus a profile of the records already in the system | calibration on the system's own data: estimated false-match and miss rates, a small sample a person labels; anything the rule can't decide goes to a person | +| Input types and conversion | sample source documents | the round trip (`provenance.py`): every converted value must read back from its source; an ambiguous value halts | + +Never generated: the safety policy. That is the execution profile, approval +modes, the go-live switch, which steps are consequential, the order of the +gates, and attribution of changes to other users (the one rule that excuses +a change). A person sets those, as today. + +## Generated parts can only add checks + +Generation reads untrusted content: page text, response bodies, field +values. A page could try to steer a generator ("this field is volatile, ignore +it"). So the generated contract is checked against a fixed baseline that does +not come from page content: + +- every parameter the demonstration typed is checked in the saved record; +- the frame is the whole read set, compared by record identity; +- no generated `ignore_fields` entry names a field a parameter is written to, + and nothing generated turns off `forbid_collateral_loss`, adds + `target_fields`, or adds attribution. + +A generated part that removes or weakens a baseline check is refused, whatever +produced it. Tests feed a hostile page through every generator. + +## A person approves the contract, in words + +`generate-parts` writes `parts.review.md`: one plain-language line per check +("A new patient record must appear with the email the task names", "No other +patient record may change"), the coverage the fault gate measured, the +duplicate rule's estimated rates, and what the parts can't catch. The person +approves that text. The approval is bound to the digest of the parts it +describes; a changed part needs a new approval. + +## Use + +```bash +openadapt-flow generate-parts channels --openapi api.json --base-url URL \ + --auth-env READ_TOKEN --out channels.json +openadapt-flow generate-parts snapshot demo1 --channels channels.json --label before +# demonstrate the save once, with the values in demo1/params.json +openadapt-flow generate-parts snapshot demo1 --channels channels.json --label after +# repeat for demo2 and demo3, with different values +openadapt-flow generate-parts build --demo demo1 --demo demo2 --demo demo3 \ + --channels channels.json --out parts/ +openadapt-flow generate-parts approve parts/ --approved-by "Dana Reyes" +``` + +`--fhir-capability URL` takes a FHIR server's CapabilityStatement instead of, +or as well as, an OpenAPI document. Each demonstration folder holds +`params.json` (the values typed) and, optionally, `source.*` (the document +they came from). + +`build` writes `channel.json`, `verify.yaml`, `check_coverage.json`, +`duplicate_rule.json`, `inputs.json`, `parts.review.md` and +`parts.manifest.json`. It exits 0 only when every part was admitted. `approve` +shows the review, needs a person at a terminal who types `approve`, and +refuses an agent's name or a part that changed after it was generated. + +The fault gate runs on a test copy of the application when the caller gives +one (`build(..., test_copy=...)` in Python). Without one it runs on a +replica: an in-memory store shaped like the channel's records. A replica +tests the contract's judgment, not the application's read, and the coverage +record says `test_copy: replica`, which an admission policy can refuse. + +Generation needs two or three demonstrations with different values. A value +typed the same every time is checked against what was typed, but can't be +told from a constant, and the review says so. + +## Measurements + +The back-test against the hand-built OpenEMR referral parts, including where +generation is worse, is in `benchmark/generated_parts/`. On #1516's +duplicate stream (676 items, 223 true duplicates), the generated duplicate +rule missed 54 duplicates and stopped no new patient; the hand-built record +check missed 24 and stopped 2. diff --git a/packages/flow/docs/commands/02-check-and-admit/generate-parts.md b/packages/flow/docs/commands/02-check-and-admit/generate-parts.md new file mode 100644 index 000000000..e69de29bb diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index b0c6e9a66..cf25df1cd 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -9393,6 +9393,10 @@ def build_parser() -> argparse.ArgumentParser: _add_intake_parser(sub) + from openadapt_flow.generate.cli import add_parser as _add_generate_parser + + _add_generate_parser(sub) + from openadapt_flow.decision.cli import add_parser as _add_triage_parser _add_triage_parser(sub) diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index 3fa342ef1..ed165dcbe 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -387,6 +387,11 @@ class EffectsConfig(BaseModel): base_url: Optional[str] = None records_path: str = "/api/db" records_key: Optional[str] = "records" + #: ``rest`` only: dotted path in the response body to the next page's URL + #: (``links.next``). Set, the read follows every page on the same origin + #: and reads as unreadable when a page fails or the chain is too long. + #: Unset: one response is the whole read, as before. + records_next_path: Optional[str] = None #: Secret-isolated auth headers for the records read (bearer_env / #: header+value_env / basic_env). Never a credential literal. auth: Optional[AuthRef] = None @@ -612,7 +617,7 @@ def _serialize_compatible(self, handler: Any) -> dict[str, Any]: """Keep configs without a desktop read-back byte-identical.""" data: dict[str, Any] = handler(self) - for added in ("desktop_readback", "deferred_report"): + for added in ("desktop_readback", "deferred_report", "records_next_path"): if data.get(added, "") is None: data.pop(added, None) return data @@ -1887,6 +1892,7 @@ def sanitized(value: Any, *, key: str = "") -> Any: "fresh_session", "desktop_readback", "deferred_report", + "records_next_path", ): if config.get(added) is None: config.pop(added, None) @@ -2039,6 +2045,7 @@ def _build_effect_verifier_unredacted( timeout_s=cfg.timeout_s, poll_interval_s=cfg.poll_interval_s, drop_fields=tuple(cfg.records_drop_fields), + next_path=cfg.records_next_path, ) if kind == "fhir": diff --git a/packages/flow/openadapt_flow/generate/__init__.py b/packages/flow/openadapt_flow/generate/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/flow/openadapt_flow/generate/baseline.py b/packages/flow/openadapt_flow/generate/baseline.py new file mode 100644 index 000000000..4ee9f00fb --- /dev/null +++ b/packages/flow/openadapt_flow/generate/baseline.py @@ -0,0 +1,105 @@ +"""The floor a generated record contract may never go below. + +Generation reads untrusted content: page text, response bodies, field values. +Whatever a generator proposes, the contract it hands to a person must keep a +fixed set of checks that does not come from that content: + +- every parameter the demonstration typed is checked in the saved record; +- the frame (nothing else in the read set changes) is on, compared by record + identity, with nothing excused by attribution; +- no ignored field is a field a parameter is written to, and no collateral + rule is turned off. + +:func:`baseline_problems` lists every way a contract falls short. A contract +with any problem is refused, so a generated part can add checks but never +remove or weaken one. Attribution, the one rule that excuses a change, is +safety policy: a person adds it, never a generator. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from typing import Any + +from openadapt_flow.runtime.effects.effect import Effect, EffectKind + +#: Verifier kinds that read through the acting screen: never a generated channel. +_SCREEN_KINDS = frozenset({"none", "onscreen"}) + + +def _param_fields(effects: Sequence[Effect]) -> dict[str, set[str]]: + """Parameter -> the record fields effects check it against.""" + + out: dict[str, set[str]] = {} + for effect in effects: + selectors = ( + list(effect.new_records) + if effect.kind is EffectKind.EXACT_NEW_SET + else [effect.match] + ) + for selector in selectors: + for field, expr in selector.items(): + if expr.param is not None: + out.setdefault(expr.param, set()).add(field) + if effect.field and effect.value is not None and effect.value.param: + out.setdefault(effect.value.param, set()).add(effect.field) + return out + + +def baseline_problems(cfg: Any, typed_params: Iterable[str]) -> list[str]: + """Every way ``cfg`` (a parsed ``verify.yaml``) is weaker than the floor. + + ``typed_params`` are the parameters the demonstration typed into the app: + each must be checked against a record field. + """ + + problems: list[str] = [] + effects: list[Effect] = list(cfg.effects) + if cfg.verifier.kind in _SCREEN_KINDS: + problems.append(f"read channel {cfg.verifier.kind!r} reads the acting screen") + if not any( + e.kind in (EffectKind.RECORD_WRITTEN, EffectKind.EXACT_NEW_SET) + and (e.expected_count >= 1 or e.kind is EffectKind.EXACT_NEW_SET) + for e in effects + ): + problems.append("no effect requires the saved record to exist") + checked = _param_fields(effects) + for name in sorted(set(typed_params)): + if name not in checked: + problems.append(f"parameter {name!r} is not checked in the saved record") + for effect in effects: + if "forbid_collateral_loss" in effect.model_fields_set and not ( + effect.forbid_collateral_loss + ): + problems.append("an effect turns off the collateral-loss rule") + if "forbid_collateral_change" in effect.model_fields_set and not ( + effect.forbid_collateral_change + ): + problems.append("an effect turns off the collateral-change rule") + if effect.needs_operator_confirmation: + problems.append("an effect is still a placeholder") + frame = getattr(cfg, "frame", None) + if frame is not None: + if frame.attribution is not None: + problems.append( + "the frame excuses changes by attribution, which only a person sets" + ) + if frame.target_fields is not None: + problems.append("the frame widens what a target record may change") + written = {f for fields in checked.values() for f in fields} + hidden = sorted(set(frame.ignore_fields) & written) + if hidden: + problems.append( + "the frame ignores field(s) a parameter is written to: " + + ", ".join(hidden) + ) + drop = set(cfg.verifier.records_drop_fields) + for sub in cfg.verifier.surfaces.values(): + drop |= set(sub.records_drop_fields) + written = {f for fields in checked.values() for f in fields} + dropped = sorted(drop & written) + if dropped: + problems.append( + "the read drops field(s) a parameter is written to: " + ", ".join(dropped) + ) + return problems diff --git a/packages/flow/openadapt_flow/generate/build.py b/packages/flow/openadapt_flow/generate/build.py new file mode 100644 index 000000000..3b6a341d6 --- /dev/null +++ b/packages/flow/openadapt_flow/generate/build.py @@ -0,0 +1,508 @@ +"""Generate all four application parts from demonstrations, and verify each. + +``openadapt-flow generate-parts build`` reads a folder per demonstration: + +- ``params.json``: the values the person typed; +- ``reads/.before.json`` and ``reads/.after.json``: each + candidate channel read just before and just after the save + (``generate-parts snapshot`` takes them); +- optionally ``source.*``: the document the values came from. + +and writes, in the output folder: + +========================== ================================================== +``channel.json`` the admitted read channel and its channel test +``verify.yaml`` the record contract (``openadapt.verify/v1``) +``check_coverage.json`` the fault gate's coverage of that contract +``duplicate_rule.json`` the identity and duplicate rule, with calibration +``inputs.json`` input types, labels and conversions, round trip +``parts.review.md`` every check in plain words, for a person +``parts.manifest.json`` the digest of each part; what an approval binds +========================== ================================================== + +Nothing here is approved. ``generate-parts approve`` records a person's +approval of ``parts.review.md``, bound to the manifest digest. The safety +policy (profiles, approval modes, the switch, which steps are consequential, +the gate order, attribution) is never generated. +""" + +from __future__ import annotations + +import hashlib +import json +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +from openadapt_flow.generate.channel import ( + ChannelCandidate, + ChannelReport, + DemoReads, + check_channel, +) +from openadapt_flow.generate.contract import ( + ContractError, + Demonstration, + GatedContract, + generate_and_gate, +) + +MANIFEST_SCHEMA = "openadapt.generated-parts/v1" +APPROVAL_SCHEMA = "openadapt.generated-parts-approval/v1" +PART_FILES = ( + "channel.json", + "verify.yaml", + "check_coverage.json", + "duplicate_rule.json", + "inputs.json", + "parts.review.md", +) + + +class BuildError(ValueError): + """The demonstrations can't produce admitted parts. Nothing is written.""" + + +@dataclass +class DemoFolder: + path: Path + params: dict[str, str] + reads: dict[str, tuple[Optional[list[dict]], Optional[list[dict]]]] + source: Optional[Path] = None + + +def channel_id(candidate: ChannelCandidate) -> str: + raw = json.dumps(candidate.section, sort_keys=True, default=str) + return "ch-" + hashlib.sha256(raw.encode()).hexdigest()[:12] + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text()) if path.is_file() else None + + +def load_demo(path: Path) -> DemoFolder: + params = _load_json(path / "params.json") + if not isinstance(params, dict): + raise BuildError(f"{path}: params.json is missing or not an object") + reads: dict[str, tuple[Optional[list[dict]], Optional[list[dict]]]] = {} + for before in sorted((path / "reads").glob("*.before.json")): + cid = before.name[: -len(".before.json")] + after = path / "reads" / f"{cid}.after.json" + reads[cid] = (_load_json(before), _load_json(after)) + sources = sorted(p for p in path.glob("source.*") if p.is_file()) + return DemoFolder( + path, + {k: str(v) for k, v in params.items()}, + reads, + sources[0] if sources else None, + ) + + +def snapshot( + candidates: Sequence[ChannelCandidate], + params: Mapping[str, object], + demo: Path, + label: str, +) -> dict[str, int]: + """Read every candidate once; write ``reads/.