diff --git a/packages/flow/benchmark/auto_repair/STOP_CAUSES.md b/packages/flow/benchmark/auto_repair/STOP_CAUSES.md new file mode 100644 index 000000000..4729a4d74 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/STOP_CAUSES.md @@ -0,0 +1,107 @@ +# Why each measured stop happened, and why repair did not resume it + +Every stop in this directory's results, classified by root cause. The +evidence for each row is in `results/causes/*.csv`, written by +`stop_causes.py` from the runs' own `report.json` and, when self-repair ran, +`self-repair/self-repair.json`: + +- `stop` is what the halted step's error says (a rule over the message); +- `blocker` is what kept repair from resuming the item; +- `unsettled` lists the steps whose effect the transaction classifier could + not settle, when the run ended `RECONCILIATION_REQUIRED`. + +The root cause sometimes needs what the benchmark knows about the change it +made (which drift mode, which injected fault). That column is filled in +here, by hand, from the benchmark's labels and the linked issues. + +A **false stop** is a stop on a run where nothing was wrong: the screen was +correct and the record would have been, or was, saved correctly. + +## Summary + +71 stops: 30 MockMed trials, 12 MockMed worklist runs, 8 OpenEMR runs, 8 +openIMIS runs and 13 silent-saves items. + +| Root cause | Stops | False stops | Resumed before | Fix | +| --- | --- | --- | --- | --- | +| A control was renamed or moved | 24 | 0 | 19 | none for the 19; the other 5 (openIMIS) never reached repair: write classification, below | +| An interstitial screen, dismissed by a "Continue" button | 13 | 0 | 0 | dismiss classified from evidence, not the label; the agent sees the screen after each click | +| A save moved behind a closed menu | 3 | 0 | 0 | the agent sees the screen after each click | +| A pixel or mined-text postcondition failed on a correct screen | 9 | 9 | 0 | semantic postconditions | +| An identity check read a label behind a modal (#1347) | 3 | 3 | 0 | the identity check itself (#1347), not repair | +| A repaired target was cropped in a transient state | 3 | 0 | 0 | a template that holds across the control's enabled and disabled states | +| A correct stop: a data fault or a new business choice | 16 | 0 | n/a | none: these go to a person | + +On top of the root cause, 13 stops (all 8 OpenEMR and 5 openIMIS) could not +reach repair at all because an earlier step had acted without proof that it +wrote nothing (**write classification**): + +| App | Steps the classifier could not settle | Why | +| --- | --- | --- | +| OpenEMR referral | `step_002` ... `step_037`: 8 × Control+a, 4 × Enter | Unreviewed special keys. #1366 classifies 11 of 12 on a recording made with network capture; `step_007` keeps a background `POST` in its window. | +| OpenEMR referral | `step_039`, "Create New Patient" (self-repair's own check) | The label reads as a write, but the click opens OpenEMR's duplicate check; "Confirm Create New Patient" (`step_040`) is the save. The workflow therefore counts two saves. | +| openIMIS claim | `step_002` (Tab), `step_012` (a picker) | Unreviewed. A person's `qualify set-risk` settles them (#1337); with that review all 3 stops reached repair. | + +## MockMed, one item per trial (`results/causes/mockmed-2026-10-19-a.csv`) + +| Mode | Stops | `stop` | `blocker` | Root cause | False stop | +| --- | --- | --- | --- | --- | --- | +| relink | 3 | `target:moved` | `none:resumed` | renamed and moved | no | +| redesign | 3 | `target:geometry_only` | `none:resumed` | renamed and moved | no | +| menu | 3 | `target:geometry_only` | `patch:proposed_control_not_a_save` | save behind a closed menu: the agent sees only "Actions" and proposes it as the save | no | +| notice | 3 | `target:not_found` | `patch:proposed_control_write_shaped` | interstitial: the agent proposes "Continue to tasks", which is refused because its label reads as a write, and "Open" is not on the screen yet | no | +| font | 3 | `postcondition:region_only` | `agent:gave_up` | `region_stable` after the font changed; the click had acted correctly | **yes** | +| zoom | 3 | `postcondition:region_only` | `agent:gave_up` | `region_stable` at 125%; fixed on main by the page-scale check (#1360) | **yes** | +| provider | 3 | `postcondition:text_and_region` | `agent:gave_up` | a new required "Ordering provider" choice the recording never made | no (correct stop) | +| ambiguous | 3 | `target:ambiguous` | `person:not_a_missing_target` | two copies of the referral | no (correct stop) | +| missing | 3 | `identity:mismatch` | `person:rejected_policy` | the referral is gone | no (correct stop) | +| empty | 3 | `identity:unreadable` | `person:rejected_policy` | no referrals | no (correct stop) | + +## MockMed worklist (`results/causes/mockmed-worklist-2026-10-19-a.csv`) + +12 rows: 10 items and the 2 test-record runs self-repair made, all +`relink` or `redesign`. The 2 repairs resumed; the other 10 rows are items +held in their group and released by its one approval (`blocker` `not_run`). +All 10 items ended correct. + +## OpenEMR referral, #1347's runs (`results/causes/openemr-1347.csv`) + +| Runs | Halted at | `stop` | Root cause | False stop | +| --- | --- | --- | --- | --- | +| `fast1` and `std1` item 26 | `step_039` | `postcondition:text` ("No matches were found." absent) | a real possible duplicate: both runs reached the same synthetic patient, whose name had been used before (#1349, fixed by #1420) | no (correct stop) | +| `fast2` 16, `par8b` 7, `std2` 8 | `step_040` | `identity:mismatch` | the identity check read "Emergency Contact:" behind the modal (#1347) | **yes** | +| `par8b` 2, `par8w1b` 9 and 20 | `step_040`, after the save | `postcondition:text_and_region` | the save's check used tab-strip text and a pixel region mined from the recording; #1402 replaced them with the chart heading | **yes** | + +Every one also ended `RECONCILIATION_REQUIRED` for the write-classification +reason above, so none reached repair. The three stops after the save are not +repair targets anyway: input reached the save. + +## openIMIS, the relabelled Save (#1331) + +| Run | Stops | `stop` | `blocker` | Root cause | +| --- | --- | --- | --- | --- | +| before #1337 (`openimis-1331-before.csv`) | 3 | `target:scroll_budget` | `reconciliation:effect_not_settled` (`step_002`, `step_012`) | relabelled Save, and write classification | +| #1337, no review (`openimis-unreviewed-2026-10-19-b.csv`) | 2 | `target:not_found` | the same | relabelled Save, and write classification | +| #1337 and a review (`openimis-reviewed-2026-10-19-b.csv`) | 3 | `target:not_found` | 2 × `probe:stopped_before_save`, 1 × `agent:gave_up` | the agent's "SAVE CLAIM" was cropped while enabled; in the probe the scroll before it runs while the Save is still disabled, so the template never matches | + +## Silent-saves 1.1.0 UI-change blocks (`results/causes/silent-saves-2026-10-19-a.csv`) + +| Items | Stops | Root cause | +| --- | --- | --- | +| 46-55 (`notice`) | 10 | interstitial and "Continue", as in MockMed | +| 76 (`relink`) | 1 | renamed and moved; repaired once, items 77-85 ran the repaired program | +| 30 (`zoom`, `wrong_record` fault) and 80 (`relink`, `optimistic` fault) | 2 | correct stops: the server's own faults | + +The zoom block's other 9 items did not stop (page-scale check). + +## Reproduce + +```bash +python benchmark/auto_repair/stop_causes.py \ + --runs-root ~/.cache/openadapt-auto-repair/mockmed-2026-10-19-a \ + --out results/causes/mockmed-2026-10-19-a.csv +``` + +The run directories are local caches (`~/.cache/openadapt-auto-repair/`, +the scoreboard's OpenEMR runs); the CSVs here are what they held on +2026-10-20. diff --git a/packages/flow/benchmark/auto_repair/results/causes/mockmed-2026-10-19-a.csv b/packages/flow/benchmark/auto_repair/results/causes/mockmed-2026-10-19-a.csv new file mode 100644 index 000000000..bff905de2 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/causes/mockmed-2026-10-19-a.csv @@ -0,0 +1,31 @@ +run,label,outcome,halted_step,delivery_attempted,risk,stop,blocker,unsettled,error +ambiguous/trial-1/item,ambiguous,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:ambiguous,person:not_a_missing_target,,Structural safety refusal for step 'step_000' (click 'Open'): DOM locator is ambiguous: candidate_count=2 — no action was admitted +ambiguous/trial-2/item,ambiguous,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:ambiguous,person:not_a_missing_target,,Structural safety refusal for step 'step_000' (click 'Open'): DOM locator is ambiguous: candidate_count=2 — no action was admitted +ambiguous/trial-3/item,ambiguous,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:ambiguous,person:not_a_missing_target,,Structural safety refusal for step 'step_000' (click 'Open'): DOM locator is ambiguous: candidate_count=2 — no action was admitted +empty/trial-1/item,empty,REJECTED_POLICY,step_000,False,reversible,identity:unreadable,person:rejected_policy,,Step 'step_000' (click 'Open') requires verified identity under the governed run policy and its target identity could not be read from the live screen (context band OCR found no usable text) — needs h +empty/trial-2/item,empty,REJECTED_POLICY,step_000,False,reversible,identity:unreadable,person:rejected_policy,,Step 'step_000' (click 'Open') requires verified identity under the governed run policy and its target identity could not be read from the live screen (context band OCR found no usable text) — needs h +empty/trial-3/item,empty,REJECTED_POLICY,step_000,False,reversible,identity:unreadable,person:rejected_policy,,Step 'step_000' (click 'Open') requires verified identity under the governed run policy and its target identity could not be read from the live screen (context band OCR found no usable text) — needs h +font/trial-1/item,font,HALTED_BEFORE_EFFECT,step_002,True,reversible,postcondition:region_only,agent:gave_up,,"Postconditions failed for step 'step_002' (click 'Triage'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 163, 177, 102) — run aborted" +font/trial-2/item,font,HALTED_BEFORE_EFFECT,step_002,True,reversible,postcondition:region_only,agent:gave_up,,"Postconditions failed for step 'step_002' (click 'Triage'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 163, 177, 102) — run aborted" +font/trial-3/item,font,HALTED_BEFORE_EFFECT,step_002,True,reversible,postcondition:region_only,agent:gave_up,,"Postconditions failed for step 'step_002' (click 'Triage'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 163, 177, 102) — run aborted" +menu/trial-1/item,menu,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,patch:proposed_control_not_a_save,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +menu/trial-2/item,menu,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,patch:proposed_control_not_a_save,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +menu/trial-3/item,menu,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,patch:proposed_control_not_a_save,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +missing/trial-1/item,missing,REJECTED_POLICY,step_000,False,reversible,identity:mismatch,person:rejected_policy,,"Identity check failed for step 'step_000' (click 'Open'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not match the recorded target's — expected '<" +missing/trial-2/item,missing,REJECTED_POLICY,step_000,False,reversible,identity:mismatch,person:rejected_policy,,"Identity check failed for step 'step_000' (click 'Open'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not match the recorded target's — expected '<" +missing/trial-3/item,missing,REJECTED_POLICY,step_000,False,reversible,identity:mismatch,person:rejected_policy,,"Identity check failed for step 'step_000' (click 'Open'): a target was found positionally (rung 'template', confidence 1.00) but its surrounding text does not match the recorded target's — expected '<" +notice/trial-1/item,notice,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +notice/trial-2/item,notice,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +notice/trial-3/item,notice,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +provider/trial-1/item,provider,HALTED_BEFORE_EFFECT,step_001,True,reversible,postcondition:text_and_region,agent:gave_up,,"Postconditions failed for step 'step_001' (click 'New Encounter'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 101, 696, 350); text_present 'EncounterType' — " +provider/trial-2/item,provider,HALTED_BEFORE_EFFECT,step_001,True,reversible,postcondition:text_and_region,agent:gave_up,,"Postconditions failed for step 'step_001' (click 'New Encounter'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 101, 696, 350); text_present 'EncounterType' — " +provider/trial-3/item,provider,HALTED_BEFORE_EFFECT,step_001,True,reversible,postcondition:text_and_region,agent:gave_up,,"Postconditions failed for step 'step_001' (click 'New Encounter'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 101, 696, 350); text_present 'EncounterType' — " +redesign/trial-1/item,redesign,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,none:resumed,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +redesign/trial-2/item,redesign,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,none:resumed,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +redesign/trial-3/item,redesign,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,none:resumed,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +relink/trial-1/item,relink,HALTED_BEFORE_EFFECT,step_001,False,reversible,target:moved,none:resumed,,Structural safety refusal for step 'step_001' (click 'New Encounter'): visual point is not an identity-bearing actionable DOM element — no action was admitted +relink/trial-2/item,relink,HALTED_BEFORE_EFFECT,step_001,False,reversible,target:moved,none:resumed,,Structural safety refusal for step 'step_001' (click 'New Encounter'): visual point is not an identity-bearing actionable DOM element — no action was admitted +relink/trial-3/item,relink,HALTED_BEFORE_EFFECT,step_001,False,reversible,target:moved,none:resumed,,Structural safety refusal for step 'step_001' (click 'New Encounter'): visual point is not an identity-bearing actionable DOM element — no action was admitted +zoom/trial-1/item,zoom,HALTED_BEFORE_EFFECT,step_000,True,reversible,postcondition:region_only,agent:gave_up,,"Postconditions failed for step 'step_000' (click 'Open'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 43, 968, 318) — run aborted" +zoom/trial-2/item,zoom,HALTED_BEFORE_EFFECT,step_000,True,reversible,postcondition:region_only,agent:gave_up,,"Postconditions failed for step 'step_000' (click 'Open'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 43, 968, 318) — run aborted" +zoom/trial-3/item,zoom,HALTED_BEFORE_EFFECT,step_000,True,reversible,postcondition:region_only,agent:gave_up,,"Postconditions failed for step 'step_000' (click 'Open'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 43, 968, 318) — run aborted" diff --git a/packages/flow/benchmark/auto_repair/results/causes/mockmed-worklist-2026-10-19-a.csv b/packages/flow/benchmark/auto_repair/results/causes/mockmed-worklist-2026-10-19-a.csv new file mode 100644 index 000000000..63d3006a4 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/causes/mockmed-worklist-2026-10-19-a.csv @@ -0,0 +1,13 @@ +run,label,outcome,halted_step,delivery_attempted,risk,stop,blocker,unsettled,error +redesign/queue/items/item-1/run-1,redesign,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,not_run,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +redesign/queue/items/item-2/run-1,redesign,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,not_run,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +redesign/queue/items/item-3/run-1,redesign,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,not_run,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +redesign/queue/items/item-4/run-1,redesign,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,not_run,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +redesign/queue/items/item-5/run-1,redesign,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,not_run,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +redesign/queue/repair-1,redesign,HALTED_BEFORE_EFFECT,step_005,False,irreversible,target:geometry_only,none:resumed,,Step 'step_005' (click 'Save Encounter') is irreversible but only resolved via the 'geometry' rung — needs human confirmation; refusing to act (v0 policy) +relink/queue/items/item-1/run-1,relink,HALTED_BEFORE_EFFECT,step_001,False,reversible,target:moved,not_run,,Structural safety refusal for step 'step_001' (click 'New Encounter'): visual point is not an identity-bearing actionable DOM element — no action was admitted +relink/queue/items/item-2/run-1,relink,HALTED_BEFORE_EFFECT,step_001,False,reversible,target:moved,not_run,,Structural safety refusal for step 'step_001' (click 'New Encounter'): visual point is not an identity-bearing actionable DOM element — no action was admitted +relink/queue/items/item-3/run-1,relink,HALTED_BEFORE_EFFECT,step_001,False,reversible,target:moved,not_run,,Structural safety refusal for step 'step_001' (click 'New Encounter'): visual point is not an identity-bearing actionable DOM element — no action was admitted +relink/queue/items/item-4/run-1,relink,HALTED_BEFORE_EFFECT,step_001,False,reversible,target:moved,not_run,,Structural safety refusal for step 'step_001' (click 'New Encounter'): visual point is not an identity-bearing actionable DOM element — no action was admitted +relink/queue/items/item-5/run-1,relink,HALTED_BEFORE_EFFECT,step_001,False,reversible,target:moved,not_run,,Structural safety refusal for step 'step_001' (click 'New Encounter'): visual point is not an identity-bearing actionable DOM element — no action was admitted +relink/queue/repair-1,relink,HALTED_BEFORE_EFFECT,step_001,False,reversible,target:moved,none:resumed,,Structural safety refusal for step 'step_001' (click 'New Encounter'): visual point is not an identity-bearing actionable DOM element — no action was admitted diff --git a/packages/flow/benchmark/auto_repair/results/causes/openemr-1347.csv b/packages/flow/benchmark/auto_repair/results/causes/openemr-1347.csv new file mode 100644 index 000000000..0ec3e9ca0 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/causes/openemr-1347.csv @@ -0,0 +1,9 @@ +run,label,outcome,halted_step,delivery_attempted,risk,stop,blocker,unsettled,error +fast1/pool/items/00026,fast1,RECONCILIATION_REQUIRED,step_039,True,reversible,postcondition:text,reconciliation:effect_not_settled,step_002 step_004 step_007 step_010 step_014 step_018 step_021 step_025 step_027 step_030 step_033 step_037,Postconditions failed for step 'step_039' (click 'Create New Patient'): expected screen state not reached (semantic drift) — failed: text_present 'No matches were found.' — run aborted +fast2/pool/items/00016,fast2,RECONCILIATION_REQUIRED,step_040,False,irreversible,identity:mismatch,reconciliation:effect_not_settled,step_002 step_004 step_007 step_010 step_014 step_018 step_021 step_025 step_027 step_030 step_033 step_037,"Identity check failed for step 'step_040' (click 'Confirm Create New Patient'): a target was found positionally (rung 'structural', confidence 1.00) but its surrounding text does not match the recorde" +par8b/pool/items/00002,par8b,RECONCILIATION_REQUIRED,step_040,True,irreversible,postcondition:text_and_region,reconciliation:effect_not_settled,step_002 step_004 step_007 step_010 step_014 step_018 step_021 step_025 step_027 step_030 step_033 step_037 step_040,"Postconditions failed for step 'step_040' (click 'Confirm Create New Patient'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 308, 1280, 451); text_present 'Tre" +par8b/pool/items/00007,par8b,RECONCILIATION_REQUIRED,step_040,False,irreversible,identity:mismatch,reconciliation:effect_not_settled,step_002 step_004 step_007 step_010 step_014 step_018 step_021 step_025 step_027 step_030 step_033 step_037,"Identity check failed for step 'step_040' (click 'Confirm Create New Patient'): a target was found positionally (rung 'structural', confidence 1.00) but its surrounding text does not match the recorde" +par8w1b/pool/items/00009,par8w1b,RECONCILIATION_REQUIRED,step_040,True,irreversible,postcondition:text_and_region,reconciliation:effect_not_settled,step_002 step_004 step_007 step_010 step_014 step_018 step_021 step_025 step_027 step_030 step_033 step_037 step_040,"Postconditions failed for step 'step_040' (click 'Confirm Create New Patient'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 308, 1280, 451); text_present 'Tre" +par8w1b/pool/items/00020,par8w1b,RECONCILIATION_REQUIRED,step_040,True,irreversible,postcondition:text_and_region,reconciliation:effect_not_settled,step_002 step_004 step_007 step_010 step_014 step_018 step_021 step_025 step_027 step_030 step_033 step_037 step_040,"Postconditions failed for step 'step_040' (click 'Confirm Create New Patient'): expected screen state not reached (semantic drift) — failed: region_stable region=(0, 308, 1280, 451); text_present 'Tre" +std1/pool/items/00026,std1,RECONCILIATION_REQUIRED,step_039,True,reversible,postcondition:text,reconciliation:effect_not_settled,step_002 step_004 step_007 step_010 step_014 step_018 step_021 step_025 step_027 step_030 step_033 step_037,Postconditions failed for step 'step_039' (click 'Create New Patient'): expected screen state not reached (semantic drift) — failed: text_present 'No matches were found.' — run aborted +std2/pool/items/00008,std2,RECONCILIATION_REQUIRED,step_040,False,irreversible,identity:mismatch,reconciliation:effect_not_settled,step_002 step_004 step_007 step_010 step_014 step_018 step_021 step_025 step_027 step_030 step_033 step_037,"Identity check failed for step 'step_040' (click 'Confirm Create New Patient'): a target was found positionally (rung 'structural', confidence 1.00) but its surrounding text does not match the recorde" diff --git a/packages/flow/benchmark/auto_repair/results/causes/openimis-1331-before.csv b/packages/flow/benchmark/auto_repair/results/causes/openimis-1331-before.csv new file mode 100644 index 000000000..9a704d075 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/causes/openimis-1331-before.csv @@ -0,0 +1,4 @@ +run,label,outcome,halted_step,delivery_attempted,risk,stop,blocker,unsettled,error +item-050-1/run,item-050-1,RECONCILIATION_REQUIRED,step_014,True,reversible,target:scroll_budget,reconciliation:effect_not_settled,step_002 step_012,"Step 'step_014' (scroll by (0, -600)): closed-loop scroll exhausted its budget (1200px of 1500px allowed, 2.5x the recorded distance) without the anchor of step 'step_015' (click recorded visual targe" +item-050-2/run,item-050-2,RECONCILIATION_REQUIRED,step_014,True,reversible,target:scroll_budget,reconciliation:effect_not_settled,step_002 step_012,"Step 'step_014' (scroll by (0, -600)): closed-loop scroll exhausted its budget (1200px of 1500px allowed, 2.5x the recorded distance) without the anchor of step 'step_015' (click recorded visual targe" +item-050-3/run,item-050-3,RECONCILIATION_REQUIRED,step_014,True,reversible,target:scroll_budget,reconciliation:effect_not_settled,step_002 step_012,"Step 'step_014' (scroll by (0, -600)): closed-loop scroll exhausted its budget (1200px of 1500px allowed, 2.5x the recorded distance) without the anchor of step 'step_015' (click recorded visual targe" diff --git a/packages/flow/benchmark/auto_repair/results/causes/openimis-reviewed-2026-10-19-b.csv b/packages/flow/benchmark/auto_repair/results/causes/openimis-reviewed-2026-10-19-b.csv new file mode 100644 index 000000000..922f512c5 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/causes/openimis-reviewed-2026-10-19-b.csv @@ -0,0 +1,4 @@ +run,label,outcome,halted_step,delivery_attempted,risk,stop,blocker,unsettled,error +item-262/run,item-262,HALTED_BEFORE_EFFECT,step_015,False,irreversible,target:not_found,agent:gave_up,,Could not resolve target for step 'step_015' (click recorded visual target): the preceding scroll 'step_014' used 1200px of its 1500px budget and the target never came into view; no deterministic rung +item-263/run,item-263,HALTED_BEFORE_EFFECT,step_015,False,irreversible,target:not_found,probe:stopped_before_save,,Could not resolve target for step 'step_015' (click recorded visual target): the preceding scroll 'step_014' used 1200px of its 1500px budget and the target never came into view; no deterministic rung +item-264/run,item-264,HALTED_BEFORE_EFFECT,step_015,False,irreversible,target:not_found,probe:stopped_before_save,,Could not resolve target for step 'step_015' (click recorded visual target): the preceding scroll 'step_014' used 1200px of its 1500px budget and the target never came into view; no deterministic rung diff --git a/packages/flow/benchmark/auto_repair/results/causes/openimis-unreviewed-2026-10-19-b.csv b/packages/flow/benchmark/auto_repair/results/causes/openimis-unreviewed-2026-10-19-b.csv new file mode 100644 index 000000000..b9b8817e1 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/causes/openimis-unreviewed-2026-10-19-b.csv @@ -0,0 +1,3 @@ +run,label,outcome,halted_step,delivery_attempted,risk,stop,blocker,unsettled,error +item-270/run,item-270,RECONCILIATION_REQUIRED,step_015,False,irreversible,target:not_found,reconciliation:effect_not_settled,step_002 step_012,Could not resolve target for step 'step_015' (click recorded visual target): the preceding scroll 'step_014' used 1200px of its 1500px budget and the target never came into view; no deterministic rung +item-271/run,item-271,RECONCILIATION_REQUIRED,step_015,False,irreversible,target:not_found,reconciliation:effect_not_settled,step_002 step_012,Could not resolve target for step 'step_015' (click recorded visual target): the preceding scroll 'step_014' used 1200px of its 1500px budget and the target never came into view; no deterministic rung diff --git a/packages/flow/benchmark/auto_repair/results/causes/silent-saves-2026-10-19-a.csv b/packages/flow/benchmark/auto_repair/results/causes/silent-saves-2026-10-19-a.csv new file mode 100644 index 000000000..5e9164723 --- /dev/null +++ b/packages/flow/benchmark/auto_repair/results/causes/silent-saves-2026-10-19-a.csv @@ -0,0 +1,14 @@ +run,label,outcome,halted_step,delivery_attempted,risk,stop,blocker,unsettled,error +item-030/run,item-030,RECONCILIATION_REQUIRED,step_005,True,irreversible,other,reconciliation:effect_not_settled,step_005,System-of-record effect verification HALTED step 'step_005' (click 'Save Encounter'): record_written refuted against the rest system of record and could not be reconciled (escalated) — [rest] record_w +item-046/run,item-046,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +item-047/run,item-047,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +item-048/run,item-048,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +item-049/run,item-049,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +item-050/run,item-050,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +item-051/run,item-051,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +item-052/run,item-052,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +item-053/run,item-053,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +item-054/run,item-054,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +item-055/run,item-055,HALTED_BEFORE_EFFECT,step_000,False,reversible,target:not_found,patch:proposed_control_write_shaped,,Could not resolve target for step 'step_000' (click 'Open'): all resolution rungs failed +item-076/run,item-076,HALTED_BEFORE_EFFECT,step_001,False,reversible,target:moved,none:resumed,,Structural safety refusal for step 'step_001' (click 'New Encounter'): visual point is not an identity-bearing actionable DOM element — no action was admitted +item-080/run,item-080,RECONCILIATION_REQUIRED,step_005,True,irreversible,other,reconciliation:effect_not_settled,step_005,System-of-record effect verification HALTED step 'step_005' (click 'Save Encounter'): record_written refuted against the rest system of record and could not be reconciled (escalated) — [rest] record_w diff --git a/packages/flow/benchmark/auto_repair/stop_causes.py b/packages/flow/benchmark/auto_repair/stop_causes.py new file mode 100644 index 000000000..216af23aa --- /dev/null +++ b/packages/flow/benchmark/auto_repair/stop_causes.py @@ -0,0 +1,191 @@ +"""Read every recorded stop and name why it stopped and why repair did not fix it. + + python benchmark/auto_repair/stop_causes.py --runs-root ROOT --out results/causes/.csv \\ + [--bundle BUNDLE] [--label-depth N] + +For every halted ``report.json`` under ``ROOT`` (the item's own run, not a +self-repair attempt or probe inside it), this writes one row with: + +- ``stop``: what the halted step's error says, by rule (:func:`stop_kind`); +- ``blocker``: what kept repair from resuming the item, from the stop's + transaction outcome and, when self-repair ran, its own evidence + (``self-repair/self-repair.json``) (:func:`repair_blocker`); +- ``unsettled``: the steps whose effect the transaction classifier could not + settle (a write may exist), when the run ended ``RECONCILIATION_REQUIRED``. + +The root cause of each group of stops, which sometimes needs the +benchmark's own knowledge of what it changed, is in ``STOP_CAUSES.md``. +No record values are copied: rows carry step ids, rule names and the first +200 characters of the engine's own messages. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +from pathlib import Path +from typing import Any, Optional + +HERE = Path(__file__).resolve().parent + +#: Directories that hold a repair's own runs, not the item's. +_INNER = {"self-repair", "probe", "halt", "resume", "campaigns"} + + +def stop_kind(error: str) -> str: + """What kind of stop the halted step's error describes.""" + + if error.startswith("Postconditions failed"): + failed = error.split("failed:", 1)[-1] + kinds = set( + re.findall( + r"\b(region_stable|text_present|text_absent|" + r"local_edit|field_value|url_changed|" + r"title_changed|new_tab_opened)\b", + failed, + ) + ) + if kinds == {"region_stable"}: + return "postcondition:region_only" + if "text_present" in kinds and "region_stable" in kinds: + return "postcondition:text_and_region" + if "text_present" in kinds: + return "postcondition:text" + return "postcondition:other" + if error.startswith("Identity check failed"): + return "identity:mismatch" + if "identity could not be read" in error: + return "identity:unreadable" + if "DOM locator is ambiguous" in error: + return "target:ambiguous" + if "not an identity-bearing actionable DOM element" in error: + return "target:moved" + if "only resolved via the 'geometry' rung" in error: + return "target:geometry_only" + if ( + error.startswith("Could not resolve target") + or "all resolution rungs failed" in error + ): + return "target:not_found" + if "closed-loop scroll exhausted" in error: + return "target:scroll_budget" + return "other" + + +def repair_blocker(outcome: Optional[str], repair: Optional[dict[str, Any]]) -> str: + """What kept the stop from being resumed by repair.""" + + if repair is not None and repair.get("status") == "resumed": + return "none:resumed" + if outcome == "RECONCILIATION_REQUIRED": + return "reconciliation:effect_not_settled" + if outcome == "REJECTED_POLICY": + return "person:rejected_policy" + if repair is None: + return "not_run" + steps = repair.get("steps") or [] + details = [str(s.get("detail") or "") for s in steps if not s.get("passed")] + text = " | ".join(details) + if "stopped before resolving its target for another reason" in text: + return "person:not_a_missing_target" + if "does not read as a save" in text: + return "patch:proposed_control_not_a_save" + if "reads as a save" in text: + return "patch:proposed_control_write_shaped" + if "the probe stopped" in text: + return "probe:stopped_before_save" + if "the agent gave up" in text: + return "agent:gave_up" + if text: + return "other:" + details[-1][:60] + return "other" + + +def _halted(report: dict[str, Any]) -> Optional[dict[str, Any]]: + for result in reversed(report.get("results") or []): + if not result.get("ok") and not result.get("skipped"): + return result + return None + + +def _unsettled(path: Path) -> list[str]: + try: + from openadapt_flow.ir import RunReport + from openadapt_flow.transaction import steps_requiring_reconciliation + + report = RunReport.model_validate_json(path.read_text()) + except Exception: # noqa: BLE001 - an old or partial report: unknown + return [] + return steps_requiring_reconciliation(report) + + +def rows_for(root: Path, label_depth: int) -> list[dict[str, Any]]: + rows = [] + for path in sorted(root.rglob("report.json")): + rel = path.parent.relative_to(root) + if _INNER & set(rel.parts): + continue + report = json.loads(path.read_text()) + halted = _halted(report) + if halted is None or report.get("success"): + continue + repair_path = path.parent / "self-repair" / "self-repair.json" + repair = json.loads(repair_path.read_text()) if repair_path.is_file() else None + outcome = report.get("transaction_outcome") + error = str(halted.get("error") or "") + rows.append( + { + "run": str(rel), + "label": "/".join(rel.parts[:label_depth]) if label_depth else "", + "outcome": outcome, + "halted_step": halted.get("step_id"), + "delivery_attempted": halted.get("delivery_attempted"), + "risk": halted.get("risk"), + "stop": stop_kind(error), + "blocker": repair_blocker(outcome, repair), + "unsettled": " ".join(_unsettled(path)) + if outcome == "RECONCILIATION_REQUIRED" + else "", + "error": error[:200], + } + ) + return rows + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--runs-root", required=True, action="append") + parser.add_argument("--out", required=True) + parser.add_argument( + "--label-depth", + type=int, + default=1, + help="how many leading path parts name the stop's group (mode, block)", + ) + args = parser.parse_args() + rows: list[dict[str, Any]] = [] + for root in args.runs_root: + rows.extend(rows_for(Path(root).expanduser(), args.label_depth)) + out = Path(args.out) + if not out.is_absolute(): + out = HERE / out + out.parent.mkdir(parents=True, exist_ok=True) + fields = list(rows[0]) if rows else ["run"] + with out.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) + groups: dict[tuple[str, str, str], int] = {} + for row in rows: + key = (row["label"], row["stop"], row["blocker"]) + groups[key] = groups.get(key, 0) + 1 + for (label, stop, blocker), count in sorted(groups.items()): + print(f"{count:3d} {label:28s} {stop:30s} {blocker}") + print(f"{len(rows)} stops -> {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())