From f5173cddbb4a667184ad53213102b8ac1864f085 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sat, 17 Oct 2026 01:19:06 -0400 Subject: [PATCH 01/12] feat(compiler): induce shadow-mined episodes into a candidate draft Add compiler/motif_induction.py. It validates an openadapt.shadow.episodes/v1 export from openadapt-capture's shadow miner (strict schema: digests only, no unknown fields), maps each episode to a linear Workflow, and runs the existing induce_program over them. The result is a MotifCandidate whose status is always candidate and whose admitted and runnable fields are pinned to False. Anchors are placeholders, so record-next always starts with recording the task once. Write-shaped clicks are irreversible with risk_review_required. Checks the episodes suggest (file written, record written, row added, page reached) are proposals and never become effects. write_candidate writes candidate.json and draft-workflow.json, never workflow.json. No model calls and no network access; tests cover both. Signed-off-by: Richard Abrich --- .../compiler/motif_induction.py | 468 ++++++++++ .../shadow/invoice_exact.episodes.json | 790 +++++++++++++++++ .../shadow/invoice_variant.episodes.json | 806 ++++++++++++++++++ .../fixtures/shadow/report_mail.episodes.json | 547 ++++++++++++ packages/flow/tests/test_motif_induction.py | 248 ++++++ 5 files changed, 2859 insertions(+) create mode 100644 packages/flow/openadapt_flow/compiler/motif_induction.py create mode 100644 packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json create mode 100644 packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json create mode 100644 packages/flow/tests/fixtures/shadow/report_mail.episodes.json create mode 100644 packages/flow/tests/test_motif_induction.py diff --git a/packages/flow/openadapt_flow/compiler/motif_induction.py b/packages/flow/openadapt_flow/compiler/motif_induction.py new file mode 100644 index 000000000..b1af7d5ff --- /dev/null +++ b/packages/flow/openadapt_flow/compiler/motif_induction.py @@ -0,0 +1,468 @@ +"""Turn shadow-mined episodes into a CANDIDATE program, never a runnable one. + +OpenAdapt Capture's shadow mode (``openadapt_capture.shadow``, off by default) +can notice that a person repeated the same cross-application sequence several +times and, if the person clicks **Review draft**, export those episodes as +``openadapt.shadow.episodes/v1`` JSON. Each step in that export is a +value-free symbol: an application, a control role, keyed digests in place of +labels and typed values, a coarse URL pattern. There are no screenshots, no +coordinates, and no plain-text values. + +This module converts the episodes into linear :class:`~openadapt_flow.ir.Workflow` +traces and hands them to the EXISTING multi-trace induction +(:func:`openadapt_flow.compiler.induction.induce_program`), which infers +parameters, loops, and branches or refuses. The result is a +:class:`MotifCandidate`: + +* ``status`` is always ``"candidate"``, ``admitted`` and ``runnable`` are + always False. Nothing here can run a step, and nothing here writes a + ``workflow.json`` bundle. +* The draft's anchors are placeholders (shadow mining keeps no pixels), so + the record-next worklist always starts with "record this task once" through + the normal record, compile, induce, and qualify path. That path is where a + person admits the program. +* Write-shaped steps (a Submit-like control, a click right before a form + submit, a save chord) are marked irreversible and ``risk_review_required``, + so certification refuses them until a person reviews them. +* A verification check is PROPOSED where the episodes show one (a file + saved, a form submitted, a row pasted into a spreadsheet). A proposal is + never an effect contract: the episodes that suggested it can't approve it. + +Deterministic and model-free; no network access. +""" + +from __future__ import annotations + +import json +import re +from collections import Counter +from pathlib import Path +from typing import Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from openadapt_flow.compiler.induction import RecordNext, TraceInput, induce_program +from openadapt_flow.ir import ActionKind, Anchor, Step, StructuralLocator, Workflow + +EPISODES_SCHEMA = "openadapt.shadow.episodes/v1" +CANDIDATE_SCHEMA: Literal["openadapt.shadow.candidate/v1"] = ( + "openadapt.shadow.candidate/v1" +) +PLACEHOLDER_TEMPLATE = "shadow-unrecorded.png" +DRAFT_FILENAME = "draft-workflow.json" +CANDIDATE_FILENAME = "candidate.json" + +_DIGEST = r"^[0-9a-f]{16}$" +_WRITE_STEMS = ("save", "submit", "send", "post", "approve", "confirm", "create", "add") +_SHEET_APPS = ("excel", "numbers", "sheets", "calc", "spreadsheet") +_SAVE_MODIFIERS = frozenset({"meta", "ctrl", "cmd", "command"}) + +ADMISSION_PATH: tuple[str, ...] = ( + "Record the task with OpenAdapt Capture (flow record or capture record).", + "Compile each recording (flow compile).", + "Induce over the recordings (flow induce), using this draft only as a guide.", + "Confirm or reject each proposed check, then qualify (flow qualify).", +) + + +class ShadowStep(BaseModel): + """One value-free symbol from the Capture export.""" + + model_config = ConfigDict(extra="forbid") + + t: float + token: str = Field(max_length=256) + kind: Literal[ + "focus", + "click", + "double_click", + "right_click", + "type", + "key", + "shortcut", + "copy", + "paste", + "navigate", + "submit", + ] + app: Optional[str] = Field(default=None, max_length=128) + window_class: Optional[str] = Field(default=None, max_length=128) + role: Optional[str] = Field(default=None, max_length=64) + label_digest: Optional[str] = Field(default=None, pattern=_DIGEST) + display_label: Optional[str] = Field( + default=None, max_length=32, pattern=r"^[^0-9@]*$" + ) + value_digest: Optional[str] = Field(default=None, pattern=_DIGEST) + value_shape: Optional[ + Literal["empty", "digits", "decimal", "date", "email", "alpha", "alnum", "text"] + ] = None + url_pattern: Optional[str] = Field(default=None, max_length=256) + key: Optional[str] = Field(default=None, max_length=32) + modifiers: list[str] = Field(default_factory=list, max_length=4) + secure: bool = False + + +class ShadowEpisode(BaseModel): + model_config = ConfigDict(extra="forbid") + + t_start: float + t_end: float + exact: bool + similarity: float = Field(ge=0.0, le=1.0) + steps: list[ShadowStep] = Field(min_length=1) + + +class ShadowEpisodesExport(BaseModel): + """The ``openadapt.shadow.episodes/v1`` document Capture writes.""" + + model_config = ConfigDict(extra="forbid") + + schema_: Literal["openadapt.shadow.episodes/v1"] = Field(alias="schema") + motif_id: str = Field(pattern=_DIGEST) + tokens: list[str] + apps: list[str] + count_in_window: int = Field(ge=0) + window_s: float + exact_count: int = Field(ge=0) + expected_by_chance: float + p_value: float + status: Literal["candidate"] + privacy: dict[str, str] + episodes: list[ShadowEpisode] = Field(min_length=1) + + +class ProposedCheck(BaseModel): + """A verification check the episodes suggest. A proposal, nothing more.""" + + kind: Literal["file_written", "record_written", "row_added", "page_reached"] + summary: str + step_id: Optional[str] = None + support: int = Field(description="Episodes in which the signal appeared.") + status: Literal["proposed"] = "proposed" + admitted: Literal[False] = False + note: str = ( + "Proposal only. The episodes that suggested this check can't approve " + "it; a person confirms or rejects it during qualification." + ) + + +class MotifCandidate(BaseModel): + """What a person reviews after clicking **Review draft**.""" + + schema_version: Literal["openadapt.shadow.candidate/v1"] = CANDIDATE_SCHEMA + motif_id: str + status: Literal["candidate"] = "candidate" + admitted: Literal[False] = False + runnable: Literal[False] = False + apps: list[str] + n_episodes: int + count_in_window: int + induction_certified: bool = Field( + description=( + "Whether induction found the episodes structurally consistent. " + "Structural only: it does not admit, qualify, or certify a run." + ) + ) + induction_report: str + draft: Optional[Workflow] = None + record_next: list[RecordNext] = Field(default_factory=list) + proposed_checks: list[ProposedCheck] = Field(default_factory=list) + admission_path: list[str] = Field(default_factory=lambda: list(ADMISSION_PATH)) + + +def _alias(digest: Optional[str]) -> str: + """Letters-only alias of a digest (induction drops digits from intents).""" + + if not digest: + return "unnamed" + return "fld-" + "".join(chr(ord("a") + int(c, 16)) for c in digest[:8]) + + +def _placeholder_anchor(role: Optional[str], name: str, *, varies: bool) -> Anchor: + """An anchor with no pixels. Shadow mining keeps no screenshots. + + A control whose identity varies per run (a list row, a table cell) gets a + structural role, so induction reads it as a which-entity selection. A + fixed control (a named button or field) is identified by its label, so + two different buttons in a row never collapse into one "button" column. + """ + + if varies: + return Anchor( + template=PLACEHOLDER_TEMPLATE, + region=(0, 0, 1, 1), + click_point=(0, 0), + structural=StructuralLocator(role=role or None, name=name), + ) + return Anchor( + template=PLACEHOLDER_TEMPLATE, + region=(0, 0, 1, 1), + click_point=(0, 0), + ocr_text=name, + ) + + +def _is_write_label(label: Optional[str]) -> bool: + if not label: + return False + low = label.lower() + return any(re.search(rf"\b{stem}\b", low) for stem in _WRITE_STEMS) + + +def _is_sheet(app: Optional[str]) -> bool: + low = (app or "").lower() + return any(name in low for name in _SHEET_APPS) + + +def _episode_steps( + episode: ShadowEpisode, +) -> tuple[list[Step], list[tuple[str, Optional[str], str]]]: + """Map one episode to IR steps plus the check signals it shows. + + A signal is ``(kind, step_id, summary)``. + """ + + steps: list[Step] = [] + signals: list[tuple[str, Optional[str], str]] = [] + symbols = episode.steps + for index, sym in enumerate(symbols): + sid = f"s{index:02d}" + app = sym.app or "unknown" + prefix = f"[{app}]" + nxt = symbols[index + 1] if index + 1 < len(symbols) else None + if sym.kind in ("click", "double_click", "right_click"): + action = { + "click": ActionKind.CLICK, + "double_click": ActionKind.DOUBLE_CLICK, + "right_click": ActionKind.RIGHT_CLICK, + }[sym.kind] + if sym.value_digest: + name = f"value:{sym.value_digest}" + what = f"{sym.role or 'control'} (varies)" + else: + name = sym.display_label or _alias(sym.label_digest) + what = f"{sym.role or 'control'} {name}" + writes = _is_write_label(sym.display_label) or ( + nxt is not None and nxt.kind == "submit" + ) + steps.append( + Step( + id=sid, + intent=f"{prefix} click {what}", + action=action, + anchor=_placeholder_anchor( + sym.role, name, varies=bool(sym.value_digest) + ), + risk="irreversible" if writes else "reversible", + risk_explanation=( + "Shadow-mined write-shaped control; a person must review it." + if writes + else None + ), + risk_review_required=writes, + ) + ) + if writes: + signals.append( + ( + "record_written", + sid, + f"A record is created or changed after '{what}' in {app}.", + ) + ) + elif sym.kind == "type": + field = sym.display_label or _alias(sym.label_digest) + intent = f"{prefix} type {sym.role or 'field'} {field}" + if sym.secure: + steps.append( + Step( + id=sid, + intent=intent, + action=ActionKind.TYPE, + secret=True, + param=f"secret_{_alias(sym.label_digest)[4:]}", + ) + ) + else: + shape = sym.value_shape or "text" + steps.append( + Step( + id=sid, + intent=intent, + action=ActionKind.TYPE, + text=f"<{shape}:{sym.value_digest or 'none'}>", + ) + ) + if nxt is not None and nxt.kind == "key" and _is_sheet(app): + signals.append(("row_added", sid, f"The sheet in {app} gains a row.")) + elif sym.kind in ("copy", "paste", "shortcut"): + mods = [m for m in sym.modifiers if m] + key = sym.key or ("c" if sym.kind == "copy" else "v") + label = {"copy": "copy", "paste": "paste"}.get(sym.kind, "press shortcut") + if mods: + steps.append( + Step( + id=sid, + intent=f"{prefix} {label} {'+'.join(mods)}+{key}", + action=ActionKind.HOTKEY, + key=key, + modifiers=mods, + ) + ) + else: + steps.append( + Step( + id=sid, + intent=f"{prefix} press {key}", + action=ActionKind.KEY, + key=key, + ) + ) + if sym.kind == "shortcut" and key == "s" and set(mods) & _SAVE_MODIFIERS: + signals.append( + ("file_written", sid, f"A file saved from {app} changes.") + ) + if ( + sym.kind == "paste" + and _is_sheet(app) + and nxt is not None + and (nxt.kind == "key") + ): + signals.append(("row_added", sid, f"The sheet in {app} gains a row.")) + elif sym.kind == "key": + steps.append( + Step( + id=sid, + intent=f"{prefix} press {sym.key or 'key'}", + action=ActionKind.KEY, + key=sym.key or "other", + ) + ) + elif sym.kind == "submit": + target = sym.url_pattern or app + signals.append( + ("page_reached", None, f"After submitting, the page matches {target}.") + ) + # focus and navigate carry context only; they are not IR actions. + return steps, signals + + +def episodes_to_workflows( + export: ShadowEpisodesExport, +) -> tuple[list[Workflow], list[ProposedCheck]]: + """Convert episodes to linear traces, and collect proposed checks.""" + + workflows: list[Workflow] = [] + support: Counter[tuple[str, str]] = Counter() + where: dict[tuple[str, str], Counter[str]] = {} + for number, episode in enumerate(export.episodes): + steps, signals = _episode_steps(episode) + workflows.append( + Workflow(name=f"shadow-episode-{export.motif_id}-{number}", steps=steps) + ) + seen = {(kind, summary) for kind, _sid, summary in signals} + support.update(seen) + for kind, sid, summary in signals: + if sid is not None: + where.setdefault((kind, summary), Counter())[sid] += 1 + majority = (len(export.episodes) + 1) // 2 + checks: list[ProposedCheck] = [] + for (kind, summary), count in sorted(support.items()): + if count < majority: + continue + ids = where.get((kind, summary)) + step_id = ( + sorted(ids.items(), key=lambda kv: (-kv[1], kv[0]))[0][0] if ids else None + ) + checks.append( + ProposedCheck( + kind=kind, # type: ignore[arg-type] + step_id=step_id, + summary=summary, + support=count, + ) + ) + return workflows, checks + + +def induce_candidate(export: ShadowEpisodesExport | dict) -> MotifCandidate: + """Run the existing induction over shadow episodes; return a candidate.""" + + if not isinstance(export, ShadowEpisodesExport): + export = ShadowEpisodesExport.model_validate(export) + workflows, checks = episodes_to_workflows(export) + traces: list[TraceInput] = list(workflows) + result = induce_program(traces) + + record_next: list[RecordNext] = [ + RecordNext( + code="shadow_full_demonstration", + instruction=( + "Record this task once with OpenAdapt Capture. Shadow mining kept " + "no screenshots or values, so the draft has nothing to act on yet." + ), + ) + ] + record_next.extend(result.record_next) + for check in checks: + record_next.append( + RecordNext( + code="shadow_confirm_check", + instruction=f"Confirm or reject the proposed check: {check.summary}", + location=check.step_id or "", + ) + ) + for name, spec in sorted(result.param_specs.items()): + record_next.append( + RecordNext( + code="shadow_name_value", + instruction=( + f"Say what the changing value '{name}' is " + f"(the draft only knows it varies)." + ), + location=name, + blocking=False, + ) + ) + + draft = None + if result.workflow is not None: + draft = result.workflow.model_copy( + update={"name": f"shadow-draft-{export.motif_id}"} + ) + return MotifCandidate( + motif_id=export.motif_id, + apps=list(export.apps), + n_episodes=len(export.episodes), + count_in_window=export.count_in_window, + induction_certified=result.certified, + induction_report=result.render(), + draft=draft, + record_next=record_next, + proposed_checks=checks, + ) + + +def write_candidate(candidate: MotifCandidate, out_dir: Path) -> Path: + """Write ``candidate.json`` and, when there is a draft, ``draft-workflow.json``. + + Never writes ``workflow.json``, so no Flow command treats the directory + as a runnable bundle. Refuses a directory that already holds one. + """ + + out = Path(out_dir) + if (out / "workflow.json").exists() or (out / "workflow.json.enc").exists(): + raise FileExistsError( + f"{out} holds a workflow bundle; refusing to mix a candidate in" + ) + out.mkdir(parents=True, exist_ok=True) + meta = candidate.model_dump(mode="json", exclude={"draft"}) + meta["draft_file"] = DRAFT_FILENAME if candidate.draft is not None else None + (out / CANDIDATE_FILENAME).write_text( + json.dumps(meta, indent=2, sort_keys=True) + "\n" + ) + if candidate.draft is not None: + (out / DRAFT_FILENAME).write_text( + candidate.draft.model_dump_json(indent=2) + "\n" + ) + return out diff --git a/packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json b/packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json new file mode 100644 index 000000000..8363ccff5 --- /dev/null +++ b/packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json @@ -0,0 +1,790 @@ +{ + "apps": [ + "Google Chrome", + "Microsoft Excel" + ], + "count_in_window": 4, + "episodes": [ + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000265.4191387234, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000266.4191387234, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_digest": "c4c11450729082bb", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000267.4191387234, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000268.4191387234, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000269.4191387234, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "87961bb30e8c3a0d", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000270.4191387234, + "token": "click|Google Chrome|textfield|87961bb3", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000271.4191387234, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000272.4191387234, + "token": "click|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000273.4191387234, + "token": "type|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": "d2a9388191e23876", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_digest": "57d88c0af57c244e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000274.4191387234, + "token": "click|Google Chrome|button|57d88c0a", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000275.4191387234, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000275.4191387234, + "t_start": 1000265.4191387234 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000570.0973327656, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000571.0973327656, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_digest": "3994e33e2632d5cc", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000572.0973327656, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000573.0973327656, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000574.0973327656, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "87961bb30e8c3a0d", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000575.0973327656, + "token": "click|Google Chrome|textfield|87961bb3", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000576.0973327656, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000577.0973327656, + "token": "click|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000578.0973327656, + "token": "type|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": "431eb9c3f9fa4935", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_digest": "57d88c0af57c244e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000579.0973327656, + "token": "click|Google Chrome|button|57d88c0a", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000580.0973327656, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000580.0973327656, + "t_start": 1000570.0973327656 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000865.0038992346, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000866.0038992346, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_digest": "857eaa216100862b", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000867.0038992346, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000868.0038992346, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000869.0038992346, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "87961bb30e8c3a0d", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000870.0038992346, + "token": "click|Google Chrome|textfield|87961bb3", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000871.0038992346, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000872.0038992346, + "token": "click|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000873.0038992346, + "token": "type|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": "f5ab28d0e13d91b8", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_digest": "57d88c0af57c244e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000874.0038992346, + "token": "click|Google Chrome|button|57d88c0a", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000875.0038992346, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000875.0038992346, + "t_start": 1000865.0038992346 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001239.5060203954, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1001240.5060203954, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_digest": "11b37ab6c3be2895", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1001241.5060203954, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001242.5060203954, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001243.5060203954, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "87961bb30e8c3a0d", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001244.5060203954, + "token": "click|Google Chrome|textfield|87961bb3", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1001245.5060203954, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001246.5060203954, + "token": "click|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001247.5060203954, + "token": "type|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": "2336b2da290c600c", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_digest": "57d88c0af57c244e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1001248.5060203954, + "token": "click|Google Chrome|button|57d88c0a", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001249.5060203954, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1001249.5060203954, + "t_start": 1001239.5060203954 + } + ], + "exact_count": 4, + "expected_by_chance": 2.0763629850095314e-16, + "motif_id": "f2f51c3e6dd5a708", + "p_value": 7.744651342950623e-65, + "privacy": { + "clipboard_contents": "never read; copy and paste come from the key chord", + "control_labels": "keyed digest; short chrome labels in plain text when enabled", + "network": "none", + "screenshots": "never taken", + "typed_values": "keyed digest only; secure fields dropped", + "urls": "scheme, host, and at most one path word", + "window_titles": "never read" + }, + "schema": "openadapt.shadow.episodes/v1", + "status": "candidate", + "tokens": [ + "focus|Microsoft Excel", + "click|Microsoft Excel|cell|-", + "copy|Microsoft Excel", + "focus|Google Chrome", + "navigate|Google Chrome|https://erp.example.com/invoices/*", + "click|Google Chrome|textfield|87961bb3", + "paste|Google Chrome", + "click|Google Chrome|textfield|64e7c628", + "type|Google Chrome|textfield|64e7c628", + "click|Google Chrome|button|57d88c0a", + "submit|Google Chrome|https://erp.example.com/invoices/*" + ], + "window_s": 86400.0 +} diff --git a/packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json b/packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json new file mode 100644 index 000000000..50487a8f7 --- /dev/null +++ b/packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json @@ -0,0 +1,806 @@ +{ + "apps": [ + "Google Chrome", + "Microsoft Excel" + ], + "count_in_window": 4, + "episodes": [ + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000265.4191387234, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000266.4191387234, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_digest": "c4c11450729082bb", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000267.4191387234, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000268.4191387234, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000269.4191387234, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "87961bb30e8c3a0d", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000270.4191387234, + "token": "click|Google Chrome|textfield|87961bb3", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000271.4191387234, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000272.4191387234, + "token": "click|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000273.4191387234, + "token": "type|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": "d2a9388191e23876", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_digest": "57d88c0af57c244e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000274.4191387234, + "token": "click|Google Chrome|button|57d88c0a", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000275.4191387234, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000275.4191387234, + "t_start": 1000265.4191387234 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000570.0973327656, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000571.0973327656, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_digest": "3994e33e2632d5cc", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000572.0973327656, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000573.0973327656, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000574.0973327656, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "87961bb30e8c3a0d", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000575.0973327656, + "token": "click|Google Chrome|textfield|87961bb3", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000576.0973327656, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000577.0973327656, + "token": "click|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000578.0973327656, + "token": "type|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": "431eb9c3f9fa4935", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_digest": "57d88c0af57c244e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000579.0973327656, + "token": "click|Google Chrome|button|57d88c0a", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000580.0973327656, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000580.0973327656, + "t_start": 1000570.0973327656 + }, + { + "exact": false, + "similarity": 0.9565, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000865.0038992346, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000866.0038992346, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_digest": "857eaa216100862b", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000867.0038992346, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000868.0038992346, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000869.0038992346, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "87961bb30e8c3a0d", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000870.0038992346, + "token": "click|Google Chrome|textfield|87961bb3", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000871.0038992346, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000872.0038992346, + "token": "click|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000873.0038992346, + "token": "type|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": "f5ab28d0e13d91b8", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Urgent", + "key": null, + "kind": "click", + "label_digest": "762cf17e6634e30a", + "modifiers": [], + "role": "checkbox", + "secure": false, + "t": 1000873.8038992346, + "token": "click|Google Chrome|checkbox|762cf17e", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_digest": "57d88c0af57c244e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000874.0038992346, + "token": "click|Google Chrome|button|57d88c0a", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000875.0038992346, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000875.0038992346, + "t_start": 1000865.0038992346 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001239.5060203954, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1001240.5060203954, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_digest": "11b37ab6c3be2895", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1001241.5060203954, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001242.5060203954, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001243.5060203954, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "87961bb30e8c3a0d", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001244.5060203954, + "token": "click|Google Chrome|textfield|87961bb3", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1001245.5060203954, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001246.5060203954, + "token": "click|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "64e7c6284cf4020f", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001247.5060203954, + "token": "type|Google Chrome|textfield|64e7c628", + "url_pattern": null, + "value_digest": "2336b2da290c600c", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_digest": "57d88c0af57c244e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1001248.5060203954, + "token": "click|Google Chrome|button|57d88c0a", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001249.5060203954, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1001249.5060203954, + "t_start": 1001239.5060203954 + } + ], + "exact_count": 3, + "expected_by_chance": 1.9581310411777164e-16, + "motif_id": "f2f51c3e6dd5a708", + "p_value": 1.2513361926197074e-48, + "privacy": { + "clipboard_contents": "never read; copy and paste come from the key chord", + "control_labels": "keyed digest; short chrome labels in plain text when enabled", + "network": "none", + "screenshots": "never taken", + "typed_values": "keyed digest only; secure fields dropped", + "urls": "scheme, host, and at most one path word", + "window_titles": "never read" + }, + "schema": "openadapt.shadow.episodes/v1", + "status": "candidate", + "tokens": [ + "focus|Microsoft Excel", + "click|Microsoft Excel|cell|-", + "copy|Microsoft Excel", + "focus|Google Chrome", + "navigate|Google Chrome|https://erp.example.com/invoices/*", + "click|Google Chrome|textfield|87961bb3", + "paste|Google Chrome", + "click|Google Chrome|textfield|64e7c628", + "type|Google Chrome|textfield|64e7c628", + "click|Google Chrome|button|57d88c0a", + "submit|Google Chrome|https://erp.example.com/invoices/*" + ], + "window_s": 86400.0 +} diff --git a/packages/flow/tests/fixtures/shadow/report_mail.episodes.json b/packages/flow/tests/fixtures/shadow/report_mail.episodes.json new file mode 100644 index 000000000..a7c189a02 --- /dev/null +++ b/packages/flow/tests/fixtures/shadow/report_mail.episodes.json @@ -0,0 +1,547 @@ +{ + "apps": [ + "Mail", + "Microsoft Excel" + ], + "count_in_window": 3, + "episodes": [ + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000238.255011055, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "s", + "kind": "shortcut", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 5000239.255011055, + "token": "shortcut|Microsoft Excel|meta+s", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000240.255011055, + "token": "focus|Mail", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "New Message", + "key": null, + "kind": "click", + "label_digest": "af4c01884ec9c100", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000241.255011055, + "token": "click|Mail|button|af4c0188", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "1bbd9c9d99169893", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000242.255011055, + "token": "click|Mail|textfield|1bbd9c9d", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "1bbd9c9d99169893", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000243.255011055, + "token": "type|Mail|textfield|1bbd9c9d", + "url_pattern": null, + "value_digest": "10b3aa0a585b2a08", + "value_shape": "email", + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "2bdb797b79f1cd8e", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000244.255011055, + "token": "click|Mail|textfield|2bdb797b", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "2bdb797b79f1cd8e", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000245.255011055, + "token": "type|Mail|textfield|2bdb797b", + "url_pattern": null, + "value_digest": "6ad789b668757434", + "value_shape": "alnum", + "window_class": null + }, + { + "app": "Mail", + "display_label": "Attach", + "key": null, + "kind": "click", + "label_digest": "aed1127ebe1853e9", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000246.255011055, + "token": "click|Mail|button|aed1127e", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "Send", + "key": null, + "kind": "click", + "label_digest": "2dc861cd16dafe9c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000247.255011055, + "token": "click|Mail|button|2dc861cd", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 5000247.255011055, + "t_start": 5000238.255011055 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000558.013902665, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "s", + "kind": "shortcut", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 5000559.013902665, + "token": "shortcut|Microsoft Excel|meta+s", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000560.013902665, + "token": "focus|Mail", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "New Message", + "key": null, + "kind": "click", + "label_digest": "af4c01884ec9c100", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000561.013902665, + "token": "click|Mail|button|af4c0188", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "1bbd9c9d99169893", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000562.013902665, + "token": "click|Mail|textfield|1bbd9c9d", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "1bbd9c9d99169893", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000563.013902665, + "token": "type|Mail|textfield|1bbd9c9d", + "url_pattern": null, + "value_digest": "5c9127c24407f866", + "value_shape": "email", + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "2bdb797b79f1cd8e", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000564.013902665, + "token": "click|Mail|textfield|2bdb797b", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "2bdb797b79f1cd8e", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000565.013902665, + "token": "type|Mail|textfield|2bdb797b", + "url_pattern": null, + "value_digest": "5a5f2f37cc4df1b7", + "value_shape": "alnum", + "window_class": null + }, + { + "app": "Mail", + "display_label": "Attach", + "key": null, + "kind": "click", + "label_digest": "aed1127ebe1853e9", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000566.013902665, + "token": "click|Mail|button|aed1127e", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "Send", + "key": null, + "kind": "click", + "label_digest": "2dc861cd16dafe9c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000567.013902665, + "token": "click|Mail|button|2dc861cd", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 5000567.013902665, + "t_start": 5000558.013902665 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000881.901317636, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "s", + "kind": "shortcut", + "label_digest": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 5000882.901317636, + "token": "shortcut|Microsoft Excel|meta+s", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "focus", + "label_digest": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000883.901317636, + "token": "focus|Mail", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "New Message", + "key": null, + "kind": "click", + "label_digest": "af4c01884ec9c100", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000884.901317636, + "token": "click|Mail|button|af4c0188", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "1bbd9c9d99169893", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000885.901317636, + "token": "click|Mail|textfield|1bbd9c9d", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "1bbd9c9d99169893", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000886.901317636, + "token": "type|Mail|textfield|1bbd9c9d", + "url_pattern": null, + "value_digest": "effc4a77c7dea6a6", + "value_shape": "email", + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_digest": "2bdb797b79f1cd8e", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000887.901317636, + "token": "click|Mail|textfield|2bdb797b", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_digest": "2bdb797b79f1cd8e", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000888.901317636, + "token": "type|Mail|textfield|2bdb797b", + "url_pattern": null, + "value_digest": "fbdea0f3782fde1b", + "value_shape": "alnum", + "window_class": null + }, + { + "app": "Mail", + "display_label": "Attach", + "key": null, + "kind": "click", + "label_digest": "aed1127ebe1853e9", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000889.901317636, + "token": "click|Mail|button|aed1127e", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "Send", + "key": null, + "kind": "click", + "label_digest": "2dc861cd16dafe9c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000890.901317636, + "token": "click|Mail|button|2dc861cd", + "url_pattern": null, + "value_digest": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 5000890.901317636, + "t_start": 5000881.901317636 + } + ], + "exact_count": 3, + "expected_by_chance": 5.1827993358769765e-15, + "motif_id": "661fe3dd51f1ed89", + "p_value": 2.3202882082950603e-44, + "privacy": { + "clipboard_contents": "never read; copy and paste come from the key chord", + "control_labels": "keyed digest; short chrome labels in plain text when enabled", + "network": "none", + "screenshots": "never taken", + "typed_values": "keyed digest only; secure fields dropped", + "urls": "scheme, host, and at most one path word", + "window_titles": "never read" + }, + "schema": "openadapt.shadow.episodes/v1", + "status": "candidate", + "tokens": [ + "focus|Microsoft Excel", + "shortcut|Microsoft Excel|meta+s", + "focus|Mail", + "click|Mail|button|af4c0188", + "click|Mail|textfield|1bbd9c9d", + "type|Mail|textfield|1bbd9c9d", + "click|Mail|textfield|2bdb797b", + "type|Mail|textfield|2bdb797b", + "click|Mail|button|aed1127e", + "click|Mail|button|2dc861cd" + ], + "window_s": 86400.0 +} diff --git a/packages/flow/tests/test_motif_induction.py b/packages/flow/tests/test_motif_induction.py new file mode 100644 index 000000000..1526b696a --- /dev/null +++ b/packages/flow/tests/test_motif_induction.py @@ -0,0 +1,248 @@ +"""Shadow-mined episodes -> the existing induction -> a CANDIDATE, never a run. + +The fixtures under ``tests/fixtures/shadow/`` are real +``openadapt.shadow.episodes/v1`` exports, produced by openadapt-capture's +shadow miner from synthetic streams +(``tests/shadow_support/generate_flow_fixtures.py`` in that repository). +""" + +from __future__ import annotations + +import ast +import copy +import json +import socket +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from openadapt_flow.compiler import motif_induction +from openadapt_flow.compiler.motif_induction import ( + CANDIDATE_FILENAME, + DRAFT_FILENAME, + PLACEHOLDER_TEMPLATE, + MotifCandidate, + ShadowEpisodesExport, + induce_candidate, + write_candidate, +) +from openadapt_flow.ir import ActionKind, Workflow +from openadapt_flow.traversal import iter_workflow_steps + +FIXTURES = Path(__file__).parent / "fixtures" / "shadow" + +# The raw values the synthetic streams typed or clicked. None may survive. +RAW_VALUES = ( + "INV-559017", + "4471.25", + "jane.doe@example.com", + "Jane Doe", + "123-45-6789", + "Quarterly report", + "invoices/INV", + "ssn=", +) + + +def _load(name: str) -> dict: + return json.loads((FIXTURES / f"{name}.episodes.json").read_text()) + + +def test_fixtures_carry_no_raw_values(): + for path in sorted(FIXTURES.glob("*.json")): + text = path.read_text() + for value in RAW_VALUES: + assert value not in text, (path.name, value) + + +def test_fixed_control_task_induces_a_candidate_draft_with_parameters(): + candidate = induce_candidate(_load("report_mail")) + assert candidate.status == "candidate" + assert candidate.admitted is False and candidate.runnable is False + assert candidate.induction_certified is True + assert candidate.n_episodes == 3 + draft = candidate.draft + assert draft is not None and draft.program is not None + assert draft.name.startswith("shadow-draft-") + assert len(draft.param_specs) == 2 + # Parameter examples are digests, never values. + assert all(v.startswith("<") and ":" in v for v in draft.params.values()) + + +def test_record_next_starts_with_a_full_demonstration(): + candidate = induce_candidate(_load("report_mail")) + first = candidate.record_next[0] + assert first.code == "shadow_full_demonstration" and first.blocking + codes = [r.code for r in candidate.record_next] + assert codes.count("shadow_confirm_check") == len(candidate.proposed_checks) + assert "shadow_name_value" in codes + + +def test_checks_are_proposals_only(): + candidate = induce_candidate(_load("report_mail")) + kinds = sorted(c.kind for c in candidate.proposed_checks) + assert kinds == ["file_written", "record_written"] + for check in candidate.proposed_checks: + assert check.status == "proposed" + assert check.admitted is False + assert "can't approve" in check.note + # No proposal became an effect contract on any step. + steps = _all_steps(candidate.draft) + assert len(steps) >= 8 and all(not s.effects for s in steps) + send = [s for s in steps if "Send" in s.intent] + assert send and send[0].risk_review_required + + +def _all_steps(workflow: Workflow | None): + return [] if workflow is None else list(iter_workflow_steps(workflow)) + + +def test_write_shaped_steps_need_review_and_no_anchor_is_real(): + export = ShadowEpisodesExport.model_validate(_load("report_mail")) + workflows, _ = motif_induction.episodes_to_workflows(export) + send = [s for s in workflows[0].steps if "Send" in s.intent] + assert send and send[0].risk == "irreversible" and send[0].risk_review_required + for step in workflows[0].steps: + if step.anchor is not None: + assert step.anchor.template == PLACEHOLDER_TEMPLATE + assert step.anchor.click_point == (0, 0) + + +def test_varying_selection_is_refused_by_existing_induction(): + candidate = induce_candidate(_load("invoice_exact")) + assert candidate.induction_certified is False + assert candidate.draft is None + codes = [r.code for r in candidate.record_next] + assert codes[0] == "shadow_full_demonstration" + assert "ambiguous_selection" in codes + assert {c.kind for c in candidate.proposed_checks} == { + "page_reached", + "record_written", + } + + +def test_variant_episode_is_read_as_an_optional_step(): + candidate = induce_candidate(_load("invoice_variant")) + assert any( + "optional step" in line for line in candidate.induction_report.splitlines() + ) + + +def test_candidate_cannot_claim_admission(): + candidate = induce_candidate(_load("report_mail")) + data = candidate.model_dump() + for field in ("admitted", "runnable"): + bad = {**data, field: True} + with pytest.raises(ValidationError): + MotifCandidate.model_validate(bad) + with pytest.raises(ValidationError): + MotifCandidate.model_validate({**data, "status": "admitted"}) + + +@pytest.mark.parametrize( + ("mutate", "why"), + [ + (lambda s: s.update(text="jane.doe@example.com"), "unknown field"), + (lambda s: s.update(display_label="Pay 1200"), "digits in a plain label"), + (lambda s: s.update(value_digest="not-a-digest"), "malformed digest"), + (lambda s: s.update(kind="screenshot"), "unknown kind"), + ], +) +def test_export_schema_refuses_raw_content(mutate, why): + data = copy.deepcopy(_load("report_mail")) + mutate(data["episodes"][0]["steps"][3]) + with pytest.raises(ValidationError): + induce_candidate(data) + + +def test_export_schema_refuses_wrong_schema_and_status(): + data = _load("report_mail") + with pytest.raises(ValidationError): + induce_candidate({**data, "schema": "openadapt.shadow.episodes/v2"}) + with pytest.raises(ValidationError): + induce_candidate({**data, "status": "admitted"}) + + +def test_write_candidate_never_writes_a_runnable_bundle(tmp_path): + candidate = induce_candidate(_load("report_mail")) + out = write_candidate(candidate, tmp_path / "cand") + assert sorted(p.name for p in out.iterdir()) == [CANDIDATE_FILENAME, DRAFT_FILENAME] + meta = json.loads((out / CANDIDATE_FILENAME).read_text()) + assert meta["status"] == "candidate" and meta["admitted"] is False + draft = Workflow.model_validate_json((out / DRAFT_FILENAME).read_text()) + assert draft.program is not None + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "workflow.json").write_text("{}") + with pytest.raises(FileExistsError): + write_candidate(candidate, bundle) + + +def test_refused_candidate_writes_no_draft(tmp_path): + candidate = induce_candidate(_load("invoice_exact")) + out = write_candidate(candidate, tmp_path / "cand") + assert [p.name for p in out.iterdir()] == [CANDIDATE_FILENAME] + + +def test_deterministic(): + a = induce_candidate(_load("report_mail")).model_dump(mode="json") + b = induce_candidate(_load("report_mail")).model_dump(mode="json") + a["draft"].pop("created_at", None) + b["draft"].pop("created_at", None) + assert a == b + + +def test_bridge_imports_no_network_module(): + banned = { + "socket", + "ssl", + "http", + "urllib", + "requests", + "httpx", + "aiohttp", + "smtplib", + } + tree = ast.parse(Path(motif_induction.__file__).read_text()) + for node in ast.walk(tree): + names = [] + if isinstance(node, ast.Import): + names = [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + names = [node.module] + for name in names: + assert name.split(".")[0] not in banned, name + + +def test_bridge_makes_zero_network_calls(tmp_path, monkeypatch): + import httpx + + attempts: list[str] = [] + + def refuse(name): + def _raise(*args, **kwargs): + attempts.append(name) + raise AssertionError(f"network access via {name}") + + return _raise + + for name in ("connect", "connect_ex", "sendto"): + monkeypatch.setattr(socket.socket, name, refuse(f"socket.{name}")) + monkeypatch.setattr(socket, "create_connection", refuse("create_connection")) + monkeypatch.setattr(socket, "getaddrinfo", refuse("getaddrinfo")) + monkeypatch.setattr(httpx.Client, "send", refuse("httpx.Client.send")) + for name in ("report_mail", "invoice_exact", "invoice_variant"): + write_candidate(induce_candidate(_load(name)), tmp_path / name) + assert attempts == [] + + +def test_hotkey_steps_keep_their_chord(): + export = ShadowEpisodesExport.model_validate(_load("report_mail")) + workflows, _ = motif_induction.episodes_to_workflows(export) + save = workflows[0].steps[0] + assert ( + save.action is ActionKind.HOTKEY + and save.key == "s" + and save.modifiers == ["meta"] + ) From bb897a7ac16a7aa1a8be9933c65d96f51cfcc689 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sat, 17 Oct 2026 01:19:06 -0400 Subject: [PATCH 02/12] docs: describe shadow-mined candidates and how a person admits one Signed-off-by: Richard Abrich --- packages/flow/docs/SHADOW_CANDIDATES.md | 84 +++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 packages/flow/docs/SHADOW_CANDIDATES.md diff --git a/packages/flow/docs/SHADOW_CANDIDATES.md b/packages/flow/docs/SHADOW_CANDIDATES.md new file mode 100644 index 000000000..df5f39c5f --- /dev/null +++ b/packages/flow/docs/SHADOW_CANDIDATES.md @@ -0,0 +1,84 @@ +# Shadow-mined candidates + +OpenAdapt Capture's shadow mode (off by default) can notice that a person +repeated the same cross-application task and, when the person clicks +**Review draft**, export the matching episodes. This page explains what Flow +does with that export. Capture's +[SHADOW_MINING.md](https://github.com/OpenAdaptAI/openadapt-capture/blob/main/docs/SHADOW_MINING.md) +covers how the episodes are mined and the privacy posture. + +The code is `openadapt_flow/compiler/motif_induction.py`. + +## What goes in + +An `openadapt.shadow.episodes/v1` JSON document. Each episode is a list of +value-free symbols: an application, a control role, keyed digests in place of +labels and typed values, a coarse URL pattern, and short plain-text labels for +buttons and menu items. There are no screenshots, coordinates, or values. + +The schema is strict. `induce_candidate` refuses a document with an unknown +field, a malformed digest, a plain-text label that contains digits or `@`, an +unknown step kind, or a status other than `candidate`. + +## What comes out + +```python +from openadapt_flow.compiler.motif_induction import induce_candidate, write_candidate + +candidate = induce_candidate(export) # dict or ShadowEpisodesExport +write_candidate(candidate, "shadow-candidates/abc") +``` + +`induce_candidate` turns each episode into a linear trace and runs the +existing multi-trace induction (`induce_program`) over them. Induction infers +parameters, loops, and optional steps, or refuses, exactly as it does for +recorded demonstrations. The result is a `MotifCandidate`: + +| Field | Meaning | +|---|---| +| `status` | Always `candidate`. | +| `admitted`, `runnable` | Always `False`. The model rejects any other value. | +| `induction_certified` | Whether induction found the episodes structurally consistent. This is a statement about shape only. | +| `draft` | The induced program, or `None` when induction refused. | +| `record_next` | What to record or decide next. The first item is always "record this task once with OpenAdapt Capture". | +| `proposed_checks` | Verification checks the episodes suggest. | +| `admission_path` | The steps that turn the candidate into a program. | + +`write_candidate` writes `candidate.json` and, when there is a draft, +`draft-workflow.json`. It never writes `workflow.json`, so no Flow command +treats the folder as a bundle, and it refuses a folder that already holds one. + +## Why a candidate can't run + +- **No anchors.** Shadow mining keeps no pixels, so every anchor in the draft + is a placeholder (`shadow-unrecorded.png` at 0,0). +- **No values.** Parameter examples are digests such as ``. +- **Writes need review.** A click on a Submit-, Save-, or Send-like control, + or a click right before a form submit, is marked `irreversible` with + `risk_review_required`, so certification refuses it until a person reviews + it. +- **Proposals aren't contracts.** A proposed check (`file_written`, + `record_written`, `row_added`, `page_reached`) has `status="proposed"` and + `admitted=False`, and is never attached to a step as an effect. The episodes + that suggested a check can't approve it. + +## How a person admits it + +1. Record the task with Capture. +2. Compile each recording with `flow compile`. +3. Run `flow induce` over the recordings, using the draft only as a guide. +4. Confirm or reject each proposed check, then run `flow qualify`. + +## Example + +The test fixtures in `tests/fixtures/shadow/` come from Capture's miner on +synthetic streams: + +- `report_mail`: save a spreadsheet, then mail it. Induction emits a draft + with two parameters (recipient and subject) and proposes two checks (the + file changes; a message is sent). +- `invoice_exact`: copy an invoice number from a spreadsheet cell into a web + form. The cell differs every time, so induction refuses with + `ambiguous_selection` ("which row?"), and the candidate has no draft. +- `invoice_variant`: the same task with an extra check box in one episode. + Induction reads it as an optional step. From f14cc62fa357e1a25e70fb7208430fd092169754 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sat, 17 Oct 2026 15:26:58 -0400 Subject: [PATCH 03/12] feat(compiler): read shadow episodes v2 and keep digests out of drafts Capture's shadow export moves to openadapt.shadow.episodes/v2: each keyed digest becomes a reference local to the export (label-a, value-c), so a draft never carries a digest that could be checked against guesses for a short identifier after the fact. induce_candidate now reads v2 only. It refuses v1 with a clear message, refuses label_digest and value_digest fields, and refuses a token with an 8+ character hex part. A scrubbed chrome label () is treated as no label. The fixtures are regenerated with Capture's v2 exporter. New tests check that no induced candidate carries a hex run other than its own export id. Signed-off-by: Richard Abrich --- packages/flow/docs/SHADOW_CANDIDATES.md | 33 ++- .../compiler/motif_induction.py | 85 +++++-- .../shadow/invoice_exact.episodes.json | 227 ++++++++--------- .../shadow/invoice_variant.episodes.json | 233 +++++++++--------- .../fixtures/shadow/report_mail.episodes.json | 187 +++++++------- packages/flow/tests/test_motif_induction.py | 30 ++- 6 files changed, 438 insertions(+), 357 deletions(-) diff --git a/packages/flow/docs/SHADOW_CANDIDATES.md b/packages/flow/docs/SHADOW_CANDIDATES.md index df5f39c5f..f4abd860b 100644 --- a/packages/flow/docs/SHADOW_CANDIDATES.md +++ b/packages/flow/docs/SHADOW_CANDIDATES.md @@ -11,14 +11,29 @@ The code is `openadapt_flow/compiler/motif_induction.py`. ## What goes in -An `openadapt.shadow.episodes/v1` JSON document. Each episode is a list of -value-free symbols: an application, a control role, keyed digests in place of -labels and typed values, a coarse URL pattern, and short plain-text labels for -buttons and menu items. There are no screenshots, coordinates, or values. - -The schema is strict. `induce_candidate` refuses a document with an unknown -field, a malformed digest, a plain-text label that contains digits or `@`, an -unknown step kind, or a status other than `candidate`. +An `openadapt.shadow.episodes/v2` JSON document. Each episode is a list of +value-free symbols: an application, a control role, a scrubbed URL pattern, +short plain-text labels for buttons and menu items made of common words, and +references in place of other labels and typed values. + +A reference such as `label-a` or `value-c` is local to one export. Two steps +with the same reference had the same label or value; that is all the export +says. Capture's keyed digests never leave Capture: its keys are destroyed +every 24 hours, and a digest that outlived its key could be matched against +guesses for short identifiers such as a record number or an SSN, so no digest +may reach a draft. + +The schema is strict. `induce_candidate` refuses: + +- a `v1` document (it carried keyed digests); +- a step with a `label_digest` or `value_digest` field, or any other unknown + field; +- a token with a hex part of eight or more characters; +- a malformed reference; +- a plain-text label that contains digits or `@`; +- an unknown step kind, or a status other than `candidate`. + +A label that Capture scrubbed (``) is treated as no label. ## What comes out @@ -52,7 +67,7 @@ treats the folder as a bundle, and it refuses a folder that already holds one. - **No anchors.** Shadow mining keeps no pixels, so every anchor in the draft is a placeholder (`shadow-unrecorded.png` at 0,0). -- **No values.** Parameter examples are digests such as ``. +- **No values.** Parameter examples are references such as ``. - **Writes need review.** A click on a Submit-, Save-, or Send-like control, or a click right before a form submit, is marked `irreversible` with `risk_review_required`, so certification refuses it until a person reviews diff --git a/packages/flow/openadapt_flow/compiler/motif_induction.py b/packages/flow/openadapt_flow/compiler/motif_induction.py index b1af7d5ff..3d4cb6f6d 100644 --- a/packages/flow/openadapt_flow/compiler/motif_induction.py +++ b/packages/flow/openadapt_flow/compiler/motif_induction.py @@ -3,10 +3,13 @@ OpenAdapt Capture's shadow mode (``openadapt_capture.shadow``, off by default) can notice that a person repeated the same cross-application sequence several times and, if the person clicks **Review draft**, export those episodes as -``openadapt.shadow.episodes/v1`` JSON. Each step in that export is a -value-free symbol: an application, a control role, keyed digests in place of -labels and typed values, a coarse URL pattern. There are no screenshots, no -coordinates, and no plain-text values. +``openadapt.shadow.episodes/v2`` JSON. Each step in that export is a +value-free symbol: an application, a control role, references local to the +export in place of labels and typed values (``label-a``, ``value-b``), a +scrubbed URL pattern. There are no screenshots, no coordinates, no plain-text +values, and no keyed digests: two steps that share a reference shared a value, +and that is all the export says about values. Version 1 exports, which carried +keyed digests, are refused, so a digest can never reach a draft. This module converts the episodes into linear :class:`~openadapt_flow.ir.Workflow` traces and hands them to the EXISTING multi-trace induction @@ -39,12 +42,13 @@ from pathlib import Path from typing import Literal, Optional -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator from openadapt_flow.compiler.induction import RecordNext, TraceInput, induce_program from openadapt_flow.ir import ActionKind, Anchor, Step, StructuralLocator, Workflow -EPISODES_SCHEMA = "openadapt.shadow.episodes/v1" +EPISODES_SCHEMA = "openadapt.shadow.episodes/v2" +REFUSED_SCHEMAS = frozenset({"openadapt.shadow.episodes/v1"}) CANDIDATE_SCHEMA: Literal["openadapt.shadow.candidate/v1"] = ( "openadapt.shadow.candidate/v1" ) @@ -53,6 +57,11 @@ CANDIDATE_FILENAME = "candidate.json" _DIGEST = r"^[0-9a-f]{16}$" +_TOKEN_DIGEST = re.compile(r"^[0-9a-f]{8,}$") +_LABEL_REF = r"^label-[a-z]{1,4}$" +_VALUE_REF = r"^value-[a-z]{1,4}$" +#: Capture's placeholder for a chrome label that looked like data. +DYNAMIC_LABEL = "" _WRITE_STEMS = ("save", "submit", "send", "post", "approve", "confirm", "create", "add") _SHEET_APPS = ("excel", "numbers", "sheets", "calc", "spreadsheet") _SAVE_MODIFIERS = frozenset({"meta", "ctrl", "cmd", "command"}) @@ -88,11 +97,11 @@ class ShadowStep(BaseModel): app: Optional[str] = Field(default=None, max_length=128) window_class: Optional[str] = Field(default=None, max_length=128) role: Optional[str] = Field(default=None, max_length=64) - label_digest: Optional[str] = Field(default=None, pattern=_DIGEST) + label_ref: Optional[str] = Field(default=None, pattern=_LABEL_REF) display_label: Optional[str] = Field( default=None, max_length=32, pattern=r"^[^0-9@]*$" ) - value_digest: Optional[str] = Field(default=None, pattern=_DIGEST) + value_ref: Optional[str] = Field(default=None, pattern=_VALUE_REF) value_shape: Optional[ Literal["empty", "digits", "decimal", "date", "email", "alpha", "alnum", "text"] ] = None @@ -102,6 +111,20 @@ class ShadowStep(BaseModel): secure: bool = False + @field_validator("token") + @classmethod + def _no_digest_in_token(cls, value: str) -> str: + if any(_TOKEN_DIGEST.match(part) for part in value.split("|")): + raise ValueError("a shadow token must not carry a keyed digest") + return value + + +def _check_tokens(tokens: list[str]) -> list[str]: + for token in tokens: + ShadowStep._no_digest_in_token(token) + return tokens + + class ShadowEpisode(BaseModel): model_config = ConfigDict(extra="forbid") @@ -113,13 +136,18 @@ class ShadowEpisode(BaseModel): class ShadowEpisodesExport(BaseModel): - """The ``openadapt.shadow.episodes/v1`` document Capture writes.""" + """The ``openadapt.shadow.episodes/v2`` document Capture writes.""" model_config = ConfigDict(extra="forbid") - schema_: Literal["openadapt.shadow.episodes/v1"] = Field(alias="schema") + schema_: Literal["openadapt.shadow.episodes/v2"] = Field(alias="schema") motif_id: str = Field(pattern=_DIGEST) tokens: list[str] + + @field_validator("tokens") + @classmethod + def _tokens_carry_no_digest(cls, value: list[str]) -> list[str]: + return _check_tokens(value) apps: list[str] count_in_window: int = Field(ge=0) window_s: float @@ -170,12 +198,20 @@ class MotifCandidate(BaseModel): admission_path: list[str] = Field(default_factory=lambda: list(ADMISSION_PATH)) -def _alias(digest: Optional[str]) -> str: - """Letters-only alias of a digest (induction drops digits from intents).""" +def _alias(ref: Optional[str]) -> str: + """Name of a control known only by its export reference.""" - if not digest: + if not ref: return "unnamed" - return "fld-" + "".join(chr(ord("a") + int(c, 16)) for c in digest[:8]) + return "fld-" + ref.split("-", 1)[1] + + +def _shown(display_label: Optional[str]) -> Optional[str]: + """A plain-text chrome label, or None for none or a scrubbed one.""" + + if not display_label or display_label == DYNAMIC_LABEL: + return None + return display_label def _placeholder_anchor(role: Optional[str], name: str, *, varies: bool) -> Anchor: @@ -236,13 +272,13 @@ def _episode_steps( "double_click": ActionKind.DOUBLE_CLICK, "right_click": ActionKind.RIGHT_CLICK, }[sym.kind] - if sym.value_digest: - name = f"value:{sym.value_digest}" + if sym.value_ref: + name = sym.value_ref what = f"{sym.role or 'control'} (varies)" else: - name = sym.display_label or _alias(sym.label_digest) + name = _shown(sym.display_label) or _alias(sym.label_ref) what = f"{sym.role or 'control'} {name}" - writes = _is_write_label(sym.display_label) or ( + writes = _is_write_label(_shown(sym.display_label)) or ( nxt is not None and nxt.kind == "submit" ) steps.append( @@ -251,7 +287,7 @@ def _episode_steps( intent=f"{prefix} click {what}", action=action, anchor=_placeholder_anchor( - sym.role, name, varies=bool(sym.value_digest) + sym.role, name, varies=bool(sym.value_ref) ), risk="irreversible" if writes else "reversible", risk_explanation=( @@ -271,7 +307,7 @@ def _episode_steps( ) ) elif sym.kind == "type": - field = sym.display_label or _alias(sym.label_digest) + field = _shown(sym.display_label) or _alias(sym.label_ref) intent = f"{prefix} type {sym.role or 'field'} {field}" if sym.secure: steps.append( @@ -280,7 +316,7 @@ def _episode_steps( intent=intent, action=ActionKind.TYPE, secret=True, - param=f"secret_{_alias(sym.label_digest)[4:]}", + param=f"secret_{_alias(sym.label_ref)[4:]}", ) ) else: @@ -290,7 +326,7 @@ def _episode_steps( id=sid, intent=intent, action=ActionKind.TYPE, - text=f"<{shape}:{sym.value_digest or 'none'}>", + text=f"<{shape}:{sym.value_ref or 'none'}>", ) ) if nxt is not None and nxt.kind == "key" and _is_sheet(app): @@ -389,6 +425,11 @@ def induce_candidate(export: ShadowEpisodesExport | dict) -> MotifCandidate: """Run the existing induction over shadow episodes; return a candidate.""" if not isinstance(export, ShadowEpisodesExport): + if isinstance(export, dict) and export.get("schema") in REFUSED_SCHEMAS: + raise ValueError( + f"{export.get('schema')} carries keyed digests and is refused; " + f"export again with a Capture that writes {EPISODES_SCHEMA}" + ) export = ShadowEpisodesExport.model_validate(export) workflows, checks = episodes_to_workflows(export) traces: list[TraceInput] = list(workflows) diff --git a/packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json b/packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json index 8363ccff5..14f37c78f 100644 --- a/packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json +++ b/packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json @@ -14,14 +14,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000265.4191387234, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -30,14 +30,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": "cell", "secure": false, "t": 1000266.4191387234, "token": "click|Microsoft Excel|cell|-", "url_pattern": null, - "value_digest": "c4c11450729082bb", + "value_ref": "value-a", "value_shape": null, "window_class": null }, @@ -46,7 +46,7 @@ "display_label": null, "key": "c", "kind": "copy", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -55,7 +55,7 @@ "t": 1000267.4191387234, "token": "copy|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -64,14 +64,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000268.4191387234, "token": "focus|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -80,14 +80,14 @@ "display_label": null, "key": null, "kind": "navigate", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000269.4191387234, "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -96,14 +96,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "87961bb30e8c3a0d", + "label_ref": "label-a", "modifiers": [], "role": "textfield", "secure": false, "t": 1000270.4191387234, - "token": "click|Google Chrome|textfield|87961bb3", + "token": "click|Google Chrome|textfield|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -112,7 +112,7 @@ "display_label": null, "key": "v", "kind": "paste", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -121,7 +121,7 @@ "t": 1000271.4191387234, "token": "paste|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -130,14 +130,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000272.4191387234, - "token": "click|Google Chrome|textfield|64e7c628", + "token": "click|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -146,14 +146,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000273.4191387234, - "token": "type|Google Chrome|textfield|64e7c628", + "token": "type|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": "d2a9388191e23876", + "value_ref": "value-b", "value_shape": "decimal", "window_class": null }, @@ -162,14 +162,14 @@ "display_label": "Submit", "key": null, "kind": "click", - "label_digest": "57d88c0af57c244e", + "label_ref": "label-c", "modifiers": [], "role": "button", "secure": false, "t": 1000274.4191387234, - "token": "click|Google Chrome|button|57d88c0a", + "token": "click|Google Chrome|button|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -178,14 +178,14 @@ "display_label": null, "key": null, "kind": "submit", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000275.4191387234, "token": "submit|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -202,14 +202,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000570.0973327656, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -218,14 +218,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": "cell", "secure": false, "t": 1000571.0973327656, "token": "click|Microsoft Excel|cell|-", "url_pattern": null, - "value_digest": "3994e33e2632d5cc", + "value_ref": "value-c", "value_shape": null, "window_class": null }, @@ -234,7 +234,7 @@ "display_label": null, "key": "c", "kind": "copy", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -243,7 +243,7 @@ "t": 1000572.0973327656, "token": "copy|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -252,14 +252,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000573.0973327656, "token": "focus|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -268,14 +268,14 @@ "display_label": null, "key": null, "kind": "navigate", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000574.0973327656, "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -284,14 +284,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "87961bb30e8c3a0d", + "label_ref": "label-a", "modifiers": [], "role": "textfield", "secure": false, "t": 1000575.0973327656, - "token": "click|Google Chrome|textfield|87961bb3", + "token": "click|Google Chrome|textfield|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -300,7 +300,7 @@ "display_label": null, "key": "v", "kind": "paste", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -309,7 +309,7 @@ "t": 1000576.0973327656, "token": "paste|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -318,14 +318,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000577.0973327656, - "token": "click|Google Chrome|textfield|64e7c628", + "token": "click|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -334,14 +334,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000578.0973327656, - "token": "type|Google Chrome|textfield|64e7c628", + "token": "type|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": "431eb9c3f9fa4935", + "value_ref": "value-d", "value_shape": "decimal", "window_class": null }, @@ -350,14 +350,14 @@ "display_label": "Submit", "key": null, "kind": "click", - "label_digest": "57d88c0af57c244e", + "label_ref": "label-c", "modifiers": [], "role": "button", "secure": false, "t": 1000579.0973327656, - "token": "click|Google Chrome|button|57d88c0a", + "token": "click|Google Chrome|button|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -366,14 +366,14 @@ "display_label": null, "key": null, "kind": "submit", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000580.0973327656, "token": "submit|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -390,14 +390,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000865.0038992346, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -406,14 +406,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": "cell", "secure": false, "t": 1000866.0038992346, "token": "click|Microsoft Excel|cell|-", "url_pattern": null, - "value_digest": "857eaa216100862b", + "value_ref": "value-e", "value_shape": null, "window_class": null }, @@ -422,7 +422,7 @@ "display_label": null, "key": "c", "kind": "copy", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -431,7 +431,7 @@ "t": 1000867.0038992346, "token": "copy|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -440,14 +440,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000868.0038992346, "token": "focus|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -456,14 +456,14 @@ "display_label": null, "key": null, "kind": "navigate", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000869.0038992346, "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -472,14 +472,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "87961bb30e8c3a0d", + "label_ref": "label-a", "modifiers": [], "role": "textfield", "secure": false, "t": 1000870.0038992346, - "token": "click|Google Chrome|textfield|87961bb3", + "token": "click|Google Chrome|textfield|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -488,7 +488,7 @@ "display_label": null, "key": "v", "kind": "paste", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -497,7 +497,7 @@ "t": 1000871.0038992346, "token": "paste|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -506,14 +506,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000872.0038992346, - "token": "click|Google Chrome|textfield|64e7c628", + "token": "click|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -522,14 +522,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000873.0038992346, - "token": "type|Google Chrome|textfield|64e7c628", + "token": "type|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": "f5ab28d0e13d91b8", + "value_ref": "value-f", "value_shape": "decimal", "window_class": null }, @@ -538,14 +538,14 @@ "display_label": "Submit", "key": null, "kind": "click", - "label_digest": "57d88c0af57c244e", + "label_ref": "label-c", "modifiers": [], "role": "button", "secure": false, "t": 1000874.0038992346, - "token": "click|Google Chrome|button|57d88c0a", + "token": "click|Google Chrome|button|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -554,14 +554,14 @@ "display_label": null, "key": null, "kind": "submit", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000875.0038992346, "token": "submit|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -578,14 +578,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1001239.5060203954, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -594,14 +594,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": "cell", "secure": false, "t": 1001240.5060203954, "token": "click|Microsoft Excel|cell|-", "url_pattern": null, - "value_digest": "11b37ab6c3be2895", + "value_ref": "value-g", "value_shape": null, "window_class": null }, @@ -610,7 +610,7 @@ "display_label": null, "key": "c", "kind": "copy", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -619,7 +619,7 @@ "t": 1001241.5060203954, "token": "copy|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -628,14 +628,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1001242.5060203954, "token": "focus|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -644,14 +644,14 @@ "display_label": null, "key": null, "kind": "navigate", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1001243.5060203954, "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -660,14 +660,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "87961bb30e8c3a0d", + "label_ref": "label-a", "modifiers": [], "role": "textfield", "secure": false, "t": 1001244.5060203954, - "token": "click|Google Chrome|textfield|87961bb3", + "token": "click|Google Chrome|textfield|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -676,7 +676,7 @@ "display_label": null, "key": "v", "kind": "paste", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -685,7 +685,7 @@ "t": 1001245.5060203954, "token": "paste|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -694,14 +694,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1001246.5060203954, - "token": "click|Google Chrome|textfield|64e7c628", + "token": "click|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -710,14 +710,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1001247.5060203954, - "token": "type|Google Chrome|textfield|64e7c628", + "token": "type|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": "2336b2da290c600c", + "value_ref": "value-h", "value_shape": "decimal", "window_class": null }, @@ -726,14 +726,14 @@ "display_label": "Submit", "key": null, "kind": "click", - "label_digest": "57d88c0af57c244e", + "label_ref": "label-c", "modifiers": [], "role": "button", "secure": false, "t": 1001248.5060203954, - "token": "click|Google Chrome|button|57d88c0a", + "token": "click|Google Chrome|button|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -742,14 +742,14 @@ "display_label": null, "key": null, "kind": "submit", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1001249.5060203954, "token": "submit|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -760,18 +760,19 @@ ], "exact_count": 4, "expected_by_chance": 2.0763629850095314e-16, - "motif_id": "f2f51c3e6dd5a708", + "motif_id": "b78791ee75702dcb", "p_value": 7.744651342950623e-65, "privacy": { "clipboard_contents": "never read; copy and paste come from the key chord", - "control_labels": "keyed digest; short chrome labels in plain text when enabled", + "control_labels": "short chrome labels of common words in plain text; others a placeholder", + "keyed_digests": "never exported; replaced by references local to this export", "network": "none", "screenshots": "never taken", - "typed_values": "keyed digest only; secure fields dropped", - "urls": "scheme, host, and at most one path word", + "typed_values": "never exported; secure fields dropped", + "urls": "work domains only: host and one scrubbed path word; other sites a marker", "window_titles": "never read" }, - "schema": "openadapt.shadow.episodes/v1", + "schema": "openadapt.shadow.episodes/v2", "status": "candidate", "tokens": [ "focus|Microsoft Excel", @@ -779,11 +780,11 @@ "copy|Microsoft Excel", "focus|Google Chrome", "navigate|Google Chrome|https://erp.example.com/invoices/*", - "click|Google Chrome|textfield|87961bb3", + "click|Google Chrome|textfield|label-a", "paste|Google Chrome", - "click|Google Chrome|textfield|64e7c628", - "type|Google Chrome|textfield|64e7c628", - "click|Google Chrome|button|57d88c0a", + "click|Google Chrome|textfield|label-b", + "type|Google Chrome|textfield|label-b", + "click|Google Chrome|button|label-c", "submit|Google Chrome|https://erp.example.com/invoices/*" ], "window_s": 86400.0 diff --git a/packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json b/packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json index 50487a8f7..c45f99099 100644 --- a/packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json +++ b/packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json @@ -14,14 +14,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000265.4191387234, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -30,14 +30,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": "cell", "secure": false, "t": 1000266.4191387234, "token": "click|Microsoft Excel|cell|-", "url_pattern": null, - "value_digest": "c4c11450729082bb", + "value_ref": "value-a", "value_shape": null, "window_class": null }, @@ -46,7 +46,7 @@ "display_label": null, "key": "c", "kind": "copy", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -55,7 +55,7 @@ "t": 1000267.4191387234, "token": "copy|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -64,14 +64,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000268.4191387234, "token": "focus|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -80,14 +80,14 @@ "display_label": null, "key": null, "kind": "navigate", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000269.4191387234, "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -96,14 +96,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "87961bb30e8c3a0d", + "label_ref": "label-a", "modifiers": [], "role": "textfield", "secure": false, "t": 1000270.4191387234, - "token": "click|Google Chrome|textfield|87961bb3", + "token": "click|Google Chrome|textfield|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -112,7 +112,7 @@ "display_label": null, "key": "v", "kind": "paste", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -121,7 +121,7 @@ "t": 1000271.4191387234, "token": "paste|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -130,14 +130,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000272.4191387234, - "token": "click|Google Chrome|textfield|64e7c628", + "token": "click|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -146,14 +146,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000273.4191387234, - "token": "type|Google Chrome|textfield|64e7c628", + "token": "type|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": "d2a9388191e23876", + "value_ref": "value-b", "value_shape": "decimal", "window_class": null }, @@ -162,14 +162,14 @@ "display_label": "Submit", "key": null, "kind": "click", - "label_digest": "57d88c0af57c244e", + "label_ref": "label-c", "modifiers": [], "role": "button", "secure": false, "t": 1000274.4191387234, - "token": "click|Google Chrome|button|57d88c0a", + "token": "click|Google Chrome|button|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -178,14 +178,14 @@ "display_label": null, "key": null, "kind": "submit", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000275.4191387234, "token": "submit|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -202,14 +202,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000570.0973327656, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -218,14 +218,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": "cell", "secure": false, "t": 1000571.0973327656, "token": "click|Microsoft Excel|cell|-", "url_pattern": null, - "value_digest": "3994e33e2632d5cc", + "value_ref": "value-c", "value_shape": null, "window_class": null }, @@ -234,7 +234,7 @@ "display_label": null, "key": "c", "kind": "copy", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -243,7 +243,7 @@ "t": 1000572.0973327656, "token": "copy|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -252,14 +252,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000573.0973327656, "token": "focus|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -268,14 +268,14 @@ "display_label": null, "key": null, "kind": "navigate", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000574.0973327656, "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -284,14 +284,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "87961bb30e8c3a0d", + "label_ref": "label-a", "modifiers": [], "role": "textfield", "secure": false, "t": 1000575.0973327656, - "token": "click|Google Chrome|textfield|87961bb3", + "token": "click|Google Chrome|textfield|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -300,7 +300,7 @@ "display_label": null, "key": "v", "kind": "paste", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -309,7 +309,7 @@ "t": 1000576.0973327656, "token": "paste|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -318,14 +318,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000577.0973327656, - "token": "click|Google Chrome|textfield|64e7c628", + "token": "click|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -334,14 +334,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000578.0973327656, - "token": "type|Google Chrome|textfield|64e7c628", + "token": "type|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": "431eb9c3f9fa4935", + "value_ref": "value-d", "value_shape": "decimal", "window_class": null }, @@ -350,14 +350,14 @@ "display_label": "Submit", "key": null, "kind": "click", - "label_digest": "57d88c0af57c244e", + "label_ref": "label-c", "modifiers": [], "role": "button", "secure": false, "t": 1000579.0973327656, - "token": "click|Google Chrome|button|57d88c0a", + "token": "click|Google Chrome|button|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -366,14 +366,14 @@ "display_label": null, "key": null, "kind": "submit", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000580.0973327656, "token": "submit|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -390,14 +390,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000865.0038992346, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -406,14 +406,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": "cell", "secure": false, "t": 1000866.0038992346, "token": "click|Microsoft Excel|cell|-", "url_pattern": null, - "value_digest": "857eaa216100862b", + "value_ref": "value-e", "value_shape": null, "window_class": null }, @@ -422,7 +422,7 @@ "display_label": null, "key": "c", "kind": "copy", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -431,7 +431,7 @@ "t": 1000867.0038992346, "token": "copy|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -440,14 +440,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000868.0038992346, "token": "focus|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -456,14 +456,14 @@ "display_label": null, "key": null, "kind": "navigate", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000869.0038992346, "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -472,14 +472,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "87961bb30e8c3a0d", + "label_ref": "label-a", "modifiers": [], "role": "textfield", "secure": false, "t": 1000870.0038992346, - "token": "click|Google Chrome|textfield|87961bb3", + "token": "click|Google Chrome|textfield|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -488,7 +488,7 @@ "display_label": null, "key": "v", "kind": "paste", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -497,7 +497,7 @@ "t": 1000871.0038992346, "token": "paste|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -506,14 +506,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000872.0038992346, - "token": "click|Google Chrome|textfield|64e7c628", + "token": "click|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -522,14 +522,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1000873.0038992346, - "token": "type|Google Chrome|textfield|64e7c628", + "token": "type|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": "f5ab28d0e13d91b8", + "value_ref": "value-f", "value_shape": "decimal", "window_class": null }, @@ -538,14 +538,14 @@ "display_label": "Urgent", "key": null, "kind": "click", - "label_digest": "762cf17e6634e30a", + "label_ref": "label-d", "modifiers": [], "role": "checkbox", "secure": false, "t": 1000873.8038992346, - "token": "click|Google Chrome|checkbox|762cf17e", + "token": "click|Google Chrome|checkbox|label-d", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -554,14 +554,14 @@ "display_label": "Submit", "key": null, "kind": "click", - "label_digest": "57d88c0af57c244e", + "label_ref": "label-c", "modifiers": [], "role": "button", "secure": false, "t": 1000874.0038992346, - "token": "click|Google Chrome|button|57d88c0a", + "token": "click|Google Chrome|button|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -570,14 +570,14 @@ "display_label": null, "key": null, "kind": "submit", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1000875.0038992346, "token": "submit|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -594,14 +594,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1001239.5060203954, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -610,14 +610,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": "cell", "secure": false, "t": 1001240.5060203954, "token": "click|Microsoft Excel|cell|-", "url_pattern": null, - "value_digest": "11b37ab6c3be2895", + "value_ref": "value-g", "value_shape": null, "window_class": null }, @@ -626,7 +626,7 @@ "display_label": null, "key": "c", "kind": "copy", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -635,7 +635,7 @@ "t": 1001241.5060203954, "token": "copy|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -644,14 +644,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1001242.5060203954, "token": "focus|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -660,14 +660,14 @@ "display_label": null, "key": null, "kind": "navigate", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1001243.5060203954, "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -676,14 +676,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "87961bb30e8c3a0d", + "label_ref": "label-a", "modifiers": [], "role": "textfield", "secure": false, "t": 1001244.5060203954, - "token": "click|Google Chrome|textfield|87961bb3", + "token": "click|Google Chrome|textfield|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -692,7 +692,7 @@ "display_label": null, "key": "v", "kind": "paste", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -701,7 +701,7 @@ "t": 1001245.5060203954, "token": "paste|Google Chrome", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -710,14 +710,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1001246.5060203954, - "token": "click|Google Chrome|textfield|64e7c628", + "token": "click|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -726,14 +726,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "64e7c6284cf4020f", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 1001247.5060203954, - "token": "type|Google Chrome|textfield|64e7c628", + "token": "type|Google Chrome|textfield|label-b", "url_pattern": null, - "value_digest": "2336b2da290c600c", + "value_ref": "value-h", "value_shape": "decimal", "window_class": null }, @@ -742,14 +742,14 @@ "display_label": "Submit", "key": null, "kind": "click", - "label_digest": "57d88c0af57c244e", + "label_ref": "label-c", "modifiers": [], "role": "button", "secure": false, "t": 1001248.5060203954, - "token": "click|Google Chrome|button|57d88c0a", + "token": "click|Google Chrome|button|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -758,14 +758,14 @@ "display_label": null, "key": null, "kind": "submit", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 1001249.5060203954, "token": "submit|Google Chrome|https://erp.example.com/invoices/*", "url_pattern": "https://erp.example.com/invoices/*", - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -776,18 +776,19 @@ ], "exact_count": 3, "expected_by_chance": 1.9581310411777164e-16, - "motif_id": "f2f51c3e6dd5a708", + "motif_id": "b78791ee75702dcb", "p_value": 1.2513361926197074e-48, "privacy": { "clipboard_contents": "never read; copy and paste come from the key chord", - "control_labels": "keyed digest; short chrome labels in plain text when enabled", + "control_labels": "short chrome labels of common words in plain text; others a placeholder", + "keyed_digests": "never exported; replaced by references local to this export", "network": "none", "screenshots": "never taken", - "typed_values": "keyed digest only; secure fields dropped", - "urls": "scheme, host, and at most one path word", + "typed_values": "never exported; secure fields dropped", + "urls": "work domains only: host and one scrubbed path word; other sites a marker", "window_titles": "never read" }, - "schema": "openadapt.shadow.episodes/v1", + "schema": "openadapt.shadow.episodes/v2", "status": "candidate", "tokens": [ "focus|Microsoft Excel", @@ -795,11 +796,11 @@ "copy|Microsoft Excel", "focus|Google Chrome", "navigate|Google Chrome|https://erp.example.com/invoices/*", - "click|Google Chrome|textfield|87961bb3", + "click|Google Chrome|textfield|label-a", "paste|Google Chrome", - "click|Google Chrome|textfield|64e7c628", - "type|Google Chrome|textfield|64e7c628", - "click|Google Chrome|button|57d88c0a", + "click|Google Chrome|textfield|label-b", + "type|Google Chrome|textfield|label-b", + "click|Google Chrome|button|label-c", "submit|Google Chrome|https://erp.example.com/invoices/*" ], "window_s": 86400.0 diff --git a/packages/flow/tests/fixtures/shadow/report_mail.episodes.json b/packages/flow/tests/fixtures/shadow/report_mail.episodes.json index a7c189a02..16402382a 100644 --- a/packages/flow/tests/fixtures/shadow/report_mail.episodes.json +++ b/packages/flow/tests/fixtures/shadow/report_mail.episodes.json @@ -14,14 +14,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 5000238.255011055, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -30,7 +30,7 @@ "display_label": null, "key": "s", "kind": "shortcut", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -39,7 +39,7 @@ "t": 5000239.255011055, "token": "shortcut|Microsoft Excel|meta+s", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -48,14 +48,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 5000240.255011055, "token": "focus|Mail", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -64,14 +64,14 @@ "display_label": "New Message", "key": null, "kind": "click", - "label_digest": "af4c01884ec9c100", + "label_ref": "label-a", "modifiers": [], "role": "button", "secure": false, "t": 5000241.255011055, - "token": "click|Mail|button|af4c0188", + "token": "click|Mail|button|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -80,14 +80,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "1bbd9c9d99169893", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 5000242.255011055, - "token": "click|Mail|textfield|1bbd9c9d", + "token": "click|Mail|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -96,14 +96,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "1bbd9c9d99169893", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 5000243.255011055, - "token": "type|Mail|textfield|1bbd9c9d", + "token": "type|Mail|textfield|label-b", "url_pattern": null, - "value_digest": "10b3aa0a585b2a08", + "value_ref": "value-a", "value_shape": "email", "window_class": null }, @@ -112,14 +112,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "2bdb797b79f1cd8e", + "label_ref": "label-c", "modifiers": [], "role": "textfield", "secure": false, "t": 5000244.255011055, - "token": "click|Mail|textfield|2bdb797b", + "token": "click|Mail|textfield|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -128,14 +128,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "2bdb797b79f1cd8e", + "label_ref": "label-c", "modifiers": [], "role": "textfield", "secure": false, "t": 5000245.255011055, - "token": "type|Mail|textfield|2bdb797b", + "token": "type|Mail|textfield|label-c", "url_pattern": null, - "value_digest": "6ad789b668757434", + "value_ref": "value-b", "value_shape": "alnum", "window_class": null }, @@ -144,14 +144,14 @@ "display_label": "Attach", "key": null, "kind": "click", - "label_digest": "aed1127ebe1853e9", + "label_ref": "label-d", "modifiers": [], "role": "button", "secure": false, "t": 5000246.255011055, - "token": "click|Mail|button|aed1127e", + "token": "click|Mail|button|label-d", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -160,14 +160,14 @@ "display_label": "Send", "key": null, "kind": "click", - "label_digest": "2dc861cd16dafe9c", + "label_ref": "label-e", "modifiers": [], "role": "button", "secure": false, "t": 5000247.255011055, - "token": "click|Mail|button|2dc861cd", + "token": "click|Mail|button|label-e", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -184,14 +184,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 5000558.013902665, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -200,7 +200,7 @@ "display_label": null, "key": "s", "kind": "shortcut", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -209,7 +209,7 @@ "t": 5000559.013902665, "token": "shortcut|Microsoft Excel|meta+s", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -218,14 +218,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 5000560.013902665, "token": "focus|Mail", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -234,14 +234,14 @@ "display_label": "New Message", "key": null, "kind": "click", - "label_digest": "af4c01884ec9c100", + "label_ref": "label-a", "modifiers": [], "role": "button", "secure": false, "t": 5000561.013902665, - "token": "click|Mail|button|af4c0188", + "token": "click|Mail|button|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -250,14 +250,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "1bbd9c9d99169893", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 5000562.013902665, - "token": "click|Mail|textfield|1bbd9c9d", + "token": "click|Mail|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -266,14 +266,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "1bbd9c9d99169893", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 5000563.013902665, - "token": "type|Mail|textfield|1bbd9c9d", + "token": "type|Mail|textfield|label-b", "url_pattern": null, - "value_digest": "5c9127c24407f866", + "value_ref": "value-c", "value_shape": "email", "window_class": null }, @@ -282,14 +282,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "2bdb797b79f1cd8e", + "label_ref": "label-c", "modifiers": [], "role": "textfield", "secure": false, "t": 5000564.013902665, - "token": "click|Mail|textfield|2bdb797b", + "token": "click|Mail|textfield|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -298,14 +298,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "2bdb797b79f1cd8e", + "label_ref": "label-c", "modifiers": [], "role": "textfield", "secure": false, "t": 5000565.013902665, - "token": "type|Mail|textfield|2bdb797b", + "token": "type|Mail|textfield|label-c", "url_pattern": null, - "value_digest": "5a5f2f37cc4df1b7", + "value_ref": "value-d", "value_shape": "alnum", "window_class": null }, @@ -314,14 +314,14 @@ "display_label": "Attach", "key": null, "kind": "click", - "label_digest": "aed1127ebe1853e9", + "label_ref": "label-d", "modifiers": [], "role": "button", "secure": false, "t": 5000566.013902665, - "token": "click|Mail|button|aed1127e", + "token": "click|Mail|button|label-d", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -330,14 +330,14 @@ "display_label": "Send", "key": null, "kind": "click", - "label_digest": "2dc861cd16dafe9c", + "label_ref": "label-e", "modifiers": [], "role": "button", "secure": false, "t": 5000567.013902665, - "token": "click|Mail|button|2dc861cd", + "token": "click|Mail|button|label-e", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -354,14 +354,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 5000881.901317636, "token": "focus|Microsoft Excel", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -370,7 +370,7 @@ "display_label": null, "key": "s", "kind": "shortcut", - "label_digest": null, + "label_ref": null, "modifiers": [ "meta" ], @@ -379,7 +379,7 @@ "t": 5000882.901317636, "token": "shortcut|Microsoft Excel|meta+s", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -388,14 +388,14 @@ "display_label": null, "key": null, "kind": "focus", - "label_digest": null, + "label_ref": null, "modifiers": [], "role": null, "secure": false, "t": 5000883.901317636, "token": "focus|Mail", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -404,14 +404,14 @@ "display_label": "New Message", "key": null, "kind": "click", - "label_digest": "af4c01884ec9c100", + "label_ref": "label-a", "modifiers": [], "role": "button", "secure": false, "t": 5000884.901317636, - "token": "click|Mail|button|af4c0188", + "token": "click|Mail|button|label-a", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -420,14 +420,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "1bbd9c9d99169893", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 5000885.901317636, - "token": "click|Mail|textfield|1bbd9c9d", + "token": "click|Mail|textfield|label-b", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -436,14 +436,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "1bbd9c9d99169893", + "label_ref": "label-b", "modifiers": [], "role": "textfield", "secure": false, "t": 5000886.901317636, - "token": "type|Mail|textfield|1bbd9c9d", + "token": "type|Mail|textfield|label-b", "url_pattern": null, - "value_digest": "effc4a77c7dea6a6", + "value_ref": "value-e", "value_shape": "email", "window_class": null }, @@ -452,14 +452,14 @@ "display_label": null, "key": null, "kind": "click", - "label_digest": "2bdb797b79f1cd8e", + "label_ref": "label-c", "modifiers": [], "role": "textfield", "secure": false, "t": 5000887.901317636, - "token": "click|Mail|textfield|2bdb797b", + "token": "click|Mail|textfield|label-c", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -468,14 +468,14 @@ "display_label": null, "key": null, "kind": "type", - "label_digest": "2bdb797b79f1cd8e", + "label_ref": "label-c", "modifiers": [], "role": "textfield", "secure": false, "t": 5000888.901317636, - "token": "type|Mail|textfield|2bdb797b", + "token": "type|Mail|textfield|label-c", "url_pattern": null, - "value_digest": "fbdea0f3782fde1b", + "value_ref": "value-f", "value_shape": "alnum", "window_class": null }, @@ -484,14 +484,14 @@ "display_label": "Attach", "key": null, "kind": "click", - "label_digest": "aed1127ebe1853e9", + "label_ref": "label-d", "modifiers": [], "role": "button", "secure": false, "t": 5000889.901317636, - "token": "click|Mail|button|aed1127e", + "token": "click|Mail|button|label-d", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null }, @@ -500,14 +500,14 @@ "display_label": "Send", "key": null, "kind": "click", - "label_digest": "2dc861cd16dafe9c", + "label_ref": "label-e", "modifiers": [], "role": "button", "secure": false, "t": 5000890.901317636, - "token": "click|Mail|button|2dc861cd", + "token": "click|Mail|button|label-e", "url_pattern": null, - "value_digest": null, + "value_ref": null, "value_shape": null, "window_class": null } @@ -518,30 +518,31 @@ ], "exact_count": 3, "expected_by_chance": 5.1827993358769765e-15, - "motif_id": "661fe3dd51f1ed89", + "motif_id": "56a7933b9d8c5256", "p_value": 2.3202882082950603e-44, "privacy": { "clipboard_contents": "never read; copy and paste come from the key chord", - "control_labels": "keyed digest; short chrome labels in plain text when enabled", + "control_labels": "short chrome labels of common words in plain text; others a placeholder", + "keyed_digests": "never exported; replaced by references local to this export", "network": "none", "screenshots": "never taken", - "typed_values": "keyed digest only; secure fields dropped", - "urls": "scheme, host, and at most one path word", + "typed_values": "never exported; secure fields dropped", + "urls": "work domains only: host and one scrubbed path word; other sites a marker", "window_titles": "never read" }, - "schema": "openadapt.shadow.episodes/v1", + "schema": "openadapt.shadow.episodes/v2", "status": "candidate", "tokens": [ "focus|Microsoft Excel", "shortcut|Microsoft Excel|meta+s", "focus|Mail", - "click|Mail|button|af4c0188", - "click|Mail|textfield|1bbd9c9d", - "type|Mail|textfield|1bbd9c9d", - "click|Mail|textfield|2bdb797b", - "type|Mail|textfield|2bdb797b", - "click|Mail|button|aed1127e", - "click|Mail|button|2dc861cd" + "click|Mail|button|label-a", + "click|Mail|textfield|label-b", + "type|Mail|textfield|label-b", + "click|Mail|textfield|label-c", + "type|Mail|textfield|label-c", + "click|Mail|button|label-d", + "click|Mail|button|label-e" ], "window_s": 86400.0 } diff --git a/packages/flow/tests/test_motif_induction.py b/packages/flow/tests/test_motif_induction.py index 1526b696a..0000b5e6c 100644 --- a/packages/flow/tests/test_motif_induction.py +++ b/packages/flow/tests/test_motif_induction.py @@ -1,7 +1,7 @@ """Shadow-mined episodes -> the existing induction -> a CANDIDATE, never a run. The fixtures under ``tests/fixtures/shadow/`` are real -``openadapt.shadow.episodes/v1`` exports, produced by openadapt-capture's +``openadapt.shadow.episodes/v2`` exports, produced by openadapt-capture's shadow miner from synthetic streams (``tests/shadow_support/generate_flow_fixtures.py`` in that repository). """ @@ -66,7 +66,7 @@ def test_fixed_control_task_induces_a_candidate_draft_with_parameters(): assert draft is not None and draft.program is not None assert draft.name.startswith("shadow-draft-") assert len(draft.param_specs) == 2 - # Parameter examples are digests, never values. + # Parameter examples are export-local references, never values or digests. assert all(v.startswith("<") and ":" in v for v in draft.params.values()) @@ -145,7 +145,10 @@ def test_candidate_cannot_claim_admission(): [ (lambda s: s.update(text="jane.doe@example.com"), "unknown field"), (lambda s: s.update(display_label="Pay 1200"), "digits in a plain label"), - (lambda s: s.update(value_digest="not-a-digest"), "malformed digest"), + (lambda s: s.update(value_ref="not-a-ref"), "malformed reference"), + (lambda s: s.update(value_digest="0123456789abcdef"), "a keyed digest field"), + (lambda s: s.update(label_digest="0123456789abcdef"), "a keyed digest field"), + (lambda s: s.update(token="click|Mail|button|0a1b2c3d"), "a digest in a token"), (lambda s: s.update(kind="screenshot"), "unknown kind"), ], ) @@ -159,11 +162,30 @@ def test_export_schema_refuses_raw_content(mutate, why): def test_export_schema_refuses_wrong_schema_and_status(): data = _load("report_mail") with pytest.raises(ValidationError): - induce_candidate({**data, "schema": "openadapt.shadow.episodes/v2"}) + induce_candidate({**data, "schema": "openadapt.shadow.episodes/v3"}) with pytest.raises(ValidationError): induce_candidate({**data, "status": "admitted"}) +def test_v1_exports_with_keyed_digests_are_refused(): + data = _load("report_mail") + with pytest.raises(ValueError, match="keyed digests"): + induce_candidate({**data, "schema": "openadapt.shadow.episodes/v1"}) + with pytest.raises(ValidationError): + induce_candidate({**data, "tokens": ["focus|Mail", "click|Mail|button|0a1b2c3d"]}) + + +@pytest.mark.parametrize("name", ["report_mail", "invoice_exact", "invoice_variant"]) +def test_draft_carries_no_digest(name): + import re + + candidate = induce_candidate(_load(name)) + blob = candidate.model_dump_json() + # No 8+ character hex run except the export's own id. + runs = set(re.findall(r"(? Date: Sat, 17 Oct 2026 19:48:48 -0400 Subject: [PATCH 04/12] style(compiler): format motif induction under the monorepo's ruff format check Signed-off-by: Richard Abrich --- packages/flow/openadapt_flow/compiler/motif_induction.py | 2 +- packages/flow/tests/test_motif_induction.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/flow/openadapt_flow/compiler/motif_induction.py b/packages/flow/openadapt_flow/compiler/motif_induction.py index 3d4cb6f6d..5a848c670 100644 --- a/packages/flow/openadapt_flow/compiler/motif_induction.py +++ b/packages/flow/openadapt_flow/compiler/motif_induction.py @@ -110,7 +110,6 @@ class ShadowStep(BaseModel): modifiers: list[str] = Field(default_factory=list, max_length=4) secure: bool = False - @field_validator("token") @classmethod def _no_digest_in_token(cls, value: str) -> str: @@ -148,6 +147,7 @@ class ShadowEpisodesExport(BaseModel): @classmethod def _tokens_carry_no_digest(cls, value: list[str]) -> list[str]: return _check_tokens(value) + apps: list[str] count_in_window: int = Field(ge=0) window_s: float diff --git a/packages/flow/tests/test_motif_induction.py b/packages/flow/tests/test_motif_induction.py index 0000b5e6c..ed758822e 100644 --- a/packages/flow/tests/test_motif_induction.py +++ b/packages/flow/tests/test_motif_induction.py @@ -172,7 +172,9 @@ def test_v1_exports_with_keyed_digests_are_refused(): with pytest.raises(ValueError, match="keyed digests"): induce_candidate({**data, "schema": "openadapt.shadow.episodes/v1"}) with pytest.raises(ValidationError): - induce_candidate({**data, "tokens": ["focus|Mail", "click|Mail|button|0a1b2c3d"]}) + induce_candidate( + {**data, "tokens": ["focus|Mail", "click|Mail|button|0a1b2c3d"]} + ) @pytest.mark.parametrize("name", ["report_mail", "invoice_exact", "invoice_variant"]) From b782a61669c2d4be23536bb12b325e28dad5d187 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 17 Oct 2026 19:49:27 -0400 Subject: [PATCH 05/12] chore(compiler): list the shadow episode fixtures in public-artifacts.json Regenerated with check_release_consistency.py --write-public-artifact-inventory on a clean export. Three entries, all under tests/fixtures/shadow. Signed-off-by: Richard Abrich --- packages/flow/public-artifacts.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 0493b0091..666866c64 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -6215,6 +6215,18 @@ "path": "tests/fixtures/runtime-validation-canonical-vector.json", "sha256": "a238b23846705f081603c7fe8a66ee16a88281ad21acfd0d047052716a49b912" }, + { + "path": "tests/fixtures/shadow/invoice_exact.episodes.json", + "sha256": "e555e3913c02df609cdf490872fad30574a1c438b2c110b0ff3345e1a31c26ce" + }, + { + "path": "tests/fixtures/shadow/invoice_variant.episodes.json", + "sha256": "cc2d9508eac86e6115485025525e7529b4830f720eb28c21e258622e7a70ec79" + }, + { + "path": "tests/fixtures/shadow/report_mail.episodes.json", + "sha256": "4e3f1fed5b5bfba9220fd37d53eff1bea2655df4903fb6a59ec215bc3e2d2ec0" + }, { "path": "tests/fixtures/terminal_verification_v2_delivery_consumer_vector.json", "sha256": "f2e32472ad562b5b67d95d3d9cafaf08b4a2adb9fb5ff3cda2813307beb61c96" From 6ccee661811489499ad8495bd2e72bd2c90d4c59 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 17 Oct 2026 21:24:59 -0400 Subject: [PATCH 06/12] feat(admit): one command from recordings or a shadow candidate to accepted checks `openadapt-flow admit ` (or `--candidate ID`) compiles each recording, induces one program from two or more, proposes the qualification checks, and stops once with a short summary. On accept it applies the checks through accept_proposal, which re-runs the oracle gate, and reseals the bundle. `admit decide DIR` decides later at a terminal. The decision record names who prepared and who decided (kind, id, route). An agent can't decide an admission it prepared, the regulated pack needs a person, and a decision binds the exact summary digest and bundle digest. Induction refusals, HALTed proposals and missing recordings stop with nothing to accept. Shadow-suggested checks are shown but never applied, and no admission is signed. Signed-off-by: Richard Abrich --- packages/flow/openadapt_flow/__main__.py | 3 + .../flow/openadapt_flow/admit_pipeline.py | 521 ++++++++++++++++++ packages/flow/openadapt_flow/cli_admit.py | 200 ++++++- .../compiler/motif_induction.py | 7 +- packages/flow/tests/test_admit_pipeline.py | 312 +++++++++++ 5 files changed, 1037 insertions(+), 6 deletions(-) create mode 100644 packages/flow/openadapt_flow/admit_pipeline.py create mode 100644 packages/flow/tests/test_admit_pipeline.py diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 34cca1e34..8a43ba42a 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -10191,6 +10191,9 @@ def main(argv: Sequence[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) argv = _qualify_default_form(argv) + from openadapt_flow.cli_admit import admit_default_form + + argv = admit_default_form(argv) args = build_parser().parse_args(argv) try: return args.func(args) diff --git a/packages/flow/openadapt_flow/admit_pipeline.py b/packages/flow/openadapt_flow/admit_pipeline.py new file mode 100644 index 000000000..304471fd9 --- /dev/null +++ b/packages/flow/openadapt_flow/admit_pipeline.py @@ -0,0 +1,521 @@ +"""One admission command: compile, induce, propose checks, stop once, qualify. + +``openadapt-flow admit `` (or ``--candidate ID`` for a +shadow-mined candidate) chains what used to be four manual steps: + +1. compile each recording (:func:`~openadapt_flow.compiler.compile_recording`); +2. induce one program over them when there are two or more + (:func:`~openadapt_flow.compiler.induction.induce_program`); +3. propose the qualification checks from the compiled demo + (:func:`~openadapt_flow.qualification_proposal.propose_qualification`); +4. stop **once**, with a short summary, for a decision; on accept, apply the + checks through :func:`~openadapt_flow.qualification_proposal.accept_proposal`, + which re-runs the oracle gate, and reseal the bundle. + +The decision can come from a person at a terminal, or from a delegated agent +approver over MCP (``openadapt-agent serve --delegate-approvals``). Either +way the decision record names who decided, by kind, id and route. + +What it never does: + +* weaken a gate. Induction that refuses, a proposal that HALTs, and an oracle + gate that fails all stop the command with nothing to accept. Accepting runs + the same ``accept_proposal`` as ``qualify accept``; certification still needs + ``qualify campaign`` to pass. +* let the preparer decide as an agent. An agent can't accept an admission it + prepared, an agent can't decide under the ``regulated`` policy pack, and a + decision binds the exact summary (``pending_sha256``) and bundle digest it + was shown. +* treat shadow-suggested checks as evidence. They are listed for the person + or approver to read; only checks mined from real recordings are applied. +* sign an admission. No local-dev or production admission is minted here. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal, Optional, Sequence + +from pydantic import BaseModel, ConfigDict, Field + +PENDING_FILE = "admission-pending.json" +DECISION_FILE = "admission-decision.json" +PROPOSAL_FILE = "qualification-proposal.draft.json" +BUNDLE_DIR = "bundle" +COMPILED_DIR = "compiled" +CANDIDATE_RECORDINGS_DIR = "recordings" +PENDING_SCHEMA: Literal["openadapt.admission-pending/v1"] = ( + "openadapt.admission-pending/v1" +) +DECISION_SCHEMA: Literal["openadapt.admission-decision/v1"] = ( + "openadapt.admission-decision/v1" +) +#: Opaque decider ids, the same shape Flow accepts for a delegated approver. +DECIDER_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$" +#: Policy packs under which only a person may decide. +PERSON_ONLY_PACKS = frozenset({"regulated"}) + +PendingStatus = Literal["needs_decision", "needs_recording", "halted"] + + +class AdmissionError(ValueError): + """The admission can't proceed; nothing was accepted.""" + + +class Decider(BaseModel): + """Who prepared or decided an admission.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: Literal["person", "agent"] + id: str = Field(pattern=DECIDER_ID_PATTERN) + route: Literal["cli", "mcp"] + + def label(self) -> str: + return f"{self.kind}:{self.id} via {self.route}" + + +class PendingAdmission(BaseModel): + """What the decider is shown, written before the single stop.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["openadapt.admission-pending/v1"] = PENDING_SCHEMA + admission_id: str = Field(pattern=r"^[0-9a-f]{32}$") + status: PendingStatus + halt_reason: Optional[str] = None + workflow_name: str + policy_pack: str + prepared_by: Decider + created_at: str + recordings: int = 0 + induced: bool = False + candidate_motif_id: Optional[str] = None + n_steps: int = 0 + n_writes: int = 0 + pins: list[dict[str, str]] = Field(default_factory=list) + checks: list[dict[str, Any]] = Field(default_factory=list) + failure_cases: int = 0 + shadow_suggestions: list[dict[str, Any]] = Field(default_factory=list) + record_next: list[dict[str, Any]] = Field(default_factory=list) + bundle_content_digest: Optional[str] = None + proposal_sha256: Optional[str] = None + + def sha256(self) -> str: + return _sha(self.model_dump(mode="json")) + + +class AdmissionDecision(BaseModel): + """The decision record. Names who decided, and binds what they saw.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["openadapt.admission-decision/v1"] = DECISION_SCHEMA + admission_id: str + decision: Literal["accept", "reject"] + reason: Optional[str] = None + decided_by: Decider + prepared_by: Decider + decided_at: str + pending_sha256: str + bundle_content_digest: Optional[str] + proposal_sha256: Optional[str] + accepted_bundle_content_digest: Optional[str] = None + oracle_gate_passed: Optional[bool] = None + next: list[str] = Field(default_factory=list) + + +def _sha(payload: Any) -> str: + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _bundle_digest(bundle: Path) -> str: + from openadapt_flow.bundle_validation import ( + compute_content_digest, + compute_file_hashes, + ) + from openadapt_flow.ir import Workflow + + workflow = Workflow.load(bundle) + return compute_content_digest(workflow, compute_file_hashes(workflow, bundle)) + + +def _is_bundle(path: Path) -> bool: + return (path / "workflow.json").is_file() or (path / "workflow.json.enc").is_file() + + +def resolve_candidate(candidate: str, root: Path | str) -> Path: + """A candidate id (``motif_id``) under ``root``, or a candidate folder.""" + + from openadapt_flow.compiler.motif_induction import CANDIDATE_FILENAME + + direct = Path(candidate) + if (direct / CANDIDATE_FILENAME).is_file(): + return direct + if not re.match(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$", candidate): + raise AdmissionError("a candidate id is one folder name under the root") + folder = Path(root) / candidate + if not (folder / CANDIDATE_FILENAME).is_file(): + raise AdmissionError(f"no shadow candidate {candidate!r} under {root}") + return folder + + +def _load_candidate(folder: Path) -> dict[str, Any]: + from openadapt_flow.compiler.motif_induction import CANDIDATE_FILENAME + + data = json.loads((folder / CANDIDATE_FILENAME).read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("admitted") is not False: + raise AdmissionError("candidate.json is not an unadmitted shadow candidate") + return data + + +def _checks_from_proposal(proposal: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for raw in proposal.pin("effect").get("payload", {}).get("effects") or []: + out.append( + { + "step_id": raw.get("step_id"), + "kind": raw.get("kind"), + "tier": raw.get("tier"), + } + ) + for raw in proposal.pin("identity").get("payload", {}).get("policies") or []: + out.append( + { + "step_id": raw.get("step_id"), + "kind": f"identity:{raw.get('enforcement')}", + } + ) + return out + + +def prepare_admission( + inputs: Sequence[Path | str], + *, + out: Path | str, + name: str, + prepared_by: Decider, + candidate: Optional[Path | str] = None, + policy_pack: str = "community", + admission_id: Optional[str] = None, +) -> PendingAdmission: + """Run compile, induce and the check proposal; write the single stop. + + ``inputs`` are recording directories, or already-compiled bundles. With + ``candidate`` and no inputs, the recordings are read from the candidate's + ``recordings/`` folder; with none there, the result is + ``needs_recording`` and carries the candidate's record-next list. + """ + + from openadapt_flow.compiler import compile_recording + from openadapt_flow.compiler.induction import induce_program + from openadapt_flow.ir import Workflow + from openadapt_flow.qualification_proposal import ( + propose_qualification, + write_proposal, + ) + from openadapt_flow.traversal import iter_workflow_steps + + target = Path(out) + if (target / PENDING_FILE).exists() or (target / DECISION_FILE).exists(): + raise AdmissionError(f"{target} already holds an admission; use a new --out") + target.mkdir(parents=True, exist_ok=True) + + paths = [Path(item) for item in inputs] + motif_id: Optional[str] = None + suggestions: list[dict[str, Any]] = [] + candidate_next: list[dict[str, Any]] = [] + if candidate is not None: + folder = Path(candidate) + data = _load_candidate(folder) + motif_id = str(data.get("motif_id") or folder.name) + suggestions = [ + { + "kind": item.get("kind"), + "summary": item.get("summary"), + "step_id": item.get("step_id"), + "note": "suggested by shadow episodes; not evidence", + } + for item in data.get("proposed_checks") or [] + ] + candidate_next = list(data.get("record_next") or []) + if not paths: + rec_root = folder / CANDIDATE_RECORDINGS_DIR + if rec_root.is_dir(): + paths = sorted(p for p in rec_root.iterdir() if p.is_dir()) + + base: dict[str, Any] = { + "admission_id": admission_id or uuid.uuid4().hex, + "workflow_name": name, + "policy_pack": policy_pack, + "prepared_by": prepared_by, + "created_at": _now(), + "candidate_motif_id": motif_id, + "shadow_suggestions": suggestions, + } + + def stop(status: PendingStatus, reason: str, **extra: Any) -> PendingAdmission: + pending = PendingAdmission(status=status, halt_reason=reason, **base, **extra) + _write_json(target / PENDING_FILE, pending.model_dump(mode="json")) + return pending + + if not paths: + return stop( + "needs_recording", + "no recording yet: record this task once with OpenAdapt Capture", + record_next=candidate_next + or [{"code": "record", "instruction": "Record the task once."}], + ) + + compiled: list[Path] = [] + recording_meta: Optional[Path] = None + for index, path in enumerate(paths): + if not path.is_dir(): + raise AdmissionError(f"{path} is not a directory") + if recording_meta is None and (path / "meta.json").is_file(): + recording_meta = path + if _is_bundle(path): + compiled.append(path) + continue + bundle = target / COMPILED_DIR / f"{index:02d}" + compile_recording(path, bundle, name=name) + compiled.append(bundle) + + bundle_dir = target / BUNDLE_DIR + induced = False + if len(compiled) == 1: + workflow = Workflow.load(compiled[0]) + else: + result = induce_program([Workflow.load(p) for p in compiled]) + if not result.certified or result.workflow is None: + return stop( + "halted", + "induction refused: the recordings don't pin down one program", + recordings=len(compiled), + record_next=[ + item.model_dump(mode="json") for item in result.record_next + ], + ) + workflow = result.workflow + induced = True + workflow = workflow.model_copy(update={"name": name}) + _copy_templates(compiled, bundle_dir) + workflow.save(bundle_dir) + workflow = Workflow.load(bundle_dir) + + proposal = propose_qualification( + workflow, recording_dir=recording_meta, policy_pack=policy_pack + ) + write_proposal(proposal, target / PROPOSAL_FILE) + steps = list(iter_workflow_steps(workflow)) + detail: dict[str, Any] = { + "recordings": len(compiled), + "induced": induced, + "n_steps": len(steps), + "n_writes": sum(1 for step in steps if step.risk == "irreversible"), + "pins": [ + {"kind": str(pin.get("kind")), "status": str(pin.get("status"))} + for pin in proposal.pins + ], + "checks": _checks_from_proposal(proposal), + "failure_cases": len(proposal.failure_matrix), + "bundle_content_digest": _bundle_digest(bundle_dir), + "proposal_sha256": proposal.sha256(), + } + if proposal.status == "halted": + return stop( + "halted", + proposal.halt_reason or "qualification proposal HALTed", + **detail, + ) + pending = PendingAdmission(status="needs_decision", **base, **detail) + _write_json(target / PENDING_FILE, pending.model_dump(mode="json")) + return pending + + +def _copy_templates(compiled: Sequence[Path], bundle_dir: Path) -> None: + """Carry each compiled bundle's crops; the first bundle wins a name clash.""" + + import shutil + + for source in compiled: + templates = source / "templates" + if not templates.is_dir(): + continue + for item in sorted(templates.rglob("*")): + if not item.is_file(): + continue + dest = bundle_dir / "templates" / item.relative_to(templates) + if dest.exists(): + continue + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(item, dest) + + +def load_pending(out: Path | str) -> PendingAdmission: + path = Path(out) / PENDING_FILE + if not path.is_file(): + raise AdmissionError(f"no pending admission in {out}") + return PendingAdmission.model_validate_json(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", "utf-8") + + +def check_decider(pending: PendingAdmission, decided_by: Decider) -> None: + """Refuse self-approval and agent decisions where only a person may decide.""" + + if decided_by.kind != "agent": + return + if pending.policy_pack in PERSON_ONLY_PACKS: + raise AdmissionError( + f"the {pending.policy_pack!r} policy pack needs a person to decide" + ) + prepared = pending.prepared_by + if prepared.kind == "agent" and prepared.id == decided_by.id: + raise AdmissionError( + "self-approval refused: the agent that prepared this admission " + "can't decide it" + ) + + +def decide_admission( + out: Path | str, + *, + pending_sha256: str, + decision: Literal["accept", "reject"], + decided_by: Decider, + reason: Optional[str] = None, +) -> AdmissionDecision: + """Record one decision on the pending admission, and apply it on accept.""" + + from openadapt_flow.ir import Workflow + from openadapt_flow.qualification_proposal import ( + QualificationProposalError, + accept_proposal, + load_proposal, + save_accepted_bundle, + ) + + target = Path(out) + pending = load_pending(target) + if (target / DECISION_FILE).exists(): + raise AdmissionError("this admission was already decided") + if pending.sha256() != pending_sha256: + raise AdmissionError( + "the pending admission changed since it was shown; show it again" + ) + if pending.status != "needs_decision": + raise AdmissionError( + f"nothing to decide: the admission is {pending.status}" + + (f" ({pending.halt_reason})" if pending.halt_reason else "") + ) + check_decider(pending, decided_by) + if decision not in ("accept", "reject"): + raise AdmissionError("decision must be accept or reject") + if decision == "reject" and not reason: + raise AdmissionError("a rejection needs a reason") + bundle = target / BUNDLE_DIR + from openadapt_flow.bundle_validation import BundleIntegrityError + + try: + current = _bundle_digest(bundle) + except (OSError, ValueError, BundleIntegrityError) as exc: + raise AdmissionError( + f"the bundle changed since the summary was written: {exc}" + ) from exc + if current != pending.bundle_content_digest: + raise AdmissionError("the bundle changed since the summary was written") + + record: dict[str, Any] = { + "admission_id": pending.admission_id, + "decision": decision, + "reason": reason, + "decided_by": decided_by, + "prepared_by": pending.prepared_by, + "decided_at": _now(), + "pending_sha256": pending_sha256, + "bundle_content_digest": pending.bundle_content_digest, + "proposal_sha256": pending.proposal_sha256, + } + if decision == "accept": + proposal = load_proposal(target / PROPOSAL_FILE) + if proposal.sha256() != pending.proposal_sha256: + raise AdmissionError("the check proposal changed since it was shown") + workflow = Workflow.load(bundle) + try: + accepted = accept_proposal(workflow, proposal) + except QualificationProposalError as exc: + raise AdmissionError(str(exc)) from exc + save_accepted_bundle(workflow, bundle, proposal=accepted) + gate = accepted.oracle_gate or {} + record.update( + accepted_bundle_content_digest=_bundle_digest(bundle), + oracle_gate_passed=bool(gate.get("passed")), + next=[ + "qualify campaign: run the qualification cases before this " + "program can be certified" + ], + ) + result = AdmissionDecision(**record) + payload = result.model_dump(mode="json") + _write_json(target / DECISION_FILE, payload) + if decision == "accept": + _write_json(bundle / DECISION_FILE, payload) + return result + + +def render_summary(pending: PendingAdmission) -> str: + """The short summary shown at the single stop.""" + + lines = [f"Admission {pending.admission_id[:12]}: {pending.workflow_name}"] + if pending.candidate_motif_id: + lines.append(f" from shadow candidate {pending.candidate_motif_id}") + lines.append( + f" {pending.recordings} recording(s)" + + (", induced into one program" if pending.induced else "") + + f"; {pending.n_steps} step(s), {pending.n_writes} write(s)" + ) + if pending.status != "needs_decision": + lines.append(f" STOPPED ({pending.status}): {pending.halt_reason}") + for item in pending.record_next: + lines.append(f" - record next: {item.get('instruction')}") + return "\n".join(lines) + "\n" + lines.append(" Proposed checks (applied on accept):") + for check in pending.checks: + tier = f", tier {check['tier']}" if check.get("tier") is not None else "" + lines.append(f" - {check.get('step_id')}: {check.get('kind')}{tier}") + lines.append(f" {pending.failure_cases} fault case(s) to qualify against") + if pending.shadow_suggestions: + lines.append(" Suggested by shadow episodes (not evidence, not applied):") + for item in pending.shadow_suggestions: + lines.append(f" - {item.get('kind')}: {item.get('summary')}") + lines.append(f" Prepared by {pending.prepared_by.label()}") + return "\n".join(lines) + "\n" + + +__all__ = [ + "AdmissionDecision", + "AdmissionError", + "Decider", + "PendingAdmission", + "check_decider", + "decide_admission", + "load_pending", + "prepare_admission", + "render_summary", + "resolve_candidate", +] diff --git a/packages/flow/openadapt_flow/cli_admit.py b/packages/flow/openadapt_flow/cli_admit.py index 504c9b377..9f7e9e1bc 100644 --- a/packages/flow/openadapt_flow/cli_admit.py +++ b/packages/flow/openadapt_flow/cli_admit.py @@ -1,11 +1,27 @@ -"""CLI: ``openadapt-flow admit status``.""" +"""CLI: ``openadapt-flow admit``. + +- ``admit `` (or ``admit --candidate ID``): compile, induce, + propose the checks, stop once with a short summary, and on ``accept`` apply + the checks and reseal the bundle. Same as ``admit run``. +- ``admit decide DIR``: decide an admission that stopped without a terminal. +- ``admit status``: read the published release-admission ledger. + +A person decides at an interactive terminal. A delegated agent approver +decides over MCP (``openadapt-agent serve --delegate-approvals``); see +:mod:`openadapt_flow.admit_pipeline`. +""" from __future__ import annotations import argparse +import getpass import json +import re import sys +from pathlib import Path +from typing import Callable, Optional +from openadapt_flow.cli_hints import command from openadapt_flow.release_admission import ( ADMITTED, DEFAULT_LEDGER_URL, @@ -15,16 +31,78 @@ status_report, ) +#: Verbs of ``admit``; anything else after ``admit`` means ``admit run``. +ADMIT_VERBS = ("run", "decide", "status") +#: Exit code when the command stopped for a decision or a recording. +EXIT_STOPPED = 3 + + +def admit_default_form(argv: list[str]) -> list[str]: + """``admit `` and ``admit --candidate ID`` mean ``admit run``.""" + + if argv[:1] == ["admit"] and len(argv) >= 2: + first = argv[1] + if first not in ADMIT_VERBS and first not in ("-h", "--help"): + return ["admit", "run", *argv[1:]] + return argv + def register_admit_parser(sub: argparse._SubParsersAction) -> None: parser = sub.add_parser( "admit", help=( - "Read the published release-admission ledger. " - "Does not sign or issue an admission." + "Turn recordings (or a shadow candidate) into a program with checks: " + "compile, induce, propose checks, stop once for a decision. " + "`admit status` reads the release-admission ledger." ), ) verbs = parser.add_subparsers(dest="admit_cmd", required=True) + run = verbs.add_parser( + "run", + help=( + "Compile, induce, propose checks, and stop once for a person's " + "decision (the default form: `admit `)" + ), + ) + run.add_argument( + "recordings", + nargs="*", + help="Recording directories (or compiled bundles) of the same task", + ) + run.add_argument( + "--candidate", + default=None, + metavar="ID", + help=( + "A shadow candidate id under --candidates-root, or its folder. Its " + "recordings/ folder is used when no recordings are given" + ), + ) + run.add_argument( + "--candidates-root", + default="shadow-candidates", + help="Where shadow candidates are kept (default: ./shadow-candidates)", + ) + run.add_argument("--out", required=True, help="New admission directory") + run.add_argument("--name", required=True, help="Workflow name") + run.add_argument( + "--policy-pack", + choices=("community", "cloud", "regulated"), + default="community", + ) + run.add_argument( + "--by", + default=None, + help="Your name for the record (default: your login name)", + ) + run.set_defaults(func=cmd_admit_run) + decide = verbs.add_parser( + "decide", + help="Show a stopped admission's summary and decide it (a person, at a terminal)", + ) + decide.add_argument("admission", help="Admission directory from `admit`") + decide.add_argument("--by", default=None, help="Your name for the record") + decide.set_defaults(func=cmd_admit_decide) status = verbs.add_parser( "status", help=( @@ -68,3 +146,119 @@ def cmd_admit_status(args: argparse.Namespace) -> int: if args.check and report["flow"]["state"] != ADMITTED: return 1 return 0 + + +def _person(by: Optional[str]) -> "object": + from openadapt_flow.admit_pipeline import DECIDER_ID_PATTERN, Decider + + name = by or getpass.getuser() or "person" + name = re.sub(r"[^A-Za-z0-9._:@/-]", "-", name).lstrip("-._:@/") or "person" + if not re.match(DECIDER_ID_PATTERN, name): + raise SystemExit("admit: --by must be a short name") + return Decider(kind="person", id=name, route="cli") + + +def _interactive() -> bool: + return sys.stdin.isatty() and sys.stdout.isatty() + + +def _ask_once( + out: Path, + by: Optional[str], + *, + read: Callable[[str], str] = input, + interactive: Optional[bool] = None, +) -> int: + """The single stop: show the summary, take one decision from a person.""" + + from openadapt_flow.admit_pipeline import ( + AdmissionError, + decide_admission, + load_pending, + render_summary, + ) + + pending = load_pending(out) + sys.stdout.write(render_summary(pending)) + if pending.status != "needs_decision": + return EXIT_STOPPED if pending.status == "needs_recording" else 2 + if not (_interactive() if interactive is None else interactive): + print( + "Stopped for a decision. A person decides with " + f"`{command(f'admit decide {out}')}` at a terminal; a delegated " + "agent approver decides with the decide_admission MCP tool." + ) + return EXIT_STOPPED + answer = read("Type accept to apply these checks, or reject: ").strip().lower() + reason = None + if answer == "reject": + reason = read("Why? ").strip() or "rejected at the terminal" + elif answer != "accept": + print( + f"No decision recorded. Decide later with `{command(f'admit decide {out}')}`." + ) + return EXIT_STOPPED + try: + result = decide_admission( + out, + pending_sha256=pending.sha256(), + decision="accept" if answer == "accept" else "reject", + decided_by=_person(by), # type: ignore[arg-type] + reason=reason, + ) + except AdmissionError as exc: + print(f"admit: {exc}", file=sys.stderr) + return 2 + if result.decision == "reject": + print(f"Rejected by {result.decided_by.label()}. Nothing was applied.") + return 1 + print( + f"Accepted by {result.decided_by.label()}. Checks applied to {out / 'bundle'}." + ) + bundle = out / "bundle" + print(f"Next: `{command(f'qualify {bundle} --verify verify.yaml')}`") + return 0 + + +def cmd_admit_run(args: argparse.Namespace) -> int: + from openadapt_flow.admit_pipeline import ( + AdmissionError, + prepare_admission, + resolve_candidate, + ) + + if not args.recordings and not args.candidate: + raise SystemExit("admit: give recording directories or --candidate ID") + try: + candidate = ( + resolve_candidate(args.candidate, args.candidates_root) + if args.candidate + else None + ) + prepare_admission( + args.recordings, + out=args.out, + name=args.name, + prepared_by=_person(args.by), # type: ignore[arg-type] + candidate=candidate, + policy_pack=args.policy_pack, + ) + except AdmissionError as exc: + print(f"admit: {exc}", file=sys.stderr) + return 2 + return _ask_once(Path(args.out), args.by) + + +def cmd_admit_decide(args: argparse.Namespace) -> int: + from openadapt_flow.admit_pipeline import AdmissionError + + if not _interactive(): + raise SystemExit( + "admit decide: a person decides at an interactive terminal; a " + "delegated agent approver uses the decide_admission MCP tool" + ) + try: + return _ask_once(Path(args.admission), args.by, interactive=True) + except AdmissionError as exc: + print(f"admit: {exc}", file=sys.stderr) + return 2 diff --git a/packages/flow/openadapt_flow/compiler/motif_induction.py b/packages/flow/openadapt_flow/compiler/motif_induction.py index 5a848c670..8e76575f6 100644 --- a/packages/flow/openadapt_flow/compiler/motif_induction.py +++ b/packages/flow/openadapt_flow/compiler/motif_induction.py @@ -68,9 +68,10 @@ ADMISSION_PATH: tuple[str, ...] = ( "Record the task with OpenAdapt Capture (flow record or capture record).", - "Compile each recording (flow compile).", - "Induce over the recordings (flow induce), using this draft only as a guide.", - "Confirm or reject each proposed check, then qualify (flow qualify).", + "Run flow admit --candidate : it compiles, induces and proposes " + "checks, then stops once for a person (or a delegated agent approver) to " + "accept or reject them.", + "Qualify the accepted bundle (flow qualify).", ) diff --git a/packages/flow/tests/test_admit_pipeline.py b/packages/flow/tests/test_admit_pipeline.py new file mode 100644 index 000000000..aadc3cf74 --- /dev/null +++ b/packages/flow/tests/test_admit_pipeline.py @@ -0,0 +1,312 @@ +"""``openadapt-flow admit``: one command, one stop, a record of who decided. + +Made-up values only. The fixture is the tiny qualified-write workflow from +test_qualification_proposal: one irreversible save with a system-of-record +effect, so the check proposal is a draft rather than a HALT. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from openadapt_flow import admit_pipeline as ap +from openadapt_flow.__main__ import main +from openadapt_flow.cli_admit import EXIT_STOPPED, _ask_once, admit_default_form +from openadapt_flow.compiler.motif_induction import induce_candidate, write_candidate +from openadapt_flow.ir import Workflow +from tests.test_qualification_proposal import _png, _workflow + +SHADOW = Path(__file__).parent / "fixtures" / "shadow" +PERSON = ap.Decider(kind="person", id="pat", route="cli") +AGENT_A = ap.Decider(kind="agent", id="agent:intake-bot", route="mcp") +AGENT_B = ap.Decider(kind="agent", id="agent:reviewer", route="mcp") + + +def _recording(root: Path, name: str = "rec1", *, effects: bool = True) -> Path: + """A compiled bundle with its recording's meta.json beside it.""" + + folder = root / name + (folder / "templates").mkdir(parents=True) + (folder / "templates" / "save.png").write_bytes(_png()) + workflow = _workflow(effects=effects) + workflow.surface = "web" + workflow.save(folder) + (folder / "meta.json").write_text( + json.dumps( + { + "app_url": "http://127.0.0.1:9/", + "application": "MockMed", + "application_version": "tutorial", + "viewport": [1280, 800], + "surface": "web", + "params": dict(workflow.params), + } + ), + encoding="utf-8", + ) + return folder + + +def _prepare(tmp_path: Path, by: ap.Decider = PERSON, **kwargs) -> ap.PendingAdmission: + return ap.prepare_admission( + [_recording(tmp_path)], + out=tmp_path / "admission", + name="qualified-write", + prepared_by=by, + **kwargs, + ) + + +def test_one_recording_stops_once_with_the_proposed_checks(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + assert pending.status == "needs_decision" + assert pending.recordings == 1 and pending.induced is False + assert pending.n_writes == 1 + assert {"step_id": "save", "kind": "field_equals", "tier": 1} in pending.checks + assert pending.failure_cases >= 1 + assert [pin["status"] for pin in pending.pins] == ["proposed"] * 4 + summary = ap.render_summary(pending) + assert "Proposed checks" in summary and "save: field_equals" in summary + assert "Prepared by person:pat via cli" in summary + # Nothing is applied before the decision. + assert Workflow.load(tmp_path / "admission" / "bundle").qualification is None + assert not (tmp_path / "admission" / ap.DECISION_FILE).exists() + + +def test_person_accept_applies_checks_and_records_who_decided(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + result = ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + assert result.decided_by == PERSON and result.prepared_by == PERSON + assert result.oracle_gate_passed is True + assert result.accepted_bundle_content_digest != pending.bundle_content_digest + bundle = Workflow.load(tmp_path / "admission" / "bundle") + assert bundle.qualification is not None + assert "fault-break-it" in {case.id for case in bundle.qualification.cases} + record = json.loads( + (tmp_path / "admission" / "bundle" / ap.DECISION_FILE).read_text() + ) + assert record["decided_by"] == {"kind": "person", "id": "pat", "route": "cli"} + assert "qualify campaign" in record["next"][0] + # No admission is signed here. + assert not ( + tmp_path / "admission" / "bundle" / "qualification-admission.local-dev.json" + ).exists() + + +def test_an_agent_cannot_decide_what_it_prepared(tmp_path: Path) -> None: + pending = _prepare(tmp_path, by=AGENT_A) + with pytest.raises(ap.AdmissionError, match="self-approval"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_A, + ) + assert not (tmp_path / "admission" / ap.DECISION_FILE).exists() + result = ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + ) + assert result.decided_by == AGENT_B and result.prepared_by == AGENT_A + + +def test_regulated_pack_needs_a_person() -> None: + pending = ap.PendingAdmission( + admission_id="0" * 32, + status="needs_decision", + workflow_name="w", + policy_pack="regulated", + prepared_by=PERSON, + created_at="2026-10-17T00:00:00+00:00", + ) + with pytest.raises(ap.AdmissionError, match="needs a person"): + ap.check_decider(pending, AGENT_B) + ap.check_decider(pending, PERSON) + + +def test_decision_binds_the_summary_and_the_bundle(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + out = tmp_path / "admission" + with pytest.raises(ap.AdmissionError, match="changed since it was shown"): + ap.decide_admission( + out, pending_sha256="0" * 64, decision="accept", decided_by=PERSON + ) + (out / "bundle" / "templates" / "save.png").write_bytes(b"not the same crop") + with pytest.raises(ap.AdmissionError, match="bundle changed"): + ap.decide_admission( + out, pending_sha256=pending.sha256(), decision="accept", decided_by=PERSON + ) + + +def test_one_decision_only_and_reject_needs_a_reason(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + out = tmp_path / "admission" + with pytest.raises(ap.AdmissionError, match="reason"): + ap.decide_admission( + out, pending_sha256=pending.sha256(), decision="reject", decided_by=PERSON + ) + result = ap.decide_admission( + out, + pending_sha256=pending.sha256(), + decision="reject", + decided_by=PERSON, + reason="wrong checks", + ) + assert result.decision == "reject" + assert Workflow.load(out / "bundle").qualification is None + with pytest.raises(ap.AdmissionError, match="already decided"): + ap.decide_admission( + out, pending_sha256=pending.sha256(), decision="accept", decided_by=PERSON + ) + + +def test_a_halted_proposal_has_nothing_to_accept(tmp_path: Path) -> None: + pending = ap.prepare_admission( + [_recording(tmp_path, effects=False)], + out=tmp_path / "admission", + name="qualified-write", + prepared_by=PERSON, + ) + assert pending.status == "halted" + assert "effect pin is missing" in (pending.halt_reason or "") + with pytest.raises(ap.AdmissionError, match="nothing to decide"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + + +def test_refused_induction_stops_without_a_bundle(tmp_path, monkeypatch) -> None: + from openadapt_flow.compiler import induction + + monkeypatch.setattr( + induction, + "induce_program", + lambda traces: SimpleNamespace(certified=False, workflow=None, record_next=[]), + ) + pending = ap.prepare_admission( + [_recording(tmp_path, "rec1"), _recording(tmp_path, "rec2")], + out=tmp_path / "admission", + name="qualified-write", + prepared_by=PERSON, + ) + assert pending.status == "halted" + assert "induction refused" in (pending.halt_reason or "") + assert not (tmp_path / "admission" / "bundle").exists() + + +def test_two_recordings_are_induced_into_one_program(tmp_path: Path) -> None: + pending = ap.prepare_admission( + [_recording(tmp_path, "rec1"), _recording(tmp_path, "rec2")], + out=tmp_path / "admission", + name="qualified-write", + prepared_by=PERSON, + ) + assert pending.recordings == 2 + assert pending.status == "needs_decision" + assert pending.induced is True + + +def test_raw_recordings_are_compiled(tmp_path, monkeypatch) -> None: + from openadapt_flow import compiler + + source = _recording(tmp_path, "compiled-source") + raw = tmp_path / "raw" + raw.mkdir() + (raw / "meta.json").write_text((source / "meta.json").read_text()) + calls = [] + + def fake_compile(recording, out, *, name): + calls.append((Path(recording), name)) + import shutil + + shutil.copytree(source, out) + return Workflow.load(out) + + monkeypatch.setattr(compiler, "compile_recording", fake_compile) + pending = ap.prepare_admission( + [raw], out=tmp_path / "admission", name="qualified-write", prepared_by=PERSON + ) + assert calls == [(raw, "qualified-write")] + assert pending.status == "needs_decision" + assert (tmp_path / "admission" / "compiled" / "00" / "workflow.json").is_file() + + +def test_candidate_without_a_recording_asks_for_one(tmp_path: Path) -> None: + export = json.loads((SHADOW / "report_mail.episodes.json").read_text()) + candidate = induce_candidate(export) + folder = write_candidate(candidate, tmp_path / "shadow-candidates" / "c1") + pending = ap.prepare_admission( + [], + out=tmp_path / "admission", + name="report-mail", + prepared_by=PERSON, + candidate=ap.resolve_candidate("c1", tmp_path / "shadow-candidates"), + ) + assert folder.name == "c1" + assert pending.status == "needs_recording" + assert pending.record_next[0]["code"] == "shadow_full_demonstration" + assert pending.shadow_suggestions + assert all("not evidence" in item["note"] for item in pending.shadow_suggestions) + + +def test_candidate_suggestions_are_shown_but_never_applied(tmp_path: Path) -> None: + export = json.loads((SHADOW / "report_mail.episodes.json").read_text()) + folder = write_candidate(induce_candidate(export), tmp_path / "cand") + _recording(folder / "recordings", "rec1") + pending = ap.prepare_admission( + [], + out=tmp_path / "admission", + name="qualified-write", + prepared_by=PERSON, + candidate=folder, + ) + assert pending.status == "needs_decision" + assert "not evidence, not applied" in ap.render_summary(pending) + # Only checks mined from the recording are proposed. + kinds = {check["kind"] for check in pending.checks} + assert not kinds & {"file_written", "row_added", "page_reached"} + + +def test_cli_default_form_stops_without_a_terminal(tmp_path, capsys) -> None: + assert admit_default_form(["admit", "a", "b"]) == ["admit", "run", "a", "b"] + assert admit_default_form(["admit", "status"]) == ["admit", "status"] + assert admit_default_form(["admit", "--candidate", "x"])[1] == "run" + rec = _recording(tmp_path) + out = tmp_path / "admission" + code = main(["admit", str(rec), "--out", str(out), "--name", "w", "--by", "pat"]) + assert code == EXIT_STOPPED + text = capsys.readouterr().out + assert "Proposed checks" in text and "admit decide" in text + assert not (out / ap.DECISION_FILE).exists() + + +def test_cli_single_stop_accepts_once_at_a_terminal(tmp_path, capsys) -> None: + _prepare(tmp_path) + answers = iter(["accept"]) + code = _ask_once( + tmp_path / "admission", "pat", read=lambda _: next(answers), interactive=True + ) + assert code == 0 + assert "Accepted by person:pat via cli" in capsys.readouterr().out + record = json.loads((tmp_path / "admission" / ap.DECISION_FILE).read_text()) + assert record["decided_by"]["kind"] == "person" + + +def test_cli_decide_refuses_without_a_terminal(tmp_path) -> None: + _prepare(tmp_path) + with pytest.raises(SystemExit, match="interactive terminal"): + main(["admit", "decide", str(tmp_path / "admission")]) From 73052715ea4835537a634c5c4f7379c37470cceb Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 17 Oct 2026 21:25:00 -0400 Subject: [PATCH 07/12] feat(agent): prepare_admission, list_admissions and decide_admission MCP tools `openadapt-agent serve --admissions DIR` exposes the admit command over MCP. Recordings and candidates are named by folder under DIR, never by path. Without --delegate-approvals, decide_admission asks a person in the client's form (person:mcp-form). With it, the agent decides as the delegated approver (agent:ID). Either way the self-approval, regulated-pack and digest-binding rules in openadapt_flow.admit_pipeline apply. Signed-off-by: Richard Abrich --- packages/agent/README.md | 1 + packages/agent/tests/test_admission_tools.py | 230 +++++++++++++++ .../flow/openadapt_flow/agent/admission.py | 278 ++++++++++++++++++ packages/flow/openadapt_flow/agent/bridge.py | 43 +++ packages/flow/openadapt_flow/agent/cli.py | 25 +- packages/flow/openadapt_flow/agent/mcp.py | 37 +++ 6 files changed, 613 insertions(+), 1 deletion(-) create mode 100644 packages/agent/tests/test_admission_tools.py create mode 100644 packages/flow/openadapt_flow/agent/admission.py diff --git a/packages/agent/README.md b/packages/agent/README.md index 4bed34b06..cc7044ff2 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -280,6 +280,7 @@ Flow's [execution profiles](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/p | `--timeout SECONDS` | How long one run may work (default 600). Time a person spends approving a save doesn't count. A run stopped at the limit is `not_sure_if_saved`. | | `--approval-mode MODE` | Whether each save waits for approval. Without it, Flow decides: under `standard` and `regulated` a run stops before each save until it's approved (`each_item`). `not_gated` turns that off for every run this server starts; each result records it under `technical.approval_mode`. | | `--delegate-approvals AGENT_ID` | Lets your agent answer a waiting save as `AGENT_ID` (an opaque id such as `agent:intake-bot`), with `list_approval_requests` and `answer_approval_request`. Flow accepts the answer only where the run's approver policy allows an agent for that step, and records the agent as the approver. By default only a person may answer. Needs an openadapt-flow with delegated approvers (not released yet). | +| `--admissions DIR` | Adds `prepare_admission`, `list_admissions` and `decide_admission`, the same steps as `openadapt-flow admit`, over recordings in `DIR/recordings` and shadow candidates in `DIR/shadow-candidates`. A person decides in your client's form. With `--delegate-approvals`, your agent decides as that approver instead, but never on an admission it prepared, and never under the `regulated` policy pack. The record names who decided. | | `--headed` | Shows the browser. A person needs it to continue or skip a stopped run in a web app. | | `--extra-run-arg ARG`, `--flow-cli CMD` | Change how Flow is called. Either one can relax Flow's checks, for example `--extra-run-arg=--allow-unencrypted`. | | `--allow-protected-export`, `--allow-synthetic-recorded-defaults`, `--allow-model-grounding` | Off by default. See [Security](#security). | diff --git a/packages/agent/tests/test_admission_tools.py b/packages/agent/tests/test_admission_tools.py new file mode 100644 index 000000000..4732a9cdf --- /dev/null +++ b/packages/agent/tests/test_admission_tools.py @@ -0,0 +1,230 @@ +"""MCP admission tools: prepare, list, decide. Made-up values only. + +The tools are off unless the operator passes --admissions. A person decides +in the client's form; with --delegate-approvals the agent decides as that +approver. An agent never decides an admission it prepared, and the record +names who decided. +""" + +from __future__ import annotations + +import io +import json +from pathlib import Path + +import anyio +import mcp.types as types +import pytest +from PIL import Image + +from openadapt_agent.admission import AdmissionDesk +from openadapt_agent.bridge import AgentBridge, BridgeError +from openadapt_agent.mcp import build_server +from openadapt_flow.ir import ( + ActionKind, + Anchor, + Postcondition, + PostconditionKind, + Step, + Workflow, +) +from openadapt_flow.runtime.effects import Effect, EffectKind, ValueExpr + +TOOLS = {"prepare_admission", "list_admissions", "decide_admission"} + + +def _png() -> bytes: + buffer = io.BytesIO() + Image.new("RGB", (40, 20), "white").save(buffer, format="PNG") + return buffer.getvalue() + + +def _recording(root: Path, name: str = "rec1") -> None: + folder = root / "recordings" / name + (folder / "templates").mkdir(parents=True) + (folder / "templates" / "save.png").write_bytes(_png()) + workflow = Workflow( + name="qualified-write", + params={"record_id": "example", "note": "example"}, + surface="web", + steps=[ + Step( + id="save", + intent="Save the record", + action=ActionKind.CLICK, + anchor=Anchor( + template="templates/save.png", + region=(10, 10, 40, 20), + click_point=(30, 20), + ocr_text="Save", + structured_identity="record identity", + ), + expect=[ + Postcondition(kind=PostconditionKind.TEXT_PRESENT, text="Saved") + ], + effects=[ + Effect( + kind=EffectKind.FIELD_EQUALS, + match={"id": ValueExpr(param="record_id")}, + field="note", + value=ValueExpr(param="note"), + idempotency_key=ValueExpr(param="record_id"), + risk="irreversible", + ) + ], + risk="irreversible", + identity_armed=True, + ) + ], + ) + workflow.save(folder) + (folder / "meta.json").write_text( + json.dumps( + { + "app_url": "http://127.0.0.1:9/", + "application": "MockMed", + "application_version": "tutorial", + "viewport": [1280, 800], + "surface": "web", + "params": dict(workflow.params), + } + ) + ) + + +def _bridge(bundles_root, runner_config, root, *, delegate=None): + desk = AdmissionDesk(root, delegate_id=delegate) + return AgentBridge(bundles_root, runner_config, admission_desk=desk) + + +def _names(bridge): + return {spec.name for spec in bridge.list_tool_specs()} + + +def test_tools_are_off_without_admissions(bundles_root, runner_config): + bridge = AgentBridge(bundles_root, runner_config) + assert not _names(bridge) & TOOLS + with pytest.raises(BridgeError, match="off"): + bridge.dispatch("list_admissions", {}) + + +def test_prepare_stops_once_and_names_the_preparer(bundles_root, runner_config, tmp_path): + _recording(tmp_path) + bridge = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:intake-bot") + assert TOOLS <= _names(bridge) + view = bridge.dispatch("prepare_admission", {"recordings": ["rec1"], "name": "w"}) + assert view["status"] == "needs_decision" + assert view["prepared_by"] == {"kind": "agent", "id": "agent:intake-bot", "route": "mcp"} + assert {"step_id": "save", "kind": "field_equals", "tier": 1} in view["checks"] + assert view["decided"] is False + listed = bridge.dispatch("list_admissions", {})["admissions"] + assert [row["admission"] for row in listed] == [view["admission"]] + + +def test_an_agent_cannot_decide_what_it_prepared(bundles_root, runner_config, tmp_path): + _recording(tmp_path) + bridge = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:intake-bot") + view = bridge.dispatch("prepare_admission", {"recordings": ["rec1"], "name": "w"}) + args = { + "admission": view["admission"], + "pending_sha256": view["pending_sha256"], + "decision": "accept", + } + with pytest.raises(BridgeError, match="self-approval"): + bridge.dispatch("decide_admission", args) + reviewer = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:reviewer") + result = reviewer.dispatch("decide_admission", args) + assert result["decided_by"] == {"kind": "agent", "id": "agent:reviewer", "route": "mcp"} + assert result["prepared_by"]["id"] == "agent:intake-bot" + assert result["oracle_gate_passed"] is True + record = json.loads( + (tmp_path / "admissions" / view["admission"] / "admission-decision.json").read_text() + ) + assert record["decided_by"]["id"] == "agent:reviewer" + + +def test_without_a_delegate_the_bridge_never_decides(bundles_root, runner_config, tmp_path): + _recording(tmp_path) + bridge = _bridge(bundles_root, runner_config, tmp_path) + view = bridge.dispatch("prepare_admission", {"recordings": ["rec1"], "name": "w"}) + with pytest.raises(BridgeError, match="person must decide"): + bridge.dispatch( + "decide_admission", + { + "admission": view["admission"], + "pending_sha256": view["pending_sha256"], + "decision": "accept", + }, + ) + + +def test_paths_stay_inside_the_admissions_folder(bundles_root, runner_config, tmp_path): + bridge = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:x") + for bad in ("../etc", "/tmp", "a/b"): + with pytest.raises(BridgeError): + bridge.dispatch("prepare_admission", {"recordings": [bad], "name": "w"}) + + +def test_regulated_pack_refuses_an_agent(bundles_root, runner_config, tmp_path): + _recording(tmp_path) + person_server = _bridge(bundles_root, runner_config, tmp_path) + view = person_server.dispatch( + "prepare_admission", + {"recordings": ["rec1"], "name": "w", "policy_pack": "regulated"}, + ) + if view["status"] != "needs_decision": + pytest.skip("regulated pack halted the proposal for this fixture") + agent = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:reviewer") + with pytest.raises(BridgeError, match="needs a person"): + agent.dispatch( + "decide_admission", + { + "admission": view["admission"], + "pending_sha256": view["pending_sha256"], + "decision": "accept", + }, + ) + + +def _wire(model): + return model.model_dump(by_alias=True, mode="json") + + +@pytest.mark.parametrize("answer", ["accept", "decline", "unsupported"]) +def test_person_decides_in_the_client_form( + answer, bundles_root, runner_config, tmp_path, mcp_client +): + _recording(tmp_path) + bridge = _bridge(bundles_root, runner_config, tmp_path) + view = bridge.dispatch("prepare_admission", {"recordings": ["rec1"], "name": "w"}) + prompts = [] + + async def elicit(context, params): + prompts.append(_wire(params)) + return types.ElicitResult(action=answer, content={"confirmed": answer == "accept"}) + + async def call(): + kwargs = {} if answer == "unsupported" else {"elicitation_callback": elicit} + async with mcp_client(build_server(bridge), **kwargs) as client: + return _wire( + await client.call_tool( + "decide_admission", + { + "admission": view["admission"], + "pending_sha256": view["pending_sha256"], + "decision": "accept", + }, + ) + ) + + result = anyio.run(call) + decision = tmp_path / "admissions" / view["admission"] / "admission-decision.json" + if answer == "accept": + assert result["isError"] is False + payload = json.loads(result["content"][0]["text"]) + assert payload["decided_by"] == {"kind": "person", "id": "mcp-form", "route": "mcp"} + assert "Proposed checks" in prompts[0]["message"] + assert decision.is_file() + return + assert result["isError"] is True + assert not decision.exists() diff --git a/packages/flow/openadapt_flow/agent/admission.py b/packages/flow/openadapt_flow/agent/admission.py new file mode 100644 index 000000000..f83b87dc3 --- /dev/null +++ b/packages/flow/openadapt_flow/agent/admission.py @@ -0,0 +1,278 @@ +"""MCP tools for ``openadapt-flow admit``: prepare, list, and decide. + +The operator turns these on with ``openadapt-agent serve --admissions DIR``. +Recordings are named by folder under ``DIR/recordings`` and shadow +candidates by id under ``DIR/shadow-candidates``; an agent never passes a +path. Admissions are written under ``DIR/admissions/``. + +Who decides: + +* With ``--delegate-approvals AGENT_ID`` the calling agent decides as that + delegated approver, and the record says ``agent:AGENT_ID via mcp``. +* Without it, ``decide_admission`` asks a person in the client's own form + (MCP elicitation) and records ``person:mcp-form via mcp``. A model-supplied + argument never stands in for that answer. + +What an agent can't do: decide an admission it prepared (the record names +the preparer), decide under the ``regulated`` policy pack, or decide without +the exact ``pending_sha256`` it was shown. Accepting runs the same gates as +``qualify accept``; nothing here signs an admission. +""" + +from __future__ import annotations + +import re +import uuid +from pathlib import Path +from typing import Any, Optional + +from openadapt_flow.admit_pipeline import ( + PENDING_FILE, + AdmissionError, + Decider, + decide_admission, + load_pending, + prepare_admission, + render_summary, +) + +TOOL_NAMES = ("prepare_admission", "list_admissions", "decide_admission") +_FOLDER = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" +_ADMISSION = r"^[0-9a-f]{32}$" +_SHA = r"^[0-9a-f]{64}$" +REJECT_REASONS = ("wrong_checks", "wrong_steps", "not_this_task", "other") +PERSON_VIA_FORM = "mcp-form" + +PREPARE_DESCRIPTION = ( + "Turn recordings of one task into a program with proposed checks, like " + "`openadapt-flow admit`: compile each recording, induce one program over " + "two or more, and propose the checks. It stops there and returns a short " + "summary and pending_sha256. Nothing is applied until someone else " + "decides with decide_admission; you can't decide an admission you prepared." +) +LIST_DESCRIPTION = ( + "List admissions on this server with their status, summary and pending_sha256." +) +DECIDE_DESCRIPTION = ( + "Accept or reject one admission that stopped for a decision. Send the " + "admission and pending_sha256 from list_admissions. If the operator " + "started this server with --delegate-approvals you decide as that " + "delegated approver; otherwise a person decides in your client's form. " + "Refused for an admission this agent prepared and under the regulated " + "policy pack. The record names who decided." +) + + +class AdmissionToolError(ValueError): + pass + + +class AdmissionDesk: + """The admission tools over one operator-chosen folder.""" + + def __init__(self, root: Path | str, *, delegate_id: Optional[str] = None): + self.root = Path(root).expanduser().resolve() + self.delegate_id = delegate_id + + # -- paths ------------------------------------------------------------- + + def _child(self, area: str, name: Any, pattern: str = _FOLDER) -> Path: + if not isinstance(name, str) or not re.match(pattern, name): + raise AdmissionToolError(f"{area} names one folder") + base = self.root / area + path = base / name + if base.is_symlink() or path.is_symlink(): + raise AdmissionToolError(f"{area} folder is unavailable") + resolved = path.resolve() + if base.resolve() not in resolved.parents or not resolved.is_dir(): + raise AdmissionToolError(f"no such folder in {area}") + return resolved + + def _preparer(self) -> Decider: + return Decider(kind="agent", id=self.delegate_id or "mcp-client", route="mcp") + + # -- specs ------------------------------------------------------------- + + def specs(self) -> list[dict[str, Any]]: + return [ + { + "name": "prepare_admission", + "description": PREPARE_DESCRIPTION, + "input_schema": { + "type": "object", + "properties": { + "recordings": { + "type": "array", + "items": {"type": "string", "pattern": _FOLDER}, + "maxItems": 20, + }, + "candidate_id": {"type": "string", "pattern": _FOLDER}, + "name": {"type": "string", "pattern": _FOLDER}, + "policy_pack": { + "type": "string", + "enum": ["community", "cloud", "regulated"], + }, + }, + "required": ["name"], + "additionalProperties": False, + }, + "annotations": { + "readOnlyHint": False, + "destructiveHint": False, + "idempotentHint": False, + "openWorldHint": False, + }, + }, + { + "name": "list_admissions", + "description": LIST_DESCRIPTION, + "input_schema": { + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + "annotations": { + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False, + }, + }, + { + "name": "decide_admission", + "description": DECIDE_DESCRIPTION, + "input_schema": { + "type": "object", + "properties": { + "admission": {"type": "string", "pattern": _ADMISSION}, + "pending_sha256": {"type": "string", "pattern": _SHA}, + "decision": {"type": "string", "enum": ["accept", "reject"]}, + "reject_reason": { + "type": "string", + "enum": list(REJECT_REASONS), + }, + }, + "required": ["admission", "pending_sha256", "decision"], + "additionalProperties": False, + }, + "annotations": { + "readOnlyHint": False, + "destructiveHint": True, + "idempotentHint": False, + "openWorldHint": False, + }, + }, + ] + + def needs_person(self) -> bool: + """Whether decide_admission must ask a person in the client's form.""" + + return self.delegate_id is None + + # -- tools ------------------------------------------------------------- + + def _view(self, folder: Path) -> dict[str, Any]: + pending = load_pending(folder) + return { + "admission": folder.name, + "status": pending.status, + "halt_reason": pending.halt_reason, + "workflow_name": pending.workflow_name, + "policy_pack": pending.policy_pack, + "prepared_by": pending.prepared_by.model_dump(mode="json"), + "checks": pending.checks, + "shadow_suggestions": [ + {"kind": item.get("kind"), "note": item.get("note")} + for item in pending.shadow_suggestions + ], + "record_next": [item.get("code") for item in pending.record_next], + "decided": (folder / "admission-decision.json").exists(), + "summary": render_summary(pending), + "pending_sha256": pending.sha256(), + } + + def prepare(self, arguments: dict[str, Any]) -> dict[str, Any]: + recordings = [ + self._child("recordings", item) + for item in arguments.get("recordings") or [] + ] + candidate = None + if arguments.get("candidate_id") is not None: + candidate = self._child("shadow-candidates", arguments["candidate_id"]) + if not recordings and candidate is None: + raise AdmissionToolError("give recordings or a candidate_id") + admission_id = uuid.uuid4().hex + out = self.root / "admissions" / admission_id + out.parent.mkdir(parents=True, exist_ok=True) + try: + prepare_admission( + recordings, + out=out, + name=arguments["name"], + prepared_by=self._preparer(), + candidate=candidate, + policy_pack=arguments.get("policy_pack") or "community", + admission_id=admission_id, + ) + except AdmissionError as exc: + raise AdmissionToolError(str(exc)) from exc + return self._view(out) + + def list(self) -> dict[str, Any]: + base = self.root / "admissions" + rows = [] + if base.is_dir() and not base.is_symlink(): + for folder in sorted(base.iterdir()): + if ( + re.match(_ADMISSION, folder.name) + and not folder.is_symlink() + and (folder / PENDING_FILE).is_file() + ): + rows.append(self._view(folder)) + return {"admissions": rows} + + def confirmation_message(self, arguments: dict[str, Any]) -> str: + folder = self._child("admissions", arguments.get("admission"), _ADMISSION) + verb = "accept" if arguments.get("decision") == "accept" else "reject" + return ( + render_summary(load_pending(folder)) + + f"\nConfirm that you, a person, {verb} this admission." + ) + + def decide( + self, arguments: dict[str, Any], *, human_confirmed: bool = False + ) -> dict[str, Any]: + folder = self._child("admissions", arguments.get("admission"), _ADMISSION) + decision = arguments.get("decision") + reason = arguments.get("reject_reason") + if (decision == "reject") != (reason is not None): + raise AdmissionToolError("a reject needs exactly one reject_reason") + if human_confirmed: + decider = Decider(kind="person", id=PERSON_VIA_FORM, route="mcp") + elif self.delegate_id is not None: + decider = Decider(kind="agent", id=self.delegate_id, route="mcp") + else: + raise AdmissionToolError( + "a person must decide this admission in the client's form" + ) + try: + result = decide_admission( + folder, + pending_sha256=str(arguments.get("pending_sha256")), + decision=decision, + decided_by=decider, + reason=reason, + ) + except AdmissionError as exc: + raise AdmissionToolError(str(exc)) from exc + return { + "admission": folder.name, + "decision": result.decision, + "decided_by": result.decided_by.model_dump(mode="json"), + "prepared_by": result.prepared_by.model_dump(mode="json"), + "oracle_gate_passed": result.oracle_gate_passed, + "next": result.next, + } + + +__all__ = ["AdmissionDesk", "AdmissionToolError", "TOOL_NAMES"] diff --git a/packages/flow/openadapt_flow/agent/bridge.py b/packages/flow/openadapt_flow/agent/bridge.py index 8ca8ac4be..1086471e7 100644 --- a/packages/flow/openadapt_flow/agent/bridge.py +++ b/packages/flow/openadapt_flow/agent/bridge.py @@ -171,8 +171,12 @@ def __init__( mode: Optional[str] = None, store: Optional[RunStore] = None, repair_desk: Optional[Any] = None, + admission_desk: Optional[Any] = None, ): self.repair_desk = repair_desk + #: ``openadapt_flow.agent.admission.AdmissionDesk`` when the operator + #: started the server with ``--admissions DIR``. + self.admission_desk = admission_desk self.bundles_dir = Path(bundles_dir) if bundles_dir is not None else None self.sandbox = sandbox self.allow_run = allow_run or sandbox is not None @@ -353,8 +357,43 @@ def _contract_specs(self) -> list[ToolSpec]: ) specs.extend(self._repair_specs()) specs.extend(self._approval_specs()) + specs.extend(self._admission_specs()) return specs + def _admission_specs(self) -> list[ToolSpec]: + """Listed only when the operator passed ``--admissions DIR``.""" + if self.admission_desk is None: + return [] + return [ + ToolSpec( + name=raw["name"], + description=raw["description"], + input_schema=raw["input_schema"], + annotations=raw["annotations"], + ) + for raw in self.admission_desk.specs() + ] + + def dispatch_admission( + self, name: str, arguments: dict, *, human_confirmed: bool = False + ) -> dict: + """Run one admission tool. ``human_confirmed`` only from the MCP form.""" + from openadapt_flow.agent.admission import AdmissionToolError + + desk = self.admission_desk + if desk is None: + raise BridgeError("admission tools are off on this server") + try: + if name == "prepare_admission": + return desk.prepare(arguments) + if name == "list_admissions": + return desk.list() + if name == "decide_admission": + return desk.decide(arguments, human_confirmed=human_confirmed) + except AdmissionToolError as exc: + raise BridgeError(str(exc)) from exc + raise BridgeError("unknown tool name") + def _delegate_id(self) -> Optional[str]: if not self.allow_run or self.sandbox is not None: return None @@ -687,6 +726,10 @@ def dispatch(self, name: str, arguments: Optional[dict]) -> dict: if not any(spec.name == name for spec in self._approval_specs()): raise BridgeError("unknown tool name") return self._approval_dispatch(name, arguments) + if name in ("prepare_admission", "list_admissions", "decide_admission"): + # decide_admission from here is the delegated-agent path only; the + # person path comes through dispatch_admission after the MCP form. + return self.dispatch_admission(name, arguments) if name in ("get_repair_screen", "teach_repair"): if not self._repair_tools_enabled(): raise BridgeError("repair tools are off on this server") diff --git a/packages/flow/openadapt_flow/agent/cli.py b/packages/flow/openadapt_flow/agent/cli.py index 64ba8cfe1..deab0c799 100644 --- a/packages/flow/openadapt_flow/agent/cli.py +++ b/packages/flow/openadapt_flow/agent/cli.py @@ -27,7 +27,7 @@ import shlex import sys from pathlib import Path -from typing import Optional, Sequence +from typing import Any, Optional, Sequence from openadapt_flow.agent import __version__ from openadapt_flow.agent.contract import APPROVAL_MODES @@ -242,6 +242,19 @@ def build_parser() -> argparse.ArgumentParser: "Flow's report record that. Callers can't change it." ), ) + p.add_argument( + "--admissions", + default=None, + metavar="DIR", + help=( + "Add prepare_admission, list_admissions and decide_admission: the " + "`openadapt-flow admit` command over recordings in DIR/recordings " + "and shadow candidates in DIR/shadow-candidates. A person decides " + "in the client's form, or, with --delegate-approvals, the agent " + "decides as that approver. An agent can't decide an admission it " + "prepared." + ), + ) p.add_argument( "--delegate-approvals", default=None, @@ -719,6 +732,7 @@ def _cmd_serve(args: argparse.Namespace) -> int: allow_recorded_defaults=args.allow_synthetic_recorded_defaults, mode=mode, repair_desk=_repair_desk(args, runner_config.runs_dir), + admission_desk=_admission_desk(args), ) n = len(bridge.workflows) print( @@ -795,6 +809,15 @@ def _cmd_emit_skill(args: argparse.Namespace) -> int: return 0 +def _admission_desk(args: argparse.Namespace) -> Optional[Any]: + """The admission tools, only when the operator passed ``--admissions``.""" + if getattr(args, "admissions", None) is None: + return None + from openadapt_flow.agent.admission import AdmissionDesk + + return AdmissionDesk(args.admissions, delegate_id=args.delegate_approvals) + + def _repair_desk(args: argparse.Namespace, runs_dir: Path): from openadapt_flow.agent.repair import ( LazyBackendDemonstrator, diff --git a/packages/flow/openadapt_flow/agent/mcp.py b/packages/flow/openadapt_flow/agent/mcp.py index 6eff4ff2c..f31c4cd93 100644 --- a/packages/flow/openadapt_flow/agent/mcp.py +++ b/packages/flow/openadapt_flow/agent/mcp.py @@ -137,6 +137,30 @@ async def _confirm_attended_action(context: Any, name: str) -> None: ) +async def _confirm_admission_decision( + context: Any, bridge: AgentBridge, arguments: dict[str, Any] +) -> None: + """A person decides an admission in the client's form (no delegate).""" + from openadapt_flow.agent.admission import AdmissionToolError + + try: + message = bridge.admission_desk.confirmation_message(arguments) + except AdmissionToolError as exc: + raise BridgeError(str(exc)) from exc + await _confirm_with_person( + context, + message, + title="I am a person and I make this decision", + unsupported=( + "deciding an admission needs a person in an MCP client with form " + "elicitation, or a server started with --delegate-approvals; " + "nothing was decided. A person can also run `openadapt-flow admit " + "decide` at a terminal" + ), + declined="the person declined or cancelled; nothing was decided", + ) + + async def _confirm_admit(context: Any, authoring: AuthoringBridge) -> None: """A person, not the calling agent, accepts a compiled draft.""" await _confirm_with_person( @@ -266,8 +290,17 @@ async def _call_tool( or "tool arguments do not match the input schema" ) from None human_confirmed = False + admission_by_person = ( + name == "decide_admission" + and bridge is not None + and bridge.admission_desk is not None + and bridge.admission_desk.needs_person() + ) if name in ATTENDED_TOOLS: await _confirm_attended_action(context, name) + elif admission_by_person: + assert bridge is not None + await _confirm_admission_decision(context, bridge, arguments or {}) elif name == "admit" and authoring is not None and authoring.handles(name): await _confirm_admit(context, authoring) human_confirmed = True @@ -280,6 +313,10 @@ def call() -> dict[str, Any]: return authoring.dispatch(name, payload) if bridge is None: raise BridgeError("unknown tool name") + if admission_by_person: + return bridge.dispatch_admission( + name, payload, human_confirmed=True + ) return bridge.dispatch(name, payload) # CLI runs and filesystem projections are blocking. Live attended From 0ae4fe305ad337f0fbb58b5f753b6dd4e16feb13 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 17 Oct 2026 21:25:00 -0400 Subject: [PATCH 08/12] docs(shadow): admit a candidate with one command, by a person or a delegated agent Signed-off-by: Richard Abrich --- packages/flow/docs/SHADOW_CANDIDATES.md | 84 +++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 5 deletions(-) diff --git a/packages/flow/docs/SHADOW_CANDIDATES.md b/packages/flow/docs/SHADOW_CANDIDATES.md index f4abd860b..175cc6252 100644 --- a/packages/flow/docs/SHADOW_CANDIDATES.md +++ b/packages/flow/docs/SHADOW_CANDIDATES.md @@ -77,12 +77,86 @@ treats the folder as a bundle, and it refuses a folder that already holds one. `admitted=False`, and is never attached to a step as an effect. The episodes that suggested a check can't approve it. -## How a person admits it +## How a candidate is admitted -1. Record the task with Capture. -2. Compile each recording with `flow compile`. -3. Run `flow induce` over the recordings, using the draft only as a guide. -4. Confirm or reject each proposed check, then run `flow qualify`. +Record the task once with Capture. Then run one command: + +```bash +openadapt flow admit --candidate abc --out admissions/abc --name report-mail +# or, with your own recordings of the same task: +openadapt flow admit rec-1 rec-2 --out admissions/report-mail --name report-mail +``` + +(`openadapt-flow admit` without the launcher.) `--candidate` reads +`shadow-candidates/abc/` and the recordings in its `recordings/` folder. +The command compiles each recording, induces one program from two or more +recordings, and proposes checks. Then it stops **once** and prints a short +summary: + +```text +Admission 3f9c0a1b2d4e: report-mail + from shadow candidate abc + 2 recording(s), induced into one program; 6 step(s), 1 write(s) + Proposed checks (applied on accept): + - send: field_equals, tier 1 + - send: identity:canonical_ladder + 4 fault case(s) to qualify against + Suggested by shadow episodes (not evidence, not applied): + - file_written: the spreadsheet is saved +Type accept to apply these checks, or reject: +``` + +`accept` applies the checks with the same gates as `qualify accept`, +including the oracle gate, and reseals the bundle. Then qualify it as usual +with `openadapt flow qualify admissions/abc/bundle --verify verify.yaml`. +Nothing certifies until that campaign passes. + +The command stops early and asks for nothing when there is nothing to +accept: + +- no recording yet (it prints the candidate's record-next list); +- induction refuses; +- the check proposal HALTs. + +Without a terminal, it writes the summary to `admission-pending.json` and +exits 3. A person decides later with `openadapt flow admit decide DIR`. + +### Letting an agent decide + +The same steps are MCP tools when the operator starts the agent server with +`--admissions DIR`: + +```bash +openadapt-agent serve --mode production --bundles ./bundles \ + --admissions ./admit --delegate-approvals agent:reviewer +``` + +- `prepare_admission` takes recording folder names under + `DIR/recordings` or a `candidate_id`, and returns the summary and a + `pending_sha256`. +- `list_admissions` lists admissions and their status. +- `decide_admission` takes the admission, the `pending_sha256` and + `accept` or `reject`. + +Without `--delegate-approvals`, `decide_admission` asks a person in the +client's own form, and the record says `person:mcp-form via mcp`. With it, +the agent decides as the delegated approver from +[#1190](https://github.com/OpenAdaptAI/OpenAdapt/pull/1190) and +[#1192](https://github.com/OpenAdaptAI/OpenAdapt/pull/1192), and the record +says `agent:reviewer via mcp`. + +The rules are the same for every decider: + +- **Who decided is recorded.** `admission-decision.json`, in the admission + folder and in the accepted bundle, names who prepared and who decided, by + kind, id and route. +- **No self-approval.** An agent can't decide an admission it prepared. +- **The `regulated` policy pack needs a person.** +- **A decision covers exactly what was shown.** It names the + `pending_sha256` of the summary, and the bundle must still match its + digest. There is one decision per admission. +- **No gate is relaxed.** Shadow suggestions are never applied, a HALTed + proposal can't be accepted, and no admission is signed here. ## Example From 3e8eb8591443d69df04a8c8966317a3473598c0e Mon Sep 17 00:00:00 2001 From: Richard Abrich <774615+abrichr@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:37:42 -0400 Subject: [PATCH 09/12] feat(admit): admission sets problems_only under a person-signed grant An accepted admission now admits the workflow to the problems_only approval mode: its runs stop only for problems (a failed check, a changed screen, a step outside what was qualified) instead of before every save. openadapt-flow run reads this from the bundle's admission-decision.json and prints who admitted it. A flag or runtime.approval_mode still wins. The admitter is a person, or a delegated agent acting under an approver grant a registered person signed (#1234's openadapt-flow/approver-grant/v1), replacing the unsigned delegate decision. The grant must verify, name the agent, cover the workflow and let an agent approve consequential saves. The decision records the grant's digest, its signer and the signed grant itself. The mode applies only to the exact admitted bundle, and an agent's admission only while its grant still verifies. problems_only keeps its own gates, so every save still waits for a person until the switch is earned. MCP: openadapt-agent serve --admissions DIR --delegate-approvals ID --approver-grant FILE. --- packages/agent/README.md | 3 +- packages/agent/tests/test_admission_tools.py | 61 ++++- .../flow/docs/APPROVAL_AND_VERIFICATION.md | 6 + packages/flow/docs/SHADOW_CANDIDATES.md | 36 ++- packages/flow/openadapt_flow/__main__.py | 32 +++ .../flow/openadapt_flow/admit_pipeline.py | 244 +++++++++++++++++- .../flow/openadapt_flow/agent/admission.py | 42 ++- packages/flow/openadapt_flow/agent/cli.py | 26 +- packages/flow/tests/test_admit_pipeline.py | 197 +++++++++++++- 9 files changed, 614 insertions(+), 33 deletions(-) diff --git a/packages/agent/README.md b/packages/agent/README.md index cc7044ff2..94996204e 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -280,7 +280,8 @@ Flow's [execution profiles](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/p | `--timeout SECONDS` | How long one run may work (default 600). Time a person spends approving a save doesn't count. A run stopped at the limit is `not_sure_if_saved`. | | `--approval-mode MODE` | Whether each save waits for approval. Without it, Flow decides: under `standard` and `regulated` a run stops before each save until it's approved (`each_item`). `not_gated` turns that off for every run this server starts; each result records it under `technical.approval_mode`. | | `--delegate-approvals AGENT_ID` | Lets your agent answer a waiting save as `AGENT_ID` (an opaque id such as `agent:intake-bot`), with `list_approval_requests` and `answer_approval_request`. Flow accepts the answer only where the run's approver policy allows an agent for that step, and records the agent as the approver. By default only a person may answer. Needs an openadapt-flow with delegated approvers (not released yet). | -| `--admissions DIR` | Adds `prepare_admission`, `list_admissions` and `decide_admission`, the same steps as `openadapt-flow admit`, over recordings in `DIR/recordings` and shadow candidates in `DIR/shadow-candidates`. A person decides in your client's form. With `--delegate-approvals`, your agent decides as that approver instead, but never on an admission it prepared, and never under the `regulated` policy pack. The record names who decided. | +| `--admissions DIR` | Adds `prepare_admission`, `list_admissions` and `decide_admission`, the same steps as `openadapt-flow admit`, over recordings in `DIR/recordings` and shadow candidates in `DIR/shadow-candidates`. A person decides in your client's form. With `--delegate-approvals` and `--approver-grant`, your agent decides as that approver instead, under a grant a person signed that names it. It never decides an admission it prepared, and never under the `regulated` policy pack. Accepting sets the workflow's runs to `problems_only`. The record names who decided, and for your agent the grant and who signed it. | +| `--approver-grant FILE` | With `--admissions` and `--delegate-approvals`: the approver grant a person signed (`openadapt-flow approver-grant sign`) that lets your agent decide admissions. Flow checks it on every decision. Without it only a person decides. | | `--headed` | Shows the browser. A person needs it to continue or skip a stopped run in a web app. | | `--extra-run-arg ARG`, `--flow-cli CMD` | Change how Flow is called. Either one can relax Flow's checks, for example `--extra-run-arg=--allow-unencrypted`. | | `--allow-protected-export`, `--allow-synthetic-recorded-defaults`, `--allow-model-grounding` | Off by default. See [Security](#security). | diff --git a/packages/agent/tests/test_admission_tools.py b/packages/agent/tests/test_admission_tools.py index 4732a9cdf..9704214ca 100644 --- a/packages/agent/tests/test_admission_tools.py +++ b/packages/agent/tests/test_admission_tools.py @@ -1,9 +1,10 @@ """MCP admission tools: prepare, list, decide. Made-up values only. The tools are off unless the operator passes --admissions. A person decides -in the client's form; with --delegate-approvals the agent decides as that -approver. An agent never decides an admission it prepared, and the record -names who decided. +in the client's form; with --delegate-approvals and a person-signed +--approver-grant the agent decides as that approver. An agent never decides +an admission it prepared or without a grant that verifies, and the record +names who decided (and, for an agent, the grant and who signed it). """ from __future__ import annotations @@ -59,9 +60,7 @@ def _recording(root: Path, name: str = "rec1") -> None: ocr_text="Save", structured_identity="record identity", ), - expect=[ - Postcondition(kind=PostconditionKind.TEXT_PRESENT, text="Saved") - ], + expect=[Postcondition(kind=PostconditionKind.TEXT_PRESENT, text="Saved")], effects=[ Effect( kind=EffectKind.FIELD_EQUALS, @@ -92,8 +91,40 @@ def _recording(root: Path, name: str = "rec1") -> None: ) -def _bridge(bundles_root, runner_config, root, *, delegate=None): - desk = AdmissionDesk(root, delegate_id=delegate) +def _grant_file(root: Path, delegates=("agent:reviewer",)) -> Path: + """A grant a made-up registered person signed, letting agents approve saves.""" + + from datetime import datetime, timedelta, timezone + + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + from openadapt_flow.approval_mode.approver import ApproverPolicyV1, StepClass + from openadapt_flow.approval_mode.signed_policy import ApproverGrant, sign_grant + from openadapt_flow.jit.authority import public_hex, register_principal + + key = Ed25519PrivateKey.generate() + register_principal( + root / "keys", + role="policy_signer", + principal_id="person-0001", + public_key_hex=public_hex(key), + ) + allowed = {c.value: ["person"] for c in StepClass} + allowed[StepClass.CONSEQUENTIAL.value] = ["person", "agent"] + start = datetime.now(timezone.utc) - timedelta(minutes=1) + grant = ApproverGrant( + grant_id="grant-0001", + signer="person-0001", + signed_at=start.isoformat(), + expires_at=(start + timedelta(days=30)).isoformat(), + approver_policy=ApproverPolicyV1.model_validate({"allowed": allowed}), + delegates=delegates, + ) + return sign_grant(grant, key).save(root / "grant.json") + + +def _bridge(bundles_root, runner_config, root, *, delegate=None, grant=None): + desk = AdmissionDesk(root, delegate_id=delegate, approver_grant=grant, key_dir=root / "keys") return AgentBridge(bundles_root, runner_config, admission_desk=desk) @@ -132,9 +163,19 @@ def test_an_agent_cannot_decide_what_it_prepared(bundles_root, runner_config, tm } with pytest.raises(BridgeError, match="self-approval"): bridge.dispatch("decide_admission", args) - reviewer = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:reviewer") + ungranted = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:reviewer") + with pytest.raises(BridgeError, match="approver_grant_missing"): + ungranted.dispatch("decide_admission", args) + grant = _grant_file(tmp_path) + reviewer = _bridge( + bundles_root, runner_config, tmp_path, delegate="agent:reviewer", grant=grant + ) result = reviewer.dispatch("decide_admission", args) - assert result["decided_by"] == {"kind": "agent", "id": "agent:reviewer", "route": "mcp"} + assert result["decided_by"]["kind"] == "agent" + assert result["decided_by"]["id"] == "agent:reviewer" + assert result["decided_by"]["on_behalf_of"] == "person-0001" + assert result["decided_by"]["policy_ref"].startswith("sha256:") + assert result["approval_mode"] == "problems_only" assert result["prepared_by"]["id"] == "agent:intake-bot" assert result["oracle_gate_passed"] is True record = json.loads( diff --git a/packages/flow/docs/APPROVAL_AND_VERIFICATION.md b/packages/flow/docs/APPROVAL_AND_VERIFICATION.md index cde7042de..bf72d4fce 100644 --- a/packages/flow/docs/APPROVAL_AND_VERIFICATION.md +++ b/packages/flow/docs/APPROVAL_AND_VERIFICATION.md @@ -197,6 +197,12 @@ must be under the target (5% at 90% by default). With no artifact or no audit labels it raises `EligibilityUnavailable` and the workflow stays per item. `openadapt-flow triage eligibility` prints the result. +An accepted `openadapt-flow admit` sets `problems_only` as the admitted +bundle's default approval mode (see `docs/SHADOW_CANDIDATES.md`). An agent +admits only under a person-signed grant that lets an agent approve +consequential saves. The gates above still apply: an admitted workflow sends +every save to a person until it has earned the switch. + ## Admitting a demonstrated read-back (planned) `openadapt-flow check admit` (branch `feat/demonstrated-readback-verifier`, diff --git a/packages/flow/docs/SHADOW_CANDIDATES.md b/packages/flow/docs/SHADOW_CANDIDATES.md index 175cc6252..95b08bcf7 100644 --- a/packages/flow/docs/SHADOW_CANDIDATES.md +++ b/packages/flow/docs/SHADOW_CANDIDATES.md @@ -128,7 +128,8 @@ The same steps are MCP tools when the operator starts the agent server with ```bash openadapt-agent serve --mode production --bundles ./bundles \ - --admissions ./admit --delegate-approvals agent:reviewer + --admissions ./admit --delegate-approvals agent:reviewer \ + --approver-grant ./reviewer-grant.json ``` - `prepare_admission` takes recording folder names under @@ -140,10 +141,15 @@ openadapt-agent serve --mode production --bundles ./bundles \ Without `--delegate-approvals`, `decide_admission` asks a person in the client's own form, and the record says `person:mcp-form via mcp`. With it, -the agent decides as the delegated approver from -[#1190](https://github.com/OpenAdaptAI/OpenAdapt/pull/1190) and -[#1192](https://github.com/OpenAdaptAI/OpenAdapt/pull/1192), and the record -says `agent:reviewer via mcp`. +the agent decides as the delegated approver, but only under the approver +grant in `--approver-grant` that a registered person signed +(`openadapt-flow approver-grant sign`; see +[`APPROVAL_AND_VERIFICATION.md`](APPROVAL_AND_VERIFICATION.md)). Flow checks +the grant on every decision: it must verify against the signer registry, name +the agent, cover the workflow, and let an agent approve consequential saves. +Without such a grant the agent's decision is refused. The record says +`agent:reviewer via mcp under grant signed by ` and keeps the +signed grant. The rules are the same for every decider: @@ -158,6 +164,26 @@ The rules are the same for every decider: - **No gate is relaxed.** Shadow suggestions are never applied, a HALTed proposal can't be accepted, and no admission is signed here. +### What admission changes for runs + +Accepting admits the workflow to the `problems_only` approval mode. Its runs +no longer wait before every save. They stop only when a check fails, the +screen changed, or a step falls outside what was qualified, and then a person +decides. `openadapt-flow run` reads this from `admission-decision.json` in the +bundle and prints who admitted it. Four things keep it narrow: + +- A `--approval-mode` flag or the deployment's `runtime.approval_mode` still + wins. +- It applies only to the exact admitted bundle. After any change to the + bundle, each save waits for a person again. +- An agent's admission applies only while its signed grant still verifies. + If the grant expires, is swapped or no longer names the agent, each save + waits for a person again. +- `problems_only` keeps its own gates (see "Earning only problems" in + [`APPROVAL_AND_VERIFICATION.md`](APPROVAL_AND_VERIFICATION.md)). Until the + workflow passes the readiness test and its switch is on, every save still + waits for a person, and repeated problems switch it back. + ## Example The test fixtures in `tests/fixtures/shadow/` come from Capture's miner on diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 2ce78f9d3..a9bad0cf4 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -2278,9 +2278,41 @@ def _resolve_run_approval_mode(args: argparse.Namespace, cfg, profile_value): requested = getattr(args, "approval_mode", None) if requested is None: requested = getattr(getattr(cfg, "runtime", None), "approval_mode", None) + if requested is None: + requested = _admitted_approval_mode(args) return resolve_approval_mode(requested) +def _admitted_approval_mode(args: argparse.Namespace) -> Optional[str]: + """The approval mode the bundle's admission gives it, or ``None``. + + An accepted ``openadapt-flow admit`` sets ``problems_only`` for that exact + bundle. A flag or ``runtime.approval_mode`` still wins. A bundle changed + since admission, or an agent's admission whose person-signed grant no + longer verifies, falls back to the default (a person approves each save) + and says why. + """ + + bundle = getattr(args, "bundle", None) + if not bundle: + return None + from openadapt_flow.admit_pipeline import admitted_approval_mode + + admitted = admitted_approval_mode(Path(bundle)) + if admitted.refusal is not None: + print( + f"admission approval mode not used ({admitted.refusal}): a person " + "approves each save." + ) + return None + if admitted.approval_mode is not None and admitted.admitted_by is not None: + print( + f"Admitted by {admitted.admitted_by.label()}: this workflow uses " + f"{admitted.approval_mode}." + ) + return admitted.approval_mode + + def _approval_wait_seconds(args: argparse.Namespace) -> float: from openadapt_flow.approval_mode.item_approval import ( DEFAULT_APPROVAL_WAIT_S, diff --git a/packages/flow/openadapt_flow/admit_pipeline.py b/packages/flow/openadapt_flow/admit_pipeline.py index 304471fd9..a99b09db3 100644 --- a/packages/flow/openadapt_flow/admit_pipeline.py +++ b/packages/flow/openadapt_flow/admit_pipeline.py @@ -13,8 +13,18 @@ which re-runs the oracle gate, and reseal the bundle. The decision can come from a person at a terminal, or from a delegated agent -approver over MCP (``openadapt-agent serve --delegate-approvals``). Either -way the decision record names who decided, by kind, id and route. +approver over MCP (``openadapt-agent serve --delegate-approvals``) acting +under an approver grant a person signed (``openadapt-flow approver-grant +sign``). Either way the decision record names who decided, by kind, id and +route, and for an agent the grant's digest and the person who signed it. + +Accepting admits the workflow to the ``problems_only`` approval mode: its runs +stop only for problems (a failed check, a changed screen, a step outside what +was qualified) instead of before every save. :func:`admitted_approval_mode` +is what ``openadapt-flow run`` reads. It applies only to the exact bundle that +was admitted, and for an agent's admission only while the grant still +verifies. ``problems_only`` keeps all of its own gates: until the switch is +earned and on, every save still waits for a person. What it never does: @@ -29,6 +39,9 @@ * treat shadow-suggested checks as evidence. They are listed for the person or approver to read; only checks mined from real recordings are applied. * sign an admission. No local-dev or production admission is minted here. +* let an agent admit without a person's signature. An agent decides only + under a verified grant that names it, covers this workflow, and lets an + agent approve consequential saves; anything less is refused. """ from __future__ import annotations @@ -41,7 +54,7 @@ from pathlib import Path from typing import Any, Literal, Optional, Sequence -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_serializer, model_validator PENDING_FILE = "admission-pending.json" DECISION_FILE = "admission-decision.json" @@ -59,6 +72,9 @@ DECIDER_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$" #: Policy packs under which only a person may decide. PERSON_ONLY_PACKS = frozenset({"regulated"}) +#: The approval mode an accepted admission gives the workflow's runs. +ADMITTED_APPROVAL_MODE: Literal["problems_only"] = "problems_only" +_SHA256 = r"^sha256:[0-9a-f]{64}$" PendingStatus = Literal["needs_decision", "needs_recording", "halted"] @@ -75,9 +91,36 @@ class Decider(BaseModel): kind: Literal["person", "agent"] id: str = Field(pattern=DECIDER_ID_PATTERN) route: Literal["cli", "mcp"] + #: For an agent that decided: the digest of the person-signed approver + #: grant it decided under. Left out of the record for a person. + policy_ref: Optional[str] = Field(default=None, pattern=_SHA256) + #: For an agent that decided: the registered person who signed that grant. + on_behalf_of: Optional[str] = Field(default=None, pattern=DECIDER_ID_PATTERN) + + @model_serializer(mode="wrap") + def _compact(self, handler: Any) -> dict[str, Any]: + data: dict[str, Any] = handler(self) + for key in ("policy_ref", "on_behalf_of"): + if data.get(key) is None: + data.pop(key, None) + return data + + @model_validator(mode="after") + def _grant_only_for_an_agent(self) -> "Decider": + if self.kind == "person" and ( + self.policy_ref is not None or self.on_behalf_of is not None + ): + raise ValueError("a person decides for themselves, under no grant") + if (self.policy_ref is None) != (self.on_behalf_of is None): + raise ValueError("a grant names both its digest and its signer") + return self def label(self) -> str: - return f"{self.kind}:{self.id} via {self.route}" + text = f"{self.kind}:{self.id} via {self.route}" + if self.policy_ref is not None: + short = self.policy_ref.removeprefix("sha256:")[:12] + text += f" under grant {short} signed by {self.on_behalf_of}" + return text class PendingAdmission(BaseModel): @@ -128,6 +171,13 @@ class AdmissionDecision(BaseModel): accepted_bundle_content_digest: Optional[str] = None oracle_gate_passed: Optional[bool] = None next: list[str] = Field(default_factory=list) + #: The workflow the admission names, for the grant's workflow scope. + workflow_name: Optional[str] = None + #: Set on accept: the approval mode the admitted workflow's runs use. + approval_mode: Optional[Literal["problems_only"]] = None + #: The person-signed grant an agent decided under, kept whole so a run + #: re-verifies it. ``None`` for a person's decision. + approver_grant: Optional[dict[str, Any]] = None def _sha(payload: Any) -> str: @@ -375,11 +425,88 @@ def _write_json(path: Path, payload: dict[str, Any]) -> None: path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", "utf-8") -def check_decider(pending: PendingAdmission, decided_by: Decider) -> None: - """Refuse self-approval and agent decisions where only a person may decide.""" +def _agent_identity(decided_by: Decider) -> Any: + from openadapt_flow.approval_mode.approver import ( + ApproverAuth, + ApproverIdentityV1, + ApproverKind, + ) + + return ApproverIdentityV1( + kind=ApproverKind.AGENT, + auth=( + ApproverAuth.MCP_SESSION + if decided_by.route == "mcp" + else ApproverAuth.LOCAL_CLI + ), + id=decided_by.id, + ) + + +def _grant_refusal( + signed: Any, + agent: Decider, + workflow_name: str, + *, + key_dir: Optional[Path | str], + now: Optional[datetime] = None, +) -> tuple[Optional[Any], Optional[str]]: + """``(verified grant, None)`` or ``(None, closed reason)``.""" + + from openadapt_flow.approval_mode.approver import StepClass + from openadapt_flow.approval_mode.signed_policy import ( + GrantRefused, + SignedApproverGrant, + verify_grant, + ) + + if signed is None: + return None, "approver_grant_missing" + try: + grant = ( + signed + if isinstance(signed, SignedApproverGrant) + else SignedApproverGrant.model_validate(signed) + ) + except ValueError: + return None, "approver_grant_unreadable" + if key_dir is None: + from openadapt_flow.jit.lifecycle import default_key_dir + + key_dir = default_key_dir() + try: + verified = verify_grant(grant, key_dir, now=now) + except GrantRefused as exc: + return None, exc.code + if not verified.names(agent.id): + return None, "delegate_not_in_grant" + if not verified.covers(workflow_name): + return None, "workflow_not_in_grant" + # Admitting to problems_only lets consequential saves go ahead without a + # person. An agent may grant that only where the person's grant already + # lets an agent approve a consequential save itself. + if verified.policy.refusal(StepClass.CONSEQUENTIAL, _agent_identity(agent)): + return None, "grant_does_not_allow_agent_saves" + if agent.policy_ref is not None and agent.policy_ref != verified.policy_ref: + return None, "policy_ref_mismatch" + return verified, None + + +def check_decider( + pending: PendingAdmission, + decided_by: Decider, + *, + approver_grant: Any = None, + key_dir: Optional[Path | str] = None, +) -> Decider: + """Refuse self-approval, and any agent decision a person didn't sign for. + + Returns the decider as recorded: an agent gains the digest of the grant + it decided under and the person who signed it. + """ if decided_by.kind != "agent": - return + return decided_by if pending.policy_pack in PERSON_ONLY_PACKS: raise AdmissionError( f"the {pending.policy_pack!r} policy pack needs a person to decide" @@ -390,6 +517,18 @@ def check_decider(pending: PendingAdmission, decided_by: Decider) -> None: "self-approval refused: the agent that prepared this admission " "can't decide it" ) + verified, refusal = _grant_refusal( + approver_grant, decided_by, pending.workflow_name, key_dir=key_dir + ) + if verified is None: + raise AdmissionError( + f"an agent admits a workflow only under an approver grant a person " + f"signed that names it, covers this workflow and lets an agent " + f"approve saves ({refusal}); see `openadapt-flow approver-grant sign`" + ) + return decided_by.model_copy( + update={"policy_ref": verified.policy_ref, "on_behalf_of": verified.signer} + ) def decide_admission( @@ -399,8 +538,17 @@ def decide_admission( decision: Literal["accept", "reject"], decided_by: Decider, reason: Optional[str] = None, + approver_grant: Any = None, + key_dir: Optional[Path | str] = None, ) -> AdmissionDecision: - """Record one decision on the pending admission, and apply it on accept.""" + """Record one decision on the pending admission, and apply it on accept. + + An agent decides only with ``approver_grant``: a person-signed + :class:`~openadapt_flow.approval_mode.signed_policy.SignedApproverGrant` + that verifies against the signer registry in ``key_dir`` (default + ``~/.openadapt/jit``). Accepting sets the workflow's approval mode to + ``problems_only``. + """ from openadapt_flow.ir import Workflow from openadapt_flow.qualification_proposal import ( @@ -423,7 +571,9 @@ def decide_admission( f"nothing to decide: the admission is {pending.status}" + (f" ({pending.halt_reason})" if pending.halt_reason else "") ) - check_decider(pending, decided_by) + decided_by = check_decider( + pending, decided_by, approver_grant=approver_grant, key_dir=key_dir + ) if decision not in ("accept", "reject"): raise AdmissionError("decision must be accept or reject") if decision == "reject" and not reason: @@ -450,7 +600,17 @@ def decide_admission( "pending_sha256": pending_sha256, "bundle_content_digest": pending.bundle_content_digest, "proposal_sha256": pending.proposal_sha256, + "workflow_name": pending.workflow_name, } + if decided_by.kind == "agent": + from openadapt_flow.approval_mode.signed_policy import SignedApproverGrant + + signed = ( + approver_grant + if isinstance(approver_grant, SignedApproverGrant) + else SignedApproverGrant.model_validate(approver_grant) + ) + record["approver_grant"] = signed.model_dump(mode="json", by_alias=True) if decision == "accept": proposal = load_proposal(target / PROPOSAL_FILE) if proposal.sha256() != pending.proposal_sha256: @@ -465,9 +625,13 @@ def decide_admission( record.update( accepted_bundle_content_digest=_bundle_digest(bundle), oracle_gate_passed=bool(gate.get("passed")), + approval_mode=ADMITTED_APPROVAL_MODE, next=[ "qualify campaign: run the qualification cases before this " - "program can be certified" + "program can be certified", + "runs use the problems_only approval mode: they stop only for " + "problems once the switch is earned; until then each save " + "still waits for a person", ], ) result = AdmissionDecision(**record) @@ -478,6 +642,63 @@ def decide_admission( return result +class AdmittedMode(BaseModel): + """What a run reads from the bundle's admission decision.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + #: ``problems_only`` when the admission applies; ``None`` otherwise. + approval_mode: Optional[Literal["problems_only"]] = None + admitted_by: Optional[Decider] = None + #: Why a recorded admission doesn't apply, as a closed code. + refusal: Optional[str] = None + + +def admitted_approval_mode( + bundle: Path | str, + *, + key_dir: Optional[Path | str] = None, + now: Optional[datetime] = None, +) -> AdmittedMode: + """The approval mode an accepted admission gives this exact bundle. + + No admission, or a rejected one: no mode (the run's default applies). A + recorded admission for a bundle that has since changed, or an agent's + admission whose grant no longer verifies, names the reason and gives no + mode. Never raises. + """ + + path = Path(bundle) / DECISION_FILE + if not path.is_file() or path.is_symlink(): + return AdmittedMode() + try: + record = AdmissionDecision.model_validate_json(path.read_text("utf-8")) + except (OSError, ValueError): + return AdmittedMode(refusal="admission_unreadable") + if record.decision != "accept" or record.approval_mode is None: + return AdmittedMode() + try: + current = _bundle_digest(Path(bundle)) + except Exception: # noqa: BLE001 - an unreadable bundle never widens a run + return AdmittedMode(refusal="admission_bundle_unreadable") + if current != record.accepted_bundle_content_digest: + return AdmittedMode(refusal="admission_bundle_changed") + decider = record.decided_by + if decider.kind == "agent": + if record.workflow_name is None or decider.policy_ref is None: + return AdmittedMode(refusal="approver_grant_missing") + verified, refusal = _grant_refusal( + record.approver_grant, + decider, + record.workflow_name, + key_dir=key_dir, + now=now, + ) + if verified is None: + return AdmittedMode(refusal=refusal) + return AdmittedMode(approval_mode=record.approval_mode, admitted_by=decider) + + def render_summary(pending: PendingAdmission) -> str: """The short summary shown at the single stop.""" @@ -508,7 +729,10 @@ def render_summary(pending: PendingAdmission) -> str: __all__ = [ + "ADMITTED_APPROVAL_MODE", "AdmissionDecision", + "AdmittedMode", + "admitted_approval_mode", "AdmissionError", "Decider", "PendingAdmission", diff --git a/packages/flow/openadapt_flow/agent/admission.py b/packages/flow/openadapt_flow/agent/admission.py index f83b87dc3..abb19dbe9 100644 --- a/packages/flow/openadapt_flow/agent/admission.py +++ b/packages/flow/openadapt_flow/agent/admission.py @@ -7,16 +7,22 @@ Who decides: -* With ``--delegate-approvals AGENT_ID`` the calling agent decides as that - delegated approver, and the record says ``agent:AGENT_ID via mcp``. +* With ``--delegate-approvals AGENT_ID --approver-grant FILE`` the calling + agent decides as that delegated approver, under the person-signed grant in + FILE. Flow verifies the grant on every decision, and the record says + ``agent:AGENT_ID via mcp under grant signed by ``. Without + a grant that verifies, names AGENT_ID, covers the workflow and lets an + agent approve saves, the agent's decision is refused. * Without it, ``decide_admission`` asks a person in the client's own form (MCP elicitation) and records ``person:mcp-form via mcp``. A model-supplied argument never stands in for that answer. What an agent can't do: decide an admission it prepared (the record names the preparer), decide under the ``regulated`` policy pack, or decide without -the exact ``pending_sha256`` it was shown. Accepting runs the same gates as -``qualify accept``; nothing here signs an admission. +the exact ``pending_sha256`` it was shown, or decide without a grant a person +signed. Accepting runs the same gates as ``qualify accept`` and admits the +workflow to the ``problems_only`` approval mode; nothing here signs an +admission. """ from __future__ import annotations @@ -70,9 +76,32 @@ class AdmissionToolError(ValueError): class AdmissionDesk: """The admission tools over one operator-chosen folder.""" - def __init__(self, root: Path | str, *, delegate_id: Optional[str] = None): + def __init__( + self, + root: Path | str, + *, + delegate_id: Optional[str] = None, + approver_grant: Optional[Path | str] = None, + key_dir: Optional[Path | str] = None, + ): self.root = Path(root).expanduser().resolve() self.delegate_id = delegate_id + #: The person-signed grant the delegate decides under. Read on every + #: decision so a replaced or expired file stops working at once. + self.approver_grant = ( + Path(approver_grant).expanduser() if approver_grant else None + ) + self.key_dir = key_dir + + def _grant(self) -> Any: + if self.approver_grant is None: + return None + import json + + try: + return json.loads(self.approver_grant.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} # -- paths ------------------------------------------------------------- @@ -262,6 +291,8 @@ def decide( decision=decision, decided_by=decider, reason=reason, + approver_grant=self._grant() if decider.kind == "agent" else None, + key_dir=self.key_dir, ) except AdmissionError as exc: raise AdmissionToolError(str(exc)) from exc @@ -271,6 +302,7 @@ def decide( "decided_by": result.decided_by.model_dump(mode="json"), "prepared_by": result.prepared_by.model_dump(mode="json"), "oracle_gate_passed": result.oracle_gate_passed, + "approval_mode": result.approval_mode, "next": result.next, } diff --git a/packages/flow/openadapt_flow/agent/cli.py b/packages/flow/openadapt_flow/agent/cli.py index 258886cd5..12a418667 100644 --- a/packages/flow/openadapt_flow/agent/cli.py +++ b/packages/flow/openadapt_flow/agent/cli.py @@ -255,6 +255,18 @@ def build_parser() -> argparse.ArgumentParser: "prepared." ), ) + p.add_argument( + "--approver-grant", + default=None, + metavar="FILE", + help=( + "With --admissions and --delegate-approvals: the approver grant a " + "person signed (openadapt-flow approver-grant sign) that lets " + "AGENT_ID decide admissions. It must name AGENT_ID, cover the " + "workflow and let an agent approve saves; Flow verifies it on " + "every decision. Without it only a person decides." + ), + ) p.add_argument( "--delegate-approvals", default=None, @@ -567,6 +579,14 @@ def _cmd_serve(args: argparse.Namespace) -> int: file=sys.stderr, ) return 2 + if getattr(args, "approver_grant", None) is not None and ( + args.delegate_approvals is None or getattr(args, "admissions", None) is None + ): + print( + "serve: --approver-grant needs --admissions and --delegate-approvals", + file=sys.stderr, + ) + return 2 if args.delegate_approvals is not None: import re as _re @@ -817,7 +837,11 @@ def _admission_desk(args: argparse.Namespace) -> Optional[Any]: return None from openadapt_flow.agent.admission import AdmissionDesk - return AdmissionDesk(args.admissions, delegate_id=args.delegate_approvals) + return AdmissionDesk( + args.admissions, + delegate_id=args.delegate_approvals, + approver_grant=getattr(args, "approver_grant", None), + ) def _repair_desk(args: argparse.Namespace, runs_dir: Path): diff --git a/packages/flow/tests/test_admit_pipeline.py b/packages/flow/tests/test_admit_pipeline.py index aadc3cf74..e300bf06b 100644 --- a/packages/flow/tests/test_admit_pipeline.py +++ b/packages/flow/tests/test_admit_pipeline.py @@ -15,9 +15,11 @@ from openadapt_flow import admit_pipeline as ap from openadapt_flow.__main__ import main +from openadapt_flow.approval_mode.approver import ApproverPolicyV1, StepClass from openadapt_flow.cli_admit import EXIT_STOPPED, _ask_once, admit_default_form from openadapt_flow.compiler.motif_induction import induce_candidate, write_candidate from openadapt_flow.ir import Workflow +from tests._approver_grant_support import key_dir, signed_grant from tests.test_qualification_proposal import _png, _workflow SHADOW = Path(__file__).parent / "fixtures" / "shadow" @@ -26,6 +28,17 @@ AGENT_B = ap.Decider(kind="agent", id="agent:reviewer", route="mcp") +def _agent_policy() -> ApproverPolicyV1: + allowed = {c.value: ["person"] for c in StepClass} + allowed[StepClass.CONSEQUENTIAL.value] = ["person", "agent"] + return ApproverPolicyV1.model_validate({"allowed": allowed}) + + +def _grant(tmp_path: Path, **kwargs): + kwargs.setdefault("delegates", ("agent:reviewer",)) + return signed_grant(tmp_path, _agent_policy(), **kwargs) + + def _recording(root: Path, name: str = "rec1", *, effects: bool = True) -> Path: """A compiled bundle with its recording's meta.json beside it.""" @@ -95,6 +108,8 @@ def test_person_accept_applies_checks_and_records_who_decided(tmp_path: Path) -> (tmp_path / "admission" / "bundle" / ap.DECISION_FILE).read_text() ) assert record["decided_by"] == {"kind": "person", "id": "pat", "route": "cli"} + assert record["approval_mode"] == "problems_only" + assert record["approver_grant"] is None assert "qualify campaign" in record["next"][0] # No admission is signed here. assert not ( @@ -112,13 +127,168 @@ def test_an_agent_cannot_decide_what_it_prepared(tmp_path: Path) -> None: decided_by=AGENT_A, ) assert not (tmp_path / "admission" / ap.DECISION_FILE).exists() + grant = _grant(tmp_path) result = ap.decide_admission( tmp_path / "admission", pending_sha256=pending.sha256(), decision="accept", decided_by=AGENT_B, + approver_grant=grant, + key_dir=key_dir(tmp_path), + ) + assert result.decided_by.id == AGENT_B.id and result.prepared_by == AGENT_A + # The record names the grant and the person who signed it. + assert result.decided_by.policy_ref == grant.grant.digest() + assert result.decided_by.on_behalf_of == grant.grant.signer + assert result.approval_mode == "problems_only" + assert "under grant" in result.decided_by.label() + + +@pytest.mark.parametrize( + "grant_kwargs, refusal", + [ + (None, "approver_grant_missing"), + ({"delegates": ("agent:someone-else",)}, "delegate_not_in_grant"), + ({"workflows": ("another-workflow",)}, "workflow_not_in_grant"), + ({"days": 1, "signed_at": "expired"}, "approver_grant_expired"), + ], +) +def test_an_agent_admits_only_under_a_person_signed_grant( + tmp_path: Path, grant_kwargs, refusal +) -> None: + from datetime import datetime, timedelta, timezone + + pending = _prepare(tmp_path) + grant = None + if grant_kwargs is not None: + kwargs = dict(grant_kwargs) + if kwargs.get("signed_at") == "expired": + kwargs["signed_at"] = datetime.now(timezone.utc) - timedelta(days=3) + grant = _grant(tmp_path, **kwargs) + with pytest.raises(ap.AdmissionError, match=refusal): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=grant, + key_dir=key_dir(tmp_path), + ) + assert not (tmp_path / "admission" / ap.DECISION_FILE).exists() + + +def test_a_tampered_or_person_only_grant_admits_nothing(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + grant = _grant(tmp_path) + tampered = grant.model_dump(mode="json", by_alias=True) + tampered["grant"]["delegates"] = ["agent:reviewer", "agent:intruder"] + with pytest.raises(ap.AdmissionError, match="approver_grant_signature_invalid"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=tampered, + key_dir=key_dir(tmp_path), + ) + # A grant that lets an agent approve only read-only steps can't admit a + # workflow whose saves would then go ahead without a person. + allowed = {c.value: ["person"] for c in StepClass} + allowed[StepClass.READ_ONLY.value] = ["person", "agent"] + narrow = signed_grant( + tmp_path, + ApproverPolicyV1.model_validate({"allowed": allowed}), + delegates=("agent:reviewer",), + ) + with pytest.raises(ap.AdmissionError, match="grant_does_not_allow_agent_saves"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=narrow, + key_dir=key_dir(tmp_path), + ) + + +def test_a_person_cannot_claim_a_grant() -> None: + with pytest.raises(ValueError): + ap.Decider( + kind="person", + id="pat", + route="cli", + policy_ref="sha256:" + "a" * 64, + on_behalf_of="person-0001", + ) + + +def test_an_admitted_bundle_runs_only_problems(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + bundle = tmp_path / "admission" / "bundle" + assert ap.admitted_approval_mode(bundle) == ap.AdmittedMode() + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + admitted = ap.admitted_approval_mode(bundle) + assert admitted.approval_mode == "problems_only" + assert admitted.admitted_by == PERSON + # Changing the bundle after admission drops back to the default. + workflow = Workflow.load(bundle) + workflow.name = "edited-after-admission" + workflow.save(bundle) + changed = ap.admitted_approval_mode(bundle) + assert changed.approval_mode is None + assert changed.refusal == "admission_bundle_changed" + + +def test_a_rejected_admission_sets_no_mode(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="reject", + decided_by=PERSON, + reason="wrong program", + ) + assert ap.admitted_approval_mode(tmp_path / "admission" / "bundle") == ( + ap.AdmittedMode() + ) + + +def test_an_agent_admission_stops_applying_when_its_grant_does(tmp_path: Path) -> None: + from datetime import datetime, timedelta, timezone + + pending = _prepare(tmp_path, by=AGENT_A) + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=_grant(tmp_path, days=2), + key_dir=key_dir(tmp_path), + ) + bundle = tmp_path / "admission" / "bundle" + keys = key_dir(tmp_path) + assert ap.admitted_approval_mode(bundle, key_dir=keys).approval_mode == ( + "problems_only" + ) + later = datetime.now(timezone.utc) + timedelta(days=3) + expired = ap.admitted_approval_mode(bundle, key_dir=keys, now=later) + assert expired.approval_mode is None + assert expired.refusal == "approver_grant_expired" + # A grant swapped into the record no longer matches the recorded digest. + record_path = bundle / ap.DECISION_FILE + record = json.loads(record_path.read_text()) + record["approver_grant"] = _grant(tmp_path, days=5).model_dump( + mode="json", by_alias=True + ) + record_path.write_text(json.dumps(record)) + assert ap.admitted_approval_mode(bundle, key_dir=keys).refusal == ( + "policy_ref_mismatch" ) - assert result.decided_by == AGENT_B and result.prepared_by == AGENT_A def test_regulated_pack_needs_a_person() -> None: @@ -310,3 +480,28 @@ def test_cli_decide_refuses_without_a_terminal(tmp_path) -> None: _prepare(tmp_path) with pytest.raises(SystemExit, match="interactive terminal"): main(["admit", "decide", str(tmp_path / "admission")]) + + +def test_run_uses_the_admitted_mode_unless_told_otherwise(tmp_path, capsys) -> None: + from openadapt_flow.__main__ import _resolve_run_approval_mode + + pending = _prepare(tmp_path) + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + bundle = str(tmp_path / "admission" / "bundle") + no_cfg = SimpleNamespace(runtime=SimpleNamespace(approval_mode=None)) + args = SimpleNamespace(approval_mode=None, bundle=bundle) + assert _resolve_run_approval_mode(args, no_cfg, None).value == "problems_only" + assert "Admitted by person:pat via cli" in capsys.readouterr().out + # A flag or the deployment's runtime.approval_mode still wins. + flagged = SimpleNamespace(approval_mode="each_item", bundle=bundle) + assert _resolve_run_approval_mode(flagged, no_cfg, None).value == "each_item" + deployed = SimpleNamespace(runtime=SimpleNamespace(approval_mode="each_item")) + assert _resolve_run_approval_mode(args, deployed, None).value == "each_item" + # Without an admission the default is a person approving each save. + bare = SimpleNamespace(approval_mode=None, bundle=str(tmp_path)) + assert _resolve_run_approval_mode(bare, no_cfg, None).value == "each_item" From dc6a1e474e5886d2b31f4b49280fd4b5474e623a Mon Sep 17 00:00:00 2001 From: Richard Abrich <774615+abrichr@users.noreply.github.com> Date: Sun, 18 Oct 2026 00:16:25 -0400 Subject: [PATCH 10/12] docs(admit): an admitted workflow still needs read-back; per-save approval stays the strictest setting --- packages/flow/docs/SHADOW_CANDIDATES.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/flow/docs/SHADOW_CANDIDATES.md b/packages/flow/docs/SHADOW_CANDIDATES.md index 95b08bcf7..e94e8de2a 100644 --- a/packages/flow/docs/SHADOW_CANDIDATES.md +++ b/packages/flow/docs/SHADOW_CANDIDATES.md @@ -169,7 +169,11 @@ The rules are the same for every decider: Accepting admits the workflow to the `problems_only` approval mode. Its runs no longer wait before every save. They stop only when a check fails, the screen changed, or a step falls outside what was qualified, and then a person -decides. `openadapt-flow run` reads this from `admission-decision.json` in the +decides. Read-back is still required: a save counts as done without a person +only when OpenAdapt read the saved record back from your system's data and it +matched. Per-save approval (`each_item`, with one-tap approval from a phone +paired to the Desktop decision portal) stays available as the strictest +setting. `openadapt-flow run` reads this from `admission-decision.json` in the bundle and prints who admitted it. Four things keep it narrow: - A `--approval-mode` flag or the deployment's `runtime.approval_mode` still From 7a9aeb6429f246f96e30f475610af14514ff4218 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 12:29:15 -0400 Subject: [PATCH 11/12] fix(admit): sign admission records, keep screen text from agents Review fixes for the admission pipeline before merge: - Sign admission-pending.json and admission-decision.json with the local admission key (the JIT key directory), as JIT admissions on main are. admitted_approval_mode refuses an unsigned or edited decision record (admission_signature_invalid), so write access to a bundle can no longer set problems_only, and an edited pending file can't change who prepared an admission. - An agent can't decide an admission an unidentified MCP client (agent:mcp-client) prepared; that may be the same agent. A person can. - The summaries the MCP tools return are value-free: record-next codes, not induction's record-next lines, which can quote recorded labels, and no halt reason or shadow-suggestion text. - Test that an admitted bundle's problems_only still sends each save to a person until the signed switch is on. Co-Authored-By: Claude Opus 5.5 --- packages/agent/tests/test_admission_tools.py | 41 ++++ packages/flow/docs/SHADOW_CANDIDATES.md | 20 +- .../flow/openadapt_flow/admit_pipeline.py | 115 +++++++++-- .../flow/openadapt_flow/agent/admission.py | 39 +++- packages/flow/tests/test_admit_pipeline.py | 188 ++++++++++++++++++ 5 files changed, 373 insertions(+), 30 deletions(-) diff --git a/packages/agent/tests/test_admission_tools.py b/packages/agent/tests/test_admission_tools.py index 9704214ca..fe150b178 100644 --- a/packages/agent/tests/test_admission_tools.py +++ b/packages/agent/tests/test_admission_tools.py @@ -199,6 +199,47 @@ def test_without_a_delegate_the_bridge_never_decides(bundles_root, runner_config ) +def test_a_delegate_cannot_decide_what_an_unidentified_client_prepared( + bundles_root, runner_config, tmp_path +): + _recording(tmp_path) + # Prepared on a server without --delegate-approvals: the preparer is + # "mcp-client", which may be the same agent that holds the delegate id. + view = _bridge(bundles_root, runner_config, tmp_path).dispatch( + "prepare_admission", {"recordings": ["rec1"], "name": "w"} + ) + assert view["prepared_by"]["id"] == "mcp-client" + reviewer = _bridge( + bundles_root, + runner_config, + tmp_path, + delegate="agent:reviewer", + grant=_grant_file(tmp_path), + ) + with pytest.raises(BridgeError, match="unidentified"): + reviewer.dispatch( + "decide_admission", + { + "admission": view["admission"], + "pending_sha256": view["pending_sha256"], + "decision": "accept", + }, + ) + assert not (tmp_path / "admissions" / view["admission"] / "admission-decision.json").exists() + + +def test_a_tampered_pending_file_is_listed_but_never_shown(bundles_root, runner_config, tmp_path): + _recording(tmp_path) + bridge = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:intake-bot") + view = bridge.dispatch("prepare_admission", {"recordings": ["rec1"], "name": "w"}) + path = tmp_path / "admissions" / view["admission"] / "admission-pending.json" + data = json.loads(path.read_text()) + data["prepared_by"] = {"kind": "person", "id": "pat", "route": "cli"} + path.write_text(json.dumps(data)) + listed = bridge.dispatch("list_admissions", {})["admissions"] + assert listed == [{"admission": view["admission"], "status": "unverified"}] + + def test_paths_stay_inside_the_admissions_folder(bundles_root, runner_config, tmp_path): bridge = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:x") for bad in ("../etc", "/tmp", "a/b"): diff --git a/packages/flow/docs/SHADOW_CANDIDATES.md b/packages/flow/docs/SHADOW_CANDIDATES.md index e94e8de2a..0aa455642 100644 --- a/packages/flow/docs/SHADOW_CANDIDATES.md +++ b/packages/flow/docs/SHADOW_CANDIDATES.md @@ -156,13 +156,25 @@ The rules are the same for every decider: - **Who decided is recorded.** `admission-decision.json`, in the admission folder and in the accepted bundle, names who prepared and who decided, by kind, id and route. -- **No self-approval.** An agent can't decide an admission it prepared. +- **No self-approval.** An agent can't decide an admission it prepared. A + server started without `--delegate-approvals` records its preparer as + `agent:mcp-client`, which names no one; only a person decides those. +- **Signed by this machine.** `admission-pending.json` and + `admission-decision.json` are signed with the local admission key (the JIT + key directory, `$OPENADAPT_JIT_KEY_DIR` or `~/.openadapt/jit`). An edited or + hand-written file is refused, so write access to the admission folder or + the bundle is not enough to change who prepared or decided. Keep the key + directory away from the agents it governs. +- **The agent sees codes, not screen text.** The summaries the MCP tools + return carry step ids, check kinds, counts and record-next codes. Induction's + record-next lines and halt reasons can quote labels from the recordings, so + they are shown only at the terminal. - **The `regulated` policy pack needs a person.** - **A decision covers exactly what was shown.** It names the `pending_sha256` of the summary, and the bundle must still match its digest. There is one decision per admission. - **No gate is relaxed.** Shadow suggestions are never applied, a HALTed - proposal can't be accepted, and no admission is signed here. + proposal can't be accepted, and no release admission is signed here. ### What admission changes for runs @@ -174,10 +186,12 @@ only when OpenAdapt read the saved record back from your system's data and it matched. Per-save approval (`each_item`, with one-tap approval from a phone paired to the Desktop decision portal) stays available as the strictest setting. `openadapt-flow run` reads this from `admission-decision.json` in the -bundle and prints who admitted it. Four things keep it narrow: +bundle and prints who admitted it. Five things keep it narrow: - A `--approval-mode` flag or the deployment's `runtime.approval_mode` still wins. +- The decision record must carry a valid signature by the local admission + key. A record without one, or edited after signing, gives no mode. - It applies only to the exact admitted bundle. After any change to the bundle, each save waits for a person again. - An agent's admission applies only while its signed grant still verifies. diff --git a/packages/flow/openadapt_flow/admit_pipeline.py b/packages/flow/openadapt_flow/admit_pipeline.py index a99b09db3..d7610e177 100644 --- a/packages/flow/openadapt_flow/admit_pipeline.py +++ b/packages/flow/openadapt_flow/admit_pipeline.py @@ -38,7 +38,15 @@ was shown. * treat shadow-suggested checks as evidence. They are listed for the person or approver to read; only checks mined from real recordings are applied. -* sign an admission. No local-dev or production admission is minted here. +* sign a release admission. No local-dev or production admission is minted + here. The pending summary and the decision record are signed with this + machine's local admission key (the JIT key directory, kept away from the + agents it governs), so write access to the admission folder or the bundle + is not enough to forge a decision or change who prepared it. +* send recorded screen text to an agent. The summary an MCP client gets + (:func:`render_summary` with ``value_free=True``) carries codes, counts and + step ids only; induction's record-next lines can quote recorded labels and + stay at the terminal. * let an agent admit without a person's signature. An agent decides only under a verified grant that names it, covers this workflow, and lets an agent approve consequential saves; anything less is refused. @@ -72,6 +80,10 @@ DECIDER_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$" #: Policy packs under which only a person may decide. PERSON_ONLY_PACKS = frozenset({"regulated"}) +#: The preparer id for an MCP client that runs without --delegate-approvals. +#: It names no one, so it may be the same agent that later decides; only a +#: person decides what it prepared. +UNIDENTIFIED_AGENT = "mcp-client" #: The approval mode an accepted admission gives the workflow's runs. ADMITTED_APPROVAL_MODE: Literal["problems_only"] = "problems_only" _SHA256 = r"^sha256:[0-9a-f]{64}$" @@ -260,6 +272,7 @@ def prepare_admission( candidate: Optional[Path | str] = None, policy_pack: str = "community", admission_id: Optional[str] = None, + key_dir: Optional[Path | str] = None, ) -> PendingAdmission: """Run compile, induce and the check proposal; write the single stop. @@ -267,6 +280,9 @@ def prepare_admission( ``candidate`` and no inputs, the recordings are read from the candidate's ``recordings/`` folder; with none there, the result is ``needs_recording`` and carries the candidate's record-next list. + + The pending file is signed with the local admission key in ``key_dir`` + (default ``~/.openadapt/jit``). """ from openadapt_flow.compiler import compile_recording @@ -318,7 +334,7 @@ def prepare_admission( def stop(status: PendingStatus, reason: str, **extra: Any) -> PendingAdmission: pending = PendingAdmission(status=status, halt_reason=reason, **base, **extra) - _write_json(target / PENDING_FILE, pending.model_dump(mode="json")) + _write_signed(target / PENDING_FILE, pending.model_dump(mode="json"), key_dir) return pending if not paths: @@ -391,7 +407,7 @@ def stop(status: PendingStatus, reason: str, **extra: Any) -> PendingAdmission: **detail, ) pending = PendingAdmission(status="needs_decision", **base, **detail) - _write_json(target / PENDING_FILE, pending.model_dump(mode="json")) + _write_signed(target / PENDING_FILE, pending.model_dump(mode="json"), key_dir) return pending @@ -414,17 +430,60 @@ def _copy_templates(compiled: Sequence[Path], bundle_dir: Path) -> None: shutil.copy2(item, dest) -def load_pending(out: Path | str) -> PendingAdmission: +def load_pending( + out: Path | str, *, key_dir: Optional[Path | str] = None +) -> PendingAdmission: + """The pending admission, only if this machine's admission key signed it.""" + path = Path(out) / PENDING_FILE - if not path.is_file(): + if not path.is_file() or path.is_symlink(): raise AdmissionError(f"no pending admission in {out}") - return PendingAdmission.model_validate_json(path.read_text(encoding="utf-8")) + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise AdmissionError(f"the pending admission is unreadable: {exc}") from exc + if not _signed(data, key_dir): + raise AdmissionError( + "the pending admission is not signed by this machine's admission " + "key; prepare it again" + ) + data.pop("signature", None) + return PendingAdmission.model_validate(data) def _write_json(path: Path, payload: dict[str, Any]) -> None: path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", "utf-8") +def _key_dir(key_dir: Optional[Path | str]) -> Path: + from openadapt_flow.jit.lifecycle import default_key_dir + + return Path(key_dir) if key_dir is not None else default_key_dir() + + +def _write_signed( + path: Path, payload: dict[str, Any], key_dir: Optional[Path | str] +) -> None: + """Write ``payload`` with a signature by the local admission key.""" + + from openadapt_flow.jit.lifecycle import _sign_record + + body = {k: v for k, v in payload.items() if k != "signature"} + body["signature"] = _sign_record(body, _key_dir(key_dir)) + _write_json(path, body) + + +def _signed(data: Any, key_dir: Optional[Path | str]) -> bool: + from openadapt_flow.jit.lifecycle import _record_signed + + if not isinstance(data, dict) or not isinstance(data.get("signature"), str): + return False + try: + return _record_signed(data, _key_dir(key_dir)) + except (OSError, ValueError): + return False + + def _agent_identity(decided_by: Decider) -> Any: from openadapt_flow.approval_mode.approver import ( ApproverAuth, @@ -512,6 +571,11 @@ def check_decider( f"the {pending.policy_pack!r} policy pack needs a person to decide" ) prepared = pending.prepared_by + if prepared.kind == "agent" and prepared.id == UNIDENTIFIED_AGENT: + raise AdmissionError( + "an unidentified MCP client prepared this admission, so it may be " + "the same agent; a person decides it" + ) if prepared.kind == "agent" and prepared.id == decided_by.id: raise AdmissionError( "self-approval refused: the agent that prepared this admission " @@ -559,7 +623,7 @@ def decide_admission( ) target = Path(out) - pending = load_pending(target) + pending = load_pending(target, key_dir=key_dir) if (target / DECISION_FILE).exists(): raise AdmissionError("this admission was already decided") if pending.sha256() != pending_sha256: @@ -636,9 +700,9 @@ def decide_admission( ) result = AdmissionDecision(**record) payload = result.model_dump(mode="json") - _write_json(target / DECISION_FILE, payload) + _write_signed(target / DECISION_FILE, payload, key_dir) if decision == "accept": - _write_json(bundle / DECISION_FILE, payload) + _write_signed(bundle / DECISION_FILE, payload, key_dir) return result @@ -663,18 +727,25 @@ def admitted_approval_mode( """The approval mode an accepted admission gives this exact bundle. No admission, or a rejected one: no mode (the run's default applies). A - recorded admission for a bundle that has since changed, or an agent's - admission whose grant no longer verifies, names the reason and gives no - mode. Never raises. + record this machine's admission key didn't sign, a recorded admission for + a bundle that has since changed, or an agent's admission whose grant no + longer verifies, names the reason and gives no mode. Never raises. """ path = Path(bundle) / DECISION_FILE if not path.is_file() or path.is_symlink(): return AdmittedMode() try: - record = AdmissionDecision.model_validate_json(path.read_text("utf-8")) + data = json.loads(path.read_text("utf-8")) except (OSError, ValueError): return AdmittedMode(refusal="admission_unreadable") + if not _signed(data, key_dir): + return AdmittedMode(refusal="admission_signature_invalid") + try: + data.pop("signature", None) + record = AdmissionDecision.model_validate(data) + except ValueError: + return AdmittedMode(refusal="admission_unreadable") if record.decision != "accept" or record.approval_mode is None: return AdmittedMode() try: @@ -699,8 +770,13 @@ def admitted_approval_mode( return AdmittedMode(approval_mode=record.approval_mode, admitted_by=decider) -def render_summary(pending: PendingAdmission) -> str: - """The short summary shown at the single stop.""" +def render_summary(pending: PendingAdmission, *, value_free: bool = False) -> str: + """The short summary shown at the single stop. + + ``value_free`` is the form for an MCP client: it leaves out the halt + reason, the record-next lines and shadow-suggestion text, which can quote + labels or dialog text from the recordings, and keeps their codes. + """ lines = [f"Admission {pending.admission_id[:12]}: {pending.workflow_name}"] if pending.candidate_motif_id: @@ -711,6 +787,11 @@ def render_summary(pending: PendingAdmission) -> str: + f"; {pending.n_steps} step(s), {pending.n_writes} write(s)" ) if pending.status != "needs_decision": + if value_free: + lines.append(f" STOPPED ({pending.status})") + for item in pending.record_next: + lines.append(f" - record next: {item.get('code')}") + return "\n".join(lines) + "\n" lines.append(f" STOPPED ({pending.status}): {pending.halt_reason}") for item in pending.record_next: lines.append(f" - record next: {item.get('instruction')}") @@ -723,7 +804,8 @@ def render_summary(pending: PendingAdmission) -> str: if pending.shadow_suggestions: lines.append(" Suggested by shadow episodes (not evidence, not applied):") for item in pending.shadow_suggestions: - lines.append(f" - {item.get('kind')}: {item.get('summary')}") + text = "" if value_free else f": {item.get('summary')}" + lines.append(f" - {item.get('kind')}{text}") lines.append(f" Prepared by {pending.prepared_by.label()}") return "\n".join(lines) + "\n" @@ -742,4 +824,5 @@ def render_summary(pending: PendingAdmission) -> str: "prepare_admission", "render_summary", "resolve_candidate", + "UNIDENTIFIED_AGENT", ] diff --git a/packages/flow/openadapt_flow/agent/admission.py b/packages/flow/openadapt_flow/agent/admission.py index abb19dbe9..9562ff04e 100644 --- a/packages/flow/openadapt_flow/agent/admission.py +++ b/packages/flow/openadapt_flow/agent/admission.py @@ -18,11 +18,17 @@ argument never stands in for that answer. What an agent can't do: decide an admission it prepared (the record names -the preparer), decide under the ``regulated`` policy pack, or decide without -the exact ``pending_sha256`` it was shown, or decide without a grant a person -signed. Accepting runs the same gates as ``qualify accept`` and admits the -workflow to the ``problems_only`` approval mode; nothing here signs an -admission. +the preparer), decide one an unidentified client prepared (a server without +``--delegate-approvals`` records its preparer as ``mcp-client``), decide +under the ``regulated`` policy pack, decide without the exact +``pending_sha256`` it was shown, or decide without a grant a person signed. +Accepting runs the same gates as ``qualify accept`` and admits the workflow to +the ``problems_only`` approval mode. + +What an agent sees: codes, counts and step ids. The summaries these tools +return are value-free (:func:`~openadapt_flow.admit_pipeline.render_summary` +with ``value_free=True``), because the client may send them to a hosted +model and induction's record-next lines can quote recorded labels. """ from __future__ import annotations @@ -34,6 +40,7 @@ from openadapt_flow.admit_pipeline import ( PENDING_FILE, + UNIDENTIFIED_AGENT, AdmissionError, Decider, decide_admission, @@ -118,7 +125,9 @@ def _child(self, area: str, name: Any, pattern: str = _FOLDER) -> Path: return resolved def _preparer(self) -> Decider: - return Decider(kind="agent", id=self.delegate_id or "mcp-client", route="mcp") + return Decider( + kind="agent", id=self.delegate_id or UNIDENTIFIED_AGENT, route="mcp" + ) # -- specs ------------------------------------------------------------- @@ -201,11 +210,10 @@ def needs_person(self) -> bool: # -- tools ------------------------------------------------------------- def _view(self, folder: Path) -> dict[str, Any]: - pending = load_pending(folder) + pending = load_pending(folder, key_dir=self.key_dir) return { "admission": folder.name, "status": pending.status, - "halt_reason": pending.halt_reason, "workflow_name": pending.workflow_name, "policy_pack": pending.policy_pack, "prepared_by": pending.prepared_by.model_dump(mode="json"), @@ -216,7 +224,7 @@ def _view(self, folder: Path) -> dict[str, Any]: ], "record_next": [item.get("code") for item in pending.record_next], "decided": (folder / "admission-decision.json").exists(), - "summary": render_summary(pending), + "summary": render_summary(pending, value_free=True), "pending_sha256": pending.sha256(), } @@ -242,6 +250,7 @@ def prepare(self, arguments: dict[str, Any]) -> dict[str, Any]: candidate=candidate, policy_pack=arguments.get("policy_pack") or "community", admission_id=admission_id, + key_dir=self.key_dir, ) except AdmissionError as exc: raise AdmissionToolError(str(exc)) from exc @@ -257,14 +266,22 @@ def list(self) -> dict[str, Any]: and not folder.is_symlink() and (folder / PENDING_FILE).is_file() ): - rows.append(self._view(folder)) + try: + rows.append(self._view(folder)) + except AdmissionError: + # Unsigned or unreadable: listed as such, never shown. + rows.append({"admission": folder.name, "status": "unverified"}) return {"admissions": rows} def confirmation_message(self, arguments: dict[str, Any]) -> str: folder = self._child("admissions", arguments.get("admission"), _ADMISSION) verb = "accept" if arguments.get("decision") == "accept" else "reject" + try: + pending = load_pending(folder, key_dir=self.key_dir) + except AdmissionError as exc: + raise AdmissionToolError(str(exc)) from exc return ( - render_summary(load_pending(folder)) + render_summary(pending, value_free=True) + f"\nConfirm that you, a person, {verb} this admission." ) diff --git a/packages/flow/tests/test_admit_pipeline.py b/packages/flow/tests/test_admit_pipeline.py index e300bf06b..d0b142526 100644 --- a/packages/flow/tests/test_admit_pipeline.py +++ b/packages/flow/tests/test_admit_pipeline.py @@ -28,6 +28,13 @@ AGENT_B = ap.Decider(kind="agent", id="agent:reviewer", route="mcp") +@pytest.fixture(autouse=True) +def _local_admission_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The local admission key lives in the test's key directory, not ~.""" + + monkeypatch.setenv("OPENADAPT_JIT_KEY_DIR", str(key_dir(tmp_path))) + + def _agent_policy() -> ApproverPolicyV1: allowed = {c.value: ["person"] for c in StepClass} allowed[StepClass.CONSEQUENTIAL.value] = ["person", "agent"] @@ -286,6 +293,13 @@ def test_an_agent_admission_stops_applying_when_its_grant_does(tmp_path: Path) - mode="json", by_alias=True ) record_path.write_text(json.dumps(record)) + assert ap.admitted_approval_mode(bundle, key_dir=keys).refusal == ( + "admission_signature_invalid" + ) + # Even re-signed by someone holding the local key, the swapped grant + # doesn't match the digest the decision recorded. + record.pop("signature") + ap._write_signed(record_path, record, keys) assert ap.admitted_approval_mode(bundle, key_dir=keys).refusal == ( "policy_ref_mismatch" ) @@ -505,3 +519,177 @@ def test_run_uses_the_admitted_mode_unless_told_otherwise(tmp_path, capsys) -> N # Without an admission the default is a person approving each save. bare = SimpleNamespace(approval_mode=None, bundle=str(tmp_path)) assert _resolve_run_approval_mode(bare, no_cfg, None).value == "each_item" + + +# -- fail-closed review of #1212 --------------------------------------------- + + +def _forged_decision(bundle: Path, pending: ap.PendingAdmission) -> dict: + return { + "schema_version": ap.DECISION_SCHEMA, + "admission_id": pending.admission_id, + "decision": "accept", + "decided_by": {"kind": "person", "id": "pat", "route": "cli"}, + "prepared_by": {"kind": "person", "id": "pat", "route": "cli"}, + "decided_at": "2026-01-01T00:00:00+00:00", + "pending_sha256": pending.sha256(), + "bundle_content_digest": pending.bundle_content_digest, + "proposal_sha256": pending.proposal_sha256, + "accepted_bundle_content_digest": ap._bundle_digest(bundle), + "approval_mode": "problems_only", + "workflow_name": pending.workflow_name, + } + + +def test_a_decision_file_written_without_the_admission_key_admits_nothing( + tmp_path: Path, +) -> None: + pending = _prepare(tmp_path) + bundle = tmp_path / "admission" / "bundle" + path = bundle / ap.DECISION_FILE + path.write_text(json.dumps(_forged_decision(bundle, pending))) + forged = ap.admitted_approval_mode(bundle) + assert forged.approval_mode is None + assert forged.refusal == "admission_signature_invalid" + # A signature by another key is no better. + record = _forged_decision(bundle, pending) + ap._write_signed(path, record, tmp_path / "other-keys") + assert ap.admitted_approval_mode(bundle).refusal == "admission_signature_invalid" + + +def test_a_signed_decision_edited_afterwards_admits_nothing(tmp_path: Path) -> None: + pending = _prepare(tmp_path, by=AGENT_A) + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=_grant(tmp_path), + key_dir=key_dir(tmp_path), + ) + bundle = tmp_path / "admission" / "bundle" + path = bundle / ap.DECISION_FILE + record = json.loads(path.read_text()) + # An agent's admission relabelled as a person's, to skip the grant check. + record["decided_by"] = {"kind": "person", "id": "pat", "route": "cli"} + record["approver_grant"] = None + path.write_text(json.dumps(record)) + edited = ap.admitted_approval_mode(bundle) + assert edited.approval_mode is None + assert edited.refusal == "admission_signature_invalid" + + +def test_an_edited_pending_file_is_refused(tmp_path: Path) -> None: + _prepare(tmp_path, by=AGENT_B) + path = tmp_path / "admission" / ap.PENDING_FILE + data = json.loads(path.read_text()) + # Relabel the preparer so the agent that prepared it could decide it. + data["prepared_by"] = {"kind": "person", "id": "pat", "route": "cli"} + path.write_text(json.dumps(data)) + with pytest.raises(ap.AdmissionError, match="not signed"): + ap.load_pending(tmp_path / "admission") + data.pop("signature") + edited = ap.PendingAdmission.model_validate(data) + with pytest.raises(ap.AdmissionError, match="not signed"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=edited.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=_grant(tmp_path), + key_dir=key_dir(tmp_path), + ) + assert not (tmp_path / "admission" / ap.DECISION_FILE).exists() + + +def test_an_agent_cannot_decide_what_an_unidentified_client_prepared( + tmp_path: Path, +) -> None: + unidentified = ap.Decider(kind="agent", id=ap.UNIDENTIFIED_AGENT, route="mcp") + pending = _prepare(tmp_path, by=unidentified) + with pytest.raises(ap.AdmissionError, match="unidentified"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=_grant(tmp_path), + key_dir=key_dir(tmp_path), + ) + # A person still can. + result = ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + assert result.decided_by == PERSON + + +def test_the_value_free_summary_quotes_no_recorded_text() -> None: + recorded = "Sample Label Alpha" + halted = ap.PendingAdmission( + admission_id="0" * 32, + status="halted", + halt_reason=f"induction refused near {recorded}", + workflow_name="w", + policy_pack="community", + prepared_by=PERSON, + created_at="2026-01-01T00:00:00+00:00", + record_next=[ + { + "code": "ambiguous_branch", + "instruction": f"Record each of {recorded} vs Other under a " + "detectable condition.", + } + ], + ) + assert recorded in ap.render_summary(halted) + agent_view = ap.render_summary(halted, value_free=True) + assert recorded not in agent_view + assert "ambiguous_branch" in agent_view + ready = halted.model_copy( + update={ + "status": "needs_decision", + "halt_reason": None, + "record_next": [], + "shadow_suggestions": [ + {"kind": "file_saved", "summary": f"saved {recorded}.xlsx"} + ], + } + ) + assert recorded not in ap.render_summary(ready, value_free=True) + + +def test_an_admitted_bundle_still_asks_a_person_until_the_switch_is_on( + tmp_path: Path, capsys +) -> None: + """Admission picks problems_only; the switch record still gates each save.""" + + from openadapt_flow.__main__ import _resolve_run_approval_mode + from openadapt_flow.approval_mode.gate import ApprovalGate + from openadapt_flow.approval_mode.model import ItemDecisionKind + from tests._approval_mode_support import BUNDLE_DIGEST, good_item + from tests.test_approval_mode_gate import Harness + + pending = _prepare(tmp_path) + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + args = SimpleNamespace( + approval_mode=None, bundle=str(tmp_path / "admission" / "bundle") + ) + no_cfg = SimpleNamespace(runtime=SimpleNamespace(approval_mode=None)) + mode = _resolve_run_approval_mode(args, no_cfg, None) + assert mode.value == "problems_only" + # No signed switch record: the gate sends the save to a person. + gate = ApprovalGate(mode, item=good_item()) + gate.begin_item( + run_id="r", bundle_content_digest=BUNDLE_DIGEST, profile="regulated" + ) + verdict = gate.before_save(Harness(tmp_path / "gate").save_context()) + assert not verdict.proceed + assert verdict.decision.kind is ItemDecisionKind.APPROVAL_REQUIRED From a89d8a88d9524bc1a20790f12e930c5d14cb3dd7 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 12:29:15 -0400 Subject: [PATCH 12/12] docs(flow): how induce, teach-branch and shadow mining learn from many runs Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/CAPABILITIES.md | 2 + packages/flow/docs/LEARN_FROM_MANY_RUNS.md | 130 +++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 packages/flow/docs/LEARN_FROM_MANY_RUNS.md diff --git a/packages/flow/docs/CAPABILITIES.md b/packages/flow/docs/CAPABILITIES.md index e175d2888..f56380ba0 100644 --- a/packages/flow/docs/CAPABILITIES.md +++ b/packages/flow/docs/CAPABILITIES.md @@ -47,6 +47,8 @@ at replaying one. These capabilities layer onto the same $0, model-free runtime. program` emits a program when the traces agree, and a `record-next:` worklist of missing demonstrations when a consequential branch or loop stays underdetermined. The healthy replay path still makes no model call. + [`LEARN_FROM_MANY_RUNS.md`](LEARN_FROM_MANY_RUNS.md) covers `induce`, + `teach-branch` and shadow mining together, with each one's status. - **Effect verification against the system of record.** The read-back checks our own work: a wrong field, a wrong record, a duplicate submit, or a partial save can all look like success on screen. A step diff --git a/packages/flow/docs/LEARN_FROM_MANY_RUNS.md b/packages/flow/docs/LEARN_FROM_MANY_RUNS.md new file mode 100644 index 000000000..f28324a7e --- /dev/null +++ b/packages/flow/docs/LEARN_FROM_MANY_RUNS.md @@ -0,0 +1,130 @@ +# Learning one program from many runs + +OpenAdapt doesn't have to compile a single linear demonstration. Three paths +turn several runs of the same task into one program with branches. Each path +refuses when the runs don't settle a question, and none of them puts a new +branch into service without a person, or an agent under a policy a person +signed, admitting it. + +| Path | Input | Output | Status | +|---|---|---|---| +| `induce` | Two or more recordings of the task | One program with parameters, loops and branches, or a refusal with a list of what to record next | Released (openadapt-flow 2.2.0 and earlier). Tested in CI on controlled demonstrations. | +| `teach-branch` | Two or more halted runs fixed the same way, plus optional normal runs | A guarded branch, written as a repair candidate | Released (openadapt-flow 2.2.0). Tested in CI on controlled, synthetic runs. | +| Shadow mining + `admit` | Episodes that Capture's shadow mode noticed in a person's real work | A candidate draft, then a program once the task is recorded and admitted | Capture side released (openadapt-capture 1.3.0, off by default). Flow side merged, not released. Tested only on synthetic episodes. | + +None of these has field measurements yet. We have not measured how many real +items take a branch that no recording showed, or how many of those branches +these paths recover. + +## What happens at a branch the program never saw + +The runtime stops. A step whose screen, record or result doesn't match what +was qualified halts before any save, and the item goes to a person. The +program never guesses a path. The three paths below are how those halts, and +more recordings, become branches. + +## `induce`: several recordings, one program + +```bash +openadapt-flow induce rec-1 rec-2 rec-3 --out program +``` + +`induce_program` aligns the recordings and works out what is the same in all +of them (fixed steps), what changes (parameters), what repeats (loops), and +where they split (branches, each with the screen condition that tells them +apart). It is deterministic and makes no model calls. + +When the recordings don't settle a question, it refuses instead of guessing: +`certified` is false and it prints a `record-next` worklist, such as "record +each of these two paths under a condition you can see" or "record the path +where this dialog doesn't appear". Record those and run it again. + +`admit rec-1 rec-2` runs the same induction as its first step, then proposes +the checks and stops once for a decision (see +[SHADOW_CANDIDATES.md](SHADOW_CANDIDATES.md#how-a-candidate-is-admitted)). + +## `teach-branch`: several halts fixed the same way + +```bash +openadapt-flow teach-branch --bundle program \ + --example runs/halt-1 fix-1.json --example runs/halt-2 fix-2.json \ + --normal-run runs/ok-1 --out candidate +``` + +A single fixed halt (`teach`) can only key on what that one screen showed, +which may be a name or a reference number. `teach-branch` needs at least two +halts at the same step, fixed with the same corrective steps. It keys the +branch on screen text that every example showed and no earlier screen did. +If a normal run also shows that text, it narrows the guard with a run +parameter that separates the two cases, or refuses when none does. + +The result goes through the same learning loop as any repair (induction, +regression gate, held-out check) and is written as a repair candidate. It is +not active. Activation follows the repair lifecycle: qualification runs, +approval and a canary (see [REPAIR_LIFECYCLE.md](REPAIR_LIFECYCLE.md)). + +## Shadow mining: many real runs, watched in the background + +Capture's shadow mode is off until the person acknowledges a notice. While on, +it keeps value-free symbols (application, control role, scrubbed URL pattern, +short chrome labels, and references in place of values) in a 24-hour encrypted +buffer on the computer, and opens no network connection. When it sees the same +sequence repeat, it offers a draft. If the person clicks **Review draft**, it +exports the episodes as `openadapt.shadow.episodes/v2`. + +Flow's `induce_candidate` turns each episode into a trace and runs the same +`induce_program` over them, so differences between episodes become +parameters, optional steps and branches, or a refusal. The output is a +candidate: `admitted` and `runnable` are always false, anchors are +placeholders (shadow mining keeps no pixels), and suggested checks are listed +as suggestions, never applied. Today a script calls `induce_candidate` and +`write_candidate`; there is no CLI command for this step yet. + +A candidate becomes a program only through `admit`: + +1. Record the task once with Capture, so the program has real anchors. +2. `openadapt-flow admit --candidate ID --out DIR --name NAME` compiles the + recordings, induces one program, proposes checks, and stops once. +3. A person accepts at a terminal or in the MCP client's form. An agent can + accept only with `--delegate-approvals` and an approver grant that a + registered person signed, which names the agent, covers the workflow, and + lets an agent approve saves. An agent can't accept an admission it + prepared, or one an unidentified MCP client prepared, and can't accept + under the `regulated` policy pack. +4. Accepting applies the checks through the same oracle gate as + `qualify accept`. It runs nothing. `qualify campaign` must still pass + before the program is certified. + +Accepting sets the `problems_only` approval mode for that exact bundle: runs +stop only for problems instead of before every save. That mode keeps its own +readiness gate. Until the workflow has earned the switch and it is on, every +save still waits for a person. A changed bundle, an expired grant, or a +decision record not signed by the machine's admission key drops back to a +person approving each save. Per-save approval stays available as the +strictest setting. + +### Privacy + +- Shadow episodes carry no screenshots, coordinates, typed values or keyed + digests; Flow refuses a v1 export (which had digests), any unknown field, a + label containing digits or `@`, and any hex-like token. +- Everything above runs on the computer. Mined episodes and candidates stay in + local folders. +- The MCP admission tools return codes, counts and step ids only. Induction's + record-next lines can quote labels from real recordings, so they appear + only at the terminal. +- Recordings made for admission are ordinary Capture recordings, and `admit` + sends none of them anywhere. Scrubbing of what Flow writes from them + follows [PRIVACY.md](PRIVACY.md): pin `OPENADAPT_FLOW_SCRUB=on` with the + `privacy` extra so a missing openadapt-privacy fails the run. Under the + default `auto`, without the extra, Flow writes plaintext and warns. + +## Code + +- `openadapt_flow/compiler/induction.py`: `induce_program` +- `openadapt_flow/learning/branch_teach.py`: `teach_branch` +- `openadapt_flow/compiler/motif_induction.py`: `induce_candidate`, + `write_candidate` +- `openadapt_flow/admit_pipeline.py` and `openadapt_flow/cli_admit.py`: + `admit` +- `openadapt_flow/agent/admission.py`: the MCP admission tools