diff --git a/packages/agent/README.md b/packages/agent/README.md index 4bb2701b4..29c07f103 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -282,6 +282,8 @@ Flow's [execution profiles](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/p | `--timeout SECONDS` | How long one run may work (default 600). Time a person spends approving a save doesn't count. A run stopped at the limit is `not_sure_if_saved`. | | `--approval-mode MODE` | Whether each save waits for approval. Without it, Flow decides, and a run stops before each save until it's approved (`each_item`). `not_gated` turns that off for every run this server starts; each result records it under `technical.approval_mode`. | | `--delegate-approvals AGENT_ID` | Lets your agent answer a waiting save as `AGENT_ID` (an opaque id such as `agent:intake-bot`), with `list_approval_requests` and `answer_approval_request`. Flow accepts the answer only where the run's approver policy allows an agent for that step, and records the agent as the approver. By default only a person may answer. Needs openadapt-flow 2.2.0 or later. | +| `--admissions DIR` | Adds `prepare_admission`, `list_admissions` and `decide_admission`, the same steps as `openadapt-flow admit`, over recordings in `DIR/recordings` and shadow candidates in `DIR/shadow-candidates`. A person decides in your client's form. With `--delegate-approvals` and `--approver-grant`, your agent decides as that approver instead, under a grant a person signed that names it. It never decides an admission it prepared or one prepared on a server without `--delegate-approvals`, and never under the `regulated` policy pack. Accepting sets the workflow's runs to `problems_only`. The record names who decided, and for your agent the grant and who signed it. Not released yet. | +| `--approver-grant FILE` | With `--admissions` and `--delegate-approvals`: the approver grant a person signed (`openadapt-flow approver-grant sign`) that lets your agent decide admissions. Flow checks it on every decision. Without it only a person decides. | | `--headed` | Shows the browser. A person needs it to continue or skip a stopped run in a web app. | | `--extra-run-arg ARG`, `--flow-cli CMD` | Change how Flow is called. Either one can relax Flow's checks, for example `--extra-run-arg=--allow-unencrypted`. | | `--allow-protected-export`, `--allow-synthetic-recorded-defaults`, `--allow-model-grounding` | Off by default. See [Security](#security). | diff --git a/packages/agent/tests/test_admission_tools.py b/packages/agent/tests/test_admission_tools.py new file mode 100644 index 000000000..fe150b178 --- /dev/null +++ b/packages/agent/tests/test_admission_tools.py @@ -0,0 +1,312 @@ +"""MCP admission tools: prepare, list, decide. Made-up values only. + +The tools are off unless the operator passes --admissions. A person decides +in the client's form; with --delegate-approvals and a person-signed +--approver-grant the agent decides as that approver. An agent never decides +an admission it prepared or without a grant that verifies, and the record +names who decided (and, for an agent, the grant and who signed it). +""" + +from __future__ import annotations + +import io +import json +from pathlib import Path + +import anyio +import mcp.types as types +import pytest +from PIL import Image + +from openadapt_agent.admission import AdmissionDesk +from openadapt_agent.bridge import AgentBridge, BridgeError +from openadapt_agent.mcp import build_server +from openadapt_flow.ir import ( + ActionKind, + Anchor, + Postcondition, + PostconditionKind, + Step, + Workflow, +) +from openadapt_flow.runtime.effects import Effect, EffectKind, ValueExpr + +TOOLS = {"prepare_admission", "list_admissions", "decide_admission"} + + +def _png() -> bytes: + buffer = io.BytesIO() + Image.new("RGB", (40, 20), "white").save(buffer, format="PNG") + return buffer.getvalue() + + +def _recording(root: Path, name: str = "rec1") -> None: + folder = root / "recordings" / name + (folder / "templates").mkdir(parents=True) + (folder / "templates" / "save.png").write_bytes(_png()) + workflow = Workflow( + name="qualified-write", + params={"record_id": "example", "note": "example"}, + surface="web", + steps=[ + Step( + id="save", + intent="Save the record", + action=ActionKind.CLICK, + anchor=Anchor( + template="templates/save.png", + region=(10, 10, 40, 20), + click_point=(30, 20), + ocr_text="Save", + structured_identity="record identity", + ), + expect=[Postcondition(kind=PostconditionKind.TEXT_PRESENT, text="Saved")], + effects=[ + Effect( + kind=EffectKind.FIELD_EQUALS, + match={"id": ValueExpr(param="record_id")}, + field="note", + value=ValueExpr(param="note"), + idempotency_key=ValueExpr(param="record_id"), + risk="irreversible", + ) + ], + risk="irreversible", + identity_armed=True, + ) + ], + ) + workflow.save(folder) + (folder / "meta.json").write_text( + json.dumps( + { + "app_url": "http://127.0.0.1:9/", + "application": "MockMed", + "application_version": "tutorial", + "viewport": [1280, 800], + "surface": "web", + "params": dict(workflow.params), + } + ) + ) + + +def _grant_file(root: Path, delegates=("agent:reviewer",)) -> Path: + """A grant a made-up registered person signed, letting agents approve saves.""" + + from datetime import datetime, timedelta, timezone + + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + from openadapt_flow.approval_mode.approver import ApproverPolicyV1, StepClass + from openadapt_flow.approval_mode.signed_policy import ApproverGrant, sign_grant + from openadapt_flow.jit.authority import public_hex, register_principal + + key = Ed25519PrivateKey.generate() + register_principal( + root / "keys", + role="policy_signer", + principal_id="person-0001", + public_key_hex=public_hex(key), + ) + allowed = {c.value: ["person"] for c in StepClass} + allowed[StepClass.CONSEQUENTIAL.value] = ["person", "agent"] + start = datetime.now(timezone.utc) - timedelta(minutes=1) + grant = ApproverGrant( + grant_id="grant-0001", + signer="person-0001", + signed_at=start.isoformat(), + expires_at=(start + timedelta(days=30)).isoformat(), + approver_policy=ApproverPolicyV1.model_validate({"allowed": allowed}), + delegates=delegates, + ) + return sign_grant(grant, key).save(root / "grant.json") + + +def _bridge(bundles_root, runner_config, root, *, delegate=None, grant=None): + desk = AdmissionDesk(root, delegate_id=delegate, approver_grant=grant, key_dir=root / "keys") + return AgentBridge(bundles_root, runner_config, admission_desk=desk) + + +def _names(bridge): + return {spec.name for spec in bridge.list_tool_specs()} + + +def test_tools_are_off_without_admissions(bundles_root, runner_config): + bridge = AgentBridge(bundles_root, runner_config) + assert not _names(bridge) & TOOLS + with pytest.raises(BridgeError, match="off"): + bridge.dispatch("list_admissions", {}) + + +def test_prepare_stops_once_and_names_the_preparer(bundles_root, runner_config, tmp_path): + _recording(tmp_path) + bridge = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:intake-bot") + assert TOOLS <= _names(bridge) + view = bridge.dispatch("prepare_admission", {"recordings": ["rec1"], "name": "w"}) + assert view["status"] == "needs_decision" + assert view["prepared_by"] == {"kind": "agent", "id": "agent:intake-bot", "route": "mcp"} + assert {"step_id": "save", "kind": "field_equals", "tier": 1} in view["checks"] + assert view["decided"] is False + listed = bridge.dispatch("list_admissions", {})["admissions"] + assert [row["admission"] for row in listed] == [view["admission"]] + + +def test_an_agent_cannot_decide_what_it_prepared(bundles_root, runner_config, tmp_path): + _recording(tmp_path) + bridge = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:intake-bot") + view = bridge.dispatch("prepare_admission", {"recordings": ["rec1"], "name": "w"}) + args = { + "admission": view["admission"], + "pending_sha256": view["pending_sha256"], + "decision": "accept", + } + with pytest.raises(BridgeError, match="self-approval"): + bridge.dispatch("decide_admission", args) + ungranted = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:reviewer") + with pytest.raises(BridgeError, match="approver_grant_missing"): + ungranted.dispatch("decide_admission", args) + grant = _grant_file(tmp_path) + reviewer = _bridge( + bundles_root, runner_config, tmp_path, delegate="agent:reviewer", grant=grant + ) + result = reviewer.dispatch("decide_admission", args) + assert result["decided_by"]["kind"] == "agent" + assert result["decided_by"]["id"] == "agent:reviewer" + assert result["decided_by"]["on_behalf_of"] == "person-0001" + assert result["decided_by"]["policy_ref"].startswith("sha256:") + assert result["approval_mode"] == "problems_only" + assert result["prepared_by"]["id"] == "agent:intake-bot" + assert result["oracle_gate_passed"] is True + record = json.loads( + (tmp_path / "admissions" / view["admission"] / "admission-decision.json").read_text() + ) + assert record["decided_by"]["id"] == "agent:reviewer" + + +def test_without_a_delegate_the_bridge_never_decides(bundles_root, runner_config, tmp_path): + _recording(tmp_path) + bridge = _bridge(bundles_root, runner_config, tmp_path) + view = bridge.dispatch("prepare_admission", {"recordings": ["rec1"], "name": "w"}) + with pytest.raises(BridgeError, match="person must decide"): + bridge.dispatch( + "decide_admission", + { + "admission": view["admission"], + "pending_sha256": view["pending_sha256"], + "decision": "accept", + }, + ) + + +def test_a_delegate_cannot_decide_what_an_unidentified_client_prepared( + bundles_root, runner_config, tmp_path +): + _recording(tmp_path) + # Prepared on a server without --delegate-approvals: the preparer is + # "mcp-client", which may be the same agent that holds the delegate id. + view = _bridge(bundles_root, runner_config, tmp_path).dispatch( + "prepare_admission", {"recordings": ["rec1"], "name": "w"} + ) + assert view["prepared_by"]["id"] == "mcp-client" + reviewer = _bridge( + bundles_root, + runner_config, + tmp_path, + delegate="agent:reviewer", + grant=_grant_file(tmp_path), + ) + with pytest.raises(BridgeError, match="unidentified"): + reviewer.dispatch( + "decide_admission", + { + "admission": view["admission"], + "pending_sha256": view["pending_sha256"], + "decision": "accept", + }, + ) + assert not (tmp_path / "admissions" / view["admission"] / "admission-decision.json").exists() + + +def test_a_tampered_pending_file_is_listed_but_never_shown(bundles_root, runner_config, tmp_path): + _recording(tmp_path) + bridge = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:intake-bot") + view = bridge.dispatch("prepare_admission", {"recordings": ["rec1"], "name": "w"}) + path = tmp_path / "admissions" / view["admission"] / "admission-pending.json" + data = json.loads(path.read_text()) + data["prepared_by"] = {"kind": "person", "id": "pat", "route": "cli"} + path.write_text(json.dumps(data)) + listed = bridge.dispatch("list_admissions", {})["admissions"] + assert listed == [{"admission": view["admission"], "status": "unverified"}] + + +def test_paths_stay_inside_the_admissions_folder(bundles_root, runner_config, tmp_path): + bridge = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:x") + for bad in ("../etc", "/tmp", "a/b"): + with pytest.raises(BridgeError): + bridge.dispatch("prepare_admission", {"recordings": [bad], "name": "w"}) + + +def test_regulated_pack_refuses_an_agent(bundles_root, runner_config, tmp_path): + _recording(tmp_path) + person_server = _bridge(bundles_root, runner_config, tmp_path) + view = person_server.dispatch( + "prepare_admission", + {"recordings": ["rec1"], "name": "w", "policy_pack": "regulated"}, + ) + if view["status"] != "needs_decision": + pytest.skip("regulated pack halted the proposal for this fixture") + agent = _bridge(bundles_root, runner_config, tmp_path, delegate="agent:reviewer") + with pytest.raises(BridgeError, match="needs a person"): + agent.dispatch( + "decide_admission", + { + "admission": view["admission"], + "pending_sha256": view["pending_sha256"], + "decision": "accept", + }, + ) + + +def _wire(model): + return model.model_dump(by_alias=True, mode="json") + + +@pytest.mark.parametrize("answer", ["accept", "decline", "unsupported"]) +def test_person_decides_in_the_client_form( + answer, bundles_root, runner_config, tmp_path, mcp_client +): + _recording(tmp_path) + bridge = _bridge(bundles_root, runner_config, tmp_path) + view = bridge.dispatch("prepare_admission", {"recordings": ["rec1"], "name": "w"}) + prompts = [] + + async def elicit(context, params): + prompts.append(_wire(params)) + return types.ElicitResult(action=answer, content={"confirmed": answer == "accept"}) + + async def call(): + kwargs = {} if answer == "unsupported" else {"elicitation_callback": elicit} + async with mcp_client(build_server(bridge), **kwargs) as client: + return _wire( + await client.call_tool( + "decide_admission", + { + "admission": view["admission"], + "pending_sha256": view["pending_sha256"], + "decision": "accept", + }, + ) + ) + + result = anyio.run(call) + decision = tmp_path / "admissions" / view["admission"] / "admission-decision.json" + if answer == "accept": + assert result["isError"] is False + payload = json.loads(result["content"][0]["text"]) + assert payload["decided_by"] == {"kind": "person", "id": "mcp-form", "route": "mcp"} + assert "Proposed checks" in prompts[0]["message"] + assert decision.is_file() + return + assert result["isError"] is True + assert not decision.exists() diff --git a/packages/flow/docs/APPROVAL_AND_VERIFICATION.md b/packages/flow/docs/APPROVAL_AND_VERIFICATION.md index 56c1a642f..c2fb02738 100644 --- a/packages/flow/docs/APPROVAL_AND_VERIFICATION.md +++ b/packages/flow/docs/APPROVAL_AND_VERIFICATION.md @@ -208,6 +208,12 @@ must be under the target (5% at 90% by default). With no artifact or no audit labels it raises `EligibilityUnavailable` and the workflow stays per item. `openadapt-flow triage eligibility` prints the result. +An accepted `openadapt-flow admit` sets `problems_only` as the admitted +bundle's default approval mode (see `docs/SHADOW_CANDIDATES.md`). An agent +admits only under a person-signed grant that lets an agent approve +consequential saves. The gates above still apply: an admitted workflow sends +every save to a person until it has earned the switch. + ## Admitting a demonstrated read-back (planned) `openadapt-flow check admit` (branch `feat/demonstrated-readback-verifier`, diff --git a/packages/flow/docs/CAPABILITIES.md b/packages/flow/docs/CAPABILITIES.md index e175d2888..f56380ba0 100644 --- a/packages/flow/docs/CAPABILITIES.md +++ b/packages/flow/docs/CAPABILITIES.md @@ -47,6 +47,8 @@ at replaying one. These capabilities layer onto the same $0, model-free runtime. program` emits a program when the traces agree, and a `record-next:` worklist of missing demonstrations when a consequential branch or loop stays underdetermined. The healthy replay path still makes no model call. + [`LEARN_FROM_MANY_RUNS.md`](LEARN_FROM_MANY_RUNS.md) covers `induce`, + `teach-branch` and shadow mining together, with each one's status. - **Effect verification against the system of record.** The read-back checks our own work: a wrong field, a wrong record, a duplicate submit, or a partial save can all look like success on screen. A step diff --git a/packages/flow/docs/LEARN_FROM_MANY_RUNS.md b/packages/flow/docs/LEARN_FROM_MANY_RUNS.md new file mode 100644 index 000000000..f28324a7e --- /dev/null +++ b/packages/flow/docs/LEARN_FROM_MANY_RUNS.md @@ -0,0 +1,130 @@ +# Learning one program from many runs + +OpenAdapt doesn't have to compile a single linear demonstration. Three paths +turn several runs of the same task into one program with branches. Each path +refuses when the runs don't settle a question, and none of them puts a new +branch into service without a person, or an agent under a policy a person +signed, admitting it. + +| Path | Input | Output | Status | +|---|---|---|---| +| `induce` | Two or more recordings of the task | One program with parameters, loops and branches, or a refusal with a list of what to record next | Released (openadapt-flow 2.2.0 and earlier). Tested in CI on controlled demonstrations. | +| `teach-branch` | Two or more halted runs fixed the same way, plus optional normal runs | A guarded branch, written as a repair candidate | Released (openadapt-flow 2.2.0). Tested in CI on controlled, synthetic runs. | +| Shadow mining + `admit` | Episodes that Capture's shadow mode noticed in a person's real work | A candidate draft, then a program once the task is recorded and admitted | Capture side released (openadapt-capture 1.3.0, off by default). Flow side merged, not released. Tested only on synthetic episodes. | + +None of these has field measurements yet. We have not measured how many real +items take a branch that no recording showed, or how many of those branches +these paths recover. + +## What happens at a branch the program never saw + +The runtime stops. A step whose screen, record or result doesn't match what +was qualified halts before any save, and the item goes to a person. The +program never guesses a path. The three paths below are how those halts, and +more recordings, become branches. + +## `induce`: several recordings, one program + +```bash +openadapt-flow induce rec-1 rec-2 rec-3 --out program +``` + +`induce_program` aligns the recordings and works out what is the same in all +of them (fixed steps), what changes (parameters), what repeats (loops), and +where they split (branches, each with the screen condition that tells them +apart). It is deterministic and makes no model calls. + +When the recordings don't settle a question, it refuses instead of guessing: +`certified` is false and it prints a `record-next` worklist, such as "record +each of these two paths under a condition you can see" or "record the path +where this dialog doesn't appear". Record those and run it again. + +`admit rec-1 rec-2` runs the same induction as its first step, then proposes +the checks and stops once for a decision (see +[SHADOW_CANDIDATES.md](SHADOW_CANDIDATES.md#how-a-candidate-is-admitted)). + +## `teach-branch`: several halts fixed the same way + +```bash +openadapt-flow teach-branch --bundle program \ + --example runs/halt-1 fix-1.json --example runs/halt-2 fix-2.json \ + --normal-run runs/ok-1 --out candidate +``` + +A single fixed halt (`teach`) can only key on what that one screen showed, +which may be a name or a reference number. `teach-branch` needs at least two +halts at the same step, fixed with the same corrective steps. It keys the +branch on screen text that every example showed and no earlier screen did. +If a normal run also shows that text, it narrows the guard with a run +parameter that separates the two cases, or refuses when none does. + +The result goes through the same learning loop as any repair (induction, +regression gate, held-out check) and is written as a repair candidate. It is +not active. Activation follows the repair lifecycle: qualification runs, +approval and a canary (see [REPAIR_LIFECYCLE.md](REPAIR_LIFECYCLE.md)). + +## Shadow mining: many real runs, watched in the background + +Capture's shadow mode is off until the person acknowledges a notice. While on, +it keeps value-free symbols (application, control role, scrubbed URL pattern, +short chrome labels, and references in place of values) in a 24-hour encrypted +buffer on the computer, and opens no network connection. When it sees the same +sequence repeat, it offers a draft. If the person clicks **Review draft**, it +exports the episodes as `openadapt.shadow.episodes/v2`. + +Flow's `induce_candidate` turns each episode into a trace and runs the same +`induce_program` over them, so differences between episodes become +parameters, optional steps and branches, or a refusal. The output is a +candidate: `admitted` and `runnable` are always false, anchors are +placeholders (shadow mining keeps no pixels), and suggested checks are listed +as suggestions, never applied. Today a script calls `induce_candidate` and +`write_candidate`; there is no CLI command for this step yet. + +A candidate becomes a program only through `admit`: + +1. Record the task once with Capture, so the program has real anchors. +2. `openadapt-flow admit --candidate ID --out DIR --name NAME` compiles the + recordings, induces one program, proposes checks, and stops once. +3. A person accepts at a terminal or in the MCP client's form. An agent can + accept only with `--delegate-approvals` and an approver grant that a + registered person signed, which names the agent, covers the workflow, and + lets an agent approve saves. An agent can't accept an admission it + prepared, or one an unidentified MCP client prepared, and can't accept + under the `regulated` policy pack. +4. Accepting applies the checks through the same oracle gate as + `qualify accept`. It runs nothing. `qualify campaign` must still pass + before the program is certified. + +Accepting sets the `problems_only` approval mode for that exact bundle: runs +stop only for problems instead of before every save. That mode keeps its own +readiness gate. Until the workflow has earned the switch and it is on, every +save still waits for a person. A changed bundle, an expired grant, or a +decision record not signed by the machine's admission key drops back to a +person approving each save. Per-save approval stays available as the +strictest setting. + +### Privacy + +- Shadow episodes carry no screenshots, coordinates, typed values or keyed + digests; Flow refuses a v1 export (which had digests), any unknown field, a + label containing digits or `@`, and any hex-like token. +- Everything above runs on the computer. Mined episodes and candidates stay in + local folders. +- The MCP admission tools return codes, counts and step ids only. Induction's + record-next lines can quote labels from real recordings, so they appear + only at the terminal. +- Recordings made for admission are ordinary Capture recordings, and `admit` + sends none of them anywhere. Scrubbing of what Flow writes from them + follows [PRIVACY.md](PRIVACY.md): pin `OPENADAPT_FLOW_SCRUB=on` with the + `privacy` extra so a missing openadapt-privacy fails the run. Under the + default `auto`, without the extra, Flow writes plaintext and warns. + +## Code + +- `openadapt_flow/compiler/induction.py`: `induce_program` +- `openadapt_flow/learning/branch_teach.py`: `teach_branch` +- `openadapt_flow/compiler/motif_induction.py`: `induce_candidate`, + `write_candidate` +- `openadapt_flow/admit_pipeline.py` and `openadapt_flow/cli_admit.py`: + `admit` +- `openadapt_flow/agent/admission.py`: the MCP admission tools diff --git a/packages/flow/docs/SHADOW_CANDIDATES.md b/packages/flow/docs/SHADOW_CANDIDATES.md new file mode 100644 index 000000000..0aa455642 --- /dev/null +++ b/packages/flow/docs/SHADOW_CANDIDATES.md @@ -0,0 +1,217 @@ +# Shadow-mined candidates + +OpenAdapt Capture's shadow mode (off by default) can notice that a person +repeated the same cross-application task and, when the person clicks +**Review draft**, export the matching episodes. This page explains what Flow +does with that export. Capture's +[SHADOW_MINING.md](https://github.com/OpenAdaptAI/openadapt-capture/blob/main/docs/SHADOW_MINING.md) +covers how the episodes are mined and the privacy posture. + +The code is `openadapt_flow/compiler/motif_induction.py`. + +## What goes in + +An `openadapt.shadow.episodes/v2` JSON document. Each episode is a list of +value-free symbols: an application, a control role, a scrubbed URL pattern, +short plain-text labels for buttons and menu items made of common words, and +references in place of other labels and typed values. + +A reference such as `label-a` or `value-c` is local to one export. Two steps +with the same reference had the same label or value; that is all the export +says. Capture's keyed digests never leave Capture: its keys are destroyed +every 24 hours, and a digest that outlived its key could be matched against +guesses for short identifiers such as a record number or an SSN, so no digest +may reach a draft. + +The schema is strict. `induce_candidate` refuses: + +- a `v1` document (it carried keyed digests); +- a step with a `label_digest` or `value_digest` field, or any other unknown + field; +- a token with a hex part of eight or more characters; +- a malformed reference; +- a plain-text label that contains digits or `@`; +- an unknown step kind, or a status other than `candidate`. + +A label that Capture scrubbed (``) is treated as no label. + +## What comes out + +```python +from openadapt_flow.compiler.motif_induction import induce_candidate, write_candidate + +candidate = induce_candidate(export) # dict or ShadowEpisodesExport +write_candidate(candidate, "shadow-candidates/abc") +``` + +`induce_candidate` turns each episode into a linear trace and runs the +existing multi-trace induction (`induce_program`) over them. Induction infers +parameters, loops, and optional steps, or refuses, exactly as it does for +recorded demonstrations. The result is a `MotifCandidate`: + +| Field | Meaning | +|---|---| +| `status` | Always `candidate`. | +| `admitted`, `runnable` | Always `False`. The model rejects any other value. | +| `induction_certified` | Whether induction found the episodes structurally consistent. This is a statement about shape only. | +| `draft` | The induced program, or `None` when induction refused. | +| `record_next` | What to record or decide next. The first item is always "record this task once with OpenAdapt Capture". | +| `proposed_checks` | Verification checks the episodes suggest. | +| `admission_path` | The steps that turn the candidate into a program. | + +`write_candidate` writes `candidate.json` and, when there is a draft, +`draft-workflow.json`. It never writes `workflow.json`, so no Flow command +treats the folder as a bundle, and it refuses a folder that already holds one. + +## Why a candidate can't run + +- **No anchors.** Shadow mining keeps no pixels, so every anchor in the draft + is a placeholder (`shadow-unrecorded.png` at 0,0). +- **No values.** Parameter examples are references such as ``. +- **Writes need review.** A click on a Submit-, Save-, or Send-like control, + or a click right before a form submit, is marked `irreversible` with + `risk_review_required`, so certification refuses it until a person reviews + it. +- **Proposals aren't contracts.** A proposed check (`file_written`, + `record_written`, `row_added`, `page_reached`) has `status="proposed"` and + `admitted=False`, and is never attached to a step as an effect. The episodes + that suggested a check can't approve it. + +## How a candidate is admitted + +Record the task once with Capture. Then run one command: + +```bash +openadapt flow admit --candidate abc --out admissions/abc --name report-mail +# or, with your own recordings of the same task: +openadapt flow admit rec-1 rec-2 --out admissions/report-mail --name report-mail +``` + +(`openadapt-flow admit` without the launcher.) `--candidate` reads +`shadow-candidates/abc/` and the recordings in its `recordings/` folder. +The command compiles each recording, induces one program from two or more +recordings, and proposes checks. Then it stops **once** and prints a short +summary: + +```text +Admission 3f9c0a1b2d4e: report-mail + from shadow candidate abc + 2 recording(s), induced into one program; 6 step(s), 1 write(s) + Proposed checks (applied on accept): + - send: field_equals, tier 1 + - send: identity:canonical_ladder + 4 fault case(s) to qualify against + Suggested by shadow episodes (not evidence, not applied): + - file_written: the spreadsheet is saved +Type accept to apply these checks, or reject: +``` + +`accept` applies the checks with the same gates as `qualify accept`, +including the oracle gate, and reseals the bundle. Then qualify it as usual +with `openadapt flow qualify admissions/abc/bundle --verify verify.yaml`. +Nothing certifies until that campaign passes. + +The command stops early and asks for nothing when there is nothing to +accept: + +- no recording yet (it prints the candidate's record-next list); +- induction refuses; +- the check proposal HALTs. + +Without a terminal, it writes the summary to `admission-pending.json` and +exits 3. A person decides later with `openadapt flow admit decide DIR`. + +### Letting an agent decide + +The same steps are MCP tools when the operator starts the agent server with +`--admissions DIR`: + +```bash +openadapt-agent serve --mode production --bundles ./bundles \ + --admissions ./admit --delegate-approvals agent:reviewer \ + --approver-grant ./reviewer-grant.json +``` + +- `prepare_admission` takes recording folder names under + `DIR/recordings` or a `candidate_id`, and returns the summary and a + `pending_sha256`. +- `list_admissions` lists admissions and their status. +- `decide_admission` takes the admission, the `pending_sha256` and + `accept` or `reject`. + +Without `--delegate-approvals`, `decide_admission` asks a person in the +client's own form, and the record says `person:mcp-form via mcp`. With it, +the agent decides as the delegated approver, but only under the approver +grant in `--approver-grant` that a registered person signed +(`openadapt-flow approver-grant sign`; see +[`APPROVAL_AND_VERIFICATION.md`](APPROVAL_AND_VERIFICATION.md)). Flow checks +the grant on every decision: it must verify against the signer registry, name +the agent, cover the workflow, and let an agent approve consequential saves. +Without such a grant the agent's decision is refused. The record says +`agent:reviewer via mcp under grant signed by ` and keeps the +signed grant. + +The rules are the same for every decider: + +- **Who decided is recorded.** `admission-decision.json`, in the admission + folder and in the accepted bundle, names who prepared and who decided, by + kind, id and route. +- **No self-approval.** An agent can't decide an admission it prepared. A + server started without `--delegate-approvals` records its preparer as + `agent:mcp-client`, which names no one; only a person decides those. +- **Signed by this machine.** `admission-pending.json` and + `admission-decision.json` are signed with the local admission key (the JIT + key directory, `$OPENADAPT_JIT_KEY_DIR` or `~/.openadapt/jit`). An edited or + hand-written file is refused, so write access to the admission folder or + the bundle is not enough to change who prepared or decided. Keep the key + directory away from the agents it governs. +- **The agent sees codes, not screen text.** The summaries the MCP tools + return carry step ids, check kinds, counts and record-next codes. Induction's + record-next lines and halt reasons can quote labels from the recordings, so + they are shown only at the terminal. +- **The `regulated` policy pack needs a person.** +- **A decision covers exactly what was shown.** It names the + `pending_sha256` of the summary, and the bundle must still match its + digest. There is one decision per admission. +- **No gate is relaxed.** Shadow suggestions are never applied, a HALTed + proposal can't be accepted, and no release admission is signed here. + +### What admission changes for runs + +Accepting admits the workflow to the `problems_only` approval mode. Its runs +no longer wait before every save. They stop only when a check fails, the +screen changed, or a step falls outside what was qualified, and then a person +decides. Read-back is still required: a save counts as done without a person +only when OpenAdapt read the saved record back from your system's data and it +matched. Per-save approval (`each_item`, with one-tap approval from a phone +paired to the Desktop decision portal) stays available as the strictest +setting. `openadapt-flow run` reads this from `admission-decision.json` in the +bundle and prints who admitted it. Five things keep it narrow: + +- A `--approval-mode` flag or the deployment's `runtime.approval_mode` still + wins. +- The decision record must carry a valid signature by the local admission + key. A record without one, or edited after signing, gives no mode. +- It applies only to the exact admitted bundle. After any change to the + bundle, each save waits for a person again. +- An agent's admission applies only while its signed grant still verifies. + If the grant expires, is swapped or no longer names the agent, each save + waits for a person again. +- `problems_only` keeps its own gates (see "Earning only problems" in + [`APPROVAL_AND_VERIFICATION.md`](APPROVAL_AND_VERIFICATION.md)). Until the + workflow passes the readiness test and its switch is on, every save still + waits for a person, and repeated problems switch it back. + +## Example + +The test fixtures in `tests/fixtures/shadow/` come from Capture's miner on +synthetic streams: + +- `report_mail`: save a spreadsheet, then mail it. Induction emits a draft + with two parameters (recipient and subject) and proposes two checks (the + file changes; a message is sent). +- `invoice_exact`: copy an invoice number from a spreadsheet cell into a web + form. The cell differs every time, so induction refuses with + `ambiguous_selection` ("which row?"), and the candidate has no draft. +- `invoice_variant`: the same task with an extra check box in one episode. + Induction reads it as an optional step. diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 77040e094..4ed012b6d 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -2304,9 +2304,41 @@ def _resolve_run_approval_mode(args: argparse.Namespace, cfg, profile_value): requested = getattr(args, "approval_mode", None) if requested is None: requested = getattr(getattr(cfg, "runtime", None), "approval_mode", None) + if requested is None: + requested = _admitted_approval_mode(args) return resolve_approval_mode(requested) +def _admitted_approval_mode(args: argparse.Namespace) -> Optional[str]: + """The approval mode the bundle's admission gives it, or ``None``. + + An accepted ``openadapt-flow admit`` sets ``problems_only`` for that exact + bundle. A flag or ``runtime.approval_mode`` still wins. A bundle changed + since admission, or an agent's admission whose person-signed grant no + longer verifies, falls back to the default (a person approves each save) + and says why. + """ + + bundle = getattr(args, "bundle", None) + if not bundle: + return None + from openadapt_flow.admit_pipeline import admitted_approval_mode + + admitted = admitted_approval_mode(Path(bundle)) + if admitted.refusal is not None: + print( + f"admission approval mode not used ({admitted.refusal}): a person " + "approves each save." + ) + return None + if admitted.approval_mode is not None and admitted.admitted_by is not None: + print( + f"Admitted by {admitted.admitted_by.label()}: this workflow uses " + f"{admitted.approval_mode}." + ) + return admitted.approval_mode + + def _approval_wait_seconds(args: argparse.Namespace) -> float: from openadapt_flow.approval_mode.item_approval import ( DEFAULT_APPROVAL_WAIT_S, @@ -10373,6 +10405,9 @@ def main(argv: Sequence[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) argv = _qualify_default_form(argv) + from openadapt_flow.cli_admit import admit_default_form + + argv = admit_default_form(argv) # Strict no-outbound mode (OPENADAPT_NO_OUTBOUND=1) is installed before # any command runs, so no code path can open a connection off the machine. from openadapt_flow import no_outbound diff --git a/packages/flow/openadapt_flow/admit_pipeline.py b/packages/flow/openadapt_flow/admit_pipeline.py new file mode 100644 index 000000000..d7610e177 --- /dev/null +++ b/packages/flow/openadapt_flow/admit_pipeline.py @@ -0,0 +1,828 @@ +"""One admission command: compile, induce, propose checks, stop once, qualify. + +``openadapt-flow admit `` (or ``--candidate ID`` for a +shadow-mined candidate) chains what used to be four manual steps: + +1. compile each recording (:func:`~openadapt_flow.compiler.compile_recording`); +2. induce one program over them when there are two or more + (:func:`~openadapt_flow.compiler.induction.induce_program`); +3. propose the qualification checks from the compiled demo + (:func:`~openadapt_flow.qualification_proposal.propose_qualification`); +4. stop **once**, with a short summary, for a decision; on accept, apply the + checks through :func:`~openadapt_flow.qualification_proposal.accept_proposal`, + which re-runs the oracle gate, and reseal the bundle. + +The decision can come from a person at a terminal, or from a delegated agent +approver over MCP (``openadapt-agent serve --delegate-approvals``) acting +under an approver grant a person signed (``openadapt-flow approver-grant +sign``). Either way the decision record names who decided, by kind, id and +route, and for an agent the grant's digest and the person who signed it. + +Accepting admits the workflow to the ``problems_only`` approval mode: its runs +stop only for problems (a failed check, a changed screen, a step outside what +was qualified) instead of before every save. :func:`admitted_approval_mode` +is what ``openadapt-flow run`` reads. It applies only to the exact bundle that +was admitted, and for an agent's admission only while the grant still +verifies. ``problems_only`` keeps all of its own gates: until the switch is +earned and on, every save still waits for a person. + +What it never does: + +* weaken a gate. Induction that refuses, a proposal that HALTs, and an oracle + gate that fails all stop the command with nothing to accept. Accepting runs + the same ``accept_proposal`` as ``qualify accept``; certification still needs + ``qualify campaign`` to pass. +* let the preparer decide as an agent. An agent can't accept an admission it + prepared, an agent can't decide under the ``regulated`` policy pack, and a + decision binds the exact summary (``pending_sha256``) and bundle digest it + was shown. +* treat shadow-suggested checks as evidence. They are listed for the person + or approver to read; only checks mined from real recordings are applied. +* sign a release admission. No local-dev or production admission is minted + here. The pending summary and the decision record are signed with this + machine's local admission key (the JIT key directory, kept away from the + agents it governs), so write access to the admission folder or the bundle + is not enough to forge a decision or change who prepared it. +* send recorded screen text to an agent. The summary an MCP client gets + (:func:`render_summary` with ``value_free=True``) carries codes, counts and + step ids only; induction's record-next lines can quote recorded labels and + stay at the terminal. +* let an agent admit without a person's signature. An agent decides only + under a verified grant that names it, covers this workflow, and lets an + agent approve consequential saves; anything less is refused. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal, Optional, Sequence + +from pydantic import BaseModel, ConfigDict, Field, model_serializer, model_validator + +PENDING_FILE = "admission-pending.json" +DECISION_FILE = "admission-decision.json" +PROPOSAL_FILE = "qualification-proposal.draft.json" +BUNDLE_DIR = "bundle" +COMPILED_DIR = "compiled" +CANDIDATE_RECORDINGS_DIR = "recordings" +PENDING_SCHEMA: Literal["openadapt.admission-pending/v1"] = ( + "openadapt.admission-pending/v1" +) +DECISION_SCHEMA: Literal["openadapt.admission-decision/v1"] = ( + "openadapt.admission-decision/v1" +) +#: Opaque decider ids, the same shape Flow accepts for a delegated approver. +DECIDER_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$" +#: Policy packs under which only a person may decide. +PERSON_ONLY_PACKS = frozenset({"regulated"}) +#: The preparer id for an MCP client that runs without --delegate-approvals. +#: It names no one, so it may be the same agent that later decides; only a +#: person decides what it prepared. +UNIDENTIFIED_AGENT = "mcp-client" +#: The approval mode an accepted admission gives the workflow's runs. +ADMITTED_APPROVAL_MODE: Literal["problems_only"] = "problems_only" +_SHA256 = r"^sha256:[0-9a-f]{64}$" + +PendingStatus = Literal["needs_decision", "needs_recording", "halted"] + + +class AdmissionError(ValueError): + """The admission can't proceed; nothing was accepted.""" + + +class Decider(BaseModel): + """Who prepared or decided an admission.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: Literal["person", "agent"] + id: str = Field(pattern=DECIDER_ID_PATTERN) + route: Literal["cli", "mcp"] + #: For an agent that decided: the digest of the person-signed approver + #: grant it decided under. Left out of the record for a person. + policy_ref: Optional[str] = Field(default=None, pattern=_SHA256) + #: For an agent that decided: the registered person who signed that grant. + on_behalf_of: Optional[str] = Field(default=None, pattern=DECIDER_ID_PATTERN) + + @model_serializer(mode="wrap") + def _compact(self, handler: Any) -> dict[str, Any]: + data: dict[str, Any] = handler(self) + for key in ("policy_ref", "on_behalf_of"): + if data.get(key) is None: + data.pop(key, None) + return data + + @model_validator(mode="after") + def _grant_only_for_an_agent(self) -> "Decider": + if self.kind == "person" and ( + self.policy_ref is not None or self.on_behalf_of is not None + ): + raise ValueError("a person decides for themselves, under no grant") + if (self.policy_ref is None) != (self.on_behalf_of is None): + raise ValueError("a grant names both its digest and its signer") + return self + + def label(self) -> str: + text = f"{self.kind}:{self.id} via {self.route}" + if self.policy_ref is not None: + short = self.policy_ref.removeprefix("sha256:")[:12] + text += f" under grant {short} signed by {self.on_behalf_of}" + return text + + +class PendingAdmission(BaseModel): + """What the decider is shown, written before the single stop.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["openadapt.admission-pending/v1"] = PENDING_SCHEMA + admission_id: str = Field(pattern=r"^[0-9a-f]{32}$") + status: PendingStatus + halt_reason: Optional[str] = None + workflow_name: str + policy_pack: str + prepared_by: Decider + created_at: str + recordings: int = 0 + induced: bool = False + candidate_motif_id: Optional[str] = None + n_steps: int = 0 + n_writes: int = 0 + pins: list[dict[str, str]] = Field(default_factory=list) + checks: list[dict[str, Any]] = Field(default_factory=list) + failure_cases: int = 0 + shadow_suggestions: list[dict[str, Any]] = Field(default_factory=list) + record_next: list[dict[str, Any]] = Field(default_factory=list) + bundle_content_digest: Optional[str] = None + proposal_sha256: Optional[str] = None + + def sha256(self) -> str: + return _sha(self.model_dump(mode="json")) + + +class AdmissionDecision(BaseModel): + """The decision record. Names who decided, and binds what they saw.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["openadapt.admission-decision/v1"] = DECISION_SCHEMA + admission_id: str + decision: Literal["accept", "reject"] + reason: Optional[str] = None + decided_by: Decider + prepared_by: Decider + decided_at: str + pending_sha256: str + bundle_content_digest: Optional[str] + proposal_sha256: Optional[str] + accepted_bundle_content_digest: Optional[str] = None + oracle_gate_passed: Optional[bool] = None + next: list[str] = Field(default_factory=list) + #: The workflow the admission names, for the grant's workflow scope. + workflow_name: Optional[str] = None + #: Set on accept: the approval mode the admitted workflow's runs use. + approval_mode: Optional[Literal["problems_only"]] = None + #: The person-signed grant an agent decided under, kept whole so a run + #: re-verifies it. ``None`` for a person's decision. + approver_grant: Optional[dict[str, Any]] = None + + +def _sha(payload: Any) -> str: + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _bundle_digest(bundle: Path) -> str: + from openadapt_flow.bundle_validation import ( + compute_content_digest, + compute_file_hashes, + ) + from openadapt_flow.ir import Workflow + + workflow = Workflow.load(bundle) + return compute_content_digest(workflow, compute_file_hashes(workflow, bundle)) + + +def _is_bundle(path: Path) -> bool: + return (path / "workflow.json").is_file() or (path / "workflow.json.enc").is_file() + + +def resolve_candidate(candidate: str, root: Path | str) -> Path: + """A candidate id (``motif_id``) under ``root``, or a candidate folder.""" + + from openadapt_flow.compiler.motif_induction import CANDIDATE_FILENAME + + direct = Path(candidate) + if (direct / CANDIDATE_FILENAME).is_file(): + return direct + if not re.match(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$", candidate): + raise AdmissionError("a candidate id is one folder name under the root") + folder = Path(root) / candidate + if not (folder / CANDIDATE_FILENAME).is_file(): + raise AdmissionError(f"no shadow candidate {candidate!r} under {root}") + return folder + + +def _load_candidate(folder: Path) -> dict[str, Any]: + from openadapt_flow.compiler.motif_induction import CANDIDATE_FILENAME + + data = json.loads((folder / CANDIDATE_FILENAME).read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("admitted") is not False: + raise AdmissionError("candidate.json is not an unadmitted shadow candidate") + return data + + +def _checks_from_proposal(proposal: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for raw in proposal.pin("effect").get("payload", {}).get("effects") or []: + out.append( + { + "step_id": raw.get("step_id"), + "kind": raw.get("kind"), + "tier": raw.get("tier"), + } + ) + for raw in proposal.pin("identity").get("payload", {}).get("policies") or []: + out.append( + { + "step_id": raw.get("step_id"), + "kind": f"identity:{raw.get('enforcement')}", + } + ) + return out + + +def prepare_admission( + inputs: Sequence[Path | str], + *, + out: Path | str, + name: str, + prepared_by: Decider, + candidate: Optional[Path | str] = None, + policy_pack: str = "community", + admission_id: Optional[str] = None, + key_dir: Optional[Path | str] = None, +) -> PendingAdmission: + """Run compile, induce and the check proposal; write the single stop. + + ``inputs`` are recording directories, or already-compiled bundles. With + ``candidate`` and no inputs, the recordings are read from the candidate's + ``recordings/`` folder; with none there, the result is + ``needs_recording`` and carries the candidate's record-next list. + + The pending file is signed with the local admission key in ``key_dir`` + (default ``~/.openadapt/jit``). + """ + + from openadapt_flow.compiler import compile_recording + from openadapt_flow.compiler.induction import induce_program + from openadapt_flow.ir import Workflow + from openadapt_flow.qualification_proposal import ( + propose_qualification, + write_proposal, + ) + from openadapt_flow.traversal import iter_workflow_steps + + target = Path(out) + if (target / PENDING_FILE).exists() or (target / DECISION_FILE).exists(): + raise AdmissionError(f"{target} already holds an admission; use a new --out") + target.mkdir(parents=True, exist_ok=True) + + paths = [Path(item) for item in inputs] + motif_id: Optional[str] = None + suggestions: list[dict[str, Any]] = [] + candidate_next: list[dict[str, Any]] = [] + if candidate is not None: + folder = Path(candidate) + data = _load_candidate(folder) + motif_id = str(data.get("motif_id") or folder.name) + suggestions = [ + { + "kind": item.get("kind"), + "summary": item.get("summary"), + "step_id": item.get("step_id"), + "note": "suggested by shadow episodes; not evidence", + } + for item in data.get("proposed_checks") or [] + ] + candidate_next = list(data.get("record_next") or []) + if not paths: + rec_root = folder / CANDIDATE_RECORDINGS_DIR + if rec_root.is_dir(): + paths = sorted(p for p in rec_root.iterdir() if p.is_dir()) + + base: dict[str, Any] = { + "admission_id": admission_id or uuid.uuid4().hex, + "workflow_name": name, + "policy_pack": policy_pack, + "prepared_by": prepared_by, + "created_at": _now(), + "candidate_motif_id": motif_id, + "shadow_suggestions": suggestions, + } + + def stop(status: PendingStatus, reason: str, **extra: Any) -> PendingAdmission: + pending = PendingAdmission(status=status, halt_reason=reason, **base, **extra) + _write_signed(target / PENDING_FILE, pending.model_dump(mode="json"), key_dir) + return pending + + if not paths: + return stop( + "needs_recording", + "no recording yet: record this task once with OpenAdapt Capture", + record_next=candidate_next + or [{"code": "record", "instruction": "Record the task once."}], + ) + + compiled: list[Path] = [] + recording_meta: Optional[Path] = None + for index, path in enumerate(paths): + if not path.is_dir(): + raise AdmissionError(f"{path} is not a directory") + if recording_meta is None and (path / "meta.json").is_file(): + recording_meta = path + if _is_bundle(path): + compiled.append(path) + continue + bundle = target / COMPILED_DIR / f"{index:02d}" + compile_recording(path, bundle, name=name) + compiled.append(bundle) + + bundle_dir = target / BUNDLE_DIR + induced = False + if len(compiled) == 1: + workflow = Workflow.load(compiled[0]) + else: + result = induce_program([Workflow.load(p) for p in compiled]) + if not result.certified or result.workflow is None: + return stop( + "halted", + "induction refused: the recordings don't pin down one program", + recordings=len(compiled), + record_next=[ + item.model_dump(mode="json") for item in result.record_next + ], + ) + workflow = result.workflow + induced = True + workflow = workflow.model_copy(update={"name": name}) + _copy_templates(compiled, bundle_dir) + workflow.save(bundle_dir) + workflow = Workflow.load(bundle_dir) + + proposal = propose_qualification( + workflow, recording_dir=recording_meta, policy_pack=policy_pack + ) + write_proposal(proposal, target / PROPOSAL_FILE) + steps = list(iter_workflow_steps(workflow)) + detail: dict[str, Any] = { + "recordings": len(compiled), + "induced": induced, + "n_steps": len(steps), + "n_writes": sum(1 for step in steps if step.risk == "irreversible"), + "pins": [ + {"kind": str(pin.get("kind")), "status": str(pin.get("status"))} + for pin in proposal.pins + ], + "checks": _checks_from_proposal(proposal), + "failure_cases": len(proposal.failure_matrix), + "bundle_content_digest": _bundle_digest(bundle_dir), + "proposal_sha256": proposal.sha256(), + } + if proposal.status == "halted": + return stop( + "halted", + proposal.halt_reason or "qualification proposal HALTed", + **detail, + ) + pending = PendingAdmission(status="needs_decision", **base, **detail) + _write_signed(target / PENDING_FILE, pending.model_dump(mode="json"), key_dir) + return pending + + +def _copy_templates(compiled: Sequence[Path], bundle_dir: Path) -> None: + """Carry each compiled bundle's crops; the first bundle wins a name clash.""" + + import shutil + + for source in compiled: + templates = source / "templates" + if not templates.is_dir(): + continue + for item in sorted(templates.rglob("*")): + if not item.is_file(): + continue + dest = bundle_dir / "templates" / item.relative_to(templates) + if dest.exists(): + continue + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(item, dest) + + +def load_pending( + out: Path | str, *, key_dir: Optional[Path | str] = None +) -> PendingAdmission: + """The pending admission, only if this machine's admission key signed it.""" + + path = Path(out) / PENDING_FILE + if not path.is_file() or path.is_symlink(): + raise AdmissionError(f"no pending admission in {out}") + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise AdmissionError(f"the pending admission is unreadable: {exc}") from exc + if not _signed(data, key_dir): + raise AdmissionError( + "the pending admission is not signed by this machine's admission " + "key; prepare it again" + ) + data.pop("signature", None) + return PendingAdmission.model_validate(data) + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", "utf-8") + + +def _key_dir(key_dir: Optional[Path | str]) -> Path: + from openadapt_flow.jit.lifecycle import default_key_dir + + return Path(key_dir) if key_dir is not None else default_key_dir() + + +def _write_signed( + path: Path, payload: dict[str, Any], key_dir: Optional[Path | str] +) -> None: + """Write ``payload`` with a signature by the local admission key.""" + + from openadapt_flow.jit.lifecycle import _sign_record + + body = {k: v for k, v in payload.items() if k != "signature"} + body["signature"] = _sign_record(body, _key_dir(key_dir)) + _write_json(path, body) + + +def _signed(data: Any, key_dir: Optional[Path | str]) -> bool: + from openadapt_flow.jit.lifecycle import _record_signed + + if not isinstance(data, dict) or not isinstance(data.get("signature"), str): + return False + try: + return _record_signed(data, _key_dir(key_dir)) + except (OSError, ValueError): + return False + + +def _agent_identity(decided_by: Decider) -> Any: + from openadapt_flow.approval_mode.approver import ( + ApproverAuth, + ApproverIdentityV1, + ApproverKind, + ) + + return ApproverIdentityV1( + kind=ApproverKind.AGENT, + auth=( + ApproverAuth.MCP_SESSION + if decided_by.route == "mcp" + else ApproverAuth.LOCAL_CLI + ), + id=decided_by.id, + ) + + +def _grant_refusal( + signed: Any, + agent: Decider, + workflow_name: str, + *, + key_dir: Optional[Path | str], + now: Optional[datetime] = None, +) -> tuple[Optional[Any], Optional[str]]: + """``(verified grant, None)`` or ``(None, closed reason)``.""" + + from openadapt_flow.approval_mode.approver import StepClass + from openadapt_flow.approval_mode.signed_policy import ( + GrantRefused, + SignedApproverGrant, + verify_grant, + ) + + if signed is None: + return None, "approver_grant_missing" + try: + grant = ( + signed + if isinstance(signed, SignedApproverGrant) + else SignedApproverGrant.model_validate(signed) + ) + except ValueError: + return None, "approver_grant_unreadable" + if key_dir is None: + from openadapt_flow.jit.lifecycle import default_key_dir + + key_dir = default_key_dir() + try: + verified = verify_grant(grant, key_dir, now=now) + except GrantRefused as exc: + return None, exc.code + if not verified.names(agent.id): + return None, "delegate_not_in_grant" + if not verified.covers(workflow_name): + return None, "workflow_not_in_grant" + # Admitting to problems_only lets consequential saves go ahead without a + # person. An agent may grant that only where the person's grant already + # lets an agent approve a consequential save itself. + if verified.policy.refusal(StepClass.CONSEQUENTIAL, _agent_identity(agent)): + return None, "grant_does_not_allow_agent_saves" + if agent.policy_ref is not None and agent.policy_ref != verified.policy_ref: + return None, "policy_ref_mismatch" + return verified, None + + +def check_decider( + pending: PendingAdmission, + decided_by: Decider, + *, + approver_grant: Any = None, + key_dir: Optional[Path | str] = None, +) -> Decider: + """Refuse self-approval, and any agent decision a person didn't sign for. + + Returns the decider as recorded: an agent gains the digest of the grant + it decided under and the person who signed it. + """ + + if decided_by.kind != "agent": + return decided_by + if pending.policy_pack in PERSON_ONLY_PACKS: + raise AdmissionError( + f"the {pending.policy_pack!r} policy pack needs a person to decide" + ) + prepared = pending.prepared_by + if prepared.kind == "agent" and prepared.id == UNIDENTIFIED_AGENT: + raise AdmissionError( + "an unidentified MCP client prepared this admission, so it may be " + "the same agent; a person decides it" + ) + if prepared.kind == "agent" and prepared.id == decided_by.id: + raise AdmissionError( + "self-approval refused: the agent that prepared this admission " + "can't decide it" + ) + verified, refusal = _grant_refusal( + approver_grant, decided_by, pending.workflow_name, key_dir=key_dir + ) + if verified is None: + raise AdmissionError( + f"an agent admits a workflow only under an approver grant a person " + f"signed that names it, covers this workflow and lets an agent " + f"approve saves ({refusal}); see `openadapt-flow approver-grant sign`" + ) + return decided_by.model_copy( + update={"policy_ref": verified.policy_ref, "on_behalf_of": verified.signer} + ) + + +def decide_admission( + out: Path | str, + *, + pending_sha256: str, + decision: Literal["accept", "reject"], + decided_by: Decider, + reason: Optional[str] = None, + approver_grant: Any = None, + key_dir: Optional[Path | str] = None, +) -> AdmissionDecision: + """Record one decision on the pending admission, and apply it on accept. + + An agent decides only with ``approver_grant``: a person-signed + :class:`~openadapt_flow.approval_mode.signed_policy.SignedApproverGrant` + that verifies against the signer registry in ``key_dir`` (default + ``~/.openadapt/jit``). Accepting sets the workflow's approval mode to + ``problems_only``. + """ + + from openadapt_flow.ir import Workflow + from openadapt_flow.qualification_proposal import ( + QualificationProposalError, + accept_proposal, + load_proposal, + save_accepted_bundle, + ) + + target = Path(out) + pending = load_pending(target, key_dir=key_dir) + if (target / DECISION_FILE).exists(): + raise AdmissionError("this admission was already decided") + if pending.sha256() != pending_sha256: + raise AdmissionError( + "the pending admission changed since it was shown; show it again" + ) + if pending.status != "needs_decision": + raise AdmissionError( + f"nothing to decide: the admission is {pending.status}" + + (f" ({pending.halt_reason})" if pending.halt_reason else "") + ) + decided_by = check_decider( + pending, decided_by, approver_grant=approver_grant, key_dir=key_dir + ) + if decision not in ("accept", "reject"): + raise AdmissionError("decision must be accept or reject") + if decision == "reject" and not reason: + raise AdmissionError("a rejection needs a reason") + bundle = target / BUNDLE_DIR + from openadapt_flow.bundle_validation import BundleIntegrityError + + try: + current = _bundle_digest(bundle) + except (OSError, ValueError, BundleIntegrityError) as exc: + raise AdmissionError( + f"the bundle changed since the summary was written: {exc}" + ) from exc + if current != pending.bundle_content_digest: + raise AdmissionError("the bundle changed since the summary was written") + + record: dict[str, Any] = { + "admission_id": pending.admission_id, + "decision": decision, + "reason": reason, + "decided_by": decided_by, + "prepared_by": pending.prepared_by, + "decided_at": _now(), + "pending_sha256": pending_sha256, + "bundle_content_digest": pending.bundle_content_digest, + "proposal_sha256": pending.proposal_sha256, + "workflow_name": pending.workflow_name, + } + if decided_by.kind == "agent": + from openadapt_flow.approval_mode.signed_policy import SignedApproverGrant + + signed = ( + approver_grant + if isinstance(approver_grant, SignedApproverGrant) + else SignedApproverGrant.model_validate(approver_grant) + ) + record["approver_grant"] = signed.model_dump(mode="json", by_alias=True) + if decision == "accept": + proposal = load_proposal(target / PROPOSAL_FILE) + if proposal.sha256() != pending.proposal_sha256: + raise AdmissionError("the check proposal changed since it was shown") + workflow = Workflow.load(bundle) + try: + accepted = accept_proposal(workflow, proposal) + except QualificationProposalError as exc: + raise AdmissionError(str(exc)) from exc + save_accepted_bundle(workflow, bundle, proposal=accepted) + gate = accepted.oracle_gate or {} + record.update( + accepted_bundle_content_digest=_bundle_digest(bundle), + oracle_gate_passed=bool(gate.get("passed")), + approval_mode=ADMITTED_APPROVAL_MODE, + next=[ + "qualify campaign: run the qualification cases before this " + "program can be certified", + "runs use the problems_only approval mode: they stop only for " + "problems once the switch is earned; until then each save " + "still waits for a person", + ], + ) + result = AdmissionDecision(**record) + payload = result.model_dump(mode="json") + _write_signed(target / DECISION_FILE, payload, key_dir) + if decision == "accept": + _write_signed(bundle / DECISION_FILE, payload, key_dir) + return result + + +class AdmittedMode(BaseModel): + """What a run reads from the bundle's admission decision.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + #: ``problems_only`` when the admission applies; ``None`` otherwise. + approval_mode: Optional[Literal["problems_only"]] = None + admitted_by: Optional[Decider] = None + #: Why a recorded admission doesn't apply, as a closed code. + refusal: Optional[str] = None + + +def admitted_approval_mode( + bundle: Path | str, + *, + key_dir: Optional[Path | str] = None, + now: Optional[datetime] = None, +) -> AdmittedMode: + """The approval mode an accepted admission gives this exact bundle. + + No admission, or a rejected one: no mode (the run's default applies). A + record this machine's admission key didn't sign, a recorded admission for + a bundle that has since changed, or an agent's admission whose grant no + longer verifies, names the reason and gives no mode. Never raises. + """ + + path = Path(bundle) / DECISION_FILE + if not path.is_file() or path.is_symlink(): + return AdmittedMode() + try: + data = json.loads(path.read_text("utf-8")) + except (OSError, ValueError): + return AdmittedMode(refusal="admission_unreadable") + if not _signed(data, key_dir): + return AdmittedMode(refusal="admission_signature_invalid") + try: + data.pop("signature", None) + record = AdmissionDecision.model_validate(data) + except ValueError: + return AdmittedMode(refusal="admission_unreadable") + if record.decision != "accept" or record.approval_mode is None: + return AdmittedMode() + try: + current = _bundle_digest(Path(bundle)) + except Exception: # noqa: BLE001 - an unreadable bundle never widens a run + return AdmittedMode(refusal="admission_bundle_unreadable") + if current != record.accepted_bundle_content_digest: + return AdmittedMode(refusal="admission_bundle_changed") + decider = record.decided_by + if decider.kind == "agent": + if record.workflow_name is None or decider.policy_ref is None: + return AdmittedMode(refusal="approver_grant_missing") + verified, refusal = _grant_refusal( + record.approver_grant, + decider, + record.workflow_name, + key_dir=key_dir, + now=now, + ) + if verified is None: + return AdmittedMode(refusal=refusal) + return AdmittedMode(approval_mode=record.approval_mode, admitted_by=decider) + + +def render_summary(pending: PendingAdmission, *, value_free: bool = False) -> str: + """The short summary shown at the single stop. + + ``value_free`` is the form for an MCP client: it leaves out the halt + reason, the record-next lines and shadow-suggestion text, which can quote + labels or dialog text from the recordings, and keeps their codes. + """ + + lines = [f"Admission {pending.admission_id[:12]}: {pending.workflow_name}"] + if pending.candidate_motif_id: + lines.append(f" from shadow candidate {pending.candidate_motif_id}") + lines.append( + f" {pending.recordings} recording(s)" + + (", induced into one program" if pending.induced else "") + + f"; {pending.n_steps} step(s), {pending.n_writes} write(s)" + ) + if pending.status != "needs_decision": + if value_free: + lines.append(f" STOPPED ({pending.status})") + for item in pending.record_next: + lines.append(f" - record next: {item.get('code')}") + return "\n".join(lines) + "\n" + lines.append(f" STOPPED ({pending.status}): {pending.halt_reason}") + for item in pending.record_next: + lines.append(f" - record next: {item.get('instruction')}") + return "\n".join(lines) + "\n" + lines.append(" Proposed checks (applied on accept):") + for check in pending.checks: + tier = f", tier {check['tier']}" if check.get("tier") is not None else "" + lines.append(f" - {check.get('step_id')}: {check.get('kind')}{tier}") + lines.append(f" {pending.failure_cases} fault case(s) to qualify against") + if pending.shadow_suggestions: + lines.append(" Suggested by shadow episodes (not evidence, not applied):") + for item in pending.shadow_suggestions: + text = "" if value_free else f": {item.get('summary')}" + lines.append(f" - {item.get('kind')}{text}") + lines.append(f" Prepared by {pending.prepared_by.label()}") + return "\n".join(lines) + "\n" + + +__all__ = [ + "ADMITTED_APPROVAL_MODE", + "AdmissionDecision", + "AdmittedMode", + "admitted_approval_mode", + "AdmissionError", + "Decider", + "PendingAdmission", + "check_decider", + "decide_admission", + "load_pending", + "prepare_admission", + "render_summary", + "resolve_candidate", + "UNIDENTIFIED_AGENT", +] diff --git a/packages/flow/openadapt_flow/agent/admission.py b/packages/flow/openadapt_flow/agent/admission.py new file mode 100644 index 000000000..9562ff04e --- /dev/null +++ b/packages/flow/openadapt_flow/agent/admission.py @@ -0,0 +1,327 @@ +"""MCP tools for ``openadapt-flow admit``: prepare, list, and decide. + +The operator turns these on with ``openadapt-agent serve --admissions DIR``. +Recordings are named by folder under ``DIR/recordings`` and shadow +candidates by id under ``DIR/shadow-candidates``; an agent never passes a +path. Admissions are written under ``DIR/admissions/``. + +Who decides: + +* With ``--delegate-approvals AGENT_ID --approver-grant FILE`` the calling + agent decides as that delegated approver, under the person-signed grant in + FILE. Flow verifies the grant on every decision, and the record says + ``agent:AGENT_ID via mcp under grant signed by ``. Without + a grant that verifies, names AGENT_ID, covers the workflow and lets an + agent approve saves, the agent's decision is refused. +* Without it, ``decide_admission`` asks a person in the client's own form + (MCP elicitation) and records ``person:mcp-form via mcp``. A model-supplied + argument never stands in for that answer. + +What an agent can't do: decide an admission it prepared (the record names +the preparer), decide one an unidentified client prepared (a server without +``--delegate-approvals`` records its preparer as ``mcp-client``), decide +under the ``regulated`` policy pack, decide without the exact +``pending_sha256`` it was shown, or decide without a grant a person signed. +Accepting runs the same gates as ``qualify accept`` and admits the workflow to +the ``problems_only`` approval mode. + +What an agent sees: codes, counts and step ids. The summaries these tools +return are value-free (:func:`~openadapt_flow.admit_pipeline.render_summary` +with ``value_free=True``), because the client may send them to a hosted +model and induction's record-next lines can quote recorded labels. +""" + +from __future__ import annotations + +import re +import uuid +from pathlib import Path +from typing import Any, Optional + +from openadapt_flow.admit_pipeline import ( + PENDING_FILE, + UNIDENTIFIED_AGENT, + AdmissionError, + Decider, + decide_admission, + load_pending, + prepare_admission, + render_summary, +) + +TOOL_NAMES = ("prepare_admission", "list_admissions", "decide_admission") +_FOLDER = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" +_ADMISSION = r"^[0-9a-f]{32}$" +_SHA = r"^[0-9a-f]{64}$" +REJECT_REASONS = ("wrong_checks", "wrong_steps", "not_this_task", "other") +PERSON_VIA_FORM = "mcp-form" + +PREPARE_DESCRIPTION = ( + "Turn recordings of one task into a program with proposed checks, like " + "`openadapt-flow admit`: compile each recording, induce one program over " + "two or more, and propose the checks. It stops there and returns a short " + "summary and pending_sha256. Nothing is applied until someone else " + "decides with decide_admission; you can't decide an admission you prepared." +) +LIST_DESCRIPTION = ( + "List admissions on this server with their status, summary and pending_sha256." +) +DECIDE_DESCRIPTION = ( + "Accept or reject one admission that stopped for a decision. Send the " + "admission and pending_sha256 from list_admissions. If the operator " + "started this server with --delegate-approvals you decide as that " + "delegated approver; otherwise a person decides in your client's form. " + "Refused for an admission this agent prepared and under the regulated " + "policy pack. The record names who decided." +) + + +class AdmissionToolError(ValueError): + pass + + +class AdmissionDesk: + """The admission tools over one operator-chosen folder.""" + + def __init__( + self, + root: Path | str, + *, + delegate_id: Optional[str] = None, + approver_grant: Optional[Path | str] = None, + key_dir: Optional[Path | str] = None, + ): + self.root = Path(root).expanduser().resolve() + self.delegate_id = delegate_id + #: The person-signed grant the delegate decides under. Read on every + #: decision so a replaced or expired file stops working at once. + self.approver_grant = ( + Path(approver_grant).expanduser() if approver_grant else None + ) + self.key_dir = key_dir + + def _grant(self) -> Any: + if self.approver_grant is None: + return None + import json + + try: + return json.loads(self.approver_grant.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + + # -- paths ------------------------------------------------------------- + + def _child(self, area: str, name: Any, pattern: str = _FOLDER) -> Path: + if not isinstance(name, str) or not re.match(pattern, name): + raise AdmissionToolError(f"{area} names one folder") + base = self.root / area + path = base / name + if base.is_symlink() or path.is_symlink(): + raise AdmissionToolError(f"{area} folder is unavailable") + resolved = path.resolve() + if base.resolve() not in resolved.parents or not resolved.is_dir(): + raise AdmissionToolError(f"no such folder in {area}") + return resolved + + def _preparer(self) -> Decider: + return Decider( + kind="agent", id=self.delegate_id or UNIDENTIFIED_AGENT, route="mcp" + ) + + # -- specs ------------------------------------------------------------- + + def specs(self) -> list[dict[str, Any]]: + return [ + { + "name": "prepare_admission", + "description": PREPARE_DESCRIPTION, + "input_schema": { + "type": "object", + "properties": { + "recordings": { + "type": "array", + "items": {"type": "string", "pattern": _FOLDER}, + "maxItems": 20, + }, + "candidate_id": {"type": "string", "pattern": _FOLDER}, + "name": {"type": "string", "pattern": _FOLDER}, + "policy_pack": { + "type": "string", + "enum": ["community", "cloud", "regulated"], + }, + }, + "required": ["name"], + "additionalProperties": False, + }, + "annotations": { + "readOnlyHint": False, + "destructiveHint": False, + "idempotentHint": False, + "openWorldHint": False, + }, + }, + { + "name": "list_admissions", + "description": LIST_DESCRIPTION, + "input_schema": { + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + "annotations": { + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False, + }, + }, + { + "name": "decide_admission", + "description": DECIDE_DESCRIPTION, + "input_schema": { + "type": "object", + "properties": { + "admission": {"type": "string", "pattern": _ADMISSION}, + "pending_sha256": {"type": "string", "pattern": _SHA}, + "decision": {"type": "string", "enum": ["accept", "reject"]}, + "reject_reason": { + "type": "string", + "enum": list(REJECT_REASONS), + }, + }, + "required": ["admission", "pending_sha256", "decision"], + "additionalProperties": False, + }, + "annotations": { + "readOnlyHint": False, + "destructiveHint": True, + "idempotentHint": False, + "openWorldHint": False, + }, + }, + ] + + def needs_person(self) -> bool: + """Whether decide_admission must ask a person in the client's form.""" + + return self.delegate_id is None + + # -- tools ------------------------------------------------------------- + + def _view(self, folder: Path) -> dict[str, Any]: + pending = load_pending(folder, key_dir=self.key_dir) + return { + "admission": folder.name, + "status": pending.status, + "workflow_name": pending.workflow_name, + "policy_pack": pending.policy_pack, + "prepared_by": pending.prepared_by.model_dump(mode="json"), + "checks": pending.checks, + "shadow_suggestions": [ + {"kind": item.get("kind"), "note": item.get("note")} + for item in pending.shadow_suggestions + ], + "record_next": [item.get("code") for item in pending.record_next], + "decided": (folder / "admission-decision.json").exists(), + "summary": render_summary(pending, value_free=True), + "pending_sha256": pending.sha256(), + } + + def prepare(self, arguments: dict[str, Any]) -> dict[str, Any]: + recordings = [ + self._child("recordings", item) + for item in arguments.get("recordings") or [] + ] + candidate = None + if arguments.get("candidate_id") is not None: + candidate = self._child("shadow-candidates", arguments["candidate_id"]) + if not recordings and candidate is None: + raise AdmissionToolError("give recordings or a candidate_id") + admission_id = uuid.uuid4().hex + out = self.root / "admissions" / admission_id + out.parent.mkdir(parents=True, exist_ok=True) + try: + prepare_admission( + recordings, + out=out, + name=arguments["name"], + prepared_by=self._preparer(), + candidate=candidate, + policy_pack=arguments.get("policy_pack") or "community", + admission_id=admission_id, + key_dir=self.key_dir, + ) + except AdmissionError as exc: + raise AdmissionToolError(str(exc)) from exc + return self._view(out) + + def list(self) -> dict[str, Any]: + base = self.root / "admissions" + rows = [] + if base.is_dir() and not base.is_symlink(): + for folder in sorted(base.iterdir()): + if ( + re.match(_ADMISSION, folder.name) + and not folder.is_symlink() + and (folder / PENDING_FILE).is_file() + ): + try: + rows.append(self._view(folder)) + except AdmissionError: + # Unsigned or unreadable: listed as such, never shown. + rows.append({"admission": folder.name, "status": "unverified"}) + return {"admissions": rows} + + def confirmation_message(self, arguments: dict[str, Any]) -> str: + folder = self._child("admissions", arguments.get("admission"), _ADMISSION) + verb = "accept" if arguments.get("decision") == "accept" else "reject" + try: + pending = load_pending(folder, key_dir=self.key_dir) + except AdmissionError as exc: + raise AdmissionToolError(str(exc)) from exc + return ( + render_summary(pending, value_free=True) + + f"\nConfirm that you, a person, {verb} this admission." + ) + + def decide( + self, arguments: dict[str, Any], *, human_confirmed: bool = False + ) -> dict[str, Any]: + folder = self._child("admissions", arguments.get("admission"), _ADMISSION) + decision = arguments.get("decision") + reason = arguments.get("reject_reason") + if (decision == "reject") != (reason is not None): + raise AdmissionToolError("a reject needs exactly one reject_reason") + if human_confirmed: + decider = Decider(kind="person", id=PERSON_VIA_FORM, route="mcp") + elif self.delegate_id is not None: + decider = Decider(kind="agent", id=self.delegate_id, route="mcp") + else: + raise AdmissionToolError( + "a person must decide this admission in the client's form" + ) + try: + result = decide_admission( + folder, + pending_sha256=str(arguments.get("pending_sha256")), + decision=decision, + decided_by=decider, + reason=reason, + approver_grant=self._grant() if decider.kind == "agent" else None, + key_dir=self.key_dir, + ) + except AdmissionError as exc: + raise AdmissionToolError(str(exc)) from exc + return { + "admission": folder.name, + "decision": result.decision, + "decided_by": result.decided_by.model_dump(mode="json"), + "prepared_by": result.prepared_by.model_dump(mode="json"), + "oracle_gate_passed": result.oracle_gate_passed, + "approval_mode": result.approval_mode, + "next": result.next, + } + + +__all__ = ["AdmissionDesk", "AdmissionToolError", "TOOL_NAMES"] diff --git a/packages/flow/openadapt_flow/agent/bridge.py b/packages/flow/openadapt_flow/agent/bridge.py index 8ca8ac4be..1086471e7 100644 --- a/packages/flow/openadapt_flow/agent/bridge.py +++ b/packages/flow/openadapt_flow/agent/bridge.py @@ -171,8 +171,12 @@ def __init__( mode: Optional[str] = None, store: Optional[RunStore] = None, repair_desk: Optional[Any] = None, + admission_desk: Optional[Any] = None, ): self.repair_desk = repair_desk + #: ``openadapt_flow.agent.admission.AdmissionDesk`` when the operator + #: started the server with ``--admissions DIR``. + self.admission_desk = admission_desk self.bundles_dir = Path(bundles_dir) if bundles_dir is not None else None self.sandbox = sandbox self.allow_run = allow_run or sandbox is not None @@ -353,8 +357,43 @@ def _contract_specs(self) -> list[ToolSpec]: ) specs.extend(self._repair_specs()) specs.extend(self._approval_specs()) + specs.extend(self._admission_specs()) return specs + def _admission_specs(self) -> list[ToolSpec]: + """Listed only when the operator passed ``--admissions DIR``.""" + if self.admission_desk is None: + return [] + return [ + ToolSpec( + name=raw["name"], + description=raw["description"], + input_schema=raw["input_schema"], + annotations=raw["annotations"], + ) + for raw in self.admission_desk.specs() + ] + + def dispatch_admission( + self, name: str, arguments: dict, *, human_confirmed: bool = False + ) -> dict: + """Run one admission tool. ``human_confirmed`` only from the MCP form.""" + from openadapt_flow.agent.admission import AdmissionToolError + + desk = self.admission_desk + if desk is None: + raise BridgeError("admission tools are off on this server") + try: + if name == "prepare_admission": + return desk.prepare(arguments) + if name == "list_admissions": + return desk.list() + if name == "decide_admission": + return desk.decide(arguments, human_confirmed=human_confirmed) + except AdmissionToolError as exc: + raise BridgeError(str(exc)) from exc + raise BridgeError("unknown tool name") + def _delegate_id(self) -> Optional[str]: if not self.allow_run or self.sandbox is not None: return None @@ -687,6 +726,10 @@ def dispatch(self, name: str, arguments: Optional[dict]) -> dict: if not any(spec.name == name for spec in self._approval_specs()): raise BridgeError("unknown tool name") return self._approval_dispatch(name, arguments) + if name in ("prepare_admission", "list_admissions", "decide_admission"): + # decide_admission from here is the delegated-agent path only; the + # person path comes through dispatch_admission after the MCP form. + return self.dispatch_admission(name, arguments) if name in ("get_repair_screen", "teach_repair"): if not self._repair_tools_enabled(): raise BridgeError("repair tools are off on this server") diff --git a/packages/flow/openadapt_flow/agent/cli.py b/packages/flow/openadapt_flow/agent/cli.py index d1bd31b7c..ef2ca3534 100644 --- a/packages/flow/openadapt_flow/agent/cli.py +++ b/packages/flow/openadapt_flow/agent/cli.py @@ -27,7 +27,7 @@ import shlex import sys from pathlib import Path -from typing import Optional, Sequence +from typing import Any, Optional, Sequence from openadapt_flow.agent import __version__ from openadapt_flow.agent.contract import APPROVAL_MODES @@ -242,6 +242,31 @@ def build_parser() -> argparse.ArgumentParser: "Flow's report record that. Callers can't change it." ), ) + p.add_argument( + "--admissions", + default=None, + metavar="DIR", + help=( + "Add prepare_admission, list_admissions and decide_admission: the " + "`openadapt-flow admit` command over recordings in DIR/recordings " + "and shadow candidates in DIR/shadow-candidates. A person decides " + "in the client's form, or, with --delegate-approvals, the agent " + "decides as that approver. An agent can't decide an admission it " + "prepared." + ), + ) + p.add_argument( + "--approver-grant", + default=None, + metavar="FILE", + help=( + "With --admissions and --delegate-approvals: the approver grant a " + "person signed (openadapt-flow approver-grant sign) that lets " + "AGENT_ID decide admissions. It must name AGENT_ID, cover the " + "workflow and let an agent approve saves; Flow verifies it on " + "every decision. Without it only a person decides." + ), + ) p.add_argument( "--delegate-approvals", default=None, @@ -554,6 +579,14 @@ def _cmd_serve(args: argparse.Namespace) -> int: file=sys.stderr, ) return 2 + if getattr(args, "approver_grant", None) is not None and ( + args.delegate_approvals is None or getattr(args, "admissions", None) is None + ): + print( + "serve: --approver-grant needs --admissions and --delegate-approvals", + file=sys.stderr, + ) + return 2 if args.delegate_approvals is not None: import re as _re @@ -721,6 +754,7 @@ def _cmd_serve(args: argparse.Namespace) -> int: allow_recorded_defaults=args.allow_synthetic_recorded_defaults, mode=mode, repair_desk=_repair_desk(args, runner_config.runs_dir), + admission_desk=_admission_desk(args), ) n = len(bridge.workflows) print( @@ -796,6 +830,19 @@ def _cmd_emit_skill(args: argparse.Namespace) -> int: return 0 +def _admission_desk(args: argparse.Namespace) -> Optional[Any]: + """The admission tools, only when the operator passed ``--admissions``.""" + if getattr(args, "admissions", None) is None: + return None + from openadapt_flow.agent.admission import AdmissionDesk + + return AdmissionDesk( + args.admissions, + delegate_id=args.delegate_approvals, + approver_grant=getattr(args, "approver_grant", None), + ) + + def _repair_desk(args: argparse.Namespace, runs_dir: Path): from openadapt_flow.agent.repair import ( LazyBackendDemonstrator, diff --git a/packages/flow/openadapt_flow/agent/mcp.py b/packages/flow/openadapt_flow/agent/mcp.py index 6eff4ff2c..f31c4cd93 100644 --- a/packages/flow/openadapt_flow/agent/mcp.py +++ b/packages/flow/openadapt_flow/agent/mcp.py @@ -137,6 +137,30 @@ async def _confirm_attended_action(context: Any, name: str) -> None: ) +async def _confirm_admission_decision( + context: Any, bridge: AgentBridge, arguments: dict[str, Any] +) -> None: + """A person decides an admission in the client's form (no delegate).""" + from openadapt_flow.agent.admission import AdmissionToolError + + try: + message = bridge.admission_desk.confirmation_message(arguments) + except AdmissionToolError as exc: + raise BridgeError(str(exc)) from exc + await _confirm_with_person( + context, + message, + title="I am a person and I make this decision", + unsupported=( + "deciding an admission needs a person in an MCP client with form " + "elicitation, or a server started with --delegate-approvals; " + "nothing was decided. A person can also run `openadapt-flow admit " + "decide` at a terminal" + ), + declined="the person declined or cancelled; nothing was decided", + ) + + async def _confirm_admit(context: Any, authoring: AuthoringBridge) -> None: """A person, not the calling agent, accepts a compiled draft.""" await _confirm_with_person( @@ -266,8 +290,17 @@ async def _call_tool( or "tool arguments do not match the input schema" ) from None human_confirmed = False + admission_by_person = ( + name == "decide_admission" + and bridge is not None + and bridge.admission_desk is not None + and bridge.admission_desk.needs_person() + ) if name in ATTENDED_TOOLS: await _confirm_attended_action(context, name) + elif admission_by_person: + assert bridge is not None + await _confirm_admission_decision(context, bridge, arguments or {}) elif name == "admit" and authoring is not None and authoring.handles(name): await _confirm_admit(context, authoring) human_confirmed = True @@ -280,6 +313,10 @@ def call() -> dict[str, Any]: return authoring.dispatch(name, payload) if bridge is None: raise BridgeError("unknown tool name") + if admission_by_person: + return bridge.dispatch_admission( + name, payload, human_confirmed=True + ) return bridge.dispatch(name, payload) # CLI runs and filesystem projections are blocking. Live attended diff --git a/packages/flow/openadapt_flow/cli_admit.py b/packages/flow/openadapt_flow/cli_admit.py index 504c9b377..9f7e9e1bc 100644 --- a/packages/flow/openadapt_flow/cli_admit.py +++ b/packages/flow/openadapt_flow/cli_admit.py @@ -1,11 +1,27 @@ -"""CLI: ``openadapt-flow admit status``.""" +"""CLI: ``openadapt-flow admit``. + +- ``admit `` (or ``admit --candidate ID``): compile, induce, + propose the checks, stop once with a short summary, and on ``accept`` apply + the checks and reseal the bundle. Same as ``admit run``. +- ``admit decide DIR``: decide an admission that stopped without a terminal. +- ``admit status``: read the published release-admission ledger. + +A person decides at an interactive terminal. A delegated agent approver +decides over MCP (``openadapt-agent serve --delegate-approvals``); see +:mod:`openadapt_flow.admit_pipeline`. +""" from __future__ import annotations import argparse +import getpass import json +import re import sys +from pathlib import Path +from typing import Callable, Optional +from openadapt_flow.cli_hints import command from openadapt_flow.release_admission import ( ADMITTED, DEFAULT_LEDGER_URL, @@ -15,16 +31,78 @@ status_report, ) +#: Verbs of ``admit``; anything else after ``admit`` means ``admit run``. +ADMIT_VERBS = ("run", "decide", "status") +#: Exit code when the command stopped for a decision or a recording. +EXIT_STOPPED = 3 + + +def admit_default_form(argv: list[str]) -> list[str]: + """``admit `` and ``admit --candidate ID`` mean ``admit run``.""" + + if argv[:1] == ["admit"] and len(argv) >= 2: + first = argv[1] + if first not in ADMIT_VERBS and first not in ("-h", "--help"): + return ["admit", "run", *argv[1:]] + return argv + def register_admit_parser(sub: argparse._SubParsersAction) -> None: parser = sub.add_parser( "admit", help=( - "Read the published release-admission ledger. " - "Does not sign or issue an admission." + "Turn recordings (or a shadow candidate) into a program with checks: " + "compile, induce, propose checks, stop once for a decision. " + "`admit status` reads the release-admission ledger." ), ) verbs = parser.add_subparsers(dest="admit_cmd", required=True) + run = verbs.add_parser( + "run", + help=( + "Compile, induce, propose checks, and stop once for a person's " + "decision (the default form: `admit `)" + ), + ) + run.add_argument( + "recordings", + nargs="*", + help="Recording directories (or compiled bundles) of the same task", + ) + run.add_argument( + "--candidate", + default=None, + metavar="ID", + help=( + "A shadow candidate id under --candidates-root, or its folder. Its " + "recordings/ folder is used when no recordings are given" + ), + ) + run.add_argument( + "--candidates-root", + default="shadow-candidates", + help="Where shadow candidates are kept (default: ./shadow-candidates)", + ) + run.add_argument("--out", required=True, help="New admission directory") + run.add_argument("--name", required=True, help="Workflow name") + run.add_argument( + "--policy-pack", + choices=("community", "cloud", "regulated"), + default="community", + ) + run.add_argument( + "--by", + default=None, + help="Your name for the record (default: your login name)", + ) + run.set_defaults(func=cmd_admit_run) + decide = verbs.add_parser( + "decide", + help="Show a stopped admission's summary and decide it (a person, at a terminal)", + ) + decide.add_argument("admission", help="Admission directory from `admit`") + decide.add_argument("--by", default=None, help="Your name for the record") + decide.set_defaults(func=cmd_admit_decide) status = verbs.add_parser( "status", help=( @@ -68,3 +146,119 @@ def cmd_admit_status(args: argparse.Namespace) -> int: if args.check and report["flow"]["state"] != ADMITTED: return 1 return 0 + + +def _person(by: Optional[str]) -> "object": + from openadapt_flow.admit_pipeline import DECIDER_ID_PATTERN, Decider + + name = by or getpass.getuser() or "person" + name = re.sub(r"[^A-Za-z0-9._:@/-]", "-", name).lstrip("-._:@/") or "person" + if not re.match(DECIDER_ID_PATTERN, name): + raise SystemExit("admit: --by must be a short name") + return Decider(kind="person", id=name, route="cli") + + +def _interactive() -> bool: + return sys.stdin.isatty() and sys.stdout.isatty() + + +def _ask_once( + out: Path, + by: Optional[str], + *, + read: Callable[[str], str] = input, + interactive: Optional[bool] = None, +) -> int: + """The single stop: show the summary, take one decision from a person.""" + + from openadapt_flow.admit_pipeline import ( + AdmissionError, + decide_admission, + load_pending, + render_summary, + ) + + pending = load_pending(out) + sys.stdout.write(render_summary(pending)) + if pending.status != "needs_decision": + return EXIT_STOPPED if pending.status == "needs_recording" else 2 + if not (_interactive() if interactive is None else interactive): + print( + "Stopped for a decision. A person decides with " + f"`{command(f'admit decide {out}')}` at a terminal; a delegated " + "agent approver decides with the decide_admission MCP tool." + ) + return EXIT_STOPPED + answer = read("Type accept to apply these checks, or reject: ").strip().lower() + reason = None + if answer == "reject": + reason = read("Why? ").strip() or "rejected at the terminal" + elif answer != "accept": + print( + f"No decision recorded. Decide later with `{command(f'admit decide {out}')}`." + ) + return EXIT_STOPPED + try: + result = decide_admission( + out, + pending_sha256=pending.sha256(), + decision="accept" if answer == "accept" else "reject", + decided_by=_person(by), # type: ignore[arg-type] + reason=reason, + ) + except AdmissionError as exc: + print(f"admit: {exc}", file=sys.stderr) + return 2 + if result.decision == "reject": + print(f"Rejected by {result.decided_by.label()}. Nothing was applied.") + return 1 + print( + f"Accepted by {result.decided_by.label()}. Checks applied to {out / 'bundle'}." + ) + bundle = out / "bundle" + print(f"Next: `{command(f'qualify {bundle} --verify verify.yaml')}`") + return 0 + + +def cmd_admit_run(args: argparse.Namespace) -> int: + from openadapt_flow.admit_pipeline import ( + AdmissionError, + prepare_admission, + resolve_candidate, + ) + + if not args.recordings and not args.candidate: + raise SystemExit("admit: give recording directories or --candidate ID") + try: + candidate = ( + resolve_candidate(args.candidate, args.candidates_root) + if args.candidate + else None + ) + prepare_admission( + args.recordings, + out=args.out, + name=args.name, + prepared_by=_person(args.by), # type: ignore[arg-type] + candidate=candidate, + policy_pack=args.policy_pack, + ) + except AdmissionError as exc: + print(f"admit: {exc}", file=sys.stderr) + return 2 + return _ask_once(Path(args.out), args.by) + + +def cmd_admit_decide(args: argparse.Namespace) -> int: + from openadapt_flow.admit_pipeline import AdmissionError + + if not _interactive(): + raise SystemExit( + "admit decide: a person decides at an interactive terminal; a " + "delegated agent approver uses the decide_admission MCP tool" + ) + try: + return _ask_once(Path(args.admission), args.by, interactive=True) + except AdmissionError as exc: + print(f"admit: {exc}", file=sys.stderr) + return 2 diff --git a/packages/flow/openadapt_flow/compiler/motif_induction.py b/packages/flow/openadapt_flow/compiler/motif_induction.py new file mode 100644 index 000000000..8e76575f6 --- /dev/null +++ b/packages/flow/openadapt_flow/compiler/motif_induction.py @@ -0,0 +1,510 @@ +"""Turn shadow-mined episodes into a CANDIDATE program, never a runnable one. + +OpenAdapt Capture's shadow mode (``openadapt_capture.shadow``, off by default) +can notice that a person repeated the same cross-application sequence several +times and, if the person clicks **Review draft**, export those episodes as +``openadapt.shadow.episodes/v2`` JSON. Each step in that export is a +value-free symbol: an application, a control role, references local to the +export in place of labels and typed values (``label-a``, ``value-b``), a +scrubbed URL pattern. There are no screenshots, no coordinates, no plain-text +values, and no keyed digests: two steps that share a reference shared a value, +and that is all the export says about values. Version 1 exports, which carried +keyed digests, are refused, so a digest can never reach a draft. + +This module converts the episodes into linear :class:`~openadapt_flow.ir.Workflow` +traces and hands them to the EXISTING multi-trace induction +(:func:`openadapt_flow.compiler.induction.induce_program`), which infers +parameters, loops, and branches or refuses. The result is a +:class:`MotifCandidate`: + +* ``status`` is always ``"candidate"``, ``admitted`` and ``runnable`` are + always False. Nothing here can run a step, and nothing here writes a + ``workflow.json`` bundle. +* The draft's anchors are placeholders (shadow mining keeps no pixels), so + the record-next worklist always starts with "record this task once" through + the normal record, compile, induce, and qualify path. That path is where a + person admits the program. +* Write-shaped steps (a Submit-like control, a click right before a form + submit, a save chord) are marked irreversible and ``risk_review_required``, + so certification refuses them until a person reviews them. +* A verification check is PROPOSED where the episodes show one (a file + saved, a form submitted, a row pasted into a spreadsheet). A proposal is + never an effect contract: the episodes that suggested it can't approve it. + +Deterministic and model-free; no network access. +""" + +from __future__ import annotations + +import json +import re +from collections import Counter +from pathlib import Path +from typing import Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from openadapt_flow.compiler.induction import RecordNext, TraceInput, induce_program +from openadapt_flow.ir import ActionKind, Anchor, Step, StructuralLocator, Workflow + +EPISODES_SCHEMA = "openadapt.shadow.episodes/v2" +REFUSED_SCHEMAS = frozenset({"openadapt.shadow.episodes/v1"}) +CANDIDATE_SCHEMA: Literal["openadapt.shadow.candidate/v1"] = ( + "openadapt.shadow.candidate/v1" +) +PLACEHOLDER_TEMPLATE = "shadow-unrecorded.png" +DRAFT_FILENAME = "draft-workflow.json" +CANDIDATE_FILENAME = "candidate.json" + +_DIGEST = r"^[0-9a-f]{16}$" +_TOKEN_DIGEST = re.compile(r"^[0-9a-f]{8,}$") +_LABEL_REF = r"^label-[a-z]{1,4}$" +_VALUE_REF = r"^value-[a-z]{1,4}$" +#: Capture's placeholder for a chrome label that looked like data. +DYNAMIC_LABEL = "" +_WRITE_STEMS = ("save", "submit", "send", "post", "approve", "confirm", "create", "add") +_SHEET_APPS = ("excel", "numbers", "sheets", "calc", "spreadsheet") +_SAVE_MODIFIERS = frozenset({"meta", "ctrl", "cmd", "command"}) + +ADMISSION_PATH: tuple[str, ...] = ( + "Record the task with OpenAdapt Capture (flow record or capture record).", + "Run flow admit --candidate : it compiles, induces and proposes " + "checks, then stops once for a person (or a delegated agent approver) to " + "accept or reject them.", + "Qualify the accepted bundle (flow qualify).", +) + + +class ShadowStep(BaseModel): + """One value-free symbol from the Capture export.""" + + model_config = ConfigDict(extra="forbid") + + t: float + token: str = Field(max_length=256) + kind: Literal[ + "focus", + "click", + "double_click", + "right_click", + "type", + "key", + "shortcut", + "copy", + "paste", + "navigate", + "submit", + ] + app: Optional[str] = Field(default=None, max_length=128) + window_class: Optional[str] = Field(default=None, max_length=128) + role: Optional[str] = Field(default=None, max_length=64) + label_ref: Optional[str] = Field(default=None, pattern=_LABEL_REF) + display_label: Optional[str] = Field( + default=None, max_length=32, pattern=r"^[^0-9@]*$" + ) + value_ref: Optional[str] = Field(default=None, pattern=_VALUE_REF) + value_shape: Optional[ + Literal["empty", "digits", "decimal", "date", "email", "alpha", "alnum", "text"] + ] = None + url_pattern: Optional[str] = Field(default=None, max_length=256) + key: Optional[str] = Field(default=None, max_length=32) + modifiers: list[str] = Field(default_factory=list, max_length=4) + secure: bool = False + + @field_validator("token") + @classmethod + def _no_digest_in_token(cls, value: str) -> str: + if any(_TOKEN_DIGEST.match(part) for part in value.split("|")): + raise ValueError("a shadow token must not carry a keyed digest") + return value + + +def _check_tokens(tokens: list[str]) -> list[str]: + for token in tokens: + ShadowStep._no_digest_in_token(token) + return tokens + + +class ShadowEpisode(BaseModel): + model_config = ConfigDict(extra="forbid") + + t_start: float + t_end: float + exact: bool + similarity: float = Field(ge=0.0, le=1.0) + steps: list[ShadowStep] = Field(min_length=1) + + +class ShadowEpisodesExport(BaseModel): + """The ``openadapt.shadow.episodes/v2`` document Capture writes.""" + + model_config = ConfigDict(extra="forbid") + + schema_: Literal["openadapt.shadow.episodes/v2"] = Field(alias="schema") + motif_id: str = Field(pattern=_DIGEST) + tokens: list[str] + + @field_validator("tokens") + @classmethod + def _tokens_carry_no_digest(cls, value: list[str]) -> list[str]: + return _check_tokens(value) + + apps: list[str] + count_in_window: int = Field(ge=0) + window_s: float + exact_count: int = Field(ge=0) + expected_by_chance: float + p_value: float + status: Literal["candidate"] + privacy: dict[str, str] + episodes: list[ShadowEpisode] = Field(min_length=1) + + +class ProposedCheck(BaseModel): + """A verification check the episodes suggest. A proposal, nothing more.""" + + kind: Literal["file_written", "record_written", "row_added", "page_reached"] + summary: str + step_id: Optional[str] = None + support: int = Field(description="Episodes in which the signal appeared.") + status: Literal["proposed"] = "proposed" + admitted: Literal[False] = False + note: str = ( + "Proposal only. The episodes that suggested this check can't approve " + "it; a person confirms or rejects it during qualification." + ) + + +class MotifCandidate(BaseModel): + """What a person reviews after clicking **Review draft**.""" + + schema_version: Literal["openadapt.shadow.candidate/v1"] = CANDIDATE_SCHEMA + motif_id: str + status: Literal["candidate"] = "candidate" + admitted: Literal[False] = False + runnable: Literal[False] = False + apps: list[str] + n_episodes: int + count_in_window: int + induction_certified: bool = Field( + description=( + "Whether induction found the episodes structurally consistent. " + "Structural only: it does not admit, qualify, or certify a run." + ) + ) + induction_report: str + draft: Optional[Workflow] = None + record_next: list[RecordNext] = Field(default_factory=list) + proposed_checks: list[ProposedCheck] = Field(default_factory=list) + admission_path: list[str] = Field(default_factory=lambda: list(ADMISSION_PATH)) + + +def _alias(ref: Optional[str]) -> str: + """Name of a control known only by its export reference.""" + + if not ref: + return "unnamed" + return "fld-" + ref.split("-", 1)[1] + + +def _shown(display_label: Optional[str]) -> Optional[str]: + """A plain-text chrome label, or None for none or a scrubbed one.""" + + if not display_label or display_label == DYNAMIC_LABEL: + return None + return display_label + + +def _placeholder_anchor(role: Optional[str], name: str, *, varies: bool) -> Anchor: + """An anchor with no pixels. Shadow mining keeps no screenshots. + + A control whose identity varies per run (a list row, a table cell) gets a + structural role, so induction reads it as a which-entity selection. A + fixed control (a named button or field) is identified by its label, so + two different buttons in a row never collapse into one "button" column. + """ + + if varies: + return Anchor( + template=PLACEHOLDER_TEMPLATE, + region=(0, 0, 1, 1), + click_point=(0, 0), + structural=StructuralLocator(role=role or None, name=name), + ) + return Anchor( + template=PLACEHOLDER_TEMPLATE, + region=(0, 0, 1, 1), + click_point=(0, 0), + ocr_text=name, + ) + + +def _is_write_label(label: Optional[str]) -> bool: + if not label: + return False + low = label.lower() + return any(re.search(rf"\b{stem}\b", low) for stem in _WRITE_STEMS) + + +def _is_sheet(app: Optional[str]) -> bool: + low = (app or "").lower() + return any(name in low for name in _SHEET_APPS) + + +def _episode_steps( + episode: ShadowEpisode, +) -> tuple[list[Step], list[tuple[str, Optional[str], str]]]: + """Map one episode to IR steps plus the check signals it shows. + + A signal is ``(kind, step_id, summary)``. + """ + + steps: list[Step] = [] + signals: list[tuple[str, Optional[str], str]] = [] + symbols = episode.steps + for index, sym in enumerate(symbols): + sid = f"s{index:02d}" + app = sym.app or "unknown" + prefix = f"[{app}]" + nxt = symbols[index + 1] if index + 1 < len(symbols) else None + if sym.kind in ("click", "double_click", "right_click"): + action = { + "click": ActionKind.CLICK, + "double_click": ActionKind.DOUBLE_CLICK, + "right_click": ActionKind.RIGHT_CLICK, + }[sym.kind] + if sym.value_ref: + name = sym.value_ref + what = f"{sym.role or 'control'} (varies)" + else: + name = _shown(sym.display_label) or _alias(sym.label_ref) + what = f"{sym.role or 'control'} {name}" + writes = _is_write_label(_shown(sym.display_label)) or ( + nxt is not None and nxt.kind == "submit" + ) + steps.append( + Step( + id=sid, + intent=f"{prefix} click {what}", + action=action, + anchor=_placeholder_anchor( + sym.role, name, varies=bool(sym.value_ref) + ), + risk="irreversible" if writes else "reversible", + risk_explanation=( + "Shadow-mined write-shaped control; a person must review it." + if writes + else None + ), + risk_review_required=writes, + ) + ) + if writes: + signals.append( + ( + "record_written", + sid, + f"A record is created or changed after '{what}' in {app}.", + ) + ) + elif sym.kind == "type": + field = _shown(sym.display_label) or _alias(sym.label_ref) + intent = f"{prefix} type {sym.role or 'field'} {field}" + if sym.secure: + steps.append( + Step( + id=sid, + intent=intent, + action=ActionKind.TYPE, + secret=True, + param=f"secret_{_alias(sym.label_ref)[4:]}", + ) + ) + else: + shape = sym.value_shape or "text" + steps.append( + Step( + id=sid, + intent=intent, + action=ActionKind.TYPE, + text=f"<{shape}:{sym.value_ref or 'none'}>", + ) + ) + if nxt is not None and nxt.kind == "key" and _is_sheet(app): + signals.append(("row_added", sid, f"The sheet in {app} gains a row.")) + elif sym.kind in ("copy", "paste", "shortcut"): + mods = [m for m in sym.modifiers if m] + key = sym.key or ("c" if sym.kind == "copy" else "v") + label = {"copy": "copy", "paste": "paste"}.get(sym.kind, "press shortcut") + if mods: + steps.append( + Step( + id=sid, + intent=f"{prefix} {label} {'+'.join(mods)}+{key}", + action=ActionKind.HOTKEY, + key=key, + modifiers=mods, + ) + ) + else: + steps.append( + Step( + id=sid, + intent=f"{prefix} press {key}", + action=ActionKind.KEY, + key=key, + ) + ) + if sym.kind == "shortcut" and key == "s" and set(mods) & _SAVE_MODIFIERS: + signals.append( + ("file_written", sid, f"A file saved from {app} changes.") + ) + if ( + sym.kind == "paste" + and _is_sheet(app) + and nxt is not None + and (nxt.kind == "key") + ): + signals.append(("row_added", sid, f"The sheet in {app} gains a row.")) + elif sym.kind == "key": + steps.append( + Step( + id=sid, + intent=f"{prefix} press {sym.key or 'key'}", + action=ActionKind.KEY, + key=sym.key or "other", + ) + ) + elif sym.kind == "submit": + target = sym.url_pattern or app + signals.append( + ("page_reached", None, f"After submitting, the page matches {target}.") + ) + # focus and navigate carry context only; they are not IR actions. + return steps, signals + + +def episodes_to_workflows( + export: ShadowEpisodesExport, +) -> tuple[list[Workflow], list[ProposedCheck]]: + """Convert episodes to linear traces, and collect proposed checks.""" + + workflows: list[Workflow] = [] + support: Counter[tuple[str, str]] = Counter() + where: dict[tuple[str, str], Counter[str]] = {} + for number, episode in enumerate(export.episodes): + steps, signals = _episode_steps(episode) + workflows.append( + Workflow(name=f"shadow-episode-{export.motif_id}-{number}", steps=steps) + ) + seen = {(kind, summary) for kind, _sid, summary in signals} + support.update(seen) + for kind, sid, summary in signals: + if sid is not None: + where.setdefault((kind, summary), Counter())[sid] += 1 + majority = (len(export.episodes) + 1) // 2 + checks: list[ProposedCheck] = [] + for (kind, summary), count in sorted(support.items()): + if count < majority: + continue + ids = where.get((kind, summary)) + step_id = ( + sorted(ids.items(), key=lambda kv: (-kv[1], kv[0]))[0][0] if ids else None + ) + checks.append( + ProposedCheck( + kind=kind, # type: ignore[arg-type] + step_id=step_id, + summary=summary, + support=count, + ) + ) + return workflows, checks + + +def induce_candidate(export: ShadowEpisodesExport | dict) -> MotifCandidate: + """Run the existing induction over shadow episodes; return a candidate.""" + + if not isinstance(export, ShadowEpisodesExport): + if isinstance(export, dict) and export.get("schema") in REFUSED_SCHEMAS: + raise ValueError( + f"{export.get('schema')} carries keyed digests and is refused; " + f"export again with a Capture that writes {EPISODES_SCHEMA}" + ) + export = ShadowEpisodesExport.model_validate(export) + workflows, checks = episodes_to_workflows(export) + traces: list[TraceInput] = list(workflows) + result = induce_program(traces) + + record_next: list[RecordNext] = [ + RecordNext( + code="shadow_full_demonstration", + instruction=( + "Record this task once with OpenAdapt Capture. Shadow mining kept " + "no screenshots or values, so the draft has nothing to act on yet." + ), + ) + ] + record_next.extend(result.record_next) + for check in checks: + record_next.append( + RecordNext( + code="shadow_confirm_check", + instruction=f"Confirm or reject the proposed check: {check.summary}", + location=check.step_id or "", + ) + ) + for name, spec in sorted(result.param_specs.items()): + record_next.append( + RecordNext( + code="shadow_name_value", + instruction=( + f"Say what the changing value '{name}' is " + f"(the draft only knows it varies)." + ), + location=name, + blocking=False, + ) + ) + + draft = None + if result.workflow is not None: + draft = result.workflow.model_copy( + update={"name": f"shadow-draft-{export.motif_id}"} + ) + return MotifCandidate( + motif_id=export.motif_id, + apps=list(export.apps), + n_episodes=len(export.episodes), + count_in_window=export.count_in_window, + induction_certified=result.certified, + induction_report=result.render(), + draft=draft, + record_next=record_next, + proposed_checks=checks, + ) + + +def write_candidate(candidate: MotifCandidate, out_dir: Path) -> Path: + """Write ``candidate.json`` and, when there is a draft, ``draft-workflow.json``. + + Never writes ``workflow.json``, so no Flow command treats the directory + as a runnable bundle. Refuses a directory that already holds one. + """ + + out = Path(out_dir) + if (out / "workflow.json").exists() or (out / "workflow.json.enc").exists(): + raise FileExistsError( + f"{out} holds a workflow bundle; refusing to mix a candidate in" + ) + out.mkdir(parents=True, exist_ok=True) + meta = candidate.model_dump(mode="json", exclude={"draft"}) + meta["draft_file"] = DRAFT_FILENAME if candidate.draft is not None else None + (out / CANDIDATE_FILENAME).write_text( + json.dumps(meta, indent=2, sort_keys=True) + "\n" + ) + if candidate.draft is not None: + (out / DRAFT_FILENAME).write_text( + candidate.draft.model_dump_json(indent=2) + "\n" + ) + return out diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 88b418550..d26c5f4c6 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -6239,6 +6239,18 @@ "path": "tests/fixtures/runtime-validation-canonical-vector.json", "sha256": "a238b23846705f081603c7fe8a66ee16a88281ad21acfd0d047052716a49b912" }, + { + "path": "tests/fixtures/shadow/invoice_exact.episodes.json", + "sha256": "e555e3913c02df609cdf490872fad30574a1c438b2c110b0ff3345e1a31c26ce" + }, + { + "path": "tests/fixtures/shadow/invoice_variant.episodes.json", + "sha256": "cc2d9508eac86e6115485025525e7529b4830f720eb28c21e258622e7a70ec79" + }, + { + "path": "tests/fixtures/shadow/report_mail.episodes.json", + "sha256": "4e3f1fed5b5bfba9220fd37d53eff1bea2655df4903fb6a59ec215bc3e2d2ec0" + }, { "path": "tests/fixtures/terminal_verification_v2_delivery_consumer_vector.json", "sha256": "f2e32472ad562b5b67d95d3d9cafaf08b4a2adb9fb5ff3cda2813307beb61c96" diff --git a/packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json b/packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json new file mode 100644 index 000000000..14f37c78f --- /dev/null +++ b/packages/flow/tests/fixtures/shadow/invoice_exact.episodes.json @@ -0,0 +1,791 @@ +{ + "apps": [ + "Google Chrome", + "Microsoft Excel" + ], + "count_in_window": 4, + "episodes": [ + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000265.4191387234, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000266.4191387234, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_ref": "value-a", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000267.4191387234, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000268.4191387234, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000269.4191387234, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000270.4191387234, + "token": "click|Google Chrome|textfield|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000271.4191387234, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000272.4191387234, + "token": "click|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000273.4191387234, + "token": "type|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": "value-b", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000274.4191387234, + "token": "click|Google Chrome|button|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000275.4191387234, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000275.4191387234, + "t_start": 1000265.4191387234 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000570.0973327656, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000571.0973327656, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_ref": "value-c", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000572.0973327656, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000573.0973327656, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000574.0973327656, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000575.0973327656, + "token": "click|Google Chrome|textfield|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000576.0973327656, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000577.0973327656, + "token": "click|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000578.0973327656, + "token": "type|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": "value-d", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000579.0973327656, + "token": "click|Google Chrome|button|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000580.0973327656, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000580.0973327656, + "t_start": 1000570.0973327656 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000865.0038992346, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000866.0038992346, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_ref": "value-e", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000867.0038992346, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000868.0038992346, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000869.0038992346, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000870.0038992346, + "token": "click|Google Chrome|textfield|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000871.0038992346, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000872.0038992346, + "token": "click|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000873.0038992346, + "token": "type|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": "value-f", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000874.0038992346, + "token": "click|Google Chrome|button|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000875.0038992346, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000875.0038992346, + "t_start": 1000865.0038992346 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001239.5060203954, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1001240.5060203954, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_ref": "value-g", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1001241.5060203954, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001242.5060203954, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001243.5060203954, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001244.5060203954, + "token": "click|Google Chrome|textfield|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1001245.5060203954, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001246.5060203954, + "token": "click|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001247.5060203954, + "token": "type|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": "value-h", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1001248.5060203954, + "token": "click|Google Chrome|button|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001249.5060203954, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1001249.5060203954, + "t_start": 1001239.5060203954 + } + ], + "exact_count": 4, + "expected_by_chance": 2.0763629850095314e-16, + "motif_id": "b78791ee75702dcb", + "p_value": 7.744651342950623e-65, + "privacy": { + "clipboard_contents": "never read; copy and paste come from the key chord", + "control_labels": "short chrome labels of common words in plain text; others a placeholder", + "keyed_digests": "never exported; replaced by references local to this export", + "network": "none", + "screenshots": "never taken", + "typed_values": "never exported; secure fields dropped", + "urls": "work domains only: host and one scrubbed path word; other sites a marker", + "window_titles": "never read" + }, + "schema": "openadapt.shadow.episodes/v2", + "status": "candidate", + "tokens": [ + "focus|Microsoft Excel", + "click|Microsoft Excel|cell|-", + "copy|Microsoft Excel", + "focus|Google Chrome", + "navigate|Google Chrome|https://erp.example.com/invoices/*", + "click|Google Chrome|textfield|label-a", + "paste|Google Chrome", + "click|Google Chrome|textfield|label-b", + "type|Google Chrome|textfield|label-b", + "click|Google Chrome|button|label-c", + "submit|Google Chrome|https://erp.example.com/invoices/*" + ], + "window_s": 86400.0 +} diff --git a/packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json b/packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json new file mode 100644 index 000000000..c45f99099 --- /dev/null +++ b/packages/flow/tests/fixtures/shadow/invoice_variant.episodes.json @@ -0,0 +1,807 @@ +{ + "apps": [ + "Google Chrome", + "Microsoft Excel" + ], + "count_in_window": 4, + "episodes": [ + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000265.4191387234, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000266.4191387234, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_ref": "value-a", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000267.4191387234, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000268.4191387234, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000269.4191387234, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000270.4191387234, + "token": "click|Google Chrome|textfield|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000271.4191387234, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000272.4191387234, + "token": "click|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000273.4191387234, + "token": "type|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": "value-b", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000274.4191387234, + "token": "click|Google Chrome|button|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000275.4191387234, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000275.4191387234, + "t_start": 1000265.4191387234 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000570.0973327656, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000571.0973327656, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_ref": "value-c", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000572.0973327656, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000573.0973327656, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000574.0973327656, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000575.0973327656, + "token": "click|Google Chrome|textfield|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000576.0973327656, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000577.0973327656, + "token": "click|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000578.0973327656, + "token": "type|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": "value-d", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000579.0973327656, + "token": "click|Google Chrome|button|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000580.0973327656, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000580.0973327656, + "t_start": 1000570.0973327656 + }, + { + "exact": false, + "similarity": 0.9565, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000865.0038992346, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1000866.0038992346, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_ref": "value-e", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000867.0038992346, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000868.0038992346, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000869.0038992346, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000870.0038992346, + "token": "click|Google Chrome|textfield|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1000871.0038992346, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000872.0038992346, + "token": "click|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1000873.0038992346, + "token": "type|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": "value-f", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Urgent", + "key": null, + "kind": "click", + "label_ref": "label-d", + "modifiers": [], + "role": "checkbox", + "secure": false, + "t": 1000873.8038992346, + "token": "click|Google Chrome|checkbox|label-d", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1000874.0038992346, + "token": "click|Google Chrome|button|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1000875.0038992346, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1000875.0038992346, + "t_start": 1000865.0038992346 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001239.5060203954, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": null, + "modifiers": [], + "role": "cell", + "secure": false, + "t": 1001240.5060203954, + "token": "click|Microsoft Excel|cell|-", + "url_pattern": null, + "value_ref": "value-g", + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "c", + "kind": "copy", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1001241.5060203954, + "token": "copy|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001242.5060203954, + "token": "focus|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "navigate", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001243.5060203954, + "token": "navigate|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001244.5060203954, + "token": "click|Google Chrome|textfield|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": "v", + "kind": "paste", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 1001245.5060203954, + "token": "paste|Google Chrome", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001246.5060203954, + "token": "click|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 1001247.5060203954, + "token": "type|Google Chrome|textfield|label-b", + "url_pattern": null, + "value_ref": "value-h", + "value_shape": "decimal", + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": "Submit", + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "button", + "secure": false, + "t": 1001248.5060203954, + "token": "click|Google Chrome|button|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Google Chrome", + "display_label": null, + "key": null, + "kind": "submit", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 1001249.5060203954, + "token": "submit|Google Chrome|https://erp.example.com/invoices/*", + "url_pattern": "https://erp.example.com/invoices/*", + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 1001249.5060203954, + "t_start": 1001239.5060203954 + } + ], + "exact_count": 3, + "expected_by_chance": 1.9581310411777164e-16, + "motif_id": "b78791ee75702dcb", + "p_value": 1.2513361926197074e-48, + "privacy": { + "clipboard_contents": "never read; copy and paste come from the key chord", + "control_labels": "short chrome labels of common words in plain text; others a placeholder", + "keyed_digests": "never exported; replaced by references local to this export", + "network": "none", + "screenshots": "never taken", + "typed_values": "never exported; secure fields dropped", + "urls": "work domains only: host and one scrubbed path word; other sites a marker", + "window_titles": "never read" + }, + "schema": "openadapt.shadow.episodes/v2", + "status": "candidate", + "tokens": [ + "focus|Microsoft Excel", + "click|Microsoft Excel|cell|-", + "copy|Microsoft Excel", + "focus|Google Chrome", + "navigate|Google Chrome|https://erp.example.com/invoices/*", + "click|Google Chrome|textfield|label-a", + "paste|Google Chrome", + "click|Google Chrome|textfield|label-b", + "type|Google Chrome|textfield|label-b", + "click|Google Chrome|button|label-c", + "submit|Google Chrome|https://erp.example.com/invoices/*" + ], + "window_s": 86400.0 +} diff --git a/packages/flow/tests/fixtures/shadow/report_mail.episodes.json b/packages/flow/tests/fixtures/shadow/report_mail.episodes.json new file mode 100644 index 000000000..16402382a --- /dev/null +++ b/packages/flow/tests/fixtures/shadow/report_mail.episodes.json @@ -0,0 +1,548 @@ +{ + "apps": [ + "Mail", + "Microsoft Excel" + ], + "count_in_window": 3, + "episodes": [ + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000238.255011055, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "s", + "kind": "shortcut", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 5000239.255011055, + "token": "shortcut|Microsoft Excel|meta+s", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000240.255011055, + "token": "focus|Mail", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "New Message", + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000241.255011055, + "token": "click|Mail|button|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000242.255011055, + "token": "click|Mail|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000243.255011055, + "token": "type|Mail|textfield|label-b", + "url_pattern": null, + "value_ref": "value-a", + "value_shape": "email", + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000244.255011055, + "token": "click|Mail|textfield|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-c", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000245.255011055, + "token": "type|Mail|textfield|label-c", + "url_pattern": null, + "value_ref": "value-b", + "value_shape": "alnum", + "window_class": null + }, + { + "app": "Mail", + "display_label": "Attach", + "key": null, + "kind": "click", + "label_ref": "label-d", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000246.255011055, + "token": "click|Mail|button|label-d", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "Send", + "key": null, + "kind": "click", + "label_ref": "label-e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000247.255011055, + "token": "click|Mail|button|label-e", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 5000247.255011055, + "t_start": 5000238.255011055 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000558.013902665, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "s", + "kind": "shortcut", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 5000559.013902665, + "token": "shortcut|Microsoft Excel|meta+s", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000560.013902665, + "token": "focus|Mail", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "New Message", + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000561.013902665, + "token": "click|Mail|button|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000562.013902665, + "token": "click|Mail|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000563.013902665, + "token": "type|Mail|textfield|label-b", + "url_pattern": null, + "value_ref": "value-c", + "value_shape": "email", + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000564.013902665, + "token": "click|Mail|textfield|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-c", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000565.013902665, + "token": "type|Mail|textfield|label-c", + "url_pattern": null, + "value_ref": "value-d", + "value_shape": "alnum", + "window_class": null + }, + { + "app": "Mail", + "display_label": "Attach", + "key": null, + "kind": "click", + "label_ref": "label-d", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000566.013902665, + "token": "click|Mail|button|label-d", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "Send", + "key": null, + "kind": "click", + "label_ref": "label-e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000567.013902665, + "token": "click|Mail|button|label-e", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 5000567.013902665, + "t_start": 5000558.013902665 + }, + { + "exact": true, + "similarity": 1.0, + "steps": [ + { + "app": "Microsoft Excel", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000881.901317636, + "token": "focus|Microsoft Excel", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Microsoft Excel", + "display_label": null, + "key": "s", + "kind": "shortcut", + "label_ref": null, + "modifiers": [ + "meta" + ], + "role": null, + "secure": false, + "t": 5000882.901317636, + "token": "shortcut|Microsoft Excel|meta+s", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "focus", + "label_ref": null, + "modifiers": [], + "role": null, + "secure": false, + "t": 5000883.901317636, + "token": "focus|Mail", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "New Message", + "key": null, + "kind": "click", + "label_ref": "label-a", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000884.901317636, + "token": "click|Mail|button|label-a", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000885.901317636, + "token": "click|Mail|textfield|label-b", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-b", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000886.901317636, + "token": "type|Mail|textfield|label-b", + "url_pattern": null, + "value_ref": "value-e", + "value_shape": "email", + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "click", + "label_ref": "label-c", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000887.901317636, + "token": "click|Mail|textfield|label-c", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": null, + "key": null, + "kind": "type", + "label_ref": "label-c", + "modifiers": [], + "role": "textfield", + "secure": false, + "t": 5000888.901317636, + "token": "type|Mail|textfield|label-c", + "url_pattern": null, + "value_ref": "value-f", + "value_shape": "alnum", + "window_class": null + }, + { + "app": "Mail", + "display_label": "Attach", + "key": null, + "kind": "click", + "label_ref": "label-d", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000889.901317636, + "token": "click|Mail|button|label-d", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + }, + { + "app": "Mail", + "display_label": "Send", + "key": null, + "kind": "click", + "label_ref": "label-e", + "modifiers": [], + "role": "button", + "secure": false, + "t": 5000890.901317636, + "token": "click|Mail|button|label-e", + "url_pattern": null, + "value_ref": null, + "value_shape": null, + "window_class": null + } + ], + "t_end": 5000890.901317636, + "t_start": 5000881.901317636 + } + ], + "exact_count": 3, + "expected_by_chance": 5.1827993358769765e-15, + "motif_id": "56a7933b9d8c5256", + "p_value": 2.3202882082950603e-44, + "privacy": { + "clipboard_contents": "never read; copy and paste come from the key chord", + "control_labels": "short chrome labels of common words in plain text; others a placeholder", + "keyed_digests": "never exported; replaced by references local to this export", + "network": "none", + "screenshots": "never taken", + "typed_values": "never exported; secure fields dropped", + "urls": "work domains only: host and one scrubbed path word; other sites a marker", + "window_titles": "never read" + }, + "schema": "openadapt.shadow.episodes/v2", + "status": "candidate", + "tokens": [ + "focus|Microsoft Excel", + "shortcut|Microsoft Excel|meta+s", + "focus|Mail", + "click|Mail|button|label-a", + "click|Mail|textfield|label-b", + "type|Mail|textfield|label-b", + "click|Mail|textfield|label-c", + "type|Mail|textfield|label-c", + "click|Mail|button|label-d", + "click|Mail|button|label-e" + ], + "window_s": 86400.0 +} diff --git a/packages/flow/tests/test_admit_pipeline.py b/packages/flow/tests/test_admit_pipeline.py new file mode 100644 index 000000000..d0b142526 --- /dev/null +++ b/packages/flow/tests/test_admit_pipeline.py @@ -0,0 +1,695 @@ +"""``openadapt-flow admit``: one command, one stop, a record of who decided. + +Made-up values only. The fixture is the tiny qualified-write workflow from +test_qualification_proposal: one irreversible save with a system-of-record +effect, so the check proposal is a draft rather than a HALT. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from openadapt_flow import admit_pipeline as ap +from openadapt_flow.__main__ import main +from openadapt_flow.approval_mode.approver import ApproverPolicyV1, StepClass +from openadapt_flow.cli_admit import EXIT_STOPPED, _ask_once, admit_default_form +from openadapt_flow.compiler.motif_induction import induce_candidate, write_candidate +from openadapt_flow.ir import Workflow +from tests._approver_grant_support import key_dir, signed_grant +from tests.test_qualification_proposal import _png, _workflow + +SHADOW = Path(__file__).parent / "fixtures" / "shadow" +PERSON = ap.Decider(kind="person", id="pat", route="cli") +AGENT_A = ap.Decider(kind="agent", id="agent:intake-bot", route="mcp") +AGENT_B = ap.Decider(kind="agent", id="agent:reviewer", route="mcp") + + +@pytest.fixture(autouse=True) +def _local_admission_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The local admission key lives in the test's key directory, not ~.""" + + monkeypatch.setenv("OPENADAPT_JIT_KEY_DIR", str(key_dir(tmp_path))) + + +def _agent_policy() -> ApproverPolicyV1: + allowed = {c.value: ["person"] for c in StepClass} + allowed[StepClass.CONSEQUENTIAL.value] = ["person", "agent"] + return ApproverPolicyV1.model_validate({"allowed": allowed}) + + +def _grant(tmp_path: Path, **kwargs): + kwargs.setdefault("delegates", ("agent:reviewer",)) + return signed_grant(tmp_path, _agent_policy(), **kwargs) + + +def _recording(root: Path, name: str = "rec1", *, effects: bool = True) -> Path: + """A compiled bundle with its recording's meta.json beside it.""" + + folder = root / name + (folder / "templates").mkdir(parents=True) + (folder / "templates" / "save.png").write_bytes(_png()) + workflow = _workflow(effects=effects) + workflow.surface = "web" + workflow.save(folder) + (folder / "meta.json").write_text( + json.dumps( + { + "app_url": "http://127.0.0.1:9/", + "application": "MockMed", + "application_version": "tutorial", + "viewport": [1280, 800], + "surface": "web", + "params": dict(workflow.params), + } + ), + encoding="utf-8", + ) + return folder + + +def _prepare(tmp_path: Path, by: ap.Decider = PERSON, **kwargs) -> ap.PendingAdmission: + return ap.prepare_admission( + [_recording(tmp_path)], + out=tmp_path / "admission", + name="qualified-write", + prepared_by=by, + **kwargs, + ) + + +def test_one_recording_stops_once_with_the_proposed_checks(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + assert pending.status == "needs_decision" + assert pending.recordings == 1 and pending.induced is False + assert pending.n_writes == 1 + assert {"step_id": "save", "kind": "field_equals", "tier": 1} in pending.checks + assert pending.failure_cases >= 1 + assert [pin["status"] for pin in pending.pins] == ["proposed"] * 4 + summary = ap.render_summary(pending) + assert "Proposed checks" in summary and "save: field_equals" in summary + assert "Prepared by person:pat via cli" in summary + # Nothing is applied before the decision. + assert Workflow.load(tmp_path / "admission" / "bundle").qualification is None + assert not (tmp_path / "admission" / ap.DECISION_FILE).exists() + + +def test_person_accept_applies_checks_and_records_who_decided(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + result = ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + assert result.decided_by == PERSON and result.prepared_by == PERSON + assert result.oracle_gate_passed is True + assert result.accepted_bundle_content_digest != pending.bundle_content_digest + bundle = Workflow.load(tmp_path / "admission" / "bundle") + assert bundle.qualification is not None + assert "fault-break-it" in {case.id for case in bundle.qualification.cases} + record = json.loads( + (tmp_path / "admission" / "bundle" / ap.DECISION_FILE).read_text() + ) + assert record["decided_by"] == {"kind": "person", "id": "pat", "route": "cli"} + assert record["approval_mode"] == "problems_only" + assert record["approver_grant"] is None + assert "qualify campaign" in record["next"][0] + # No admission is signed here. + assert not ( + tmp_path / "admission" / "bundle" / "qualification-admission.local-dev.json" + ).exists() + + +def test_an_agent_cannot_decide_what_it_prepared(tmp_path: Path) -> None: + pending = _prepare(tmp_path, by=AGENT_A) + with pytest.raises(ap.AdmissionError, match="self-approval"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_A, + ) + assert not (tmp_path / "admission" / ap.DECISION_FILE).exists() + grant = _grant(tmp_path) + result = ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=grant, + key_dir=key_dir(tmp_path), + ) + assert result.decided_by.id == AGENT_B.id and result.prepared_by == AGENT_A + # The record names the grant and the person who signed it. + assert result.decided_by.policy_ref == grant.grant.digest() + assert result.decided_by.on_behalf_of == grant.grant.signer + assert result.approval_mode == "problems_only" + assert "under grant" in result.decided_by.label() + + +@pytest.mark.parametrize( + "grant_kwargs, refusal", + [ + (None, "approver_grant_missing"), + ({"delegates": ("agent:someone-else",)}, "delegate_not_in_grant"), + ({"workflows": ("another-workflow",)}, "workflow_not_in_grant"), + ({"days": 1, "signed_at": "expired"}, "approver_grant_expired"), + ], +) +def test_an_agent_admits_only_under_a_person_signed_grant( + tmp_path: Path, grant_kwargs, refusal +) -> None: + from datetime import datetime, timedelta, timezone + + pending = _prepare(tmp_path) + grant = None + if grant_kwargs is not None: + kwargs = dict(grant_kwargs) + if kwargs.get("signed_at") == "expired": + kwargs["signed_at"] = datetime.now(timezone.utc) - timedelta(days=3) + grant = _grant(tmp_path, **kwargs) + with pytest.raises(ap.AdmissionError, match=refusal): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=grant, + key_dir=key_dir(tmp_path), + ) + assert not (tmp_path / "admission" / ap.DECISION_FILE).exists() + + +def test_a_tampered_or_person_only_grant_admits_nothing(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + grant = _grant(tmp_path) + tampered = grant.model_dump(mode="json", by_alias=True) + tampered["grant"]["delegates"] = ["agent:reviewer", "agent:intruder"] + with pytest.raises(ap.AdmissionError, match="approver_grant_signature_invalid"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=tampered, + key_dir=key_dir(tmp_path), + ) + # A grant that lets an agent approve only read-only steps can't admit a + # workflow whose saves would then go ahead without a person. + allowed = {c.value: ["person"] for c in StepClass} + allowed[StepClass.READ_ONLY.value] = ["person", "agent"] + narrow = signed_grant( + tmp_path, + ApproverPolicyV1.model_validate({"allowed": allowed}), + delegates=("agent:reviewer",), + ) + with pytest.raises(ap.AdmissionError, match="grant_does_not_allow_agent_saves"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=narrow, + key_dir=key_dir(tmp_path), + ) + + +def test_a_person_cannot_claim_a_grant() -> None: + with pytest.raises(ValueError): + ap.Decider( + kind="person", + id="pat", + route="cli", + policy_ref="sha256:" + "a" * 64, + on_behalf_of="person-0001", + ) + + +def test_an_admitted_bundle_runs_only_problems(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + bundle = tmp_path / "admission" / "bundle" + assert ap.admitted_approval_mode(bundle) == ap.AdmittedMode() + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + admitted = ap.admitted_approval_mode(bundle) + assert admitted.approval_mode == "problems_only" + assert admitted.admitted_by == PERSON + # Changing the bundle after admission drops back to the default. + workflow = Workflow.load(bundle) + workflow.name = "edited-after-admission" + workflow.save(bundle) + changed = ap.admitted_approval_mode(bundle) + assert changed.approval_mode is None + assert changed.refusal == "admission_bundle_changed" + + +def test_a_rejected_admission_sets_no_mode(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="reject", + decided_by=PERSON, + reason="wrong program", + ) + assert ap.admitted_approval_mode(tmp_path / "admission" / "bundle") == ( + ap.AdmittedMode() + ) + + +def test_an_agent_admission_stops_applying_when_its_grant_does(tmp_path: Path) -> None: + from datetime import datetime, timedelta, timezone + + pending = _prepare(tmp_path, by=AGENT_A) + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=_grant(tmp_path, days=2), + key_dir=key_dir(tmp_path), + ) + bundle = tmp_path / "admission" / "bundle" + keys = key_dir(tmp_path) + assert ap.admitted_approval_mode(bundle, key_dir=keys).approval_mode == ( + "problems_only" + ) + later = datetime.now(timezone.utc) + timedelta(days=3) + expired = ap.admitted_approval_mode(bundle, key_dir=keys, now=later) + assert expired.approval_mode is None + assert expired.refusal == "approver_grant_expired" + # A grant swapped into the record no longer matches the recorded digest. + record_path = bundle / ap.DECISION_FILE + record = json.loads(record_path.read_text()) + record["approver_grant"] = _grant(tmp_path, days=5).model_dump( + mode="json", by_alias=True + ) + record_path.write_text(json.dumps(record)) + assert ap.admitted_approval_mode(bundle, key_dir=keys).refusal == ( + "admission_signature_invalid" + ) + # Even re-signed by someone holding the local key, the swapped grant + # doesn't match the digest the decision recorded. + record.pop("signature") + ap._write_signed(record_path, record, keys) + assert ap.admitted_approval_mode(bundle, key_dir=keys).refusal == ( + "policy_ref_mismatch" + ) + + +def test_regulated_pack_needs_a_person() -> None: + pending = ap.PendingAdmission( + admission_id="0" * 32, + status="needs_decision", + workflow_name="w", + policy_pack="regulated", + prepared_by=PERSON, + created_at="2026-10-17T00:00:00+00:00", + ) + with pytest.raises(ap.AdmissionError, match="needs a person"): + ap.check_decider(pending, AGENT_B) + ap.check_decider(pending, PERSON) + + +def test_decision_binds_the_summary_and_the_bundle(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + out = tmp_path / "admission" + with pytest.raises(ap.AdmissionError, match="changed since it was shown"): + ap.decide_admission( + out, pending_sha256="0" * 64, decision="accept", decided_by=PERSON + ) + (out / "bundle" / "templates" / "save.png").write_bytes(b"not the same crop") + with pytest.raises(ap.AdmissionError, match="bundle changed"): + ap.decide_admission( + out, pending_sha256=pending.sha256(), decision="accept", decided_by=PERSON + ) + + +def test_one_decision_only_and_reject_needs_a_reason(tmp_path: Path) -> None: + pending = _prepare(tmp_path) + out = tmp_path / "admission" + with pytest.raises(ap.AdmissionError, match="reason"): + ap.decide_admission( + out, pending_sha256=pending.sha256(), decision="reject", decided_by=PERSON + ) + result = ap.decide_admission( + out, + pending_sha256=pending.sha256(), + decision="reject", + decided_by=PERSON, + reason="wrong checks", + ) + assert result.decision == "reject" + assert Workflow.load(out / "bundle").qualification is None + with pytest.raises(ap.AdmissionError, match="already decided"): + ap.decide_admission( + out, pending_sha256=pending.sha256(), decision="accept", decided_by=PERSON + ) + + +def test_a_halted_proposal_has_nothing_to_accept(tmp_path: Path) -> None: + pending = ap.prepare_admission( + [_recording(tmp_path, effects=False)], + out=tmp_path / "admission", + name="qualified-write", + prepared_by=PERSON, + ) + assert pending.status == "halted" + assert "effect pin is missing" in (pending.halt_reason or "") + with pytest.raises(ap.AdmissionError, match="nothing to decide"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + + +def test_refused_induction_stops_without_a_bundle(tmp_path, monkeypatch) -> None: + from openadapt_flow.compiler import induction + + monkeypatch.setattr( + induction, + "induce_program", + lambda traces: SimpleNamespace(certified=False, workflow=None, record_next=[]), + ) + pending = ap.prepare_admission( + [_recording(tmp_path, "rec1"), _recording(tmp_path, "rec2")], + out=tmp_path / "admission", + name="qualified-write", + prepared_by=PERSON, + ) + assert pending.status == "halted" + assert "induction refused" in (pending.halt_reason or "") + assert not (tmp_path / "admission" / "bundle").exists() + + +def test_two_recordings_are_induced_into_one_program(tmp_path: Path) -> None: + pending = ap.prepare_admission( + [_recording(tmp_path, "rec1"), _recording(tmp_path, "rec2")], + out=tmp_path / "admission", + name="qualified-write", + prepared_by=PERSON, + ) + assert pending.recordings == 2 + assert pending.status == "needs_decision" + assert pending.induced is True + + +def test_raw_recordings_are_compiled(tmp_path, monkeypatch) -> None: + from openadapt_flow import compiler + + source = _recording(tmp_path, "compiled-source") + raw = tmp_path / "raw" + raw.mkdir() + (raw / "meta.json").write_text((source / "meta.json").read_text()) + calls = [] + + def fake_compile(recording, out, *, name): + calls.append((Path(recording), name)) + import shutil + + shutil.copytree(source, out) + return Workflow.load(out) + + monkeypatch.setattr(compiler, "compile_recording", fake_compile) + pending = ap.prepare_admission( + [raw], out=tmp_path / "admission", name="qualified-write", prepared_by=PERSON + ) + assert calls == [(raw, "qualified-write")] + assert pending.status == "needs_decision" + assert (tmp_path / "admission" / "compiled" / "00" / "workflow.json").is_file() + + +def test_candidate_without_a_recording_asks_for_one(tmp_path: Path) -> None: + export = json.loads((SHADOW / "report_mail.episodes.json").read_text()) + candidate = induce_candidate(export) + folder = write_candidate(candidate, tmp_path / "shadow-candidates" / "c1") + pending = ap.prepare_admission( + [], + out=tmp_path / "admission", + name="report-mail", + prepared_by=PERSON, + candidate=ap.resolve_candidate("c1", tmp_path / "shadow-candidates"), + ) + assert folder.name == "c1" + assert pending.status == "needs_recording" + assert pending.record_next[0]["code"] == "shadow_full_demonstration" + assert pending.shadow_suggestions + assert all("not evidence" in item["note"] for item in pending.shadow_suggestions) + + +def test_candidate_suggestions_are_shown_but_never_applied(tmp_path: Path) -> None: + export = json.loads((SHADOW / "report_mail.episodes.json").read_text()) + folder = write_candidate(induce_candidate(export), tmp_path / "cand") + _recording(folder / "recordings", "rec1") + pending = ap.prepare_admission( + [], + out=tmp_path / "admission", + name="qualified-write", + prepared_by=PERSON, + candidate=folder, + ) + assert pending.status == "needs_decision" + assert "not evidence, not applied" in ap.render_summary(pending) + # Only checks mined from the recording are proposed. + kinds = {check["kind"] for check in pending.checks} + assert not kinds & {"file_written", "row_added", "page_reached"} + + +def test_cli_default_form_stops_without_a_terminal(tmp_path, capsys) -> None: + assert admit_default_form(["admit", "a", "b"]) == ["admit", "run", "a", "b"] + assert admit_default_form(["admit", "status"]) == ["admit", "status"] + assert admit_default_form(["admit", "--candidate", "x"])[1] == "run" + rec = _recording(tmp_path) + out = tmp_path / "admission" + code = main(["admit", str(rec), "--out", str(out), "--name", "w", "--by", "pat"]) + assert code == EXIT_STOPPED + text = capsys.readouterr().out + assert "Proposed checks" in text and "admit decide" in text + assert not (out / ap.DECISION_FILE).exists() + + +def test_cli_single_stop_accepts_once_at_a_terminal(tmp_path, capsys) -> None: + _prepare(tmp_path) + answers = iter(["accept"]) + code = _ask_once( + tmp_path / "admission", "pat", read=lambda _: next(answers), interactive=True + ) + assert code == 0 + assert "Accepted by person:pat via cli" in capsys.readouterr().out + record = json.loads((tmp_path / "admission" / ap.DECISION_FILE).read_text()) + assert record["decided_by"]["kind"] == "person" + + +def test_cli_decide_refuses_without_a_terminal(tmp_path) -> None: + _prepare(tmp_path) + with pytest.raises(SystemExit, match="interactive terminal"): + main(["admit", "decide", str(tmp_path / "admission")]) + + +def test_run_uses_the_admitted_mode_unless_told_otherwise(tmp_path, capsys) -> None: + from openadapt_flow.__main__ import _resolve_run_approval_mode + + pending = _prepare(tmp_path) + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + bundle = str(tmp_path / "admission" / "bundle") + no_cfg = SimpleNamespace(runtime=SimpleNamespace(approval_mode=None)) + args = SimpleNamespace(approval_mode=None, bundle=bundle) + assert _resolve_run_approval_mode(args, no_cfg, None).value == "problems_only" + assert "Admitted by person:pat via cli" in capsys.readouterr().out + # A flag or the deployment's runtime.approval_mode still wins. + flagged = SimpleNamespace(approval_mode="each_item", bundle=bundle) + assert _resolve_run_approval_mode(flagged, no_cfg, None).value == "each_item" + deployed = SimpleNamespace(runtime=SimpleNamespace(approval_mode="each_item")) + assert _resolve_run_approval_mode(args, deployed, None).value == "each_item" + # Without an admission the default is a person approving each save. + bare = SimpleNamespace(approval_mode=None, bundle=str(tmp_path)) + assert _resolve_run_approval_mode(bare, no_cfg, None).value == "each_item" + + +# -- fail-closed review of #1212 --------------------------------------------- + + +def _forged_decision(bundle: Path, pending: ap.PendingAdmission) -> dict: + return { + "schema_version": ap.DECISION_SCHEMA, + "admission_id": pending.admission_id, + "decision": "accept", + "decided_by": {"kind": "person", "id": "pat", "route": "cli"}, + "prepared_by": {"kind": "person", "id": "pat", "route": "cli"}, + "decided_at": "2026-01-01T00:00:00+00:00", + "pending_sha256": pending.sha256(), + "bundle_content_digest": pending.bundle_content_digest, + "proposal_sha256": pending.proposal_sha256, + "accepted_bundle_content_digest": ap._bundle_digest(bundle), + "approval_mode": "problems_only", + "workflow_name": pending.workflow_name, + } + + +def test_a_decision_file_written_without_the_admission_key_admits_nothing( + tmp_path: Path, +) -> None: + pending = _prepare(tmp_path) + bundle = tmp_path / "admission" / "bundle" + path = bundle / ap.DECISION_FILE + path.write_text(json.dumps(_forged_decision(bundle, pending))) + forged = ap.admitted_approval_mode(bundle) + assert forged.approval_mode is None + assert forged.refusal == "admission_signature_invalid" + # A signature by another key is no better. + record = _forged_decision(bundle, pending) + ap._write_signed(path, record, tmp_path / "other-keys") + assert ap.admitted_approval_mode(bundle).refusal == "admission_signature_invalid" + + +def test_a_signed_decision_edited_afterwards_admits_nothing(tmp_path: Path) -> None: + pending = _prepare(tmp_path, by=AGENT_A) + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=_grant(tmp_path), + key_dir=key_dir(tmp_path), + ) + bundle = tmp_path / "admission" / "bundle" + path = bundle / ap.DECISION_FILE + record = json.loads(path.read_text()) + # An agent's admission relabelled as a person's, to skip the grant check. + record["decided_by"] = {"kind": "person", "id": "pat", "route": "cli"} + record["approver_grant"] = None + path.write_text(json.dumps(record)) + edited = ap.admitted_approval_mode(bundle) + assert edited.approval_mode is None + assert edited.refusal == "admission_signature_invalid" + + +def test_an_edited_pending_file_is_refused(tmp_path: Path) -> None: + _prepare(tmp_path, by=AGENT_B) + path = tmp_path / "admission" / ap.PENDING_FILE + data = json.loads(path.read_text()) + # Relabel the preparer so the agent that prepared it could decide it. + data["prepared_by"] = {"kind": "person", "id": "pat", "route": "cli"} + path.write_text(json.dumps(data)) + with pytest.raises(ap.AdmissionError, match="not signed"): + ap.load_pending(tmp_path / "admission") + data.pop("signature") + edited = ap.PendingAdmission.model_validate(data) + with pytest.raises(ap.AdmissionError, match="not signed"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=edited.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=_grant(tmp_path), + key_dir=key_dir(tmp_path), + ) + assert not (tmp_path / "admission" / ap.DECISION_FILE).exists() + + +def test_an_agent_cannot_decide_what_an_unidentified_client_prepared( + tmp_path: Path, +) -> None: + unidentified = ap.Decider(kind="agent", id=ap.UNIDENTIFIED_AGENT, route="mcp") + pending = _prepare(tmp_path, by=unidentified) + with pytest.raises(ap.AdmissionError, match="unidentified"): + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=AGENT_B, + approver_grant=_grant(tmp_path), + key_dir=key_dir(tmp_path), + ) + # A person still can. + result = ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + assert result.decided_by == PERSON + + +def test_the_value_free_summary_quotes_no_recorded_text() -> None: + recorded = "Sample Label Alpha" + halted = ap.PendingAdmission( + admission_id="0" * 32, + status="halted", + halt_reason=f"induction refused near {recorded}", + workflow_name="w", + policy_pack="community", + prepared_by=PERSON, + created_at="2026-01-01T00:00:00+00:00", + record_next=[ + { + "code": "ambiguous_branch", + "instruction": f"Record each of {recorded} vs Other under a " + "detectable condition.", + } + ], + ) + assert recorded in ap.render_summary(halted) + agent_view = ap.render_summary(halted, value_free=True) + assert recorded not in agent_view + assert "ambiguous_branch" in agent_view + ready = halted.model_copy( + update={ + "status": "needs_decision", + "halt_reason": None, + "record_next": [], + "shadow_suggestions": [ + {"kind": "file_saved", "summary": f"saved {recorded}.xlsx"} + ], + } + ) + assert recorded not in ap.render_summary(ready, value_free=True) + + +def test_an_admitted_bundle_still_asks_a_person_until_the_switch_is_on( + tmp_path: Path, capsys +) -> None: + """Admission picks problems_only; the switch record still gates each save.""" + + from openadapt_flow.__main__ import _resolve_run_approval_mode + from openadapt_flow.approval_mode.gate import ApprovalGate + from openadapt_flow.approval_mode.model import ItemDecisionKind + from tests._approval_mode_support import BUNDLE_DIGEST, good_item + from tests.test_approval_mode_gate import Harness + + pending = _prepare(tmp_path) + ap.decide_admission( + tmp_path / "admission", + pending_sha256=pending.sha256(), + decision="accept", + decided_by=PERSON, + ) + args = SimpleNamespace( + approval_mode=None, bundle=str(tmp_path / "admission" / "bundle") + ) + no_cfg = SimpleNamespace(runtime=SimpleNamespace(approval_mode=None)) + mode = _resolve_run_approval_mode(args, no_cfg, None) + assert mode.value == "problems_only" + # No signed switch record: the gate sends the save to a person. + gate = ApprovalGate(mode, item=good_item()) + gate.begin_item( + run_id="r", bundle_content_digest=BUNDLE_DIGEST, profile="regulated" + ) + verdict = gate.before_save(Harness(tmp_path / "gate").save_context()) + assert not verdict.proceed + assert verdict.decision.kind is ItemDecisionKind.APPROVAL_REQUIRED diff --git a/packages/flow/tests/test_motif_induction.py b/packages/flow/tests/test_motif_induction.py new file mode 100644 index 000000000..ed758822e --- /dev/null +++ b/packages/flow/tests/test_motif_induction.py @@ -0,0 +1,272 @@ +"""Shadow-mined episodes -> the existing induction -> a CANDIDATE, never a run. + +The fixtures under ``tests/fixtures/shadow/`` are real +``openadapt.shadow.episodes/v2`` exports, produced by openadapt-capture's +shadow miner from synthetic streams +(``tests/shadow_support/generate_flow_fixtures.py`` in that repository). +""" + +from __future__ import annotations + +import ast +import copy +import json +import socket +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from openadapt_flow.compiler import motif_induction +from openadapt_flow.compiler.motif_induction import ( + CANDIDATE_FILENAME, + DRAFT_FILENAME, + PLACEHOLDER_TEMPLATE, + MotifCandidate, + ShadowEpisodesExport, + induce_candidate, + write_candidate, +) +from openadapt_flow.ir import ActionKind, Workflow +from openadapt_flow.traversal import iter_workflow_steps + +FIXTURES = Path(__file__).parent / "fixtures" / "shadow" + +# The raw values the synthetic streams typed or clicked. None may survive. +RAW_VALUES = ( + "INV-559017", + "4471.25", + "jane.doe@example.com", + "Jane Doe", + "123-45-6789", + "Quarterly report", + "invoices/INV", + "ssn=", +) + + +def _load(name: str) -> dict: + return json.loads((FIXTURES / f"{name}.episodes.json").read_text()) + + +def test_fixtures_carry_no_raw_values(): + for path in sorted(FIXTURES.glob("*.json")): + text = path.read_text() + for value in RAW_VALUES: + assert value not in text, (path.name, value) + + +def test_fixed_control_task_induces_a_candidate_draft_with_parameters(): + candidate = induce_candidate(_load("report_mail")) + assert candidate.status == "candidate" + assert candidate.admitted is False and candidate.runnable is False + assert candidate.induction_certified is True + assert candidate.n_episodes == 3 + draft = candidate.draft + assert draft is not None and draft.program is not None + assert draft.name.startswith("shadow-draft-") + assert len(draft.param_specs) == 2 + # Parameter examples are export-local references, never values or digests. + assert all(v.startswith("<") and ":" in v for v in draft.params.values()) + + +def test_record_next_starts_with_a_full_demonstration(): + candidate = induce_candidate(_load("report_mail")) + first = candidate.record_next[0] + assert first.code == "shadow_full_demonstration" and first.blocking + codes = [r.code for r in candidate.record_next] + assert codes.count("shadow_confirm_check") == len(candidate.proposed_checks) + assert "shadow_name_value" in codes + + +def test_checks_are_proposals_only(): + candidate = induce_candidate(_load("report_mail")) + kinds = sorted(c.kind for c in candidate.proposed_checks) + assert kinds == ["file_written", "record_written"] + for check in candidate.proposed_checks: + assert check.status == "proposed" + assert check.admitted is False + assert "can't approve" in check.note + # No proposal became an effect contract on any step. + steps = _all_steps(candidate.draft) + assert len(steps) >= 8 and all(not s.effects for s in steps) + send = [s for s in steps if "Send" in s.intent] + assert send and send[0].risk_review_required + + +def _all_steps(workflow: Workflow | None): + return [] if workflow is None else list(iter_workflow_steps(workflow)) + + +def test_write_shaped_steps_need_review_and_no_anchor_is_real(): + export = ShadowEpisodesExport.model_validate(_load("report_mail")) + workflows, _ = motif_induction.episodes_to_workflows(export) + send = [s for s in workflows[0].steps if "Send" in s.intent] + assert send and send[0].risk == "irreversible" and send[0].risk_review_required + for step in workflows[0].steps: + if step.anchor is not None: + assert step.anchor.template == PLACEHOLDER_TEMPLATE + assert step.anchor.click_point == (0, 0) + + +def test_varying_selection_is_refused_by_existing_induction(): + candidate = induce_candidate(_load("invoice_exact")) + assert candidate.induction_certified is False + assert candidate.draft is None + codes = [r.code for r in candidate.record_next] + assert codes[0] == "shadow_full_demonstration" + assert "ambiguous_selection" in codes + assert {c.kind for c in candidate.proposed_checks} == { + "page_reached", + "record_written", + } + + +def test_variant_episode_is_read_as_an_optional_step(): + candidate = induce_candidate(_load("invoice_variant")) + assert any( + "optional step" in line for line in candidate.induction_report.splitlines() + ) + + +def test_candidate_cannot_claim_admission(): + candidate = induce_candidate(_load("report_mail")) + data = candidate.model_dump() + for field in ("admitted", "runnable"): + bad = {**data, field: True} + with pytest.raises(ValidationError): + MotifCandidate.model_validate(bad) + with pytest.raises(ValidationError): + MotifCandidate.model_validate({**data, "status": "admitted"}) + + +@pytest.mark.parametrize( + ("mutate", "why"), + [ + (lambda s: s.update(text="jane.doe@example.com"), "unknown field"), + (lambda s: s.update(display_label="Pay 1200"), "digits in a plain label"), + (lambda s: s.update(value_ref="not-a-ref"), "malformed reference"), + (lambda s: s.update(value_digest="0123456789abcdef"), "a keyed digest field"), + (lambda s: s.update(label_digest="0123456789abcdef"), "a keyed digest field"), + (lambda s: s.update(token="click|Mail|button|0a1b2c3d"), "a digest in a token"), + (lambda s: s.update(kind="screenshot"), "unknown kind"), + ], +) +def test_export_schema_refuses_raw_content(mutate, why): + data = copy.deepcopy(_load("report_mail")) + mutate(data["episodes"][0]["steps"][3]) + with pytest.raises(ValidationError): + induce_candidate(data) + + +def test_export_schema_refuses_wrong_schema_and_status(): + data = _load("report_mail") + with pytest.raises(ValidationError): + induce_candidate({**data, "schema": "openadapt.shadow.episodes/v3"}) + with pytest.raises(ValidationError): + induce_candidate({**data, "status": "admitted"}) + + +def test_v1_exports_with_keyed_digests_are_refused(): + data = _load("report_mail") + with pytest.raises(ValueError, match="keyed digests"): + induce_candidate({**data, "schema": "openadapt.shadow.episodes/v1"}) + with pytest.raises(ValidationError): + induce_candidate( + {**data, "tokens": ["focus|Mail", "click|Mail|button|0a1b2c3d"]} + ) + + +@pytest.mark.parametrize("name", ["report_mail", "invoice_exact", "invoice_variant"]) +def test_draft_carries_no_digest(name): + import re + + candidate = induce_candidate(_load(name)) + blob = candidate.model_dump_json() + # No 8+ character hex run except the export's own id. + runs = set(re.findall(r"(?