From b69a489952bb36943099ee8c0d35c079ee1bf612 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 24 Oct 2026 23:26:55 -0400 Subject: [PATCH 1/9] test(flow): characterize KEY, HOTKEY and WAIT steps on the real browser Pins today's behavior before these executors move into runtime.core: the key event and form submit, the modifier string, the refusal of a key step with no key, and a WAIT that sends no input. Co-Authored-By: Claude Opus 5.5 --- packages/flow/tests/test_core_key_actions.py | 125 +++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 packages/flow/tests/test_core_key_actions.py diff --git a/packages/flow/tests/test_core_key_actions.py b/packages/flow/tests/test_core_key_actions.py new file mode 100644 index 000000000..43ca053c9 --- /dev/null +++ b/packages/flow/tests/test_core_key_actions.py @@ -0,0 +1,125 @@ +"""KEY, HOTKEY and WAIT steps on the real browser. + +Characterization tests for the first action executors moved into +``openadapt_flow.runtime.core`` (docs/design/DECOMPOSITION.md, PR 2). They +pin what the replayer does today, through the real ``PlaywrightBackend``, so +the move can be shown to change nothing. The guarded-keyboard path is pinned +by ``test_playwright_atomic_actuation.py`` and the refusal of raw input on a +consequential identity step by ``test_replayer.py``. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from openadapt_flow.ir import ActionKind, Step, Workflow +from openadapt_flow.runtime.replayer import Replayer + +sync_api = pytest.importorskip("playwright.sync_api") + +_PAGE = """ + + +
+ +
+ + +""" + + +@pytest.fixture(scope="module") +def browser() -> Iterator[object]: + with sync_api.sync_playwright() as playwright: + chromium = playwright.chromium.launch(headless=True) + yield chromium + chromium.close() + + +def _run(browser: object, tmp_path: Path, *steps: Step) -> tuple[object, list[str]]: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + page = browser.new_page(viewport={"width": 640, "height": 320}) + try: + page.set_content(_PAGE) + page.locator("#field").focus() + bundle = tmp_path / "bundle" + bundle.mkdir() + report = Replayer(PlaywrightBackend(page), use_structural=False).run( + Workflow(name="keys", steps=list(steps)), + bundle_dir=bundle, + run_dir=tmp_path / "run", + ) + events = page.evaluate("window.events") + finally: + page.close() + return report, events + + +def test_key_step_presses_the_key_and_the_form_submits(browser, tmp_path) -> None: + report, events = _run( + browser, + tmp_path, + Step(id="k1", intent="submit", action=ActionKind.KEY, key="Enter"), + ) + + assert report.success is True + assert events == ["keydown:Enter", "submit"] + result = report.results[0] + assert result.delivery_attempted is True + assert result.actuation is None # an unguarded press records no actuation kind + assert result.delivery_receipt is None + + +def test_hotkey_step_joins_modifiers_and_key(browser, tmp_path) -> None: + report, events = _run( + browser, + tmp_path, + Step( + id="h1", + intent="select all", + action=ActionKind.HOTKEY, + key="a", + modifiers=["Shift"], + ), + ) + + assert report.success is True + # Playwright presses the lowercase key with Shift held. + assert events == ["keydown:Shift+Shift", "keydown:Shift+a"] + + +def test_key_step_without_a_key_fails_without_input(browser, tmp_path) -> None: + report, events = _run( + browser, + tmp_path, + Step(id="k0", intent="press nothing", action=ActionKind.KEY, key=None), + ) + + assert report.success is False + assert events == [] + result = report.results[0] + assert result.error == "Step 'k0' (press nothing) is a key step with no key" + assert result.delivery_attempted is False + + +def test_wait_step_sends_no_input(browser, tmp_path) -> None: + report, events = _run( + browser, + tmp_path, + Step(id="w1", intent="let it settle", action=ActionKind.WAIT), + ) + + assert report.success is True + assert events == [] + assert report.results[0].delivery_attempted is False From 528fe883481fe53870bb0833b757b0e1c3d3e4b9 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 24 Oct 2026 23:32:19 -0400 Subject: [PATCH 2/9] refactor(flow): runtime.core with the first action executors, behind an import contract runtime.core carries out steps through a backend; governance reaches it only as gates. KEY, HOTKEY and WAIT now run there: ActionContext.authorize runs delivery authorization and then the qualification-environment check (Replayer._authorize_delivery), exactly the order the replayer used, and deliver is the replayer's own delivery bookkeeping. Behavior is unchanged: the characterization tests from the previous commit and the keyboard, guarded-keyboard and refusal suites pass. packages/flow/.importlinter, run by lint-imports in flow core lint: - runtime.core imports no runner, hosted, approval_mode, jit, replayer or qualification* module (kept with no exceptions); - the same through every import chain, except 45 listed edges by which the schema, effects, policy, repair, fleet and durable modules reach governance today. New governance imports in the core's dependencies fail. tests/test_import_boundaries.py keeps the forbidden list complete and proves every ActionKind has exactly one executor. Co-Authored-By: Claude Opus 5.5 --- .github/CODEOWNERS | 2 + .github/workflows/packages.yml | 7 + packages/flow/.importlinter | 122 ++++++++++++++++++ .../openadapt_flow/runtime/core/__init__.py | 20 +++ .../openadapt_flow/runtime/core/actions.py | 117 +++++++++++++++++ .../flow/openadapt_flow/runtime/replayer.py | 94 ++++++-------- packages/flow/tests/test_import_boundaries.py | 67 ++++++++++ 7 files changed, 372 insertions(+), 57 deletions(-) create mode 100644 packages/flow/.importlinter create mode 100644 packages/flow/openadapt_flow/runtime/core/__init__.py create mode 100644 packages/flow/openadapt_flow/runtime/core/actions.py create mode 100644 packages/flow/tests/test_import_boundaries.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4390a4020..98c45c70d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,6 +12,8 @@ /packages/flow/openadapt_flow/policies/ @abrichr /packages/flow/openadapt_flow/qualification.py @abrichr /packages/flow/openadapt_flow/runtime/replayer.py @abrichr +/packages/flow/openadapt_flow/runtime/core/ @abrichr +/packages/flow/.importlinter @abrichr /packages/flow/openadapt_flow/runtime/resolver.py @abrichr /packages/flow/openadapt_flow/submit_binding.py @abrichr /packages/flow/openadapt_flow/validation/ @abrichr diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 20556c037..6f0f29108 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -288,6 +288,11 @@ jobs: run: ruff check openadapt_flow - name: Ruff format check run: ruff format --check openadapt_flow tests + - name: Import boundaries (runtime.core imports no governance) + # Contracts in packages/flow/.importlinter; DECOMPOSITION.md, section 2. + run: | + pip install 'import-linter==2.15' + lint-imports - name: Mypy run: mypy - name: Mypy (strict, safety path only) @@ -304,6 +309,8 @@ jobs: openadapt_flow/runtime/identity_template.py \ openadapt_flow/runtime/identity_vlm.py \ openadapt_flow/runtime/replayer.py \ + openadapt_flow/runtime/core/__init__.py \ + openadapt_flow/runtime/core/actions.py \ openadapt_flow/runtime/resolver.py \ openadapt_flow/submit_binding.py \ openadapt_flow/runtime/effects/effect.py \ diff --git a/packages/flow/.importlinter b/packages/flow/.importlinter new file mode 100644 index 000000000..68c3c179d --- /dev/null +++ b/packages/flow/.importlinter @@ -0,0 +1,122 @@ +# Import boundaries for openadapt-flow, checked by `lint-imports` in CI +# (.github/workflows/packages.yml, flow core lint). See +# docs/design/DECOMPOSITION.md, section 2. +# +# runtime.core carries out steps; governance (authorization, approvals, +# qualification, profiles, runners, the replayer that wires them) is passed in +# as gates. tests/test_import_boundaries.py keeps the forbidden list complete. + +[importlinter] +root_package = openadapt_flow +include_external_packages = False + +[importlinter:contract:runtime-core-direct] +name = runtime.core imports no governance module +type = forbidden +source_modules = + openadapt_flow.runtime.core +forbidden_modules = + openadapt_flow.runner + openadapt_flow.hosted + openadapt_flow.approval_mode + openadapt_flow.jit + openadapt_flow.runtime.replayer + openadapt_flow.production_qualification + openadapt_flow.qualification + openadapt_flow.qualification_admission + openadapt_flow.qualification_admission_v2 + openadapt_flow.qualification_admission_v4 + openadapt_flow.qualification_campaign_authority + openadapt_flow.qualification_campaign_permit + openadapt_flow.qualification_dev_signer + openadapt_flow.qualification_drills + openadapt_flow.qualification_environment + openadapt_flow.qualification_faults + openadapt_flow.qualification_identity_evidence + openadapt_flow.qualification_oracle_gate + openadapt_flow.qualification_proposal + openadapt_flow.qualification_web_session + openadapt_flow.qualified_business_decisions + openadapt_flow.qualify +allow_indirect_imports = True + +[importlinter:contract:runtime-core-indirect] +# The same boundary through every chain of imports. Today the schema and +# several shared modules import governance, so every module runtime.core +# depends on reaches it. Each edge below is one of those imports. A new import +# of governance by any module runtime.core depends on fails; a new route to an +# edge already listed does not, so keep the list short. Cutting an edge removes +# its line; an empty list makes the boundary complete. +name = runtime.core reaches no governance module except through listed edges +type = forbidden +source_modules = + openadapt_flow.runtime.core +forbidden_modules = + openadapt_flow.runner + openadapt_flow.hosted + openadapt_flow.approval_mode + openadapt_flow.jit + openadapt_flow.runtime.replayer + openadapt_flow.production_qualification + openadapt_flow.qualification + openadapt_flow.qualification_admission + openadapt_flow.qualification_admission_v2 + openadapt_flow.qualification_admission_v4 + openadapt_flow.qualification_campaign_authority + openadapt_flow.qualification_campaign_permit + openadapt_flow.qualification_dev_signer + openadapt_flow.qualification_drills + openadapt_flow.qualification_environment + openadapt_flow.qualification_faults + openadapt_flow.qualification_identity_evidence + openadapt_flow.qualification_oracle_gate + openadapt_flow.qualification_proposal + openadapt_flow.qualification_web_session + openadapt_flow.qualified_business_decisions + openadapt_flow.qualify +ignore_imports = + openadapt_flow.admit_pipeline -> openadapt_flow.approval_mode.approver + openadapt_flow.admit_pipeline -> openadapt_flow.approval_mode.signed_policy + openadapt_flow.admit_pipeline -> openadapt_flow.jit.lifecycle + openadapt_flow.admit_pipeline -> openadapt_flow.qualification_proposal + openadapt_flow.admitted_composition -> openadapt_flow.qualification_admission + openadapt_flow.deployment -> openadapt_flow.qualification_environment + openadapt_flow.execution_profiles -> openadapt_flow.qualification + openadapt_flow.execution_profiles -> openadapt_flow.qualification_identity_evidence + openadapt_flow.fleet.fingerprint -> openadapt_flow.qualification_environment + openadapt_flow.fleet.signing -> openadapt_flow.qualification_admission_v2 + openadapt_flow.fleet.status -> openadapt_flow.qualification_admission_v2 + openadapt_flow.ir -> openadapt_flow.approval_mode.model + openadapt_flow.ir -> openadapt_flow.qualification + openadapt_flow.ir -> openadapt_flow.qualification_faults + openadapt_flow.policy -> openadapt_flow.qualification + openadapt_flow.repair.candidate -> openadapt_flow.qualification + openadapt_flow.repair.invariants -> openadapt_flow.qualification + openadapt_flow.repair.lifecycle -> openadapt_flow.jit.authority + openadapt_flow.repair.readmission -> openadapt_flow.jit.lifecycle + openadapt_flow.repair.readmission -> openadapt_flow.jit.signature + openadapt_flow.repair.readmission -> openadapt_flow.qualify.campaign + openadapt_flow.repair.registration -> openadapt_flow.qualification + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.approver + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.form_state + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.gate + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.item_approval + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.model + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.post + openadapt_flow.repair.self_repair -> openadapt_flow.jit.adapters + openadapt_flow.repair.self_repair -> openadapt_flow.jit.keys + openadapt_flow.repair.self_repair -> openadapt_flow.jit.lifecycle + openadapt_flow.repair.self_repair -> openadapt_flow.jit.signature + openadapt_flow.repair.standard_change -> openadapt_flow.jit.authority + openadapt_flow.run_gate -> openadapt_flow.qualification + openadapt_flow.run_gate -> openadapt_flow.qualification_faults + openadapt_flow.runtime.authorization -> openadapt_flow.qualification + openadapt_flow.runtime.authorization -> openadapt_flow.qualification_admission + openadapt_flow.runtime.durable.attended -> openadapt_flow.qualification + openadapt_flow.runtime.durable.business_decision -> openadapt_flow.qualification + openadapt_flow.runtime.durable.controller -> openadapt_flow.approval_mode.model + openadapt_flow.runtime.durable.resume -> openadapt_flow.runtime.replayer + openadapt_flow.runtime_validation -> openadapt_flow.hosted + openadapt_flow.sections -> openadapt_flow.qualification + openadapt_flow.teach_scope -> openadapt_flow.jit.adapters + openadapt_flow.terminal_verification_v2 -> openadapt_flow.qualification_admission_v2 diff --git a/packages/flow/openadapt_flow/runtime/core/__init__.py b/packages/flow/openadapt_flow/runtime/core/__init__.py new file mode 100644 index 000000000..9aaf99927 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/core/__init__.py @@ -0,0 +1,20 @@ +"""The governance-free core of step execution. + +Code here decides HOW a step is carried out through a backend. It never +decides WHETHER it may be: authorization, approvals, qualification, profiles +and durable state are gates the caller passes in (see ``ActionContext``), so +the core can be tested against a backend alone and reused by any runner. + +An import-linter contract (``packages/flow/.importlinter``) enforces the +boundary: nothing in this package may import ``runner``, ``hosted``, +``qualification*``, ``approval_mode``, ``jit`` or the replayer. See +docs/design/DECOMPOSITION.md, section 2. +""" + +from openadapt_flow.runtime.core.actions import ( + ACTION_EXECUTORS, + ActionContext, + ActionExecutor, +) + +__all__ = ["ACTION_EXECUTORS", "ActionContext", "ActionExecutor"] diff --git a/packages/flow/openadapt_flow/runtime/core/actions.py b/packages/flow/openadapt_flow/runtime/core/actions.py new file mode 100644 index 000000000..017ccdd2f --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/core/actions.py @@ -0,0 +1,117 @@ +"""Action executors: carry out one step's action through the backend. + +One executor per ``ActionKind``. An executor returns an error string when it +performed no action (or only part of one), and None when the action was +delivered. Every input it delivers goes through two callables from the +caller, in this order: + +1. ``authorize()``: every governance check that must pass at the last point + before input (delivery authorization, the qualification environment). It + returns a refusal string, or None to proceed, and may raise. +2. ``deliver(result, call)``: the caller's delivery bookkeeping around the + backend call (attempt reporting, settle seeds, acknowledgements). + +Kinds not listed in ``ACTION_EXECUTORS`` are still carried out by +``Replayer._act_without_dialog``; they move here one at a time +(docs/design/DECOMPOSITION.md, seam S4). +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any, Optional, Protocol, TypeVar, cast + +from openadapt_flow.backend import ( + GuardedKeyboardActionBackend, + RemoteActuationBackend, +) +from openadapt_flow.ir import ActionKind, Step, StepResult + +_T = TypeVar("_T") + + +class Deliver(Protocol): + def __call__(self, result: StepResult, call: Callable[[], _T]) -> _T: ... + + +@dataclass(frozen=True) +class ActionContext: + """What an executor needs: the step, the backend and the caller's gates.""" + + step: Step + result: StepResult + backend: Any + # The settled frame the step's checks ran on; guarded input binds to it. + before_png: bytes + # Governance at the last point before input: a refusal, or None. + authorize: Callable[[], Optional[str]] + deliver: Deliver + # Whether keyboard input must stay bound to the verified focus. Called + # only when an executor needs it, in the order the original code did. + requires_atomic_keyboard: Callable[[], bool] + + +ActionExecutor = Callable[[ActionContext], Optional[str]] + + +def execute_key(ctx: ActionContext) -> Optional[str]: + """KEY and HOTKEY: press one key or one modifier combination.""" + step, result = ctx.step, ctx.result + if not step.key: + return ( + f"Step '{step.id}' ({step.intent}) is a " + f"{step.action.value} step with no key" + ) + key = ( + "+".join([*step.modifiers, step.key]) + if step.action is ActionKind.HOTKEY + else step.key + ) + requires_atomic_keyboard = ctx.requires_atomic_keyboard() + guarded_key = ( + requires_atomic_keyboard + and not isinstance(ctx.backend, RemoteActuationBackend) + and isinstance(ctx.backend, GuardedKeyboardActionBackend) + ) + if guarded_key: + refusal = ctx.authorize() + if refusal is not None: + return refusal + result.delivery_receipt = ctx.deliver( + result, + lambda: cast(GuardedKeyboardActionBackend, ctx.backend).press_guarded( + key, + expected_frame_sha256=hashlib.sha256(ctx.before_png).hexdigest(), + ), + ) + result.actuation = "guarded_keyboard" + elif requires_atomic_keyboard and not isinstance( + ctx.backend, RemoteActuationBackend + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) lost its guarded keyboard " + "capability before delivery — refusing unguarded key input; " + "run aborted" + ) + else: + refusal = ctx.authorize() + if refusal is not None: + return refusal + ctx.deliver(result, lambda: ctx.backend.press(key)) + return None + + +def execute_wait(ctx: ActionContext) -> Optional[str]: + """WAIT: no input. The settle after every action does the waiting.""" + return None + + +ACTION_EXECUTORS: Mapping[ActionKind, ActionExecutor] = { + ActionKind.KEY: execute_key, + ActionKind.HOTKEY: execute_key, + ActionKind.WAIT: execute_wait, +} diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index e5df26bc3..5741d1707 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -167,6 +167,7 @@ runtime_param_text, runtime_params_for_gui, ) +from openadapt_flow.runtime.core import ACTION_EXECUTORS, ActionContext from openadapt_flow.runtime.durable.approval import StateDiverged from openadapt_flow.runtime.durable.program_checkpoint import ( TOP_GRAPH_ID, @@ -12447,6 +12448,25 @@ def _delivery_authorization_refusal( self._active_delivery_acknowledgers = tuple(acknowledgers) return refusal + def _authorize_delivery( + self, + workflow: Workflow, + params: Mapping[str, RuntimeParamScalar], + step: Step, + result: StepResult, + ) -> Optional[str]: + """Every governance check at the last point before input, in order. + + The gate the core action executors call before they deliver input: + delivery authorization first, then the qualification environment + (which raises when it is no longer current). + """ + refusal = self._delivery_authorization_refusal(workflow, params, step, result) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + return None + def _begin_approval_item( self, report: RunReport, @@ -13841,65 +13861,25 @@ def _act_without_dialog( allow_retry=not guarded_type, ) - if step.action in (ActionKind.KEY, ActionKind.HOTKEY): - if not step.key: - return ( - f"Step '{step.id}' ({step.intent}) is a " - f"{step.action.value} step with no key" - ) - key = ( - "+".join([*step.modifiers, step.key]) - if step.action is ActionKind.HOTKEY - else step.key - ) - requires_atomic_keyboard = self._requires_atomic_identity_keyboard( - step, workflow - ) - guarded_key = ( - requires_atomic_keyboard - and not isinstance(self.backend, RemoteActuationBackend) - and isinstance(self.backend, GuardedKeyboardActionBackend) - ) - if guarded_key: - refusal = self._delivery_authorization_refusal( - workflow, params, step, result - ) - if refusal is not None: - return refusal - self._require_qualification_environment_current() - result.delivery_receipt = self._deliver_backend_call( - result, - lambda: cast( - GuardedKeyboardActionBackend, self.backend - ).press_guarded( - key, - expected_frame_sha256=hashlib.sha256(before_png).hexdigest(), + executor = ACTION_EXECUTORS.get(step.action) + if executor is not None: + # KEY, HOTKEY and WAIT run in the governance-free core; the gates + # it must pass before input come from here. + return executor( + ActionContext( + step=step, + result=result, + backend=self.backend, + before_png=before_png, + authorize=lambda: self._authorize_delivery( + workflow, params, step, result + ), + deliver=self._deliver_backend_call, + requires_atomic_keyboard=lambda: ( + self._requires_atomic_identity_keyboard(step, workflow) ), ) - result.actuation = "guarded_keyboard" - elif requires_atomic_keyboard and not isinstance( - self.backend, RemoteActuationBackend - ): - result.safety_halt = True - result.failure_category = "safety_halt" - return ( - f"Step '{step.id}' ({step.intent}) lost its guarded keyboard " - "capability before delivery — refusing unguarded key input; " - "run aborted" - ) - else: - refusal = self._delivery_authorization_refusal( - workflow, params, step, result - ) - if refusal is not None: - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call(result, lambda: self.backend.press(key)) - return None - - if step.action is ActionKind.WAIT: - # WAIT means wait_settled only; the post-action settle handles it. - return None + ) if step.action is ActionKind.SCROLL: return self._act_scroll( diff --git a/packages/flow/tests/test_import_boundaries.py b/packages/flow/tests/test_import_boundaries.py new file mode 100644 index 000000000..097010795 --- /dev/null +++ b/packages/flow/tests/test_import_boundaries.py @@ -0,0 +1,67 @@ +"""The runtime.core boundary stays complete as the package grows. + +``lint-imports`` (CI, flow core lint) checks the contracts in +``.importlinter``. These tests keep the contracts themselves honest: every +governance module is on the forbidden list, and every action kind is carried +out by exactly one place. +""" + +from __future__ import annotations + +import configparser +import re +from pathlib import Path + +from openadapt_flow.ir import ActionKind +from openadapt_flow.runtime.core import ACTION_EXECUTORS + +FLOW = Path(__file__).resolve().parents[1] +PACKAGE = FLOW / "openadapt_flow" + +# Kinds the replayer still carries out itself (_act_without_dialog and the +# surface switch). Each moves into runtime.core with its own executor. +NOT_YET_IN_CORE = { + ActionKind.CLICK, + ActionKind.DOUBLE_CLICK, + ActionKind.RIGHT_CLICK, + ActionKind.DRAG, + ActionKind.TYPE, + ActionKind.SELECT_OPTION, + ActionKind.SCROLL, + ActionKind.READ, + ActionKind.SWITCH_SURFACE, +} + + +def _contract(name: str) -> configparser.SectionProxy: + config = configparser.ConfigParser() + config.read(FLOW / ".importlinter") + return config[f"importlinter:contract:{name}"] + + +def _modules(value: str) -> set[str]: + return {line.strip() for line in value.splitlines() if line.strip()} + + +def test_every_qualification_module_is_forbidden_to_the_core() -> None: + governance = { + f"openadapt_flow.{path.stem if path.is_file() else path.name}" + for path in PACKAGE.iterdir() + if re.match(r"(production_)?qualif", path.name) + and (path.suffix == ".py" or (path / "__init__.py").is_file()) + } + for name in ("runtime-core-direct", "runtime-core-indirect"): + forbidden = _modules(_contract(name)["forbidden_modules"]) + assert governance - forbidden == set(), name + + +def test_both_contracts_forbid_the_same_modules() -> None: + direct = _modules(_contract("runtime-core-direct")["forbidden_modules"]) + indirect = _modules(_contract("runtime-core-indirect")["forbidden_modules"]) + assert direct == indirect + + +def test_every_action_kind_has_exactly_one_executor() -> None: + in_core = set(ACTION_EXECUTORS) + assert in_core & NOT_YET_IN_CORE == set() + assert in_core | NOT_YET_IN_CORE == set(ActionKind) From 7e8894a2e557d06f7fa74710c13a1d1c38385ef7 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 24 Oct 2026 23:32:20 -0400 Subject: [PATCH 3/9] docs(flow): the runtime.core boundary and the duplicates to collapse Section 2.4 measures the boundary (a core module reaches 249 of 562 modules through 45 governance edges). Section 10 inventories duplicate implementations (strength scales, outcomes, admissions, certify paths, canonical JSON, runtimes) with the canonical one and the PR for each. Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/design/DECOMPOSITION.md | 61 +++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/packages/flow/docs/design/DECOMPOSITION.md b/packages/flow/docs/design/DECOMPOSITION.md index b5e5f6d9e..36b9bc64c 100644 --- a/packages/flow/docs/design/DECOMPOSITION.md +++ b/packages/flow/docs/design/DECOMPOSITION.md @@ -23,6 +23,7 @@ repeat the measurements. 7. [Coordination with work in flight](#7-coordination-with-work-in-flight) 8. [How the numbers were measured](#8-how-the-numbers-were-measured) 9. [Review decisions](#9-review-decisions) +10. [Duplicates to collapse](#10-duplicates-to-collapse) ## 1. What replayer.py is today @@ -299,7 +300,37 @@ state, and the production lifecycle work that changes them often. object. Reason: the report schema is a public contract, so its construction should be in one place. -### 2.4 Deliberately not done +### 2.4 The `runtime.core` boundary + +`openadapt_flow/runtime/core/` holds the code that carries out a step through +a backend and nothing that decides whether it may. Governance reaches it only +as gates the caller passes in: `ActionContext.authorize` runs delivery +authorization and the qualification-environment check at the last point +before input, in the order the replayer always ran them. + +Two import-linter contracts in `packages/flow/.importlinter`, run by +`lint-imports` in the flow lint job, keep it that way. Neither may import +`runner`, `hosted`, `approval_mode`, `jit`, the replayer or any +`qualification*` module: + +- **Direct.** No module in `runtime.core` imports a governance module. This + holds with no exceptions. +- **Indirect.** The same through every chain of imports, except 45 listed + edges. Measured on `origin/main` at `ecc162e12`: a module that imports only + `ir`, `backend` and the resolver reaches 249 of the package's 562 modules, + 44 of them governance. The routes run through the schema (`ir` imports + `qualification`, `qualification_faults` and `approval_mode.model`), the + effect verifier (`effects.effect` → `authorization` → `execution_profiles` + → `durable` → the replayer → `runner`), and the policy, repair, fleet and + durable modules. A new governance import in any module the core depends on + fails the contract. Each cut removes a line; PR 5 removes the `ir` edges. + +`tests/test_import_boundaries.py` keeps the forbidden list complete as +`qualification_*` modules are added, and proves every `ActionKind` is carried +out in exactly one place: `runtime.core.ACTION_EXECUTORS` (KEY, HOTKEY and +WAIT so far) or the replayer's list of kinds not yet moved. + +### 2.5 Deliberately not done - **Resolution rungs as registered strategies.** `resolver.resolve()` is already separate, and the rung order is part of the safety argument @@ -656,3 +687,31 @@ Decided in review on 2026-10-24: 4. **Resolution rungs:** no second ladder is planned, so `resolver.resolve()` stays one ladder. Checking identity against the system of record is a check source (S5), not a ladder. +5. **Core boundary:** an import-linter contract keeps `runtime.core` free of + governance imports (2.4), and the first executors move behind it at the + start of PR 2. Governance becomes gates around the core. +6. **Landing order:** #1510 (PR 1) lands last among the in-flight pull + requests, with budgets reset to `main` at merge time and the committed + golden baselines; the ratchet then becomes blocking. + +## 10. Duplicates to collapse + +Measured on `origin/main` at `ecc162e12` by reading and `grep`. "Users" counts +non-test files that reference the symbol. Each family names the canonical +implementation and the pull request in section 4.3 that collapses it. + +| Family | What exists | Canonical | PR | +|---|---|---|---| +| Check-strength scales | 8 ladders for one idea: `CheckLevel` (types, 0–5, higher is stronger, already maps the others), `VerificationTier` (1–4, lower is stronger, 46 users, on the wire), `EffectVerificationTier`, `EffectStrengthV1`, `OracleTier`, `ReadBackTier` (defined in types and hand-copied in `approval_mode/approver.py:237`), six copies of the `independent_readback \| screen_readback \| unchecked` literal, two identical `ReadGrade` literals | `CheckLevel`; `VerificationTier` stays as a wire alias | 5 | +| Run outcomes | `TransactionOutcome` (10 states), `ExecutionOutcome` and the envelope literal (5), `QualificationOutcome` (the same 5 in lower case), `ProductionExecutionOutcome`, `ExecuteTerminalOutcomeV1`, `PlainResultV1`, a parallel plain vocabulary in `desktop/outcomes.py`, free strings in the agent runner; `RunReport` carries `success`, `execution_outcome`, `outcome_envelope` and `transaction_outcome` at once | `TransactionOutcome` for the terminal state; `PlainResultV1` for people | 3 (report), 5 (schema) | +| Admission formats | Qualification admission v1 (enforced today), v2, v4; local-dev admission; JIT `admission.json`; execute admission; admit-pipeline pending and decision records; product-release admission | The v4 envelope, with dev, JIT and execute as issuer or mode variants | 5, after a cross-repository contract review | +| Certify paths | Three certify commands repeat the same policy-source resolution; Desktop `certify_bundle` evaluates qualification twice; JIT, repair readmission and the admit pipeline each sign their own records | `certify_project` and one `resolve_policy` helper | 4 | +| Signing and canonical JSON | About 45 canonical-JSON helpers. They are not byte-identical: some use `ensure_ascii=True` (`runner_capability.py:177`, `jit/lifecycle.py:192` by default), most `False`. `flow/openadapt_flow/agent/seal.py` copies `types/openadapt_types/seal.py`. Six key stores | `openadapt_types.seal.canonical_json` with an explicit `ensure_ascii`; signed formats keep a versioned helper so existing signatures still verify | 3 (agent seal copy first), 5 | +| Pagination | One pager (`runtime/list_source.py`), used only by `list_guard.py` | Nothing to collapse | — | +| Census | Not on `main`; on its feature branch a positional-id check exists in Python twice and once in JavaScript | One implementation with a generated JavaScript mirror | After that branch merges | +| Runtimes | One step interpreter (`Replayer`). Four callers start it as a subprocess, five through `deployment.build_replayer`, and about ten construct `Replayer(...)` directly with their own arguments, which can skip governance wiring | `build_replayer` for every in-process caller | 3 | + +Cheapest first: the agent seal copy, the shared literals, the copied outcome +string sets, `resolve_policy`, routing direct `Replayer(...)` callers through +`build_replayer`. Strength scales, outcomes and admissions change persisted or +signed formats and come last, each with a versioned mapping. From 32fa5c8289b1e9545996465a9592e288f7b92858 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 00:17:02 -0400 Subject: [PATCH 4/9] refactor(flow): move render scale and layout drift into runtime.core (move only) Seven Replayer methods move verbatim into runtime/core/render.py as the RenderScaleAndLayoutDrift mixin; scripts/verify_moved_code.py finds 7 moved units and no new logic. scripts/extract_mixin.py, added here, does such moves: it cuts methods verbatim, imports what they use from where the replayer got it, and writes a host class that declares for mypy only what the code reads from self, with the replayer's own annotations. Co-Authored-By: Claude Opus 5.5 --- .../openadapt_flow/runtime/core/render.py | 283 ++++++++++++ .../flow/openadapt_flow/runtime/replayer.py | 233 +--------- scripts/extract_mixin.py | 435 ++++++++++++++++++ 3 files changed, 720 insertions(+), 231 deletions(-) create mode 100644 packages/flow/openadapt_flow/runtime/core/render.py create mode 100644 scripts/extract_mixin.py diff --git a/packages/flow/openadapt_flow/runtime/core/render.py b/packages/flow/openadapt_flow/runtime/core/render.py new file mode 100644 index 000000000..1f904e3c8 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/core/render.py @@ -0,0 +1,283 @@ +# mypy: disable-error-code="empty-body" +"""Render scale and layout drift: keep recorded templates comparable to the live screen. + +Moved verbatim from ``openadapt_flow.runtime.replayer.Replayer`` by +scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1). +``_RenderScaleAndLayoutDriftHost`` declares, for mypy only, what this code reads from the +Replayer it is mixed into. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional, cast # noqa: F401 + +from openadapt_flow.backend import Backend + +if TYPE_CHECKING: + from typing import Any, Callable + + from openadapt_flow.ir import ExecutionTargetKind, Step, Workflow + from openadapt_flow.runtime.surfaces import SurfaceRouter + + +class _RenderScaleAndLayoutDriftHost: + def _active_surface_binding( + self, workflow: Workflow + ) -> tuple[Optional[ExecutionTargetKind], Optional[str]]: ... + def _asset_bytes( + self, bundle_dir: Path, rel: str, *, workflow: Optional[Workflow] = None + ) -> Optional[bytes]: ... + + _layout_fingerprints: Optional[dict[str, Any]] + _layout_target_kind: Optional[str] + _render_scale_attempted: bool + _render_scale_evidence: Optional[dict[str, Any]] + _render_scale_inner: Optional[Backend] + _surface_router: Optional[SurfaceRouter] + backend: Backend + layout_drift: Optional[bool] + layout_drift_advisor: Optional[Callable[[Any], Optional[str]]] + layout_drift_tolerances: Optional[Any] + render_scale: Optional[bool] + vision: Any + + +if TYPE_CHECKING: + _Base = _RenderScaleAndLayoutDriftHost +else: + _Base = object + + +class RenderScaleAndLayoutDrift(_Base): + """Render scale and layout drift: keep recorded templates comparable to the live screen.""" + + def _layout_drift_on(self, workflow: Workflow) -> bool: + if self.layout_drift is not None: + return self.layout_drift + kind: Optional[str] + if self._surface_router is not None: + # Multi-surface run: follow the surface the step acts on, so an + # RDP segment of a web-first bundle keeps the check. + kind, _ = self._active_surface_binding(workflow) + else: + kind = self._layout_target_kind or workflow.surface + if kind in ("rdp", "citrix"): + return True + backend = self._render_scale_inner or self.backend + return type(backend).__name__ == "FreeRDPBackend" + + def _live_session_profile(self) -> Optional[str]: + """Digest of the pinned remote-session profile, when there is one.""" + transport = getattr(self.backend, "transport", None) + profile = getattr(getattr(transport, "profile", None), "sha256", None) + return profile if isinstance(profile, str) else None + + def _session_profile_changed(self, recorded: Any) -> bool: + live = self._live_session_profile() + return bool( + recorded.profile_sha256 and live and recorded.profile_sha256 != live + ) + + def _reset_render_scale(self) -> None: + """Return to the caller's backend and forget the last estimate.""" + if self._render_scale_inner is not None: + release = getattr(self.backend, "release_frame_sources", None) + if callable(release): + release() + self.backend = self._render_scale_inner + self._render_scale_inner = None + self._render_scale_attempted = False + self._render_scale_evidence = None + + def _load_layout_fingerprints( + self, bundle_dir: Path, workflow: Workflow + ) -> dict[str, Any]: + from openadapt_flow.runtime import layout_drift as drift_mod + + if self._layout_fingerprints is None: + self._layout_fingerprints = drift_mod.load_bundle_fingerprints( + self._asset_bytes( + Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow + ) + ) + return self._layout_fingerprints + + def _maybe_scale_render( + self, + step: Step, + frame_png: bytes, + bundle_dir: Optional[Path], + workflow: Workflow, + ) -> bytes: + """Estimate the session's display scale once; return the frame to use. + + Runs at the first step that has a recorded fingerprint. A frame that + matches the fingerprint, or gives no confident estimate, is returned + unchanged and the layout-drift check judges it as before. Otherwise + the backend is replaced by a + :class:`~openadapt_flow.runtime.render_scale.RenderScaledBackend` and + the frame is returned in the recorded geometry, where the same + layout-drift check must then pass. + """ + if self._render_scale_attempted or bundle_dir is None: + return frame_png + enabled = ( + self.render_scale + if self.render_scale is not None + else self._layout_drift_on(workflow) + ) + if not enabled or self._surface_router is not None: + return frame_png + from openadapt_flow.runtime import layout_drift as drift_mod + from openadapt_flow.runtime import render_scale as scale_mod + + recorded = self._load_layout_fingerprints(Path(bundle_dir), workflow).get( + step.id + ) + if recorded is None: + return frame_png + self._render_scale_attempted = True + try: + import io + + from PIL import Image + + with Image.open(io.BytesIO(frame_png)) as img: + viewport = (int(img.width), int(img.height)) + except Exception: # noqa: BLE001 - undecodable: leave it to the drift check + return frame_png + if self._session_profile_changed(recorded): + # A pinned remote-session profile (scale, DPI, font smoothing) is + # a deployment decision; a different profile still halts below. + self._render_scale_evidence = { + "status": "no_estimate", + "reason": "the pinned remote session profile changed", + } + return frame_png + lines = self.vision.ocr(frame_png) + live = drift_mod.fingerprint(lines, viewport) + verdict = drift_mod.compare( + recorded, + live, + self.layout_drift_tolerances or drift_mod.Tolerances(), + ) + if not verdict.drifted: + self._render_scale_evidence = {"status": "recorded_geometry"} + return frame_png + salt, hashed = drift_mod.load_scale_landmarks( + self._asset_bytes( + Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow + ) + ) + landmarks = scale_mod.hashed_landmarks( + hashed.get(step.id, []) + ) + scale_mod.anchor_landmarks(workflow.steps) + pairs = scale_mod.match_landmarks(landmarks, lines, salt_hex=salt) + transform, reason = scale_mod.estimate_render_transform( + pairs, + recorded_viewport=recorded.viewport, + live_viewport=viewport, + ) + if transform is None: + self._render_scale_evidence = { + "status": "no_estimate", + "reason": reason, + "drift": list(verdict.reasons), + } + return frame_png + scaled = scale_mod.render_scaled_backend(self.backend, transform) + self._render_scale_inner = self.backend + self.backend = cast(Backend, scaled) + self._render_scale_evidence = {"status": "normalized", **transform.evidence()} + # The layout-drift check below now judges the normalized frame, so a + # font or layout change that is not a pure rescale still stops here. + return scaled.normalize(frame_png) + + def _layout_drift_error( + self, + step: Step, + frame_png: bytes, + bundle_dir: Optional[Path], + workflow: Workflow, + ) -> Optional[str]: + """Halt message when the frame drifted from the recorded fingerprint. + + Steps without a recorded fingerprint (older bundles, non-click steps) + are not checked. Model-free: the advisor, when configured, is called + only after drift is found and never changes the verdict. + """ + if bundle_dir is None or not self._layout_drift_on(workflow): + return None + from openadapt_flow.runtime import layout_drift as drift_mod + + recorded = self._load_layout_fingerprints(Path(bundle_dir), workflow).get( + step.id + ) + if recorded is None: + return None + live_profile = self._live_session_profile() + try: + import io + + from PIL import Image + + with Image.open(io.BytesIO(frame_png)) as img: + viewport = (int(img.width), int(img.height)) + except Exception: # noqa: BLE001 - an undecodable frame is drift + viewport = (0, 0) + lines = self.vision.ocr(frame_png) + tolerances = self.layout_drift_tolerances or drift_mod.Tolerances() + if self._render_scale_inner is not None and not self._session_profile_changed( + recorded + ): + # A normalized frame is judged label by label: the recorded label + # set says where each label sat and how large it was, so OCR box + # jitter on other lines cannot pass for a font change. + from openadapt_flow.runtime import render_scale as scale_mod + + salt, hashed = drift_mod.load_scale_landmarks( + self._asset_bytes( + Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow + ) + ) + details = ( + scale_mod.check_normalized_frame( + hashed.get(step.id, []), + lines, + salt_hex=salt, + text_height_ratio=tolerances.text_height_ratio, + landmark_px=tolerances.landmark_px, + ) + if salt is not None + else None + ) + if details: + return ( + f"Layout drift before step '{step.id}' ({step.intent}) on " + f"the display-scaled session: {'; '.join(details)} — the " + "screen no longer matches the recording; refusing to act." + ) + if details is not None: + return None + live = drift_mod.fingerprint( + lines, + viewport, + landmarks=list(recorded.landmarks), + profile_sha256=(live_profile if isinstance(live_profile, str) else None), + ) + try: + drift_mod.check_layout( + recorded, + live, + tolerances=tolerances, + advisor=self.layout_drift_advisor, + ) + except drift_mod.LayoutDriftHalt as halt: + advice = f" Advice: {halt.verdict.advice}" if halt.verdict.advice else "" + return ( + f"Layout drift before step '{step.id}' ({step.intent}): " + f"{drift_mod.describe(halt.verdict)} — the screen no longer " + f"matches the recording; refusing to act.{advice}" + ) + return None diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 5741d1707..669e2a71d 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -168,6 +168,7 @@ runtime_params_for_gui, ) from openadapt_flow.runtime.core import ACTION_EXECUTORS, ActionContext +from openadapt_flow.runtime.core.render import RenderScaleAndLayoutDrift from openadapt_flow.runtime.durable.approval import StateDiverged from openadapt_flow.runtime.durable.program_checkpoint import ( TOP_GRAPH_ID, @@ -735,7 +736,7 @@ class _WebFormApprovalView: ref: Any -class Replayer: +class Replayer(RenderScaleAndLayoutDrift): """Replays a Workflow against a Backend using injected vision. Args: @@ -17888,236 +17889,6 @@ def _postcondition_template(self, pc: Any, bundle_dir: Path) -> Optional[bytes]: return None return self._asset_bytes(bundle_dir, rel) - def _layout_drift_on(self, workflow: Workflow) -> bool: - if self.layout_drift is not None: - return self.layout_drift - kind: Optional[str] - if self._surface_router is not None: - # Multi-surface run: follow the surface the step acts on, so an - # RDP segment of a web-first bundle keeps the check. - kind, _ = self._active_surface_binding(workflow) - else: - kind = self._layout_target_kind or workflow.surface - if kind in ("rdp", "citrix"): - return True - backend = self._render_scale_inner or self.backend - return type(backend).__name__ == "FreeRDPBackend" - - def _live_session_profile(self) -> Optional[str]: - """Digest of the pinned remote-session profile, when there is one.""" - transport = getattr(self.backend, "transport", None) - profile = getattr(getattr(transport, "profile", None), "sha256", None) - return profile if isinstance(profile, str) else None - - def _session_profile_changed(self, recorded: Any) -> bool: - live = self._live_session_profile() - return bool( - recorded.profile_sha256 and live and recorded.profile_sha256 != live - ) - - def _reset_render_scale(self) -> None: - """Return to the caller's backend and forget the last estimate.""" - if self._render_scale_inner is not None: - release = getattr(self.backend, "release_frame_sources", None) - if callable(release): - release() - self.backend = self._render_scale_inner - self._render_scale_inner = None - self._render_scale_attempted = False - self._render_scale_evidence = None - - def _load_layout_fingerprints( - self, bundle_dir: Path, workflow: Workflow - ) -> dict[str, Any]: - from openadapt_flow.runtime import layout_drift as drift_mod - - if self._layout_fingerprints is None: - self._layout_fingerprints = drift_mod.load_bundle_fingerprints( - self._asset_bytes( - Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow - ) - ) - return self._layout_fingerprints - - def _maybe_scale_render( - self, - step: Step, - frame_png: bytes, - bundle_dir: Optional[Path], - workflow: Workflow, - ) -> bytes: - """Estimate the session's display scale once; return the frame to use. - - Runs at the first step that has a recorded fingerprint. A frame that - matches the fingerprint, or gives no confident estimate, is returned - unchanged and the layout-drift check judges it as before. Otherwise - the backend is replaced by a - :class:`~openadapt_flow.runtime.render_scale.RenderScaledBackend` and - the frame is returned in the recorded geometry, where the same - layout-drift check must then pass. - """ - if self._render_scale_attempted or bundle_dir is None: - return frame_png - enabled = ( - self.render_scale - if self.render_scale is not None - else self._layout_drift_on(workflow) - ) - if not enabled or self._surface_router is not None: - return frame_png - from openadapt_flow.runtime import layout_drift as drift_mod - from openadapt_flow.runtime import render_scale as scale_mod - - recorded = self._load_layout_fingerprints(Path(bundle_dir), workflow).get( - step.id - ) - if recorded is None: - return frame_png - self._render_scale_attempted = True - try: - import io - - from PIL import Image - - with Image.open(io.BytesIO(frame_png)) as img: - viewport = (int(img.width), int(img.height)) - except Exception: # noqa: BLE001 - undecodable: leave it to the drift check - return frame_png - if self._session_profile_changed(recorded): - # A pinned remote-session profile (scale, DPI, font smoothing) is - # a deployment decision; a different profile still halts below. - self._render_scale_evidence = { - "status": "no_estimate", - "reason": "the pinned remote session profile changed", - } - return frame_png - lines = self.vision.ocr(frame_png) - live = drift_mod.fingerprint(lines, viewport) - verdict = drift_mod.compare( - recorded, - live, - self.layout_drift_tolerances or drift_mod.Tolerances(), - ) - if not verdict.drifted: - self._render_scale_evidence = {"status": "recorded_geometry"} - return frame_png - salt, hashed = drift_mod.load_scale_landmarks( - self._asset_bytes( - Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow - ) - ) - landmarks = scale_mod.hashed_landmarks( - hashed.get(step.id, []) - ) + scale_mod.anchor_landmarks(workflow.steps) - pairs = scale_mod.match_landmarks(landmarks, lines, salt_hex=salt) - transform, reason = scale_mod.estimate_render_transform( - pairs, - recorded_viewport=recorded.viewport, - live_viewport=viewport, - ) - if transform is None: - self._render_scale_evidence = { - "status": "no_estimate", - "reason": reason, - "drift": list(verdict.reasons), - } - return frame_png - scaled = scale_mod.render_scaled_backend(self.backend, transform) - self._render_scale_inner = self.backend - self.backend = cast(Backend, scaled) - self._render_scale_evidence = {"status": "normalized", **transform.evidence()} - # The layout-drift check below now judges the normalized frame, so a - # font or layout change that is not a pure rescale still stops here. - return scaled.normalize(frame_png) - - def _layout_drift_error( - self, - step: Step, - frame_png: bytes, - bundle_dir: Optional[Path], - workflow: Workflow, - ) -> Optional[str]: - """Halt message when the frame drifted from the recorded fingerprint. - - Steps without a recorded fingerprint (older bundles, non-click steps) - are not checked. Model-free: the advisor, when configured, is called - only after drift is found and never changes the verdict. - """ - if bundle_dir is None or not self._layout_drift_on(workflow): - return None - from openadapt_flow.runtime import layout_drift as drift_mod - - recorded = self._load_layout_fingerprints(Path(bundle_dir), workflow).get( - step.id - ) - if recorded is None: - return None - live_profile = self._live_session_profile() - try: - import io - - from PIL import Image - - with Image.open(io.BytesIO(frame_png)) as img: - viewport = (int(img.width), int(img.height)) - except Exception: # noqa: BLE001 - an undecodable frame is drift - viewport = (0, 0) - lines = self.vision.ocr(frame_png) - tolerances = self.layout_drift_tolerances or drift_mod.Tolerances() - if self._render_scale_inner is not None and not self._session_profile_changed( - recorded - ): - # A normalized frame is judged label by label: the recorded label - # set says where each label sat and how large it was, so OCR box - # jitter on other lines cannot pass for a font change. - from openadapt_flow.runtime import render_scale as scale_mod - - salt, hashed = drift_mod.load_scale_landmarks( - self._asset_bytes( - Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow - ) - ) - details = ( - scale_mod.check_normalized_frame( - hashed.get(step.id, []), - lines, - salt_hex=salt, - text_height_ratio=tolerances.text_height_ratio, - landmark_px=tolerances.landmark_px, - ) - if salt is not None - else None - ) - if details: - return ( - f"Layout drift before step '{step.id}' ({step.intent}) on " - f"the display-scaled session: {'; '.join(details)} — the " - "screen no longer matches the recording; refusing to act." - ) - if details is not None: - return None - live = drift_mod.fingerprint( - lines, - viewport, - landmarks=list(recorded.landmarks), - profile_sha256=(live_profile if isinstance(live_profile, str) else None), - ) - try: - drift_mod.check_layout( - recorded, - live, - tolerances=tolerances, - advisor=self.layout_drift_advisor, - ) - except drift_mod.LayoutDriftHalt as halt: - advice = f" Advice: {halt.verdict.advice}" if halt.verdict.advice else "" - return ( - f"Layout drift before step '{step.id}' ({step.intent}): " - f"{drift_mod.describe(halt.verdict)} — the screen no longer " - f"matches the recording; refusing to act.{advice}" - ) - return None - def _asset_bytes( self, bundle_dir: Path, diff --git a/scripts/extract_mixin.py b/scripts/extract_mixin.py new file mode 100644 index 000000000..ee5b26b49 --- /dev/null +++ b/scripts/extract_mixin.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +"""Move methods of a large class into a mixin module, without editing them. + +The tool behind the move commits of the decomposition plan +(packages/flow/docs/design/DECOMPOSITION.md, section 4.1). Given a source +module, a class and a list of its methods, it: + +1. cuts each method out of the class verbatim (decorators and the comment + lines directly above it included); +2. writes them into a new mixin class in the target module, importing every + module-level name the bodies use from where the source module got it; +3. generates a host class that declares, for mypy only, exactly the + attributes and other methods the moved code reads from ``self``, with + the source class's own annotations (``Any`` when the source has none). + It is a plain class, not a ``Protocol``: mypy treats a protocol attribute + without a value as abstract and re-infers its type from the mixin's own + assignments, and implicitly abstract protocol methods could shadow the + real ones earlier in the method resolution order. The stubs' empty + bodies are allowed by a per-module ``empty-body`` exemption; +4. makes the source class inherit from the mixin. + +``--module-defs`` moves module-level functions and constants instead and +re-exports them from the source module, so their import paths keep working. + +The result must pass ``scripts/verify_moved_code.py``: identical ASTs for +everything moved, and only imports, re-exports and protocol stubs added. +Run ``ruff check --fix`` and ``ruff format`` on the touched files afterwards. + +Usage: + python scripts/extract_mixin.py SOURCE --class Replayer --target MODULE_PATH + --mixin NAME --methods m1,m2,... + python scripts/extract_mixin.py SOURCE --module-defs --target MODULE_PATH + --names n1,n2,... +""" + +from __future__ import annotations + +import argparse +import ast +import builtins +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +BUILTINS = set(dir(builtins)) + + +@dataclass +class Imports: + """Where each module-level name of the source module comes from.""" + + runtime: dict[str, str] # name -> import statement + type_checking: dict[str, str] + defined: dict[str, ast.stmt] # module-level defs in the source + + @classmethod + def of(cls, tree: ast.Module) -> Imports: + runtime: dict[str, str] = {} + type_checking: dict[str, str] = {} + defined: dict[str, ast.stmt] = {} + + def record(node: ast.stmt, into: dict[str, str]) -> None: + if isinstance(node, ast.ImportFrom): + for alias in node.names: + name = alias.asname or alias.name + text = f"from {'.' * node.level}{node.module or ''} import {alias.name}" + into[name] = text + (f" as {alias.asname}" if alias.asname else "") + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.asname: + into[alias.asname] = f"import {alias.name} as {alias.asname}" + else: + into[alias.name.split(".")[0]] = f"import {alias.name}" + + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + record(node, runtime) + elif ( + isinstance(node, ast.If) + and isinstance(node.test, ast.Name) + and node.test.id == "TYPE_CHECKING" + ): + for sub in node.body: + record(sub, type_checking) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + defined[node.name] = node + elif isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + defined[target.id] = node + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + defined[node.target.id] = node + return cls(runtime, type_checking, defined) + + +def names_in(nodes: list[ast.AST]) -> set[str]: + found: set[str] = set() + for node in nodes: + for sub in ast.walk(node): + if isinstance(sub, ast.Name): + found.add(sub.id) + elif isinstance(sub, ast.Attribute): + base = sub + while isinstance(base, ast.Attribute): + base = base.value + if isinstance(base, ast.Name): + found.add(base.id) + elif isinstance(sub, ast.Constant) and isinstance(sub.value, str): + # String annotations ("Foo", "Optional[Bar]") name types too. + for word in re.findall(r"[A-Za-z_][A-Za-z_0-9]*", sub.value): + found.add(word) + return found + + +def annotation_names(nodes: list[ast.AST]) -> set[str]: + """Names used only in annotations of the given functions.""" + found: set[str] = set() + for node in nodes: + for sub in ast.walk(node): + if isinstance(sub, ast.arg) and sub.annotation is not None: + found |= names_in([sub.annotation]) + if isinstance(sub, (ast.FunctionDef, ast.AsyncFunctionDef)) and sub.returns: + found |= names_in([sub.returns]) + if isinstance(sub, ast.AnnAssign): + found |= names_in([sub.annotation]) + return found + + +def span(lines: list[str], node: ast.AST) -> tuple[int, int]: + """0-based [start, end) of a definition with decorators and comments above.""" + start = min([node.lineno] + [d.lineno for d in getattr(node, "decorator_list", [])]) - 1 + while start > 0 and lines[start - 1].lstrip().startswith("#"): + start -= 1 + end = node.end_lineno # type: ignore[attr-defined] + return start, end + + +def runtime_names(nodes: list[ast.AST]) -> set[str]: + """Names the bodies need at run time (annotations are strings here).""" + found: set[str] = set() + for node in nodes: + assert isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + for deco in node.decorator_list: + found |= names_in([deco]) + for default in node.args.defaults + [d for d in node.args.kw_defaults if d]: + found |= names_in([default]) + for stmt in node.body: + for sub in ast.walk(stmt): + if isinstance(sub, ast.Name): + found.add(sub.id) + elif isinstance(sub, ast.AnnAssign): + pass + return found + + +def import_lines(names: set[str], imports: Imports, shared: dict[str, str]) -> tuple[list[str], list[str]]: + """Import statements for ``names``; second list is names nobody provides.""" + out, missing = [], [] + for name in sorted(names): + if name in BUILTINS or name == "self": + continue + if name in shared: + out.append(shared[name]) + elif name in imports.runtime: + out.append(imports.runtime[name]) + elif name in imports.type_checking: + out.append(imports.type_checking[name]) + elif name in imports.defined: + missing.append(name) + return sorted(set(out)), missing + + +def class_annotations(cls: ast.ClassDef) -> dict[str, str]: + """Attribute annotations the class states: class-level, then ``__init__``.""" + found: dict[str, str] = {} + for node in cls.body: + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + found[node.target.id] = ast.unparse(node.annotation) + init = next( + (n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "__init__"), + None, + ) + if init is None: + return found + params = { + a.arg: ast.unparse(a.annotation) + for a in init.args.args + init.args.kwonlyargs + if a.annotation is not None + } + for sub in ast.walk(init): + target = None + if isinstance(sub, ast.AnnAssign): + target, ann = sub.target, ast.unparse(sub.annotation) + elif isinstance(sub, ast.Assign) and len(sub.targets) == 1: + target = sub.targets[0] + value = sub.value + ann = params.get(value.id) if isinstance(value, ast.Name) else None + else: + continue + if ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "self" + and target.attr not in found + and ann + ): + found[target.attr] = ann + return found + + +def stub(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str: + copy = ast.parse(ast.unparse(node)).body[0] + assert isinstance(copy, (ast.FunctionDef, ast.AsyncFunctionDef)) + copy.body = [ast.Expr(ast.Constant(Ellipsis))] + return ast.unparse(copy) + + +def extract_methods(args: argparse.Namespace) -> None: + source_path = Path(args.source) + text = source_path.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + tree = ast.parse(text) + imports = Imports.of(tree) + cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == args.cls) + methods = { + n.name: n for n in cls.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + wanted = [m for m in args.methods.split(",") if m] + unknown = [m for m in wanted if m not in methods] + if unknown: + sys.exit(f"not methods of {args.cls}: {unknown}") + moved = [methods[m] for m in wanted] + shared = shared_imports(args.shared, imports) + + # Runtime imports for the bodies. + runtime, missing = import_lines(runtime_names(moved) - set(wanted), imports, shared) + if args.cls in runtime_names(moved): + sys.exit(f"{args.cls} is referenced by name in the moved bodies; leave those methods") + if missing: + sys.exit(f"module-level names not yet in --shared: {missing}") + + # The host protocol: what the moved code reads from self. + used: set[str] = set() + for node in moved: + for sub in ast.walk(node): + if ( + isinstance(sub, ast.Attribute) + and isinstance(sub.value, ast.Name) + and sub.value.id == "self" + and not sub.attr.startswith("__") + ): + used.add(sub.attr) + used -= set(wanted) + annotations = class_annotations(cls) + host_lines: list[str] = [] + stub_nodes: list[ast.AST] = [] + for name in sorted(used): + if name in methods: + host_lines.append(_indent(stub(methods[name]))) + stub_nodes.append(methods[name]) + else: + host_lines.append(f" {name}: {annotations.get(name, 'Any')}") + ann_text = [ast.parse(a, mode="eval").body for a in annotations.values()] + type_names = ( + annotation_names(moved) + | annotation_names(stub_nodes) + | names_in([n for n in ann_text if any(f": {ast.unparse(n)}" in h for h in host_lines)]) + ) + type_lines, type_missing = import_lines(type_names - set(wanted), imports, shared) + type_only = sorted(set(type_lines) - set(runtime)) + if type_missing: + type_only += [f"from {args.source_module} import {n}" for n in sorted(type_missing)] + + host = f"_{args.mixin}Host" + body = "".join("".join(lines[a:b]) + "\n" for a, b in (span(lines, n) for n in moved)) + module = [ + '# mypy: disable-error-code="empty-body"\n', + f'"""{args.doc}\n\nMoved verbatim from ``{args.source_module}.{args.cls}`` by\n' + "scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1).\n" + f'``{host}`` declares, for mypy only, what this code reads from the\n' + f'{args.cls} it is mixed into.\n"""\n\n', + "from __future__ import annotations\n\n", + "from typing import TYPE_CHECKING, Any # noqa: F401\n", + *[line + "\n" for line in runtime], + "\nif TYPE_CHECKING:\n", + *([f" {line}\n" for line in type_only] or [" pass\n"]), + f"\n\nclass {host}:\n", + "\n".join(host_lines) + "\n" if host_lines else " pass\n", + "\n\nif TYPE_CHECKING:\n", + f" _Base = {host}\nelse:\n _Base = object\n\n\n", + f"class {args.mixin}(_Base):\n", + f' """{args.doc.splitlines()[0]}"""\n\n', + body, + ] + target = Path(args.target) + target.parent.mkdir(parents=True, exist_ok=True) + init = target.parent / "__init__.py" + if not init.exists(): + init.write_text(f'"""{args.package_doc}"""\n', encoding="utf-8") + target.write_text("".join(module), encoding="utf-8") + + # Cut the methods out of the source class, bottom up. + for a, b in sorted((span(lines, n) for n in moved), reverse=True): + del lines[a:b] + while a < len(lines) and a > 0 and not lines[a].strip() and not lines[a - 1].strip(): + del lines[a] + out = "".join(lines) + mixin_module = _module_of(target) + header = re.search(rf"^class {args.cls}(\((?P[^)]*)\))?:", out, re.M) + assert header + bases = [b.strip() for b in (header.group("bases") or "").split(",") if b.strip()] + bases.append(args.mixin) + out = out[: header.start()] + f"class {args.cls}({', '.join(bases)}):" + out[header.end() :] + out = _add_import(out, f"from {mixin_module} import {args.mixin}") + source_path.write_text(out, encoding="utf-8") + print(f"moved {len(moved)} methods into {target}") + + +def extract_module_defs(args: argparse.Namespace) -> None: + source_path = Path(args.source) + text = source_path.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + tree = ast.parse(text) + imports = Imports.of(tree) + wanted = [n for n in args.names.split(",") if n] + nodes = [] + for name in wanted: + node = imports.defined.get(name) + if node is None: + sys.exit(f"not a module-level definition: {name}") + if node not in nodes: + nodes.append(node) + names = set() + for node in nodes: + names |= names_in([node]) + provided = {n for node in nodes for n in _defined_names(node)} + runtime, missing = import_lines(names - provided, imports, {}) + if missing: + sys.exit(f"these definitions use other module-level names; move them too: {missing}") + body = "\n\n".join("".join(lines[a:b]).rstrip("\n") for a, b in (span(lines, n) for n in nodes)) + target = Path(args.target) + target.parent.mkdir(parents=True, exist_ok=True) + init = target.parent / "__init__.py" + if not init.exists(): + init.write_text(f'"""{args.package_doc}"""\n', encoding="utf-8") + target.write_text( + f'"""{args.doc}\n\nMoved verbatim from ``{args.source_module}`` by\n' + 'scripts/extract_mixin.py; the source module re-exports every name.\n"""\n\n' + "from __future__ import annotations\n\n" + + "".join(line + "\n" for line in runtime) + + "\n\n" + + body + + "\n", + encoding="utf-8", + ) + for a, b in sorted((span(lines, n) for n in nodes), reverse=True): + del lines[a:b] + out = "".join(lines) + exported = sorted(provided) + out = _add_import( + out, + f"from {_module_of(target)} import ( # noqa: F401 re-exported\n" + + "".join(f" {n},\n" for n in exported) + + ")", + ) + source_path.write_text(out, encoding="utf-8") + print(f"moved {len(exported)} names into {target}") + + +def _defined_names(node: ast.stmt) -> list[str]: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return [node.name] + if isinstance(node, ast.Assign): + return [t.id for t in node.targets if isinstance(t, ast.Name)] + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + return [node.target.id] + return [] + + +def shared_imports(paths: list[str] | None, imports: Imports) -> dict[str, str]: + """Names other moved modules now provide: name -> import statement.""" + found: dict[str, str] = {} + for path in paths or []: + tree = ast.parse(Path(path).read_text(encoding="utf-8")) + module = _module_of(Path(path)) + for node in tree.body: + for name in _defined_names(node): + found[name] = f"from {module} import {name}" + return found + + +def _module_of(path: Path) -> str: + parts = path.with_suffix("").parts + start = next(i for i, p in enumerate(parts) if p.startswith("openadapt")) + return ".".join(parts[start:]) + + +def _indent(code: str) -> str: + return "".join(" " + line if line.strip() else line for line in code.splitlines(True)) + + +def _add_import(source: str, statement: str) -> str: + """Insert an import after the last top-level import of the module.""" + tree = ast.parse(source) + last = max( + n.end_lineno for n in tree.body if isinstance(n, (ast.Import, ast.ImportFrom)) + ) + lines = source.splitlines(keepends=True) + lines.insert(last, statement + "\n") + return "".join(lines) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("source") + parser.add_argument("--class", dest="cls", default="Replayer") + parser.add_argument("--source-module", default="openadapt_flow.runtime.replayer") + parser.add_argument("--target", required=True) + parser.add_argument("--mixin") + parser.add_argument("--methods", default="") + parser.add_argument("--module-defs", action="store_true") + parser.add_argument("--names", default="") + parser.add_argument("--shared", action="append") + parser.add_argument("--doc", default="Moved replayer code.") + parser.add_argument("--package-doc", default="Replayer concerns, one module each.") + args = parser.parse_args(argv) + if args.module_defs: + extract_module_defs(args) + else: + extract_methods(args) + + +if __name__ == "__main__": + main() From 79e5dd99db56a11814a93f29cdb587379eb132a4 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 00:32:10 -0400 Subject: [PATCH 5/9] refactor(flow): move surfaces, overlay, assets and healing into runtime/replay (move only) SurfacesAndOverlay (10 methods) and AssetsHealingAndEvidence (3) move verbatim; verify_moved_code finds 13 moved units and no new logic. They read governed-run state, so they go to runtime/replay, not runtime.core. The extractor now copies only signature decorators onto host stubs and ignores docstrings when it looks for referenced names. Co-Authored-By: Claude Opus 5.5 --- .../openadapt_flow/runtime/replay/__init__.py | 1 + .../openadapt_flow/runtime/replay/evidence.py | 146 ++++++ .../runtime/replay/surfaces_overlay.py | 382 ++++++++++++++++ .../flow/openadapt_flow/runtime/replayer.py | 416 +----------------- scripts/extract_mixin.py | 35 +- 5 files changed, 565 insertions(+), 415 deletions(-) create mode 100644 packages/flow/openadapt_flow/runtime/replay/__init__.py create mode 100644 packages/flow/openadapt_flow/runtime/replay/evidence.py create mode 100644 packages/flow/openadapt_flow/runtime/replay/surfaces_overlay.py diff --git a/packages/flow/openadapt_flow/runtime/replay/__init__.py b/packages/flow/openadapt_flow/runtime/replay/__init__.py new file mode 100644 index 000000000..d82a6e701 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/__init__.py @@ -0,0 +1 @@ +"""Replayer concerns that involve governance, one module each (docs/design/DECOMPOSITION.md, section 4).""" diff --git a/packages/flow/openadapt_flow/runtime/replay/evidence.py b/packages/flow/openadapt_flow/runtime/replay/evidence.py new file mode 100644 index 000000000..c92a4f6dd --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/evidence.py @@ -0,0 +1,146 @@ +# mypy: disable-error-code="empty-body" +"""Bundle assets, anchor healing and step evidence keys. + +Moved verbatim from ``openadapt_flow.runtime.replayer.Replayer`` by +scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1). +``_AssetsHealingAndEvidenceHost`` declares, for mypy only, what this code reads from the +Replayer it is mixed into. +""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path +from typing import TYPE_CHECKING, Any # noqa: F401 + +from openadapt_flow import sealed_set as _sealed +from openadapt_flow.runtime import heal as heal_mod +from openadapt_flow.runtime import healing as healing_mod + +if TYPE_CHECKING: + from typing import Any, Optional + + from openadapt_flow.ir import Region, Resolution, Step, Workflow + from openadapt_flow.runtime.authorization import GovernedRunAuthorization + + +class _AssetsHealingAndEvidenceHost: + def _accept_healed_anchor_in_workflow_snapshot( + self, workflow: Workflow, event: Any + ) -> None: ... + + _execution_workflow_source: Optional[Workflow] + _governed_asset_hashes: dict[str, str] + _governed_asset_mutation: Optional[str] + _governed_asset_snapshot: dict[str, bytes] + _governed_plaintext_assets: Any + governed_authorization: Optional[GovernedRunAuthorization] + vision: Any + + +if TYPE_CHECKING: + _Base = _AssetsHealingAndEvidenceHost +else: + _Base = object + + +class AssetsHealingAndEvidence(_Base): + """Bundle assets, anchor healing and step evidence keys.""" + + def _asset_bytes( + self, + bundle_dir: Path, + rel: str, + *, + workflow: Optional[Workflow] = None, + ) -> Optional[bytes]: + """Return the verified run snapshot for governed asset consumers.""" + if not rel: + # No crop: a record-selecting click rebound to another record + # (runtime.target_param) carries none. + return None + if self.governed_authorization is not None: + data = self._governed_asset_snapshot.get(rel) + expected = self._governed_asset_hashes.get(rel) + if self._governed_plaintext_assets and expected is not None: + path = Path(bundle_dir) / rel + try: + current = path.read_bytes() + except OSError: + current = b"" + if hashlib.sha256(current).hexdigest() != expected: + self._governed_asset_mutation = ( + f"governed sealed asset {rel!r} changed after admission; " + "refusing to act; run aborted" + ) + return data + if workflow is not None and workflow.encrypted: + decrypted = workflow.decrypted_template(rel) + if decrypted is not None: + return decrypted + path = Path(bundle_dir) / rel + return path.read_bytes() if path.is_file() else None + + def _heal_step( + self, + step: Step, + resolution: Resolution, + matched_region: Region, + frame_png: bytes, + workflow: Workflow, + run_dir: Path, + new_crops: dict[str, bytes], + ): + """Build, govern, and persist a repair candidate. + + A heal is a governed PATCH, not a silent bundle swap: the raw event is + wrapped in a reviewable :class:`~openadapt_flow.runtime.healing.HealPatch` + and run through the regression gate (identity + effect + risk) before + it may touch the workflow. A patch that would weaken the step's + identity band -- the reviewed context-drop bug -- is QUARANTINED + (persisted under ``run_dir/heals//patch.json`` for review) + and NOT applied; the caller halts the run. A gate-passing local repair + can update the in-memory workflow. A gate-passing repair discovered + under exact authorization remains a promotable candidate, because the + running program cannot change after admission. + """ + event, crop_png = heal_mod.build_heal_event( + step, resolution, matched_region, frame_png, self.vision + ) + outcome = healing_mod.govern_heal(step, event, run_dir=run_dir) + if outcome.promoted: + if self.governed_authorization is None: + heal_mod.apply_heal(workflow, event) + source = self._execution_workflow_source + if source is not None and source is not workflow: + heal_mod.apply_heal(source, event) + self._accept_healed_anchor_in_workflow_snapshot(workflow, event) + new_crops[step.id] = crop_png + else: + # Exact authorization binds the admitted workflow semantics. + # Keep a gate-passing repair as a reviewable candidate for a + # later qualified bundle. Never change the program that is + # currently executing under that authorization. + outcome.patch.status = "promotable" + healing_mod.persist_patch(outcome.patch, run_dir) + heal_mod.persist_heal(event, crop_png, frame_png, run_dir) + return outcome + + @staticmethod + def _new_step_evidence_key(run_dir: Path, step_id: str) -> str: + """Allocate one stable, collision-free file key for an action occurrence.""" + + safe = re.sub(r"[^A-Za-z0-9._-]+", "_", step_id).strip("._-") or "step" + if safe != step_id: + safe = f"{safe}_{hashlib.sha256(step_id.encode()).hexdigest()[:8]}" + steps_dir = Path(run_dir) / "steps" + candidate = safe + occurrence = 1 + while any( + _sealed.exists(steps_dir / f"{candidate}_{suffix}.png") + for suffix in ("before", "after", "attended-before", "attended-after") + ): + occurrence += 1 + candidate = f"{safe}__{occurrence:04d}" + return candidate diff --git a/packages/flow/openadapt_flow/runtime/replay/surfaces_overlay.py b/packages/flow/openadapt_flow/runtime/replay/surfaces_overlay.py new file mode 100644 index 000000000..688d3b269 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/surfaces_overlay.py @@ -0,0 +1,382 @@ +# mypy: disable-error-code="empty-body" +"""Surfaces and the control overlay: which backend a step acts on, and the progress shown to the person watching. + +Moved verbatim from ``openadapt_flow.runtime.replayer.Replayer`` by +scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1). +``_SurfacesAndOverlayHost`` declares, for mypy only, what this code reads from the +Replayer it is mixed into. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional, cast # noqa: F401 + +from openadapt_flow.backend import BrowserPresentationGeometryBackend +from openadapt_flow.ir import ActionKind, ExecutionTargetKind +from openadapt_flow.runtime.surfaces import SurfaceRouter + +if TYPE_CHECKING: + from pathlib import Path + + from openadapt_flow.backend import Backend + from openadapt_flow.ir import ( + ObservedSurfaceEnvironment, + Point, + Region, + Resolution, + Step, + Workflow, + ) + from openadapt_flow.runtime.authorization import GovernedRunAuthorization + from openadapt_flow.runtime.control_overlay import RuntimeControlOverlayEmitter + from openadapt_flow.runtime.replayer import _GraphStepContext + + +class _SurfacesAndOverlayHost: + _actuation_epoch: Any + + def _cancel_guarded_coordinate(self) -> None: ... + def _cancel_guarded_keyboard(self) -> None: ... + + _control_overlay_disabled: Any + _control_overlay_last_observation_png: Optional[bytes] + _durable_linear_snapshot: tuple[Any, ...] + _last_click_point: Optional[Point] + _last_click_region: Optional[Region] + _managed_dispatch_snapshot: tuple[str, str, str] | None + _settle_seed: Optional[tuple[Any, int]] + _surface_environment_observations: dict[str, ObservedSurfaceEnvironment] + + def _surface_identity_check(self, backend: Any, step: Step) -> Optional[bool]: ... + + _surface_identity_context: Any + _surface_router: Optional[SurfaceRouter] + backend: Backend + control_overlay: Optional["RuntimeControlOverlayEmitter"] + control_overlay_error: Optional[str] + governed_authorization: Optional[GovernedRunAuthorization] + poll_interval_s: float + surface_backends: dict[str, Backend] + + +if TYPE_CHECKING: + _Base = _SurfacesAndOverlayHost +else: + _Base = object + + +class SurfacesAndOverlay(_Base): + """Surfaces and the control overlay: which backend a step acts on, and the progress shown to the person watching.""" + + def _begin_control_overlay(self, profile: Optional[str]) -> None: + """Start an optional presentation rail without affecting execution.""" + + self.control_overlay_error = None + self._control_overlay_disabled = False + self._control_overlay_last_observation_png = None + if self.control_overlay is None: + return + try: + if profile is None: + raise ValueError("run report has no execution profile") + self.control_overlay.begin(profile=profile) + # Prime browser-only presentation metadata at the run boundary, + # never between final target revalidation and one-shot delivery. + # Subsequent reads are cache-only and fail closed if viewport + # dimensions changed. + if isinstance(self.backend, BrowserPresentationGeometryBackend): + self.backend.browser_presentation_viewport() + except Exception as exc: + self.control_overlay_error = type(exc).__name__ + self._control_overlay_disabled = True + + def _emit_control_overlay_phase( + self, + phase: str, + *, + current_step: Optional[int] = None, + total_steps: Optional[int] = None, + target_tracking: Optional[Any] = None, + observation_png: Optional[bytes] = None, + ) -> None: + if self.control_overlay is None or self._control_overlay_disabled: + return + try: + self.control_overlay.emit_phase( + phase, + current_step=current_step, + total_steps=total_steps, + target_tracking=target_tracking, + observation_png=observation_png, + ) + if observation_png is not None: + self._control_overlay_last_observation_png = observation_png + except Exception as exc: + self.control_overlay_error = type(exc).__name__ + self._control_overlay_disabled = True + + def _control_overlay_browser_target( + self, + step: Step, + resolution: Optional[Resolution], + matched_region: Optional[Region], + observation_png: bytes, + ) -> Optional[Any]: + """Build presentation-only geometry from the final resolved target. + + Only a backend that explicitly advertises top-level browser CSS + geometry can produce this V2 target. Missing/invalid presentation + metadata omits the rectangle without disabling the status rail or + influencing execution. + """ + + if ( + self.control_overlay is None + or self._control_overlay_disabled + or resolution is None + or not isinstance(self.backend, BrowserPresentationGeometryBackend) + ): + return None + viewport = self.backend.browser_presentation_viewport() + if viewport is None: + return None + # Visual ladder regions are screenshot-pixel coordinates. The + # reference browser runner records CSS-scale screenshots (DPR 1); for + # any other DPR, omit visual tracking until an exact pixel-to-CSS + # projection is retained. Structural handle regions are already CSS + # viewport coordinates and remain exact at any DPR. + if resolution.structural_handle is None and viewport[2] != 1.0: + return None + region = ( + resolution.structural_handle.region + if resolution.structural_handle is not None + and resolution.structural_handle.region is not None + else matched_region + ) + if region is None: + return None + action_kind = { + ActionKind.CLICK: "click", + ActionKind.DOUBLE_CLICK: "double_click", + ActionKind.RIGHT_CLICK: "right_click", + ActionKind.DRAG: "drag", + ActionKind.TYPE: "type", + ActionKind.SELECT_OPTION: "type", + ActionKind.SCROLL: "scroll", + }.get(step.action) + if action_kind is None: + return None + try: + return self.control_overlay.browser_target_tracking( + observation_png=observation_png, + region_css_px=region, + viewport=viewport, + action_kind=action_kind, + ) + except Exception: + return None + + def _emit_control_overlay_terminal(self, outcome: Optional[str]) -> None: + if self.control_overlay is None or self._control_overlay_disabled: + return + try: + if outcome is None: + raise ValueError("run report has no exact execution outcome") + self.control_overlay.emit_terminal( + outcome, + observation_png=self._control_overlay_last_observation_png, + ) + except Exception as exc: + self.control_overlay_error = type(exc).__name__ + self._control_overlay_disabled = True + + @staticmethod + def _control_overlay_progress( + workflow: Workflow, + step_index: int, + graph_ctx: Optional["_GraphStepContext"], + ) -> tuple[Optional[int], Optional[int]]: + if workflow.program is not None or graph_ctx is not None: + return None, None + return step_index + 1, len(workflow.steps) + + def _begin_surfaces( + self, + workflow: Workflow, + *, + resume: bool, + resume_from: Optional[int] = None, + resume_program: bool = False, + bundle_dir: Optional[Path] = None, + ) -> Optional[str]: + """Bind the workflow's surfaces for this run, or say why not. + + A single-surface workflow binds nothing and runs exactly as before. + A surface is a name plus an identity check, never a live handle, so a + linear resume re-binds every surface by identity from the last + checkpoint's surface observations (docs/MULTI_SURFACE.md). + """ + + self._surface_router = None + if not workflow.surfaces: + if self.surface_backends: + return ( + "surface_backends were supplied, but this workflow declares " + "no surfaces; refusing to run" + ) + return None + # Governed, qualified, durable, resumed, and managed (Cloud) runs are + # admitted: the governed authorization binds the sealed bundle, whose + # workflow.json carries the declared surface map; every step result + # records the surface it ran on; the outcome classifier requires that + # record to match the contract; a qualification case observes each + # surface against its own qualified boundary; and checkpoints carry + # each surface's last-seen identity observation, never a handle. + if resume_program: + return ( + "a multi-surface workflow program cannot be resumed yet: program " + "checkpoints do not carry surface observations; refusing to run" + ) + if self._managed_dispatch_snapshot is not None: + # An unattended run must not start when a surface it will switch + # to has no target: it would stop half-way after earlier writes. + unbound = [ + spec.id + for spec in workflow.surfaces[1:] + if not spec.is_page and spec.id not in (self.surface_backends or {}) + ] + if unbound: + return ( + "a managed dispatch of a multi-surface workflow needs a " + "backend for every backend surface in the deployment " + "profile's surfaces: map; unbound: " + + ", ".join(unbound) + + "; refusing to run" + ) + resume_active: Optional[str] = None + resume_observations: dict[str, Any] = {} + if resume and resume_from: + last = None + for checkpoint in self._durable_linear_snapshot: + if getattr(checkpoint, "step_index", None) == resume_from - 1: + last = checkpoint + if last is None or getattr(last, "active_surface", None) is None: + return ( + "the checkpoint this multi-surface run resumes from carries " + "no surface observation, so its surfaces cannot be re-bound " + "by identity; refusing to run" + ) + resume_active = last.active_surface + resume_observations = dict(last.surface_observations) + authorization = self.governed_authorization + if ( + authorization is not None + and authorization.qualification_case_id is not None + ): + project = workflow.qualification + if project is None: + return ( + "a multi-surface qualification case needs the bundle's " + "qualification project; refusing to run" + ) + for spec in workflow.surfaces[1:]: + boundary = project.surface_environments.get(spec.id) + if boundary is None: + return ( + f"the qualification project has no environment boundary " + f"for surface '{spec.id}'; every surface of a " + "multi-surface workflow must be qualified; refusing to run" + ) + if boundary.target_kind != spec.kind: + return ( + f"the qualification boundary for surface '{spec.id}' is " + f"{boundary.target_kind}, but the surface is declared " + f"{spec.kind}; refusing to run" + ) + extra = sorted( + set(project.surface_environments) + - {spec.id for spec in workflow.surfaces[1:]} + ) + if extra: + return ( + "the qualification project bounds surfaces the workflow " + "does not declare: " + ", ".join(extra) + "; refusing to run" + ) + self._surface_environment_observations = {} + try: + self._surface_identity_context = (workflow, bundle_dir) + self._surface_router = SurfaceRouter( + workflow, + self.backend, + self.surface_backends, + poll_interval_s=self.poll_interval_s, + identity_check=self._surface_identity_check, + resume_active=resume_active, + observations=resume_observations, + ) + except ValueError as exc: + return f"{exc}; refusing to run" + return None + + def _end_surfaces(self) -> None: + """Restore the primary backend after a multi-surface run.""" + + router = self._surface_router + if router is None: + return + self._surface_router = None + if self.backend is not router.primary_backend: + self._activate_surface_backend(router.primary_backend) + # Drop the qualification input-edge guard from every other surface + # backend; the primary's guard follows the single-surface lifecycle. + for backend in router.backends.values(): + if backend is router.primary_backend: + continue + clear_guard = getattr(backend, "set_qualification_input_guard", None) + if callable(clear_guard): + try: + clear_guard(None) + except Exception: + pass + + def _activate_surface_backend(self, backend: Backend) -> None: + """Make ``backend`` current and drop every cache bound to the old one. + + Nothing observed on one surface may be reused on another: the settle + seed, the on-screen read-back verifier, the last click point, and any + armed pointer/keyboard lease all belong to the surface they came from. + """ + + self._cancel_guarded_coordinate() + self._cancel_guarded_keyboard() + self.backend = backend + self._settle_seed = None + self._actuation_epoch += 1 + # The on-screen read-back verifier is bound to one backend; drop it so + # the next step builds one for the new surface. + vars(self).pop("_readback_verifier", None) + self._last_click_point = None + self._last_click_region = None + self._control_overlay_last_observation_png = None + + def _active_surface_binding( + self, workflow: Workflow + ) -> tuple[Optional[ExecutionTargetKind], Optional[str]]: + """(kind, execution mode) of the surface the run is on right now. + + Single-surface runs use the bundle-level binding, unchanged. In a + multi-surface run the remote-surface safety rules follow the ACTIVE + surface, so an RDP segment of a web-first bundle keeps them. + """ + + router = self._surface_router + if router is None: + return workflow.surface, workflow.execution_mode + from openadapt_flow.surface_selection import execution_mode_for_surface + + kind = cast(ExecutionTargetKind, router.active_kind()) + return kind, execution_mode_for_surface(kind) + + def _active_surface_is_external_remote(self, workflow: Workflow) -> bool: + kind, mode = self._active_surface_binding(workflow) + return kind in ("rdp", "citrix") and mode == "external" diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 669e2a71d..8d913497c 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -71,7 +71,6 @@ from openadapt_flow.backend import ( ActionDeliveryUncertain, Backend, - BrowserPresentationGeometryBackend, FocusedElementActuationLeaseBackend, FreshActuationReacquisitionBackend, FreshActuationRequired, @@ -157,7 +156,6 @@ ) from openadapt_flow.run_gate import is_consequential from openadapt_flow.runtime import heal as heal_mod -from openadapt_flow.runtime import healing as healing_mod from openadapt_flow.runtime import identity as identity_mod from openadapt_flow.runtime import identity_looks as looks_mod from openadapt_flow.runtime.authorization import ( @@ -206,6 +204,8 @@ predicate_uses_frame, program_predicate_evaluator_contract_sha256, ) +from openadapt_flow.runtime.replay.evidence import AssetsHealingAndEvidence +from openadapt_flow.runtime.replay.surfaces_overlay import SurfacesAndOverlay from openadapt_flow.runtime.resolver import ( OCR_MIN_RATIO, is_below_ocr, @@ -736,7 +736,7 @@ class _WebFormApprovalView: ref: Any -class Replayer(RenderScaleAndLayoutDrift): +class Replayer(RenderScaleAndLayoutDrift, SurfacesAndOverlay, AssetsHealingAndEvidence): """Replays a Workflow against a Backend using injected vision. Args: @@ -3042,128 +3042,6 @@ def _stamp_execution_outcome( transition_predicate_vision=self.vision, ) - def _begin_control_overlay(self, profile: Optional[str]) -> None: - """Start an optional presentation rail without affecting execution.""" - - self.control_overlay_error = None - self._control_overlay_disabled = False - self._control_overlay_last_observation_png = None - if self.control_overlay is None: - return - try: - if profile is None: - raise ValueError("run report has no execution profile") - self.control_overlay.begin(profile=profile) - # Prime browser-only presentation metadata at the run boundary, - # never between final target revalidation and one-shot delivery. - # Subsequent reads are cache-only and fail closed if viewport - # dimensions changed. - if isinstance(self.backend, BrowserPresentationGeometryBackend): - self.backend.browser_presentation_viewport() - except Exception as exc: - self.control_overlay_error = type(exc).__name__ - self._control_overlay_disabled = True - - def _emit_control_overlay_phase( - self, - phase: str, - *, - current_step: Optional[int] = None, - total_steps: Optional[int] = None, - target_tracking: Optional[Any] = None, - observation_png: Optional[bytes] = None, - ) -> None: - if self.control_overlay is None or self._control_overlay_disabled: - return - try: - self.control_overlay.emit_phase( - phase, - current_step=current_step, - total_steps=total_steps, - target_tracking=target_tracking, - observation_png=observation_png, - ) - if observation_png is not None: - self._control_overlay_last_observation_png = observation_png - except Exception as exc: - self.control_overlay_error = type(exc).__name__ - self._control_overlay_disabled = True - - def _control_overlay_browser_target( - self, - step: Step, - resolution: Optional[Resolution], - matched_region: Optional[Region], - observation_png: bytes, - ) -> Optional[Any]: - """Build presentation-only geometry from the final resolved target. - - Only a backend that explicitly advertises top-level browser CSS - geometry can produce this V2 target. Missing/invalid presentation - metadata omits the rectangle without disabling the status rail or - influencing execution. - """ - - if ( - self.control_overlay is None - or self._control_overlay_disabled - or resolution is None - or not isinstance(self.backend, BrowserPresentationGeometryBackend) - ): - return None - viewport = self.backend.browser_presentation_viewport() - if viewport is None: - return None - # Visual ladder regions are screenshot-pixel coordinates. The - # reference browser runner records CSS-scale screenshots (DPR 1); for - # any other DPR, omit visual tracking until an exact pixel-to-CSS - # projection is retained. Structural handle regions are already CSS - # viewport coordinates and remain exact at any DPR. - if resolution.structural_handle is None and viewport[2] != 1.0: - return None - region = ( - resolution.structural_handle.region - if resolution.structural_handle is not None - and resolution.structural_handle.region is not None - else matched_region - ) - if region is None: - return None - action_kind = { - ActionKind.CLICK: "click", - ActionKind.DOUBLE_CLICK: "double_click", - ActionKind.RIGHT_CLICK: "right_click", - ActionKind.DRAG: "drag", - ActionKind.TYPE: "type", - ActionKind.SELECT_OPTION: "type", - ActionKind.SCROLL: "scroll", - }.get(step.action) - if action_kind is None: - return None - try: - return self.control_overlay.browser_target_tracking( - observation_png=observation_png, - region_css_px=region, - viewport=viewport, - action_kind=action_kind, - ) - except Exception: - return None - - def _emit_control_overlay_terminal(self, outcome: Optional[str]) -> None: - if self.control_overlay is None or self._control_overlay_disabled: - return - try: - if outcome is None: - raise ValueError("run report has no exact execution outcome") - self.control_overlay.emit_terminal( - outcome, - observation_png=self._control_overlay_last_observation_png, - ) - except Exception as exc: - self.control_overlay_error = type(exc).__name__ - self._control_overlay_disabled = True - def _finalize_report( self, report: RunReport, @@ -3682,16 +3560,6 @@ def _record_idempotency_outcome(self, report: RunReport) -> None: run_id=self._run_id, ) - @staticmethod - def _control_overlay_progress( - workflow: Workflow, - step_index: int, - graph_ctx: Optional["_GraphStepContext"], - ) -> tuple[Optional[int], Optional[int]]: - if workflow.program is not None or graph_ctx is not None: - return None, None - return step_index + 1, len(workflow.steps) - def _profile_runtime_refusal(self, workflow: Workflow) -> Optional[str]: """Recheck load-bearing profile invariants at the actuation boundary.""" @@ -7256,124 +7124,6 @@ def _apply_effect_fault_mutation( # -- multi-surface routing (docs/MULTI_SURFACE.md) ----------------------- - def _begin_surfaces( - self, - workflow: Workflow, - *, - resume: bool, - resume_from: Optional[int] = None, - resume_program: bool = False, - bundle_dir: Optional[Path] = None, - ) -> Optional[str]: - """Bind the workflow's surfaces for this run, or say why not. - - A single-surface workflow binds nothing and runs exactly as before. - A surface is a name plus an identity check, never a live handle, so a - linear resume re-binds every surface by identity from the last - checkpoint's surface observations (docs/MULTI_SURFACE.md). - """ - - self._surface_router = None - if not workflow.surfaces: - if self.surface_backends: - return ( - "surface_backends were supplied, but this workflow declares " - "no surfaces; refusing to run" - ) - return None - # Governed, qualified, durable, resumed, and managed (Cloud) runs are - # admitted: the governed authorization binds the sealed bundle, whose - # workflow.json carries the declared surface map; every step result - # records the surface it ran on; the outcome classifier requires that - # record to match the contract; a qualification case observes each - # surface against its own qualified boundary; and checkpoints carry - # each surface's last-seen identity observation, never a handle. - if resume_program: - return ( - "a multi-surface workflow program cannot be resumed yet: program " - "checkpoints do not carry surface observations; refusing to run" - ) - if self._managed_dispatch_snapshot is not None: - # An unattended run must not start when a surface it will switch - # to has no target: it would stop half-way after earlier writes. - unbound = [ - spec.id - for spec in workflow.surfaces[1:] - if not spec.is_page and spec.id not in (self.surface_backends or {}) - ] - if unbound: - return ( - "a managed dispatch of a multi-surface workflow needs a " - "backend for every backend surface in the deployment " - "profile's surfaces: map; unbound: " - + ", ".join(unbound) - + "; refusing to run" - ) - resume_active: Optional[str] = None - resume_observations: dict[str, Any] = {} - if resume and resume_from: - last = None - for checkpoint in self._durable_linear_snapshot: - if getattr(checkpoint, "step_index", None) == resume_from - 1: - last = checkpoint - if last is None or getattr(last, "active_surface", None) is None: - return ( - "the checkpoint this multi-surface run resumes from carries " - "no surface observation, so its surfaces cannot be re-bound " - "by identity; refusing to run" - ) - resume_active = last.active_surface - resume_observations = dict(last.surface_observations) - authorization = self.governed_authorization - if ( - authorization is not None - and authorization.qualification_case_id is not None - ): - project = workflow.qualification - if project is None: - return ( - "a multi-surface qualification case needs the bundle's " - "qualification project; refusing to run" - ) - for spec in workflow.surfaces[1:]: - boundary = project.surface_environments.get(spec.id) - if boundary is None: - return ( - f"the qualification project has no environment boundary " - f"for surface '{spec.id}'; every surface of a " - "multi-surface workflow must be qualified; refusing to run" - ) - if boundary.target_kind != spec.kind: - return ( - f"the qualification boundary for surface '{spec.id}' is " - f"{boundary.target_kind}, but the surface is declared " - f"{spec.kind}; refusing to run" - ) - extra = sorted( - set(project.surface_environments) - - {spec.id for spec in workflow.surfaces[1:]} - ) - if extra: - return ( - "the qualification project bounds surfaces the workflow " - "does not declare: " + ", ".join(extra) + "; refusing to run" - ) - self._surface_environment_observations = {} - try: - self._surface_identity_context = (workflow, bundle_dir) - self._surface_router = SurfaceRouter( - workflow, - self.backend, - self.surface_backends, - poll_interval_s=self.poll_interval_s, - identity_check=self._surface_identity_check, - resume_active=resume_active, - observations=resume_observations, - ) - except ValueError as exc: - return f"{exc}; refusing to run" - return None - @timed_span("identity") def _surface_identity_check(self, backend: Any, step: Step) -> Optional[bool]: """Vision-first identity check of one live target for ``step``. @@ -7413,47 +7163,6 @@ def _surface_identity_check(self, backend: Any, step: Step) -> Optional[bool]: return False return True if evidence else None - def _end_surfaces(self) -> None: - """Restore the primary backend after a multi-surface run.""" - - router = self._surface_router - if router is None: - return - self._surface_router = None - if self.backend is not router.primary_backend: - self._activate_surface_backend(router.primary_backend) - # Drop the qualification input-edge guard from every other surface - # backend; the primary's guard follows the single-surface lifecycle. - for backend in router.backends.values(): - if backend is router.primary_backend: - continue - clear_guard = getattr(backend, "set_qualification_input_guard", None) - if callable(clear_guard): - try: - clear_guard(None) - except Exception: - pass - - def _activate_surface_backend(self, backend: Backend) -> None: - """Make ``backend`` current and drop every cache bound to the old one. - - Nothing observed on one surface may be reused on another: the settle - seed, the on-screen read-back verifier, the last click point, and any - armed pointer/keyboard lease all belong to the surface they came from. - """ - - self._cancel_guarded_coordinate() - self._cancel_guarded_keyboard() - self.backend = backend - self._settle_seed = None - self._actuation_epoch += 1 - # The on-screen read-back verifier is bound to one backend; drop it so - # the next step builds one for the new surface. - vars(self).pop("_readback_verifier", None) - self._last_click_point = None - self._last_click_region = None - self._control_overlay_last_observation_png = None - def _bind_surface_qualification_environment( self, workflow: Workflow, surface_id: str ) -> Optional[str]: @@ -7507,28 +7216,6 @@ def _bind_surface_qualification_environment( ) return refusal - def _active_surface_binding( - self, workflow: Workflow - ) -> tuple[Optional[ExecutionTargetKind], Optional[str]]: - """(kind, execution mode) of the surface the run is on right now. - - Single-surface runs use the bundle-level binding, unchanged. In a - multi-surface run the remote-surface safety rules follow the ACTIVE - surface, so an RDP segment of a web-first bundle keeps them. - """ - - router = self._surface_router - if router is None: - return workflow.surface, workflow.execution_mode - from openadapt_flow.surface_selection import execution_mode_for_surface - - kind = cast(ExecutionTargetKind, router.active_kind()) - return kind, execution_mode_for_surface(kind) - - def _active_surface_is_external_remote(self, workflow: Workflow) -> bool: - kind, mode = self._active_surface_binding(workflow) - return kind in ("rdp", "citrix") and mode == "external" - @staticmethod def _surface_refusal_result(step: Step, error: str) -> StepResult: return StepResult( @@ -17889,40 +17576,6 @@ def _postcondition_template(self, pc: Any, bundle_dir: Path) -> Optional[bytes]: return None return self._asset_bytes(bundle_dir, rel) - def _asset_bytes( - self, - bundle_dir: Path, - rel: str, - *, - workflow: Optional[Workflow] = None, - ) -> Optional[bytes]: - """Return the verified run snapshot for governed asset consumers.""" - if not rel: - # No crop: a record-selecting click rebound to another record - # (runtime.target_param) carries none. - return None - if self.governed_authorization is not None: - data = self._governed_asset_snapshot.get(rel) - expected = self._governed_asset_hashes.get(rel) - if self._governed_plaintext_assets and expected is not None: - path = Path(bundle_dir) / rel - try: - current = path.read_bytes() - except OSError: - current = b"" - if hashlib.sha256(current).hexdigest() != expected: - self._governed_asset_mutation = ( - f"governed sealed asset {rel!r} changed after admission; " - "refusing to act; run aborted" - ) - return data - if workflow is not None and workflow.encrypted: - decrypted = workflow.decrypted_template(rel) - if decrypted is not None: - return decrypted - path = Path(bundle_dir) / rel - return path.read_bytes() if path.is_file() else None - def _raise_on_governed_asset_mutation(self) -> None: """Make predicate-only mutation a non-catchable program safety halt.""" if self._governed_asset_mutation is not None: @@ -17930,51 +17583,6 @@ def _raise_on_governed_asset_mutation(self) -> None: # -- healing --------------------------------------------------------------- - def _heal_step( - self, - step: Step, - resolution: Resolution, - matched_region: Region, - frame_png: bytes, - workflow: Workflow, - run_dir: Path, - new_crops: dict[str, bytes], - ): - """Build, govern, and persist a repair candidate. - - A heal is a governed PATCH, not a silent bundle swap: the raw event is - wrapped in a reviewable :class:`~openadapt_flow.runtime.healing.HealPatch` - and run through the regression gate (identity + effect + risk) before - it may touch the workflow. A patch that would weaken the step's - identity band -- the reviewed context-drop bug -- is QUARANTINED - (persisted under ``run_dir/heals//patch.json`` for review) - and NOT applied; the caller halts the run. A gate-passing local repair - can update the in-memory workflow. A gate-passing repair discovered - under exact authorization remains a promotable candidate, because the - running program cannot change after admission. - """ - event, crop_png = heal_mod.build_heal_event( - step, resolution, matched_region, frame_png, self.vision - ) - outcome = healing_mod.govern_heal(step, event, run_dir=run_dir) - if outcome.promoted: - if self.governed_authorization is None: - heal_mod.apply_heal(workflow, event) - source = self._execution_workflow_source - if source is not None and source is not workflow: - heal_mod.apply_heal(source, event) - self._accept_healed_anchor_in_workflow_snapshot(workflow, event) - new_crops[step.id] = crop_png - else: - # Exact authorization binds the admitted workflow semantics. - # Keep a gate-passing repair as a reviewable candidate for a - # later qualified bundle. Never change the program that is - # currently executing under that authorization. - outcome.patch.status = "promotable" - healing_mod.persist_patch(outcome.patch, run_dir) - heal_mod.persist_heal(event, crop_png, frame_png, run_dir) - return outcome - def _accept_healed_anchor_in_workflow_snapshot( self, workflow: Workflow, @@ -18017,24 +17625,6 @@ def _accept_healed_anchor_in_workflow_snapshot( # -- io ---------------------------------------------------------------------- - @staticmethod - def _new_step_evidence_key(run_dir: Path, step_id: str) -> str: - """Allocate one stable, collision-free file key for an action occurrence.""" - - safe = re.sub(r"[^A-Za-z0-9._-]+", "_", step_id).strip("._-") or "step" - if safe != step_id: - safe = f"{safe}_{hashlib.sha256(step_id.encode()).hexdigest()[:8]}" - steps_dir = Path(run_dir) / "steps" - candidate = safe - occurrence = 1 - while any( - _sealed.exists(steps_dir / f"{candidate}_{suffix}.png") - for suffix in ("before", "after", "attended-before", "attended-after") - ): - occurrence += 1 - candidate = f"{safe}__{occurrence:04d}" - return candidate - @staticmethod @timed_span("artifact_write") def _save_step_png(run_dir: Path, step_id: str, suffix: str, png: bytes) -> str: diff --git a/scripts/extract_mixin.py b/scripts/extract_mixin.py index ee5b26b49..5ebdf8162 100644 --- a/scripts/extract_mixin.py +++ b/scripts/extract_mixin.py @@ -96,8 +96,17 @@ def record(node: ast.stmt, into: dict[str, str]) -> None: def names_in(nodes: list[ast.AST]) -> set[str]: found: set[str] = set() + # Docstrings mention names in prose; they are not references. + docstrings = { + id(sub.value) + for node in nodes + for sub in ast.walk(node) + if isinstance(sub, ast.Expr) and isinstance(sub.value, ast.Constant) + } for node in nodes: for sub in ast.walk(node): + if id(sub) in docstrings: + continue if isinstance(sub, ast.Name): found.add(sub.id) elif isinstance(sub, ast.Attribute): @@ -213,6 +222,13 @@ def stub(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str: copy = ast.parse(ast.unparse(node)).body[0] assert isinstance(copy, (ast.FunctionDef, ast.AsyncFunctionDef)) copy.body = [ast.Expr(ast.Constant(Ellipsis))] + # Only the decorators that change the signature belong on a stub; others + # (timing spans, caches) wrap behavior the stub does not have. + copy.decorator_list = [ + d + for d in copy.decorator_list + if isinstance(d, ast.Name) and d.id in ("staticmethod", "classmethod", "property") + ] return ast.unparse(copy) @@ -338,7 +354,10 @@ def extract_module_defs(args: argparse.Namespace) -> None: runtime, missing = import_lines(names - provided, imports, {}) if missing: sys.exit(f"these definitions use other module-level names; move them too: {missing}") - body = "\n\n".join("".join(lines[a:b]).rstrip("\n") for a, b in (span(lines, n) for n in nodes)) + # Adjacent definitions (only blank or comment lines between them) move as + # one block, so comments that cover several constants stay with them. + spans = merge_adjacent(lines, sorted(span(lines, n) for n in nodes)) + body = "\n\n\n".join("".join(lines[a:b]).strip("\n") for a, b in spans) target = Path(args.target) target.parent.mkdir(parents=True, exist_ok=True) init = target.parent / "__init__.py" @@ -354,7 +373,7 @@ def extract_module_defs(args: argparse.Namespace) -> None: + "\n", encoding="utf-8", ) - for a, b in sorted((span(lines, n) for n in nodes), reverse=True): + for a, b in reversed(spans): del lines[a:b] out = "".join(lines) exported = sorted(provided) @@ -368,6 +387,18 @@ def extract_module_defs(args: argparse.Namespace) -> None: print(f"moved {len(exported)} names into {target}") +def merge_adjacent(lines: list[str], spans: list[tuple[int, int]]) -> list[tuple[int, int]]: + merged: list[tuple[int, int]] = [] + for a, b in spans: + if merged: + gap = lines[merged[-1][1] : a] + if all(not g.strip() or g.lstrip().startswith("#") for g in gap): + merged[-1] = (merged[-1][0], b) + continue + merged.append((a, b)) + return merged + + def _defined_names(node: ast.stmt) -> list[str]: if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): return [node.name] From 6f8c17c0d5dea2e1e8f8cd6dba895aca5094ff26 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 00:32:11 -0400 Subject: [PATCH 6/9] docs(flow): host classes instead of protocols; contract-hashed modules stay put Records why a mixin's host is a plain class (protocol attributes without a value are abstract and get re-inferred; implicitly abstract protocol stubs can shadow real methods), the rule for what goes into runtime.core, and two invariants: resolver.py and program_predicates.py source bytes are part of the predicate evaluator contract, and the CLI split keeps command identities. Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/design/DECOMPOSITION.md | 56 ++++++++++++++++------ 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/packages/flow/docs/design/DECOMPOSITION.md b/packages/flow/docs/design/DECOMPOSITION.md index 36b9bc64c..604efd6df 100644 --- a/packages/flow/docs/design/DECOMPOSITION.md +++ b/packages/flow/docs/design/DECOMPOSITION.md @@ -357,6 +357,8 @@ how the pull request proves it. | Approval gates need a signed human answer | Approval tests; no change to approval code paths outside the moves (and none before #1486 merges) | | Same side effects in the same order | Golden runs record every backend call (method and arguments) and compare the trace | | Import paths keep working | `replayer.py` re-exports every moved name; a test imports every name importable from the module on `main` before the first move | +| Contract hashes over source bytes unchanged | Nothing moves out of `runtime/resolver.py` or `runtime/program_predicates.py`: the program-predicate evaluator contract hashes their source bytes (and the vision module's) | +| Command identities unchanged | The CLI split keeps `_SEALED_READY_COMMANDS` matching, because it keys on each command function's `module:name` | | Strict typing and code-owner review keep covering the moved code | Same pull request updates the strict mypy list and CODEOWNERS to cover `runtime/replay/` | ### 3.1 Golden runs @@ -401,19 +403,20 @@ mix.py:7: error: The erased type of self "host.Replayer" is not a supertype of its class "mix.IdentityMixin" [misc] ``` -Instead, each mixin declares what it needs from the host as a `Protocol` that -only mypy sees: +Instead, each mixin declares what it needs from the host in a host class +that only mypy sees as its base: ```python -class _IdentityHost(Protocol): +# mypy: disable-error-code="empty-body" +class _IdentityChecksHost: backend: Backend - vision: Vision - identity_vlm: IdentityVlm | None + vision: Any + identity_vlm: Optional[Any] pixel_verify_enabled: bool - def _asset_bytes(self, ...) -> bytes: ... + def _asset_bytes(self, bundle_dir: Path, rel: str, *, workflow: Optional[Workflow] = None) -> Optional[bytes]: ... if TYPE_CHECKING: - _Base = _IdentityHost + _Base = _IdentityChecksHost else: _Base = object @@ -422,14 +425,37 @@ class IdentityChecks(_Base): ... ``` -This was checked with mypy 1.11.2 using the strict flags from CI. The mixin -type-checks; at run time the class is a plain `type` whose method resolution -order is `Replayer`, the mixins, `object` (no `Protocol` metaclass); and when -`Replayer` assigns an attribute a type that no longer matches the declaration, -mypy reports an error. Drift is caught, not silent. - -The host protocols are also the first deliverable of the design: each one -lists exactly what its concern reads, which is the input for Stage B. +The host is a plain class, not a `Protocol`. A first version used +`Protocol` and failed in two ways under mypy 1.11 and 2.4: + +- A protocol attribute declared without a value is abstract, so a mixin + method that assigns it (`self._render_scale_inner = None`) re-infers its + type as `None`, and the replayer's own declaration then conflicts. +- Protocol methods with empty bodies are implicitly abstract. When one mixin + stubs a method that a later mixin defines, the stub comes first in the + method resolution order and mypy reports the replayer as abstract. + +A plain host class avoids both. Its stubs' empty bodies are allowed by a +per-module `empty-body` exemption, which cannot hide anything in the moved +code because moved methods are never empty. The attribute types are the +replayer's own (class-level annotations, `__init__` annotations, or the +annotation of the parameter an attribute is assigned from), with `Any` only +where the replayer states no type. + +This was checked with mypy 2.4 using the strict flags from CI, on the real +moves in PR 2: the mixins type-check; at run time the class is a plain +`type` whose method resolution order is `Replayer`, the mixins, `object`; +and when `Replayer` assigns an attribute a type that no longer matches a +host declaration, mypy reports an error. Drift is caught, not silent. + +`scripts/extract_mixin.py` generates the mixin module, its imports and its +host class. Each host lists exactly what its concern reads, which is the +input for Stage B. + +A concern goes into `runtime.core` only when its host reads no governance +state (approval, authorization, qualification, durable or profile +attributes) and its module imports no governance module, even for types. +Everything else goes into `runtime/replay/`. Stage A also reduces merge conflicts. In the October merge-conflict review, three API-step pull requests (#1370, #1371, #1372) each conflicted on From e73cd9522ce8cc5ec795e4952075a797866b2f11 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 00:34:37 -0400 Subject: [PATCH 7/9] refactor(flow): move the replayer's module-level helpers into runtime/replay/_shared (move only) 49 constants, helpers and small types move verbatim, adjacent definitions as one block so shared comments stay with them; replayer.py re-exports every name, so imports such as _DURABLE_RESUME_AUTHORITY, secret_env_var and _ProgramHalt keep working (same objects). verify_moved_code: 58 units. The extractor no longer adds module imports for names a moved body imports itself: ruff would otherwise delete the body's own import. Co-Authored-By: Claude Opus 5.5 --- .../openadapt_flow/runtime/replay/_shared.py | 527 +++++++++++++++++ .../flow/openadapt_flow/runtime/replayer.py | 555 ++---------------- scripts/extract_mixin.py | 26 +- 3 files changed, 599 insertions(+), 509 deletions(-) create mode 100644 packages/flow/openadapt_flow/runtime/replay/_shared.py diff --git a/packages/flow/openadapt_flow/runtime/replay/_shared.py b/packages/flow/openadapt_flow/runtime/replay/_shared.py new file mode 100644 index 000000000..8a446483b --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/_shared.py @@ -0,0 +1,527 @@ +"""Constants, helpers and small types the replayer's concerns share. + +Moved verbatim from ``openadapt_flow.runtime.replayer`` by +scripts/extract_mixin.py; the source module re-exports every name. +""" + +from __future__ import annotations + +import re +import struct +from dataclasses import dataclass +from typing import Any, Mapping, Optional, TypeVar + +from openadapt_flow.ir import ( + ActionKind, + LoopIterationBinding, + RunReport, + StateKind, + Step, + Workflow, +) +from openadapt_flow.network_scope import EndpointScope, endpoint_scope +from openadapt_flow.runtime.authorization import RuntimeParamScalar +from openadapt_flow.runtime.durable.program_checkpoint import ( + GraphFrame, + ProgramCheckpoint, +) + +_DeliveryResultT = TypeVar("_DeliveryResultT") +_MAX_FRESH_ACTUATION_REACQUISITIONS = 2 +# Check-to-click guard on the pixel path (``Replayer._pixel_recheck``): how +# often a changed target or identity region is re-observed before the click +# is refused, and how much a pixel must change to count (remote encoders +# shift colours by a few levels; a dialog or a new row changes far more). +_PIXEL_RECHECK_REOBSERVES = 2 +_PIXEL_RECHECK_LEVEL = 24 +_PIXEL_RECHECK_MIN_PIXELS = 4 +# How long a step that raises a browser dialog waits for it to open. +_DIALOG_WAIT_S = 5.0 + +# How long past its step's timeout a held record check may run under an +# approval mode before the held report decides the item (a person). +READBACK_DEADLINE_MARGIN_S = 120.0 +# Private module capability. The public ``run`` API cannot mint a durable +# continuation. ``runtime.durable.resume`` imports this object only after it +# has validated the pause, approval, bundle, checkpoint, and live state. +_DURABLE_RESUME_AUTHORITY = object() + + +# REGION_STABLE template check: how far the expected content may shift from +# the recorded region (real apps re-layout by a few pixels between runs), +# and the minimum template-match score to accept it. +PC_TEMPLATE_SEARCH_PAD = 80 + + +_FIELD_CHECK_LIMIT = 32 +_FIELD_VALUE_LIMIT = 256 + + +def _field_text(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (str, int, float)): + return str(value)[:_FIELD_VALUE_LIMIT] + return None + + +def effect_field_checks(effect: Any, verdict: Any, tier: Any) -> list[Any]: + """The field comparisons one effect verdict made, for the local report. + + A ``field_equals`` effect compares one field. A record effect compares + each literal ``match`` value against the first matched record. Values + that aren't plain text or numbers are left out rather than guessed. + """ + + from openadapt_flow.ir import EffectFieldCheck + + tier_value = int(tier) if tier is not None else None + checks: list[EffectFieldCheck] = [] + field = getattr(effect, "field", None) + expected_value = getattr(verdict, "expected_value", None) + if field and expected_value is not None: + checks.append( + EffectFieldCheck( + field=str(field)[:_FIELD_VALUE_LIMIT], + expected=_field_text(expected_value), + observed=_field_text(getattr(verdict, "observed_value", None)), + matched=bool(getattr(verdict, "confirmed", False)), + verification_tier=tier_value, + ) + ) + return checks + records = getattr(verdict, "matched_records", None) or [] + record = records[0] if records and isinstance(records[0], dict) else None + for name, want in (getattr(effect, "match", None) or {}).items(): + expected = _field_text(want) + if expected is None: + continue + observed = _field_text(record.get(name)) if record is not None else None + checks.append( + EffectFieldCheck( + field=str(name)[:_FIELD_VALUE_LIMIT], + expected=expected, + observed=observed, + matched=observed == expected, + verification_tier=tier_value, + ) + ) + if len(checks) >= _FIELD_CHECK_LIMIT: + break + return checks + + +#: A uniform page scale is inferred only from a click point at least this far +#: from both edges at the origin, only when both axes agree to within this +#: fraction, and only inside these bounds (and not within 5% of 1). +_PAGE_SCALE_MIN_COORD = 40 +_PAGE_SCALE_AXIS_AGREEMENT = 0.02 +_PAGE_SCALE_BOUNDS = (0.5, 2.0) +#: One pixel of click-point noise moves the estimate by a few tenths of a +#: percent, enough to drop a wide template below threshold; the check tries +#: this tight band around it. +_PAGE_SCALE_BAND = (0.99, 0.995, 1.0, 1.005, 1.01) + + +def _uniform_page_scale(step: Any, result: Any) -> Optional[float]: + """The page scale this step's click showed, or ``None``. + + The recorded click point and the point the ladder resolved this run, + divided axis by axis. A whole-page zoom multiplies both coordinates by + the same factor; a control that merely moved almost never does, so the + two axes must agree. + """ + + resolution = getattr(result, "resolution", None) + anchor = getattr(step, "anchor", None) + if resolution is None or anchor is None: + return None + try: + rx, ry = (float(v) for v in anchor.click_point) + lx, ly = (float(v) for v in resolution.point) + except (TypeError, ValueError): + return None + if min(rx, ry) < _PAGE_SCALE_MIN_COORD: + return None + sx, sy = lx / rx, ly / ry + if abs(sx - sy) > _PAGE_SCALE_AXIS_AGREEMENT * max(sx, sy): + return None + scale = (sx + sy) / 2 + low, high = _PAGE_SCALE_BOUNDS + if not low <= scale <= high or abs(scale - 1.0) < 0.05: + return None + return round(scale, 3) + + +def _frame_viewport(frame_png: bytes) -> tuple[int, int]: + """Return the viewport of THIS frame, read from the frame itself. + + ``self.backend.viewport`` is a live read. Pairing it with an + already-captured frame lets a resize or move between capture and use + reinterpret that frame under geometry it never had, which can place a + coordinate where the operator never demonstrated one. A PNG carries its own + dimensions in the IHDR chunk, so the frame can always answer for itself. + """ + + if len(frame_png) < 24 or not frame_png.startswith(b"\x89PNG\r\n\x1a\n"): + raise ValueError("frame viewport requires valid PNG bytes") + width, height = struct.unpack(">II", frame_png[16:24]) + if width <= 0 or height <= 0: + raise ValueError("frame viewport must be positive") + return int(width), int(height) + + +PC_TEMPLATE_THRESHOLD = 0.9 +# REGION_STABLE asserts recorded structure, not palette. The ordinary +# grayscale template matcher remains the first (stricter) check; this edge-map +# threshold is a color-invariant fallback. Exact v1.16.1 MockMed evidence +# measured correct theme+parameter substitution at 0.86 and a true blocking +# modal at 0.24, leaving a deliberate margin on both sides. +PC_STRUCTURAL_TEMPLATE_THRESHOLD = 0.8 + +# Typed-input verification: size of the "field region" diffed/OCRed around +# the focusing click point after a TYPE action. Generous on purpose — the +# typed text renders at the field's own left edge / first line, not at the +# click point — and clamped to the viewport. When no focusing click is +# known (keyboard-only focus moves, e.g. Tab between fields), the whole +# frame is used instead. +FIELD_REGION_SIZE = (640, 240) +FIELD_REGION_PAD = 16 + +# Typed-input verification, secret-only diff acceptance: when a SECRET typed +# value is OCR-able but OCR cannot find it, a pixel change alone is accepted +# only if the field region gained no other READABLE text — masked fields render +# dots, which OCR reads as nothing, low-confidence noise, or punctuation +# runs, while a dialog rendering over the region adds confident words. +# "Readable" therefore counts alphanumeric characters of lines OCR is +# confident about (>= MASKED_MIN_CONFIDENCE), EXCLUDING homogeneous glyph +# runs: a dot row can misread as a confident digit run (measured on the +# Linux renderer: 17 bullets -> '0000000000006' at 0.81 confidence), and +# such runs are >= MASKED_REPEAT_FRACTION one repeated character — +# no real dialog sentence is. Alnum counts are also invariant to +# segmentation differences between frames. The gain may not exceed +# MASKED_NEW_TEXT_SLACK. +MASKED_NEW_TEXT_SLACK = 3 +MASKED_MIN_CONFIDENCE = 0.6 +MASKED_REPEAT_FRACTION = 0.66 + +# Closed-loop scroll: a SCROLL step keeps scrolling by its recorded delta +# until the NEXT anchored step's anchor resolves on a settled frame, bounded +# by this multiple of the step's own recorded scroll distance. Consecutive +# SCROLL steps hand the loop to each other (each probes first and no-ops +# once the anchor is in view), so a run of N recorded scrolls has a combined +# budget of ~2.5x the total recorded distance. +SCROLL_BUDGET_FACTOR = 2.5 + +# A secret TYPE step's value is never stored in the bundle; it is read at +# replay from this environment variable (the param name upper-cased, with +# non-alphanumeric characters mapped to '_'). See ir.Step.secret. +SECRET_ENV_PREFIX = "OPENADAPT_FLOW_SECRET_" + + +def secret_env_var(param: str) -> str: + """Environment variable a secret parameter's value is read from.""" + key = "".join(ch if ch.isalnum() else "_" for ch in param).upper() + return f"{SECRET_ENV_PREFIX}{key}" + + +# Bounds for the Phase-2 program interpreter -- deterministic termination +# guarantees. A worklist loop is bounded per-loop (LoopSpec.max_iterations); the +# graph walk itself is bounded by a total step budget (so an authored cycle of +# always-true transitions can never spin forever) and subflow/loop nesting is +# depth-bounded. +PROGRAM_MAX_STEPS = 100_000 +PROGRAM_MAX_DEPTH = 64 + + +class _GraphStepContext: + """Per-action-state context the program interpreter feeds ``_run_step``. + + Supplies the two pieces the linear loop derives from list position but a + graph lacks: the previously EXECUTED action (for the click-to-focus TYPE + heuristic) and the SCROLL closed-loop successors (the next anchored action + state and the immediately following state). All optional; a None field means + "no such successor" (the scroll loop then falls back to its open-loop + gesture, exactly as at the end of a linear workflow). + """ + + __slots__ = ( + "prev_action", + "next_anchored", + "following", + "has_exception_handler", + ) + + def __init__( + self, + prev_action: Optional[ActionKind] = None, + next_anchored: Optional[Step] = None, + following: Optional[Step] = None, + has_exception_handler: bool = False, + ) -> None: + self.prev_action = prev_action + self.next_anchored = next_anchored + self.following = following + self.has_exception_handler = has_exception_handler + + +def _pipeline_safe_verifier(verifier: Any) -> bool: + """Whether a read-back may run on a side thread while the next item runs. + + Only verifiers that read the system of record over the network and never + touch the screen: REST, FHIR, GraphQL and SQL. Anything that binds to a + backend (on-screen read-backs, and the redacting and candidate wrappers, + which forward ``bind_backend``) or any other kind verifies inline. + """ + + if ( + verifier is None + or hasattr(verifier, "bind_backend") + or getattr(verifier, "backend", None) is not None + or getattr(verifier, "_backend", None) is not None + ): + return False + safe: list[type] = [] + for module, name in ( + ("openadapt_flow.runtime.effects.rest", "RestRecordVerifier"), + ("openadapt_flow.runtime.effects.fhir", "FhirEffectVerifier"), + ("openadapt_flow.runtime.effects.graphql", "GraphQLRecordVerifier"), + ("openadapt_flow.runtime.effects.sql", "SqlRecordVerifier"), + ): + try: + import importlib + + safe.append(getattr(importlib.import_module(module), name)) + except Exception: # noqa: BLE001 - an unavailable kind is not safe + continue + return isinstance(verifier, tuple(safe)) + + +class _ProgramHalt(Exception): + """Internal signal: an unrecoverable state was reached (an unhandled action + failure, a dead-end with no matching transition, a bound exceeded, or a + ``halt`` / ``escalate`` terminal). Carries the outcome + reason the run + report records. Caught only by ``_interpret_program`` -- never escapes the + Replayer.""" + + def __init__(self, outcome: str, reason: str, *, safety: bool = False) -> None: + super().__init__(reason) + self.outcome = outcome + self.reason = reason + self.safety = safety + # Populated at an ACTION-state failure before graph frames unwind. + # Empty means the halt is not eligible for a generic attended + # Continue/Skip transition. + self.program_frames: list[GraphFrame] = [] + self.program_params: dict[str, RuntimeParamScalar] = {} + self.program_history_hash: str = "" + + +def effect_endpoint_scope(verifier: Any) -> EndpointScope: + """Where an effect verifier reads its system of record. + + A verifier that names a ``base_url`` (REST, FHIR) is ``loopback`` when that + URL is on this computer and ``external`` otherwise. A verifier without one + is ``unknown``, which keeps a network substrate counted as egress. + """ + base_url = getattr(verifier, "base_url", None) + return endpoint_scope(base_url) if isinstance(base_url, str) else "unknown" + + +def _record_loop_iteration( + report: RunReport, + loop_state_id: str, + row_index: int, + row: Mapping[str, RuntimeParamScalar], +) -> None: + """Record the row values one loop iteration binds, before it runs. + + ``report.params`` is the run's base scope. A row field overrides it for + that iteration, so the audit report needs each row's own values to state + what the run typed. + """ + report.loop_iterations.append( + LoopIterationBinding( + loop_state_id=loop_state_id, + row_index=row_index, + params=dict(row), + ) + ) + + +def _loop_iterations_from_checkpoints( + checkpoints: list[ProgramCheckpoint], +) -> list[LoopIterationBinding]: + """Rebuild the loop rows that earlier legs of a resumed run bound. + + A resumed report carries the earlier legs' results, so it also needs the + row values those results typed. Each checkpoint's frame stack names the + loop row it ran in. A row is listed once, at its first checkpoint, keyed by + its position in the frame stack so a nested loop's rows under different + outer rows stay distinct. + """ + seen: set[tuple[tuple[str, str, int], ...]] = set() + bindings: list[LoopIterationBinding] = [] + for checkpoint in checkpoints: + path: tuple[tuple[str, str, int], ...] = () + for frame in checkpoint.frames: + cursor = frame.loop + if cursor is None: + path = (*path, (frame.graph_id, "", -1)) + continue + path = (*path, (frame.graph_id, cursor.loop_state_id, cursor.row_index)) + if path in seen or not 0 <= cursor.row_index < len(cursor.rows): + continue + seen.add(path) + bindings.append( + LoopIterationBinding( + loop_state_id=cursor.loop_state_id, + row_index=cursor.row_index, + params=dict(cursor.rows[cursor.row_index]), + ) + ) + return bindings + + +def _all_workflow_steps(workflow: Workflow): + """Yield every ``Step`` in a workflow for whole-bundle audits. + + Linear bundle: the ``steps`` list. Program bundle: every ``action`` state's + step across the top-level program graph AND all subflows (a loop body's + steps included). Order is stable but not execution order -- used only for + static coverage counting, not for running. + """ + if workflow.program is None: + yield from workflow.steps + return + graphs = [workflow.program, *workflow.subflows.values()] + for graph in graphs: + for state in graph.states.values(): + if state.kind is StateKind.ACTION and state.step is not None: + yield state.step + + +def _effect_read_wording(tier: Any) -> tuple[str, str]: + """Name an effect check in a halt message by its evidence tier. + + A separate-session record-page read (tier 2) and a screen read-back + (tiers 3 and 4) are not system-of-record reads, so their halt messages + must not call them that. Untyped legacy verifiers keep the old wording. + """ + + from openadapt_flow.verification import VerificationTier + + if tier is VerificationTier.INDEPENDENT_SESSION: + return "Effect verification", "record page read in a separate session" + if tier in ( + VerificationTier.PERSISTED_STATE_REACQUISITION, + VerificationTier.IMMEDIATE_SCREEN, + ): + return "Effect verification", "screen read-back" + return "System-of-record effect verification", "system of record" + + +class EgressNotPermitted(RuntimeError): + """Raised when an egress-capable model component (a grounder / identity-VLM + / state-verifier that could send a screenshot off the box) is wired without + the operator's explicit ``allow_model_grounding`` opt-in (PHI audit REM-3). + """ + + +#: How far past its demonstrated position (as a fraction of the frame's extent +#: along the scroll axis) a scroll target may still be before the closed-loop +#: scroll counts it as ready. +SCROLL_POSITION_SLACK_FRACTION = 0.25 + +#: Rungs that show the target itself on screen: what closed-loop scroll +#: readiness accepts, and all a step accepts after its scroll missed it. +_DIRECT_TARGET_RUNGS = frozenset({"structural", "template", "template_global", "ocr"}) + +#: Steps that own the halt when the closed-loop scroll before them never found +#: their target (see ``Replayer._act_scroll``). +_SCROLL_HANDOFF_ACTIONS = frozenset( + {ActionKind.CLICK, ActionKind.DOUBLE_CLICK, ActionKind.RIGHT_CLICK} +) + + +def _scroll_target_positioned( + recorded: tuple[int, int], + resolved: tuple[int, int], + scroll_delta: tuple[int, int], + frame_png: bytes, +) -> bool: + """Is the resolved target no further along the scroll than recorded?""" + + dx, dy = scroll_delta + w, h = _frame_viewport(frame_png) + if dy: + slack = SCROLL_POSITION_SLACK_FRACTION * h + ahead = (resolved[1] - recorded[1]) if dy > 0 else (recorded[1] - resolved[1]) + if ahead > slack: + return False + if dx: + slack = SCROLL_POSITION_SLACK_FRACTION * w + ahead = (resolved[0] - recorded[0]) if dx > 0 else (recorded[0] - resolved[0]) + if ahead > slack: + return False + return True + + +def _identity_tokens(text: Any) -> set[str]: + return {t for t in re.split(r"[^0-9a-z]+", str(text or "").lower()) if t} + + +def identity_read_contradicts(check: Any) -> bool: + """A ``mismatch`` read that saw text the expected identity doesn't have. + + "mismatch" also covers a read that is merely incomplete (columns cut + off). Only a read that shows a token the expected record lacks (another + MRN, a different spelling or date) is evidence of a different record. + """ + + if getattr(check, "status", None) != "mismatch": + return False + expected = _identity_tokens(getattr(check, "expected", None)) + observed = _identity_tokens(getattr(check, "observed", None)) + if not expected or not observed: + return True # nothing to compare: keep the refusal + return not observed <= expected + + +def better_identity_read(first: Any, retry: Any) -> Any: + """The identity read to keep from a first read and its upscaled re-read. + + A read that contradicts the expected record (:func:`identity_read_contradicts`) + is final: a second read of the same screen must never turn it into + ``verified``. A mismatch that is only incomplete can be rescued by a + re-read that verifies. Otherwise ``verified`` beats the "cannot certify" + outcomes, then a mismatch, then ``abstain``, then a blank ``unreadable``; + a tie goes to the read that covered more of the expected identity. + """ + + if identity_read_contradicts(first): + return first + if identity_read_contradicts(retry): + return retry + rank = {"unreadable": 0, "abstain": 1, "mismatch": 2, "verified": 3} + if (rank[retry.status], retry.coverage) > (rank[first.status], first.coverage): + return retry + return first + + +#: Server answers kept per step (a save makes one or a few write requests). +_MAX_SERVER_RESPONSES = 64 + + +@dataclass(frozen=True) +class _WebFormApprovalView: + """An API step's prepared request, as a person is asked to approve it.""" + + prepared: Any + #: :func:`~openadapt_flow.runtime.actuators.web_form.operation_digest`. + digest: str + rows: list[tuple[str, str]] + png: bytes + ref: Any diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 8d913497c..1b5e36e5c 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -44,12 +44,10 @@ import math import os import re -import struct import threading import time import uuid from copy import deepcopy -from dataclasses import dataclass from datetime import date, datetime, timezone from pathlib import Path from types import SimpleNamespace @@ -61,7 +59,6 @@ Mapping, Optional, Sequence, - TypeVar, Union, cast, ) @@ -109,7 +106,6 @@ IdentitySignalEvidence, Interstitial, InterstitialActionResult, - LoopIterationBinding, LoopSpec, ObservedSurfaceEnvironment, PixelIdentityEvidence, @@ -136,7 +132,6 @@ Workflow, predicate_contract_sha256, ) -from openadapt_flow.network_scope import EndpointScope, endpoint_scope from openadapt_flow.privacy import scrub_image_bytes as _scrub_png from openadapt_flow.privacy import scrub_text as _scrub_phi from openadapt_flow.qualification_environment import ( @@ -204,6 +199,57 @@ predicate_uses_frame, program_predicate_evaluator_contract_sha256, ) +from openadapt_flow.runtime.replay._shared import ( # noqa: F401 re-exported + _DIALOG_WAIT_S, + _DIRECT_TARGET_RUNGS, + _DURABLE_RESUME_AUTHORITY, + _FIELD_CHECK_LIMIT, + _FIELD_VALUE_LIMIT, + _MAX_FRESH_ACTUATION_REACQUISITIONS, + _MAX_SERVER_RESPONSES, + _PAGE_SCALE_AXIS_AGREEMENT, + _PAGE_SCALE_BAND, + _PAGE_SCALE_BOUNDS, + _PAGE_SCALE_MIN_COORD, + _PIXEL_RECHECK_LEVEL, + _PIXEL_RECHECK_MIN_PIXELS, + _PIXEL_RECHECK_REOBSERVES, + _SCROLL_HANDOFF_ACTIONS, + FIELD_REGION_PAD, + FIELD_REGION_SIZE, + MASKED_MIN_CONFIDENCE, + MASKED_NEW_TEXT_SLACK, + MASKED_REPEAT_FRACTION, + PC_STRUCTURAL_TEMPLATE_THRESHOLD, + PC_TEMPLATE_SEARCH_PAD, + PC_TEMPLATE_THRESHOLD, + PROGRAM_MAX_DEPTH, + PROGRAM_MAX_STEPS, + READBACK_DEADLINE_MARGIN_S, + SCROLL_BUDGET_FACTOR, + SCROLL_POSITION_SLACK_FRACTION, + SECRET_ENV_PREFIX, + EgressNotPermitted, + _all_workflow_steps, + _DeliveryResultT, + _effect_read_wording, + _field_text, + _frame_viewport, + _GraphStepContext, + _identity_tokens, + _loop_iterations_from_checkpoints, + _pipeline_safe_verifier, + _ProgramHalt, + _record_loop_iteration, + _scroll_target_positioned, + _uniform_page_scale, + _WebFormApprovalView, + better_identity_read, + effect_endpoint_scope, + effect_field_checks, + identity_read_contradicts, + secret_env_var, +) from openadapt_flow.runtime.replay.evidence import AssetsHealingAndEvidence from openadapt_flow.runtime.replay.surfaces_overlay import SurfacesAndOverlay from openadapt_flow.runtime.resolver import ( @@ -223,26 +269,6 @@ ) from openadapt_flow.vision.ocr import OcrResolutionRefused -_DeliveryResultT = TypeVar("_DeliveryResultT") -_MAX_FRESH_ACTUATION_REACQUISITIONS = 2 -# Check-to-click guard on the pixel path (``Replayer._pixel_recheck``): how -# often a changed target or identity region is re-observed before the click -# is refused, and how much a pixel must change to count (remote encoders -# shift colours by a few levels; a dialog or a new row changes far more). -_PIXEL_RECHECK_REOBSERVES = 2 -_PIXEL_RECHECK_LEVEL = 24 -_PIXEL_RECHECK_MIN_PIXELS = 4 -# How long a step that raises a browser dialog waits for it to open. -_DIALOG_WAIT_S = 5.0 - -# How long past its step's timeout a held record check may run under an -# approval mode before the held report decides the item (a person). -READBACK_DEADLINE_MARGIN_S = 120.0 -# Private module capability. The public ``run`` API cannot mint a durable -# continuation. ``runtime.durable.resume`` imports this object only after it -# has validated the pause, approval, bundle, checkpoint, and live state. -_DURABLE_RESUME_AUTHORITY = object() - if TYPE_CHECKING: from openadapt_flow.approval_mode.gate import ApprovalGate from openadapt_flow.production_qualification import ProductionQualificationGuard @@ -256,485 +282,6 @@ ) from openadapt_flow.transaction import IdempotencyLedger -# REGION_STABLE template check: how far the expected content may shift from -# the recorded region (real apps re-layout by a few pixels between runs), -# and the minimum template-match score to accept it. -PC_TEMPLATE_SEARCH_PAD = 80 - - -_FIELD_CHECK_LIMIT = 32 -_FIELD_VALUE_LIMIT = 256 - - -def _field_text(value: Any) -> Optional[str]: - if value is None: - return None - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, (str, int, float)): - return str(value)[:_FIELD_VALUE_LIMIT] - return None - - -def effect_field_checks(effect: Any, verdict: Any, tier: Any) -> list[Any]: - """The field comparisons one effect verdict made, for the local report. - - A ``field_equals`` effect compares one field. A record effect compares - each literal ``match`` value against the first matched record. Values - that aren't plain text or numbers are left out rather than guessed. - """ - - from openadapt_flow.ir import EffectFieldCheck - - tier_value = int(tier) if tier is not None else None - checks: list[EffectFieldCheck] = [] - field = getattr(effect, "field", None) - expected_value = getattr(verdict, "expected_value", None) - if field and expected_value is not None: - checks.append( - EffectFieldCheck( - field=str(field)[:_FIELD_VALUE_LIMIT], - expected=_field_text(expected_value), - observed=_field_text(getattr(verdict, "observed_value", None)), - matched=bool(getattr(verdict, "confirmed", False)), - verification_tier=tier_value, - ) - ) - return checks - records = getattr(verdict, "matched_records", None) or [] - record = records[0] if records and isinstance(records[0], dict) else None - for name, want in (getattr(effect, "match", None) or {}).items(): - expected = _field_text(want) - if expected is None: - continue - observed = _field_text(record.get(name)) if record is not None else None - checks.append( - EffectFieldCheck( - field=str(name)[:_FIELD_VALUE_LIMIT], - expected=expected, - observed=observed, - matched=observed == expected, - verification_tier=tier_value, - ) - ) - if len(checks) >= _FIELD_CHECK_LIMIT: - break - return checks - - -#: A uniform page scale is inferred only from a click point at least this far -#: from both edges at the origin, only when both axes agree to within this -#: fraction, and only inside these bounds (and not within 5% of 1). -_PAGE_SCALE_MIN_COORD = 40 -_PAGE_SCALE_AXIS_AGREEMENT = 0.02 -_PAGE_SCALE_BOUNDS = (0.5, 2.0) -#: One pixel of click-point noise moves the estimate by a few tenths of a -#: percent, enough to drop a wide template below threshold; the check tries -#: this tight band around it. -_PAGE_SCALE_BAND = (0.99, 0.995, 1.0, 1.005, 1.01) - - -def _uniform_page_scale(step: Any, result: Any) -> Optional[float]: - """The page scale this step's click showed, or ``None``. - - The recorded click point and the point the ladder resolved this run, - divided axis by axis. A whole-page zoom multiplies both coordinates by - the same factor; a control that merely moved almost never does, so the - two axes must agree. - """ - - resolution = getattr(result, "resolution", None) - anchor = getattr(step, "anchor", None) - if resolution is None or anchor is None: - return None - try: - rx, ry = (float(v) for v in anchor.click_point) - lx, ly = (float(v) for v in resolution.point) - except (TypeError, ValueError): - return None - if min(rx, ry) < _PAGE_SCALE_MIN_COORD: - return None - sx, sy = lx / rx, ly / ry - if abs(sx - sy) > _PAGE_SCALE_AXIS_AGREEMENT * max(sx, sy): - return None - scale = (sx + sy) / 2 - low, high = _PAGE_SCALE_BOUNDS - if not low <= scale <= high or abs(scale - 1.0) < 0.05: - return None - return round(scale, 3) - - -def _frame_viewport(frame_png: bytes) -> tuple[int, int]: - """Return the viewport of THIS frame, read from the frame itself. - - ``self.backend.viewport`` is a live read. Pairing it with an - already-captured frame lets a resize or move between capture and use - reinterpret that frame under geometry it never had, which can place a - coordinate where the operator never demonstrated one. A PNG carries its own - dimensions in the IHDR chunk, so the frame can always answer for itself. - """ - - if len(frame_png) < 24 or not frame_png.startswith(b"\x89PNG\r\n\x1a\n"): - raise ValueError("frame viewport requires valid PNG bytes") - width, height = struct.unpack(">II", frame_png[16:24]) - if width <= 0 or height <= 0: - raise ValueError("frame viewport must be positive") - return int(width), int(height) - - -PC_TEMPLATE_THRESHOLD = 0.9 -# REGION_STABLE asserts recorded structure, not palette. The ordinary -# grayscale template matcher remains the first (stricter) check; this edge-map -# threshold is a color-invariant fallback. Exact v1.16.1 MockMed evidence -# measured correct theme+parameter substitution at 0.86 and a true blocking -# modal at 0.24, leaving a deliberate margin on both sides. -PC_STRUCTURAL_TEMPLATE_THRESHOLD = 0.8 - -# Typed-input verification: size of the "field region" diffed/OCRed around -# the focusing click point after a TYPE action. Generous on purpose — the -# typed text renders at the field's own left edge / first line, not at the -# click point — and clamped to the viewport. When no focusing click is -# known (keyboard-only focus moves, e.g. Tab between fields), the whole -# frame is used instead. -FIELD_REGION_SIZE = (640, 240) -FIELD_REGION_PAD = 16 - -# Typed-input verification, secret-only diff acceptance: when a SECRET typed -# value is OCR-able but OCR cannot find it, a pixel change alone is accepted -# only if the field region gained no other READABLE text — masked fields render -# dots, which OCR reads as nothing, low-confidence noise, or punctuation -# runs, while a dialog rendering over the region adds confident words. -# "Readable" therefore counts alphanumeric characters of lines OCR is -# confident about (>= MASKED_MIN_CONFIDENCE), EXCLUDING homogeneous glyph -# runs: a dot row can misread as a confident digit run (measured on the -# Linux renderer: 17 bullets -> '0000000000006' at 0.81 confidence), and -# such runs are >= MASKED_REPEAT_FRACTION one repeated character — -# no real dialog sentence is. Alnum counts are also invariant to -# segmentation differences between frames. The gain may not exceed -# MASKED_NEW_TEXT_SLACK. -MASKED_NEW_TEXT_SLACK = 3 -MASKED_MIN_CONFIDENCE = 0.6 -MASKED_REPEAT_FRACTION = 0.66 - -# Closed-loop scroll: a SCROLL step keeps scrolling by its recorded delta -# until the NEXT anchored step's anchor resolves on a settled frame, bounded -# by this multiple of the step's own recorded scroll distance. Consecutive -# SCROLL steps hand the loop to each other (each probes first and no-ops -# once the anchor is in view), so a run of N recorded scrolls has a combined -# budget of ~2.5x the total recorded distance. -SCROLL_BUDGET_FACTOR = 2.5 - -# A secret TYPE step's value is never stored in the bundle; it is read at -# replay from this environment variable (the param name upper-cased, with -# non-alphanumeric characters mapped to '_'). See ir.Step.secret. -SECRET_ENV_PREFIX = "OPENADAPT_FLOW_SECRET_" - - -def secret_env_var(param: str) -> str: - """Environment variable a secret parameter's value is read from.""" - key = "".join(ch if ch.isalnum() else "_" for ch in param).upper() - return f"{SECRET_ENV_PREFIX}{key}" - - -# Bounds for the Phase-2 program interpreter -- deterministic termination -# guarantees. A worklist loop is bounded per-loop (LoopSpec.max_iterations); the -# graph walk itself is bounded by a total step budget (so an authored cycle of -# always-true transitions can never spin forever) and subflow/loop nesting is -# depth-bounded. -PROGRAM_MAX_STEPS = 100_000 -PROGRAM_MAX_DEPTH = 64 - - -class _GraphStepContext: - """Per-action-state context the program interpreter feeds ``_run_step``. - - Supplies the two pieces the linear loop derives from list position but a - graph lacks: the previously EXECUTED action (for the click-to-focus TYPE - heuristic) and the SCROLL closed-loop successors (the next anchored action - state and the immediately following state). All optional; a None field means - "no such successor" (the scroll loop then falls back to its open-loop - gesture, exactly as at the end of a linear workflow). - """ - - __slots__ = ( - "prev_action", - "next_anchored", - "following", - "has_exception_handler", - ) - - def __init__( - self, - prev_action: Optional[ActionKind] = None, - next_anchored: Optional[Step] = None, - following: Optional[Step] = None, - has_exception_handler: bool = False, - ) -> None: - self.prev_action = prev_action - self.next_anchored = next_anchored - self.following = following - self.has_exception_handler = has_exception_handler - - -def _pipeline_safe_verifier(verifier: Any) -> bool: - """Whether a read-back may run on a side thread while the next item runs. - - Only verifiers that read the system of record over the network and never - touch the screen: REST, FHIR, GraphQL and SQL. Anything that binds to a - backend (on-screen read-backs, and the redacting and candidate wrappers, - which forward ``bind_backend``) or any other kind verifies inline. - """ - - if ( - verifier is None - or hasattr(verifier, "bind_backend") - or getattr(verifier, "backend", None) is not None - or getattr(verifier, "_backend", None) is not None - ): - return False - safe: list[type] = [] - for module, name in ( - ("openadapt_flow.runtime.effects.rest", "RestRecordVerifier"), - ("openadapt_flow.runtime.effects.fhir", "FhirEffectVerifier"), - ("openadapt_flow.runtime.effects.graphql", "GraphQLRecordVerifier"), - ("openadapt_flow.runtime.effects.sql", "SqlRecordVerifier"), - ): - try: - import importlib - - safe.append(getattr(importlib.import_module(module), name)) - except Exception: # noqa: BLE001 - an unavailable kind is not safe - continue - return isinstance(verifier, tuple(safe)) - - -class _ProgramHalt(Exception): - """Internal signal: an unrecoverable state was reached (an unhandled action - failure, a dead-end with no matching transition, a bound exceeded, or a - ``halt`` / ``escalate`` terminal). Carries the outcome + reason the run - report records. Caught only by ``_interpret_program`` -- never escapes the - Replayer.""" - - def __init__(self, outcome: str, reason: str, *, safety: bool = False) -> None: - super().__init__(reason) - self.outcome = outcome - self.reason = reason - self.safety = safety - # Populated at an ACTION-state failure before graph frames unwind. - # Empty means the halt is not eligible for a generic attended - # Continue/Skip transition. - self.program_frames: list[GraphFrame] = [] - self.program_params: dict[str, RuntimeParamScalar] = {} - self.program_history_hash: str = "" - - -def effect_endpoint_scope(verifier: Any) -> EndpointScope: - """Where an effect verifier reads its system of record. - - A verifier that names a ``base_url`` (REST, FHIR) is ``loopback`` when that - URL is on this computer and ``external`` otherwise. A verifier without one - is ``unknown``, which keeps a network substrate counted as egress. - """ - base_url = getattr(verifier, "base_url", None) - return endpoint_scope(base_url) if isinstance(base_url, str) else "unknown" - - -def _record_loop_iteration( - report: RunReport, - loop_state_id: str, - row_index: int, - row: Mapping[str, RuntimeParamScalar], -) -> None: - """Record the row values one loop iteration binds, before it runs. - - ``report.params`` is the run's base scope. A row field overrides it for - that iteration, so the audit report needs each row's own values to state - what the run typed. - """ - report.loop_iterations.append( - LoopIterationBinding( - loop_state_id=loop_state_id, - row_index=row_index, - params=dict(row), - ) - ) - - -def _loop_iterations_from_checkpoints( - checkpoints: list[ProgramCheckpoint], -) -> list[LoopIterationBinding]: - """Rebuild the loop rows that earlier legs of a resumed run bound. - - A resumed report carries the earlier legs' results, so it also needs the - row values those results typed. Each checkpoint's frame stack names the - loop row it ran in. A row is listed once, at its first checkpoint, keyed by - its position in the frame stack so a nested loop's rows under different - outer rows stay distinct. - """ - seen: set[tuple[tuple[str, str, int], ...]] = set() - bindings: list[LoopIterationBinding] = [] - for checkpoint in checkpoints: - path: tuple[tuple[str, str, int], ...] = () - for frame in checkpoint.frames: - cursor = frame.loop - if cursor is None: - path = (*path, (frame.graph_id, "", -1)) - continue - path = (*path, (frame.graph_id, cursor.loop_state_id, cursor.row_index)) - if path in seen or not 0 <= cursor.row_index < len(cursor.rows): - continue - seen.add(path) - bindings.append( - LoopIterationBinding( - loop_state_id=cursor.loop_state_id, - row_index=cursor.row_index, - params=dict(cursor.rows[cursor.row_index]), - ) - ) - return bindings - - -def _all_workflow_steps(workflow: Workflow): - """Yield every ``Step`` in a workflow for whole-bundle audits. - - Linear bundle: the ``steps`` list. Program bundle: every ``action`` state's - step across the top-level program graph AND all subflows (a loop body's - steps included). Order is stable but not execution order -- used only for - static coverage counting, not for running. - """ - if workflow.program is None: - yield from workflow.steps - return - graphs = [workflow.program, *workflow.subflows.values()] - for graph in graphs: - for state in graph.states.values(): - if state.kind is StateKind.ACTION and state.step is not None: - yield state.step - - -def _effect_read_wording(tier: Any) -> tuple[str, str]: - """Name an effect check in a halt message by its evidence tier. - - A separate-session record-page read (tier 2) and a screen read-back - (tiers 3 and 4) are not system-of-record reads, so their halt messages - must not call them that. Untyped legacy verifiers keep the old wording. - """ - - from openadapt_flow.verification import VerificationTier - - if tier is VerificationTier.INDEPENDENT_SESSION: - return "Effect verification", "record page read in a separate session" - if tier in ( - VerificationTier.PERSISTED_STATE_REACQUISITION, - VerificationTier.IMMEDIATE_SCREEN, - ): - return "Effect verification", "screen read-back" - return "System-of-record effect verification", "system of record" - - -class EgressNotPermitted(RuntimeError): - """Raised when an egress-capable model component (a grounder / identity-VLM - / state-verifier that could send a screenshot off the box) is wired without - the operator's explicit ``allow_model_grounding`` opt-in (PHI audit REM-3). - """ - - -#: How far past its demonstrated position (as a fraction of the frame's extent -#: along the scroll axis) a scroll target may still be before the closed-loop -#: scroll counts it as ready. -SCROLL_POSITION_SLACK_FRACTION = 0.25 - -#: Rungs that show the target itself on screen: what closed-loop scroll -#: readiness accepts, and all a step accepts after its scroll missed it. -_DIRECT_TARGET_RUNGS = frozenset({"structural", "template", "template_global", "ocr"}) - -#: Steps that own the halt when the closed-loop scroll before them never found -#: their target (see ``Replayer._act_scroll``). -_SCROLL_HANDOFF_ACTIONS = frozenset( - {ActionKind.CLICK, ActionKind.DOUBLE_CLICK, ActionKind.RIGHT_CLICK} -) - - -def _scroll_target_positioned( - recorded: tuple[int, int], - resolved: tuple[int, int], - scroll_delta: tuple[int, int], - frame_png: bytes, -) -> bool: - """Is the resolved target no further along the scroll than recorded?""" - - dx, dy = scroll_delta - w, h = _frame_viewport(frame_png) - if dy: - slack = SCROLL_POSITION_SLACK_FRACTION * h - ahead = (resolved[1] - recorded[1]) if dy > 0 else (recorded[1] - resolved[1]) - if ahead > slack: - return False - if dx: - slack = SCROLL_POSITION_SLACK_FRACTION * w - ahead = (resolved[0] - recorded[0]) if dx > 0 else (recorded[0] - resolved[0]) - if ahead > slack: - return False - return True - - -def _identity_tokens(text: Any) -> set[str]: - return {t for t in re.split(r"[^0-9a-z]+", str(text or "").lower()) if t} - - -def identity_read_contradicts(check: Any) -> bool: - """A ``mismatch`` read that saw text the expected identity doesn't have. - - "mismatch" also covers a read that is merely incomplete (columns cut - off). Only a read that shows a token the expected record lacks (another - MRN, a different spelling or date) is evidence of a different record. - """ - - if getattr(check, "status", None) != "mismatch": - return False - expected = _identity_tokens(getattr(check, "expected", None)) - observed = _identity_tokens(getattr(check, "observed", None)) - if not expected or not observed: - return True # nothing to compare: keep the refusal - return not observed <= expected - - -def better_identity_read(first: Any, retry: Any) -> Any: - """The identity read to keep from a first read and its upscaled re-read. - - A read that contradicts the expected record (:func:`identity_read_contradicts`) - is final: a second read of the same screen must never turn it into - ``verified``. A mismatch that is only incomplete can be rescued by a - re-read that verifies. Otherwise ``verified`` beats the "cannot certify" - outcomes, then a mismatch, then ``abstain``, then a blank ``unreadable``; - a tie goes to the read that covered more of the expected identity. - """ - - if identity_read_contradicts(first): - return first - if identity_read_contradicts(retry): - return retry - rank = {"unreadable": 0, "abstain": 1, "mismatch": 2, "verified": 3} - if (rank[retry.status], retry.coverage) > (rank[first.status], first.coverage): - return retry - return first - - -#: Server answers kept per step (a save makes one or a few write requests). -_MAX_SERVER_RESPONSES = 64 - - -@dataclass(frozen=True) -class _WebFormApprovalView: - """An API step's prepared request, as a person is asked to approve it.""" - - prepared: Any - #: :func:`~openadapt_flow.runtime.actuators.web_form.operation_digest`. - digest: str - rows: list[tuple[str, str]] - png: bytes - ref: Any - class Replayer(RenderScaleAndLayoutDrift, SurfacesAndOverlay, AssetsHealingAndEvidence): """Replays a Workflow against a Backend using injected vision. diff --git a/scripts/extract_mixin.py b/scripts/extract_mixin.py index 5ebdf8162..2ee7184d1 100644 --- a/scripts/extract_mixin.py +++ b/scripts/extract_mixin.py @@ -146,8 +146,19 @@ def span(lines: list[str], node: ast.AST) -> tuple[int, int]: def runtime_names(nodes: list[ast.AST]) -> set[str]: - """Names the bodies need at run time (annotations are strings here).""" + """Names the bodies need at run time (annotations are strings here). + + Names a body imports for itself are left out: importing them at module + level as well would let ``ruff --fix`` delete the body's own import, + which is an edit, not a move. + """ found: set[str] = set() + local: set[str] = set() + for node in nodes: + for sub in ast.walk(node): + if isinstance(sub, (ast.Import, ast.ImportFrom)): + for alias in sub.names: + local.add((alias.asname or alias.name).split(".")[0]) for node in nodes: assert isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) for deco in node.decorator_list: @@ -158,9 +169,7 @@ def runtime_names(nodes: list[ast.AST]) -> set[str]: for sub in ast.walk(stmt): if isinstance(sub, ast.Name): found.add(sub.id) - elif isinstance(sub, ast.AnnAssign): - pass - return found + return found - local def import_lines(names: set[str], imports: Imports, shared: dict[str, str]) -> tuple[list[str], list[str]]: @@ -351,7 +360,14 @@ def extract_module_defs(args: argparse.Namespace) -> None: for node in nodes: names |= names_in([node]) provided = {n for node in nodes for n in _defined_names(node)} - runtime, missing = import_lines(names - provided, imports, {}) + local = { + (alias.asname or alias.name).split(".")[0] + for node in nodes + for sub in ast.walk(node) + if isinstance(sub, (ast.Import, ast.ImportFrom)) + for alias in sub.names + } + runtime, missing = import_lines(names - provided - local, imports, {}) if missing: sys.exit(f"these definitions use other module-level names; move them too: {missing}") # Adjacent definitions (only blank or comment lines between them) move as From a3196fae4ac84bfca54030d21bcddd5953c6a329 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 00:53:40 -0400 Subject: [PATCH 8/9] refactor(flow): move seven replayer concerns into runtime/replay (move only) Qualification fault injection, consequential delivery, effect checks and read-back, the API tier, acting by kind, settling and confirming move verbatim into mixins (107 units; verify_moved_code finds no edits). replayer.py goes from 18,300 to 10,926 lines. Left in place because open pull requests change them: identity checks (#1518, #1504), _resolve_step (#1504), postconditions and step evidence (#1516, #1514), the starting-state settle and qualification boundary (#1449), and the run and step drivers. Two methods that name Replayer stay too. Two tests patched module globals the moved code now reads from its new module, and patch them there as well: resolve (scroll readiness is in replay.acting, interstitials in replay.settling) and _DIALOG_WAIT_S (replay.acting). Checked: strict mypy on runtime/core, runtime/replay and replayer.py; package-wide mypy; both import contracts; 3,149 tests across the 146 files that import the replayer plus the real-browser e2e files. Co-Authored-By: Claude Opus 5.5 --- .../openadapt_flow/runtime/replay/acting.py | 1682 +++ .../openadapt_flow/runtime/replay/api_tier.py | 1261 ++ .../runtime/replay/confirming.py | 1053 ++ .../openadapt_flow/runtime/replay/delivery.py | 1179 ++ .../openadapt_flow/runtime/replay/effects.py | 961 ++ .../runtime/replay/qualification_faults.py | 405 + .../openadapt_flow/runtime/replay/settling.py | 530 + .../flow/openadapt_flow/runtime/replayer.py | 11714 ++++------------ .../tests/test_multi_surface_fail_closed.py | 8 +- packages/flow/tests/test_replayer.py | 11 +- 10 files changed, 9809 insertions(+), 8995 deletions(-) create mode 100644 packages/flow/openadapt_flow/runtime/replay/acting.py create mode 100644 packages/flow/openadapt_flow/runtime/replay/api_tier.py create mode 100644 packages/flow/openadapt_flow/runtime/replay/confirming.py create mode 100644 packages/flow/openadapt_flow/runtime/replay/delivery.py create mode 100644 packages/flow/openadapt_flow/runtime/replay/effects.py create mode 100644 packages/flow/openadapt_flow/runtime/replay/qualification_faults.py create mode 100644 packages/flow/openadapt_flow/runtime/replay/settling.py diff --git a/packages/flow/openadapt_flow/runtime/replay/acting.py b/packages/flow/openadapt_flow/runtime/replay/acting.py new file mode 100644 index 000000000..494a9f623 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/acting.py @@ -0,0 +1,1682 @@ +# mypy: disable-error-code="empty-body" +"""Acting by kind: deliver clicks, drags, typing, selections, scrolls and dialog answers. + +Moved verbatim from ``openadapt_flow.runtime.replayer.Replayer`` by +scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1). +``_ActingHost`` declares, for mypy only, what this code reads from the +Replayer it is mixed into. +""" + +from __future__ import annotations + +import hashlib +import math +import os +import time +from typing import TYPE_CHECKING, Any, Callable, Optional, cast # noqa: F401 + +from openadapt_flow.backend import ( + GuardedCoordinateActionBackend, + GuardedDragActionBackend, + GuardedKeyboardActionBackend, + GuardedRemotePointerActionBackend, + GuardedSelectOptionBackend, + RemoteActuationBackend, + RichPointerActionBackend, + SelectOptionBackend, +) +from openadapt_flow.ir import ActionKind, Point, Predicate, PredicateKind, Region +from openadapt_flow.privacy import scrub_text as _scrub_phi +from openadapt_flow.runtime.authorization import runtime_param_text +from openadapt_flow.runtime.core import ACTION_EXECUTORS, ActionContext +from openadapt_flow.runtime.replay._shared import ( + _DIALOG_WAIT_S, + _DIRECT_TARGET_RUNGS, + _SCROLL_HANDOFF_ACTIONS, + SCROLL_BUDGET_FACTOR, + _frame_viewport, + _scroll_target_positioned, + secret_env_var, +) +from openadapt_flow.runtime.resolver import resolve +from openadapt_flow.runtime.step_timing import timed_span +from openadapt_flow.vision.ocr import OcrResolutionRefused + +if TYPE_CHECKING: + from pathlib import Path + from typing import Mapping, Union + + from openadapt_flow.backend import Backend + from openadapt_flow.ir import ( + Anchor, + IdentityCheck, + Resolution, + Step, + StepResult, + Workflow, + ) + from openadapt_flow.qualification_faults import QualificationFaultDriver + from openadapt_flow.runtime.authorization import ( + GovernedRunAuthorization, + RuntimeParamScalar, + ) + from openadapt_flow.runtime.replay._shared import _GraphStepContext + + +class _ActingHost: + _asset_bytes: Any + + def _authorize_delivery( + self, + workflow: Workflow, + params: Mapping[str, RuntimeParamScalar], + step: Step, + result: StepResult, + ) -> Optional[str]: ... + + _cancel_guarded_coordinate: Any + _cancel_guarded_keyboard: Any + _deliver_backend_call: Any + _delivery_authorization_refusal: Any + + @staticmethod + def _describe_predicate(pred: Predicate) -> str: ... + + _field_reads: dict[str, Any] + _governed_asset_mutation: Optional[str] + _last_click_point: Optional[Point] + _last_click_region: Optional[Region] + + @staticmethod + def _next_anchored_step(workflow: Workflow, step_index: int) -> Optional[Step]: ... + def _pixel_recheck( + self, + step: Step, + resolution: Resolution, + checked_png: bytes, + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + bundle_dir: Path, + result: StepResult, + run_dir: Path, + ) -> Union[Resolution, str]: ... + def _predicate_holds( + self, + pred: Predicate, + frame_png: bytes, + bundle_dir: Path, + params: Mapping[str, RuntimeParamScalar], + *, + workflow: Optional[Workflow] = None, + ) -> bool: ... + + _read_clipboard: Optional[Any] + + def _read_gate_refusal(self, step: Step, result: StepResult) -> Optional[str]: ... + + _require_qualification_environment_current: Any + _requires_atomic_identity_keyboard: Any + _requires_atomic_identity_pointer: Any + + def _resolve_drag_end( + self, step: Step, screen_png: bytes, bundle_dir: Path, workflow: Workflow + ) -> tuple[Optional[Resolution], Optional[Region], Optional[str]]: ... + def _retain_visual_resolution_evidence( + self, + *, + workflow: Workflow, + anchor: Anchor, + resolution: Resolution, + matched_region: Region, + frame_png: bytes, + bundle_dir: Path, + run_dir: Path, + allow_target_ocr: bool = True, + ) -> Resolution: ... + + _revalidate_consequential_actuation: Any + _run_id: str + _scroll_missed_target: Optional[tuple[str, str]] + _selection_contract_error: Any + + @staticmethod + def _selection_region_compatible( + qualified: Region, live: Optional[Region] + ) -> bool: ... + @staticmethod + def _selection_target_continuous( + qualified: Region, focused: Point, refreshed: Point + ) -> bool: ... + + _step_is_consequential: Any + _step_needs_consequential_revalidation: Any + _step_requires_remote_guarded_actuation: Any + + def _text_value_at(self, field_point: Optional[Point]) -> Optional[str]: ... + + _translate_recorded_region: Any + _typed_remote_receipt_required: Any + + def _verify_identity( + self, + step: Step, + resolution: Resolution, + before_png: bytes, + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + bundle_dir: Optional[Path] = None, + ) -> IdentityCheck: ... + def _verify_typed_input( + self, + step: Step, + text: str, + field_point: Optional[Point], + baseline_png: bytes, + result: StepResult, + *, + workflow: Workflow, + params: Mapping[str, RuntimeParamScalar], + bundle_dir: Path, + run_dir: Path, + resolution: Optional[Resolution], + field_region: Optional[Region] = None, + baseline_field_value: Optional[str] = None, + allow_masked: bool = False, + allow_retry: bool = True, + ) -> Optional[str]: ... + + backend: Backend + governed_authorization: Optional[GovernedRunAuthorization] + poll_interval_s: float + qualification_fault_driver: Optional[QualificationFaultDriver] + settle_readiness_timeout_s: float + use_structural: bool + vision: Any + + +if TYPE_CHECKING: + _Base = _ActingHost +else: + _Base = object + + +class Acting(_Base): + """Acting by kind: deliver clicks, drags, typing, selections, scrolls and dialog answers.""" + + def _act_read( + self, + step: Step, + resolution: Optional[Resolution], + *, + workflow: Workflow, + params: Mapping[str, RuntimeParamScalar], + before_png: bytes, + result: StepResult, + ) -> Optional[str]: + """READ: read one field into ``result.field_read``. Zero model calls: + OCR and the clipboard are deterministic channels.""" + + from openadapt_flow.records.read import read_field + from openadapt_flow.records.schema import SealRef + + spec = step.read + assert spec is not None + region = self._translate_recorded_region(step, resolution, spec.region) + clipboard = self._read_clipboard + if spec.proof.double_read and clipboard is None: + result.failure_category = "runtime_failure" + return ( + f"Step '{step.id}' ({step.intent}) asks for a double read but " + "this run has no clipboard channel" + ) + if clipboard is not None: + # Focusing and copying a field is input delivery, so it passes the + # same authorization gate as any other input. + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + result.delivery_attempted = True + seal = SealRef(run_id=self._run_id) if spec.proof.seal else None + try: + field_read = read_field( + spec.field, + self.backend, + region, + proof=spec.proof, + clipboard=clipboard, + copy_keys=tuple(spec.copy_keys), + ocr=self.vision.ocr, + step_id=step.id, + seal=seal, + frame_png=before_png, + ) + except Exception as exc: # noqa: BLE001 - a read failure is a halt + result.failure_category = "runtime_failure" + return f"Step '{step.id}' ({step.intent}) could not read: {exc}" + result.field_read = field_read + result.actuation = "read" + self._field_reads[spec.field.name] = field_read + if field_read.needs_review and spec.on_review == "halt": + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) read a value that needs " + f"review ({field_read.review_reason}). Halting for a person" + ) + return None + + @timed_span("actuate") + def _act( + self, + step: Step, + resolution: Optional[Resolution], + params: Mapping[str, RuntimeParamScalar], + **kwargs: Any, + ) -> Optional[str]: + """Perform the step's action. + + A click recorded as closing its own popup (``Step.closes_surface``) + arms the page backend's one-shot expectation first + (:class:`~openadapt_flow.backend.PageCloseExpectationBackend`), so a + "page closed" report from that click counts as delivery once the page + is confirmed closed. Every other step leaves such a report uncertain. + """ + + expect = ( + getattr(self.backend, "expect_page_close", None) + if step.closes_surface + else None + ) + if not callable(expect): + return self._act_answering_dialog(step, resolution, params, **kwargs) + backend = self.backend + expect() + try: + return self._act_answering_dialog(step, resolution, params, **kwargs) + finally: + cancel = getattr(backend, "cancel_page_close", None) + if callable(cancel): + cancel() + + def _act_answering_dialog( + self, + step: Step, + resolution: Optional[Resolution], + params: Mapping[str, RuntimeParamScalar], + **kwargs: Any, + ) -> Optional[str]: + """Perform the step's action, answering the dialog it raises, if any. + + A step with a recorded ``dialog`` arms a one-shot answer on the + browser backend before its input edge. The dialog's message is its + identity: a different message is dismissed, never accepted, and the + step halts; so does a dialog that never appears. A backend that cannot + see browser dialogs refuses the step before any input + (docs/MULTI_SURFACE.md, "Dialogs"). Errors name a code + (``dialog_missing``, ``dialog_mismatch``), never the dialog's text: + the error lands in report.json and REPORT.md, and a live dialog can + show application data. + """ + + dialog = step.dialog + if dialog is None: + return self._act_without_dialog(step, resolution, params, **kwargs) + arm = getattr(self.backend, "arm_dialog", None) + take = getattr(self.backend, "take_dialogs", None) + if not callable(arm) or not callable(take): + return ( + f"Step '{step.id}' ({step.intent}) answers a browser dialog, but " + "the active backend cannot see browser dialogs; run aborted" + ) + self._arm_dialog(arm, dialog) + try: + error = self._act_without_dialog(step, resolution, params, **kwargs) + finally: + seen = take(0.0) + if error is not None: + return error + if not seen: + seen = self._wait_for_dialog(take, dialog) + if not seen: + return ( + f"Step '{step.id}' ({step.intent}): the recorded browser dialog " + "did not appear (dialog_missing); run aborted" + ) + mismatched = [entry for entry in seen if not entry.get("matched")] + if mismatched: + return ( + f"Step '{step.id}' ({step.intent}): a browser dialog did not " + "show the recorded message (dialog_mismatch); it was " + "dismissed, not answered; run aborted" + ) + return None + + @staticmethod + def _arm_dialog(arm: Callable[..., Any], dialog: Any) -> None: + if dialog.message_pattern is not None: + arm(dialog.response, None, message_pattern=dialog.message_pattern) + else: + arm(dialog.response, dialog.message) + + @timed_span("postcondition") + def _wait_for_dialog(self, take: Callable[..., Any], dialog: Any) -> list[Any]: + """Wait briefly for an armed dialog that opens after the input edge.""" + + arm = getattr(self.backend, "arm_dialog", None) + if callable(arm): + self._arm_dialog(arm, dialog) + seen = take(_DIALOG_WAIT_S) + return list(seen) if isinstance(seen, list) else [] + + def _act_without_dialog( + self, + step: Step, + resolution: Optional[Resolution], + params: Mapping[str, RuntimeParamScalar], + *, + workflow: Workflow, + step_index: int, + bundle_dir: Path, + before_png: bytes, + result: StepResult, + run_dir: Path, + graph_ctx: Optional["_GraphStepContext"] = None, + ) -> Optional[str]: + """Perform the step's action through the backend. + + Returns: + An error string (no action performed / partial) or None. + """ + if step.read_gate is not None: + refusal = self._read_gate_refusal(step, result) + if refusal is not None: + return refusal + if step.action is ActionKind.READ: + return self._act_read( + step, + resolution, + workflow=workflow, + params=params, + before_png=before_png, + result=result, + ) + if step.action in (ActionKind.CLICK, ActionKind.DOUBLE_CLICK): + assert resolution is not None # guaranteed by _resolve_step + x, y = resolution.point + native_act = getattr(self.backend, "act_structural", None) + requires_atomic_identity = self._requires_atomic_identity_pointer( + step, workflow + ) + if ( + resolution.rung == "structural" + and resolution.structural_handle is not None + and step.anchor is not None + and step.anchor.structural is not None + and callable(native_act) + ): + if ( + requires_atomic_identity + and resolution.structural_handle.target_fingerprint is None + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "identity-gated click, but its structural target has no " + "fresh actuation fingerprint — refusing raw coordinate " + "delivery; run aborted" + ) + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + structural_locator = step.anchor.structural + structural_handle = resolution.structural_handle + structural_act = cast(Callable[..., Any], native_act) + delivery_receipt = self._deliver_backend_call( + result, + lambda: structural_act( + structural_locator, + structural_handle, + double=step.action is ActionKind.DOUBLE_CLICK, + ), + ) + result.delivery_receipt = delivery_receipt + result.actuation = "uia" if delivery_receipt.native else "dom" + else: + remote_consequential = self._step_requires_remote_guarded_actuation( + step, workflow + ) + typed_remote = isinstance( + self.backend, + GuardedRemotePointerActionBackend, + ) + if ( + remote_consequential + and not typed_remote + and self._typed_remote_receipt_required() + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "remote click, but this backend cannot bind its exact " + "fresh frame and target to delivery; run aborted" + ) + if remote_consequential: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + if typed_remote: + remote_pointer = cast( + GuardedRemotePointerActionBackend, self.backend + ) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: remote_pointer.click_guarded( + x, + y, + expected_frame_sha256=hashlib.sha256( + before_png + ).hexdigest(), + double=step.action is ActionKind.DOUBLE_CLICK, + ), + ) + result.actuation = "remote_guarded" + else: + # See ``_typed_remote_receipt_required``: the exact + # fresh frame is already bound by the backend's own + # one-shot lease, armed immediately above by + # ``_revalidate_consequential_actuation``. + self._deliver_backend_call( + result, + lambda: self.backend.click( + x, + y, + double=step.action is ActionKind.DOUBLE_CLICK, + ), + ) + elif requires_atomic_identity: + if isinstance(self.backend, GuardedCoordinateActionBackend): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + coordinate_backend = cast( + GuardedCoordinateActionBackend, self.backend + ) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: coordinate_backend.act_guarded_coordinate( + x, + y, + expected_frame_sha256=hashlib.sha256( + before_png + ).hexdigest(), + double=step.action is ActionKind.DOUBLE_CLICK, + ), + ) + result.actuation = "guarded_coordinate" + else: + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "identity-gated click, but this backend cannot bind " + "identity verification to the same actuation operation " + "— refusing raw coordinate delivery; run aborted" + ) + else: + rechecked = self._pixel_recheck( + step, + resolution, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir, + ) + if isinstance(rechecked, str): + return rechecked + resolution = rechecked + x, y = resolution.point + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: self.backend.click( + x, + y, + double=step.action is ActionKind.DOUBLE_CLICK, + ), + ) + self._last_click_point = (x, y) + # Only a STRUCTURAL handle reports the element's own bounds. A + # visual rung's ``matched_region`` is where the anchor TEMPLATE + # matched, which is not the field: narrowing a following TYPE's + # verification window to it hides the typed value and false-halts + # a healthy run. + self._last_click_region = ( + resolution.structural_handle.region + if resolution.rung == "structural" + and resolution.structural_handle is not None + else None + ) + return None + + if step.action is ActionKind.RIGHT_CLICK: + assert resolution is not None # guaranteed by _resolve_step + x, y = resolution.point + requires_atomic_identity = self._requires_atomic_identity_pointer( + step, workflow + ) + remote_consequential = self._step_requires_remote_guarded_actuation( + step, workflow + ) + if remote_consequential and not isinstance( + self.backend, + GuardedRemotePointerActionBackend, + ): + if self._typed_remote_receipt_required(): + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "remote right click, but this backend cannot bind its " + "exact fresh frame and target to delivery; run aborted" + ) + # See ``_typed_remote_receipt_required``: the exact fresh frame + # is already bound by the backend's own one-shot lease, and its + # first input edge consumes it. + if not isinstance(self.backend, RichPointerActionBackend): + return ( + f"Step '{step.id}' ({step.intent}) requires a right " + "click, but this backend has no bounded right-click " + "operation" + ) + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: cast(RichPointerActionBackend, self.backend).right_click( + x, y + ), + ) + return None + if remote_consequential: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + remote_pointer = cast(GuardedRemotePointerActionBackend, self.backend) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: remote_pointer.right_click_guarded( + x, + y, + expected_frame_sha256=hashlib.sha256(before_png).hexdigest(), + ), + ) + result.actuation = "remote_guarded" + elif requires_atomic_identity: + if isinstance(self.backend, GuardedCoordinateActionBackend): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + coordinate_backend = cast( + GuardedCoordinateActionBackend, self.backend + ) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: coordinate_backend.act_guarded_coordinate( + x, + y, + expected_frame_sha256=hashlib.sha256( + before_png + ).hexdigest(), + button="right", + ), + ) + result.actuation = "guarded_coordinate" + else: + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "identity-gated right click, but this backend cannot bind " + "identity verification to delivery; run aborted" + ) + elif isinstance(self.backend, RichPointerActionBackend): + rechecked = self._pixel_recheck( + step, + resolution, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir, + ) + if isinstance(rechecked, str): + return rechecked + x, y = rechecked.point + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: cast(RichPointerActionBackend, self.backend).right_click( + x, y + ), + ) + else: + return ( + f"Step '{step.id}' ({step.intent}) requires a right click, " + "but this backend does not implement it" + ) + return None + + if step.action is ActionKind.DRAG: + assert resolution is not None # guaranteed by _resolve_step + drag_end, drag_end_region, drag_error = self._resolve_drag_end( + step, + before_png, + bundle_dir, + workflow, + ) + if ( + drag_end is not None + and drag_end_region is not None + and step.drag_end_anchor is not None + ): + drag_end = self._retain_visual_resolution_evidence( + workflow=workflow, + anchor=step.drag_end_anchor, + resolution=drag_end, + matched_region=drag_end_region, + frame_png=before_png, + bundle_dir=bundle_dir, + run_dir=run_dir, + ) + result.drag_end_resolution = drag_end + if drag_error is not None: + self._cancel_guarded_coordinate() + return drag_error + assert drag_end is not None + x, y = resolution.point + end_x, end_y = drag_end.point + requires_atomic_identity = self._requires_atomic_identity_pointer( + step, workflow + ) + structural_drag = getattr(self.backend, "drag_structural_guarded", None) + if ( + resolution.rung == "structural" + and drag_end.rung == "structural" + and resolution.structural_handle is not None + and drag_end.structural_handle is not None + and step.anchor is not None + and step.anchor.structural is not None + and step.drag_end_anchor is not None + and step.drag_end_anchor.structural is not None + and callable(structural_drag) + ): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + source_locator = step.anchor.structural + source_handle = resolution.structural_handle + destination_locator = step.drag_end_anchor.structural + destination_handle = drag_end.structural_handle + structural_drag_act = cast(Callable[..., Any], structural_drag) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: structural_drag_act( + source_locator, + source_handle, + destination_locator, + destination_handle, + ), + ) + result.actuation = "dom" + elif self._step_requires_remote_guarded_actuation(step, workflow) and ( + isinstance(self.backend, GuardedRemotePointerActionBackend) + or self._typed_remote_receipt_required() + ): + if not isinstance( + self.backend, + GuardedRemotePointerActionBackend, + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "remote drag, but this backend cannot bind both exact " + "fresh-frame endpoints to delivery; run aborted" + ) + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + remote_pointer = cast(GuardedRemotePointerActionBackend, self.backend) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: remote_pointer.drag_guarded( + x, + y, + end_x, + end_y, + expected_frame_sha256=hashlib.sha256(before_png).hexdigest(), + ), + ) + result.actuation = "remote_guarded" + elif requires_atomic_identity and not isinstance( + self.backend, RemoteActuationBackend + ): + if not isinstance(self.backend, GuardedDragActionBackend): + self._cancel_guarded_coordinate() + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "drag, but this backend cannot bind both freshly " + "resolved endpoints to delivery; run aborted" + ) + # Pixel-only source and destination evidence were resolved + # against ``before_png``. Drop any unused structural tokens, + # then bind the source coordinate and require that exact frame + # immediately before mouse-down. Fully structural browser + # drags take the stronger two-element lease above. + cancel_structural = getattr( + self.backend, + "cancel_pending_structural_guards", + None, + ) + if callable(cancel_structural): + cancel_structural() + if not isinstance(self.backend, GuardedCoordinateActionBackend): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "drag, but this backend cannot bind its freshly " + "resolved source to delivery; run aborted" + ) + try: + self.backend.arm_guarded_coordinate(x, y) + except Exception as exc: # noqa: BLE001 - backend boundary + result.safety_halt = True + result.failure_category = "safety_halt" + detail = _scrub_phi(str(exc)) or type(exc).__name__ + return ( + f"Step '{step.id}' ({step.intent}) could not bind its " + f"freshly resolved drag source: {detail}; run aborted" + ) + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + drag_backend = cast(GuardedDragActionBackend, self.backend) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: drag_backend.drag_guarded( + x, + y, + end_x, + end_y, + expected_frame_sha256=hashlib.sha256(before_png).hexdigest(), + ), + ) + result.actuation = "guarded_coordinate" + elif isinstance(self.backend, RichPointerActionBackend): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: cast(RichPointerActionBackend, self.backend).drag( + x, y, end_x, end_y + ), + ) + else: + return ( + f"Step '{step.id}' ({step.intent}) requires a drag, but " + "this backend does not implement it" + ) + return None + + if step.action in (ActionKind.TYPE, ActionKind.SELECT_OPTION): + if step.secret: + # Secret value is never in the bundle/params: inject it from + # the environment, failing fast with an actionable message. + env_var = getattr(self, "secret_env_overrides", {}).get( + step.param or "" + ) or secret_env_var(step.param or "") + text = os.environ.get(env_var, "") + if not step.param: + return ( + f"Step '{step.id}' ({step.intent}) is marked secret " + "but names no parameter" + ) + if not text: + return ( + f"Step '{step.id}' ({step.intent}) requires secret " + f"parameter '{step.param}', but the environment " + f"variable {env_var} is not set — export it with the " + "secret value and re-run" + ) + elif step.param is not None: + if step.param not in params: + return ( + f"Step '{step.id}' ({step.intent}) requires parameter " + f"'{step.param}' but it was not provided" + ) + text = runtime_param_text(params[step.param]) + elif step.text is not None: + text = step.text + else: + return ( + f"Step '{step.id}' ({step.intent}) is a TYPE step with " + "neither text nor param" + ) + identity_atomic_keyboard = self._requires_atomic_identity_keyboard( + step, workflow + ) + requires_atomic_keyboard = ( + identity_atomic_keyboard or self.qualification_fault_driver is not None + ) + if ( + requires_atomic_keyboard + and not isinstance(self.backend, RemoteActuationBackend) + and not isinstance(self.backend, GuardedKeyboardActionBackend) + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a qualification or " + "identity-gated keyboard action, but this backend cannot " + "bind the verified focus and execution context to the same " + "input operation — refusing unguarded keyboard delivery; " + "run aborted" + ) + if step.selection_commit_key is not None: + selection_contract_error = self._selection_contract_error( + step, workflow + ) + if selection_contract_error is not None: + result.safety_halt = True + result.failure_category = "safety_halt" + return selection_contract_error + # The field point: this step's own focusing click (anchored + # TYPE), or the immediately preceding step's click point (the + # recorder's click-to-focus-then-type pattern). When focus was + # moved some other way (Tab between fields), there is no known + # field point — verification diffs the whole frame and the + # retry cannot re-click. + field_point: Optional[Point] = None + field_region: Optional[Region] = None + if resolution is not None: + # Anchored TYPE: click to focus the field first. + x, y = resolution.point + native_act = getattr(self.backend, "act_structural", None) + requires_atomic_identity = self._requires_atomic_identity_pointer( + step, workflow + ) + if ( + resolution.rung == "structural" + and resolution.structural_handle is not None + and step.anchor is not None + and step.anchor.structural is not None + and callable(native_act) + ): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + structural_locator = step.anchor.structural + structural_handle = resolution.structural_handle + structural_act = cast(Callable[..., Any], native_act) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: structural_act( + structural_locator, + structural_handle, + ), + ) + result.actuation = "uia" + else: + if ( + requires_atomic_identity + and not isinstance(self.backend, RemoteActuationBackend) + and isinstance(self.backend, GuardedCoordinateActionBackend) + ): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + coordinate_backend = cast( + GuardedCoordinateActionBackend, self.backend + ) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: coordinate_backend.act_guarded_coordinate( + x, + y, + expected_frame_sha256=hashlib.sha256( + before_png + ).hexdigest(), + ), + ) + result.actuation = "guarded_coordinate" + elif requires_atomic_identity and not isinstance( + self.backend, RemoteActuationBackend + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "identity-gated focusing click, but this backend " + "cannot bind target and context verification to the " + "same coordinate operation — refusing raw coordinate " + "delivery; run aborted" + ) + else: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, lambda: self.backend.click(x, y) + ) + field_point = (x, y) + # See ``_last_click_region``: a visual rung's matched region is + # the anchor template's live position, not the field's bounds. + # SELECT_OPTION does not need it here — it always revalidates + # below and consumes that pass's fresh matched region. + field_region = ( + resolution.structural_handle.region + if resolution.rung == "structural" + and resolution.structural_handle is not None + else None + ) + # Fresh baseline AFTER the focusing click so its own focus ring + # never counts as "input landed". A consequential remote TYPE + # is a second input gesture after that click, so reacquire and + # re-run target/identity before typing; the focusing click + # consumed the first one-shot remote lease. + if self._step_needs_consequential_revalidation(step, workflow): + focused_field_point = field_point + ( + refreshed, + refreshed_region, + before_png, + remote_error, + ) = self._revalidate_consequential_actuation( + step, + resolution, + field_region, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir=run_dir, + arm_keyboard=True, + ) + if remote_error is not None: + return remote_error + if refreshed is not None: + if ( + step.action is ActionKind.SELECT_OPTION + and focused_field_point is not None + and step.anchor is not None + and not self._selection_target_continuous( + step.anchor.region, + focused_field_point, + refreshed.point, + ) + ): + self._cancel_guarded_keyboard() + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) re-resolved " + "to a different field after focus; refusing " + "option selection" + ) + assert step.anchor is not None + if refreshed_region is not None: + refreshed = self._retain_visual_resolution_evidence( + workflow=workflow, + anchor=step.anchor, + resolution=refreshed, + matched_region=refreshed_region, + frame_png=before_png, + bundle_dir=bundle_dir, + run_dir=run_dir, + allow_target_ocr=False, + ) + resolution = refreshed + result.resolution = refreshed + field_point = refreshed.point + # A visual match region is the retained template crop. It + # is not the editable control's bounds. Treating that crop + # as the typed-value readback region can exclude text that + # renders at the left edge of a wide field. Keep exact + # structural bounds when the backend supplies them; + # otherwise use the point-centred visual readback window. + if step.action is ActionKind.SELECT_OPTION: + # Composite selection binds its type-ahead and commit + # to the exact live resolved selection region. + field_region = refreshed_region + else: + field_region = ( + refreshed.structural_handle.region + if ( + refreshed is not None + and refreshed.rung == "structural" + and refreshed.structural_handle is not None + ) + else None + ) + else: + before_png = self.backend.screenshot() + elif self._prev_was_click(workflow, step_index, graph_ctx): + field_point = self._last_click_point + field_region = self._last_click_region + baseline_field_value = self._text_value_at(field_point) + if baseline_field_value is None: + baseline_field_value = self._text_value_at(None) + if step.selection_commit_key is not None: + selection_commit_key = step.selection_commit_key + assert step.selection_region is not None + assert isinstance(self.backend, SelectOptionBackend) + assert step.anchor is not None + if not self._selection_region_compatible( + step.anchor.region, field_region + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) re-resolved to " + "incompatible live field geometry; refusing option " + "selection" + ) + assert field_region is not None + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + if isinstance(self.backend, GuardedSelectOptionBackend): + assert field_point is not None + guarded_select = cast(GuardedSelectOptionBackend, self.backend) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: guarded_select.select_option_guarded( + text, + selection_commit_key, + target_point=field_point, + expected_frame_sha256=hashlib.sha256( + before_png + ).hexdigest(), + ), + ) + result.actuation = ( + "remote_guarded" + if isinstance(self.backend, RemoteActuationBackend) + else "guarded_keyboard" + ) + else: + select_backend = cast(SelectOptionBackend, self.backend) + self._deliver_backend_call( + result, + lambda: select_backend.select_option( + text, selection_commit_key + ), + ) + return None + if ( + self.qualification_fault_driver is not None + and not identity_atomic_keyboard + and isinstance(self.backend, GuardedKeyboardActionBackend) + ): + if field_point is None: + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) has no focused field " + "point for exact qualification delivery; run aborted" + ) + self._require_qualification_environment_current() + cast(GuardedKeyboardActionBackend, self.backend).arm_guarded_keyboard( + *field_point + ) + guarded_type = ( + requires_atomic_keyboard + and not isinstance(self.backend, RemoteActuationBackend) + and isinstance(self.backend, GuardedKeyboardActionBackend) + ) + if guarded_type: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: cast( + GuardedKeyboardActionBackend, self.backend + ).type_text_guarded( + text, + expected_frame_sha256=hashlib.sha256(before_png).hexdigest(), + ), + ) + result.actuation = "guarded_keyboard" + elif requires_atomic_keyboard and not isinstance( + self.backend, RemoteActuationBackend + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) lost its guarded keyboard " + "capability before delivery — refusing unguarded text input; " + "run aborted" + ) + else: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call(result, lambda: self.backend.type_text(text)) + if not text: + return None # nothing typed, nothing to verify + return self._verify_typed_input( + step, + text, + field_point, + before_png, + result, + workflow=workflow, + params=params, + bundle_dir=bundle_dir, + run_dir=run_dir, + resolution=resolution, + field_region=field_region, + baseline_field_value=baseline_field_value, + allow_masked=step.secret, + allow_retry=not guarded_type, + ) + + executor = ACTION_EXECUTORS.get(step.action) + if executor is not None: + # KEY, HOTKEY and WAIT run in the governance-free core; the gates + # it must pass before input come from here. + return executor( + ActionContext( + step=step, + result=result, + backend=self.backend, + before_png=before_png, + authorize=lambda: self._authorize_delivery( + workflow, params, step, result + ), + deliver=self._deliver_backend_call, + requires_atomic_keyboard=lambda: ( + self._requires_atomic_identity_keyboard(step, workflow) + ), + ) + ) + + if step.action is ActionKind.SCROLL: + return self._act_scroll( + step, + workflow=workflow, + step_index=step_index, + bundle_dir=bundle_dir, + run_dir=run_dir, + before_png=before_png, + params=params, + result=result, + graph_ctx=graph_ctx, + ) + + return f"Step '{step.id}' has unsupported action {step.action!r}" + + @staticmethod + def _prev_was_click( + workflow: Workflow, + step_index: int, + graph_ctx: Optional["_GraphStepContext"], + ) -> bool: + """Whether the step executed just before this TYPE step was a click. + + Linear mode reads ``workflow.steps[step_index - 1]`` exactly as before; + program mode reads the interpreter-supplied ``graph_ctx.prev_action`` + (the previously EXECUTED action state -- the graph has no positional + predecessor). The click-to-focus-then-type heuristic is thereby + identical on both paths. + """ + if graph_ctx is not None: + return graph_ctx.prev_action in ( + ActionKind.CLICK, + ActionKind.DOUBLE_CLICK, + ) + return step_index > 0 and workflow.steps[step_index - 1].action in ( + ActionKind.CLICK, + ActionKind.DOUBLE_CLICK, + ) + + @timed_span("resolve") + def _implicit_scroll_target_ready( + self, + step: Step, + frame_png: bytes, + *, + workflow: Workflow, + bundle_dir: Path, + params: Mapping[str, RuntimeParamScalar], + scroll_delta: Optional[tuple[int, int]] = None, + misses: Optional[list[str]] = None, + ) -> bool: + """Resolve an implicit scroll target and honor its armed identity. + + With ``scroll_delta``, the target must also be no further along the + scroll direction than where the demonstration had it (within + :data:`SCROLL_POSITION_SLACK_FRACTION` of the frame). A target that + is visible at the bottom edge but was clicked mid-screen in the + recording still needs the scroll: the step after the click was + recorded with that scroll position (for example, a section that + expands below the clicked header). + + A generic input crop can visually resolve in several places on a long + form. Declaring the target visible from that visual match alone makes + the preceding scroll a false no-op. Structural resolution stays + available, model grounding stays disabled, and an identity-armed target + is ready only when the same pre-action identity gate verifies it. + + When not ready, ``misses`` (if given) gets why: ``unresolved`` (no + direct deterministic rung found the target), ``offscreen`` (the page + reports the recorded element outside the viewport), ``position`` or + ``identity``. + """ + + def miss(reason: str) -> bool: + if misses is not None: + misses.append(reason) + return False + + if step.anchor is None: + return miss("unresolved") + structural = ( + self.backend + if self.use_structural and hasattr(self.backend, "locate_structural") + else None + ) + offscreen = getattr(structural, "structural_offscreen", None) + if ( + callable(offscreen) + and step.anchor.structural is not None + and offscreen(step.anchor.structural) is True + ): + # The page itself shows the unique recorded target exists below + # (or above) the viewport: not ready, keep scrolling. Stricter + # than letting a visual rung find a look-alike on screen, and it + # skips a full-frame OCR search per probe. The control exists, so + # this miss never hands the halt to the click (#1331). + return miss("offscreen") + template_png = self._asset_bytes( + bundle_dir, step.anchor.template, workflow=workflow + ) + try: + resolved = resolve( + step.anchor, + frame_png, + self.vision, + None, # scroll readiness must remain deterministic and model-free + step.intent, + template_png=template_png, + viewport=_frame_viewport(frame_png), + structural=structural, + ) + except OcrResolutionRefused: + # Readiness is a bounded, non-actuating probe. An ambiguous OCR + # candidate means that the target is not ready yet, so the scroll + # loop must continue. The later target action runs the full + # resolver again and still halts on the same ambiguity. + return miss("unresolved") + if resolved is None: + return miss("unresolved") + resolution, _matched_region = resolved + # Geometry estimates a point from surrounding landmarks; it does not + # prove that the target itself is visible. In particular, a fixed + # header can project the recorded point while the form row is still + # off-screen, turning the preceding scroll into a false no-op. Scroll + # readiness therefore requires direct deterministic target evidence. + # Model grounding is disabled above and future weaker rungs must opt in + # deliberately rather than silently acquiring readiness authority. + if resolution.rung not in _DIRECT_TARGET_RUNGS: + return miss("unresolved") + if scroll_delta is not None and not _scroll_target_positioned( + step.anchor.click_point, resolution.point, scroll_delta, frame_png + ): + return miss("position") + if not step.identity_armed: + return True + check = self._verify_identity( + step, resolution, frame_png, params, workflow, bundle_dir + ) + return check.status == "verified" or miss("identity") + + def _act_scroll( + self, + step: Step, + *, + workflow: Workflow, + step_index: int, + bundle_dir: Path, + run_dir: Path, + before_png: bytes, + params: Mapping[str, RuntimeParamScalar], + result: StepResult, + graph_ctx: Optional["_GraphStepContext"] = None, + ) -> Optional[str]: + """Execute a SCROLL step as a closed loop on a wait_until predicate. + + A recorded scroll's purpose is to bring the next target into view, so + the step scrolls by its recorded delta until a READINESS predicate holds + on a settled frame — not a fixed number of times. That predicate is the + step's own ``wait_until`` when set; otherwise it defaults to + ``ANCHOR_RESOLVES`` on the NEXT anchored step's anchor — today's closed + loop, now expressed as the first concrete ``wait_until`` predicate + rather than a special case (RFC §6). The step probes BEFORE scrolling + (a preceding SCROLL step may already have brought the target into view, + making this one a no-op) and stops as soon as the predicate holds. + + The loop is bounded: this step may scroll at most + ``SCROLL_BUDGET_FACTOR`` times its own recorded distance. On budget + exhaustion the step fails loudly — unless the immediately following + step is another SCROLL step, which inherits the loop (so a recorded + run of N scrolls shares a combined ~2.5x budget). + + Falls back to the fixed recorded delta (open-loop, one gesture) when no + readiness predicate exists (no ``wait_until`` and no later anchor) or + the recorded delta is zero. Predicate probes never call the grounder: + closed-loop scrolling must stay model-free. + + Returns: + An error string on budget exhaustion (see above) or None. + """ + dx = step.scroll_dx or 0 + dy = step.scroll_dy or 0 + # The next anchored step (default scroll stop condition): linear mode + # scans forward in ``workflow.steps``; program mode uses the + # interpreter-supplied successor (the target of this state's + # unconditional transition chain). + next_step = ( + graph_ctx.next_anchored + if graph_ctx is not None + else self._next_anchored_step(workflow, step_index) + ) + # The scroll's readiness is a wait_until predicate: an explicit + # step.wait_until wins; otherwise the next anchor's ANCHOR_RESOLVES. + stop_pred = step.wait_until + implicit_target_readiness = stop_pred is None and next_step is not None + if stop_pred is None and next_step is not None: + stop_pred = Predicate( + kind=PredicateKind.ANCHOR_RESOLVES, + anchor=next_step.anchor, + intent=next_step.intent, + ) + if stop_pred is None or (dx == 0 and dy == 0): + remote_preflight_error = self._prepare_remote_scroll_input(step, result) + if remote_preflight_error is not None: + return remote_preflight_error + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: self.backend.scroll(dx, dy), + ) + return None + + # Until a wheel edge stops moving the page, readiness also requires + # the target at (or past) its demonstrated position. + position_required = [True] + # Why each readiness probe of the next target failed; they decide who + # owns a budget-exhaustion halt (see below). + misses: list[str] = [] + + def readiness_holds(frame_png: bytes) -> bool: + if implicit_target_readiness: + assert next_step is not None + return self._implicit_scroll_target_ready( + next_step, + frame_png, + workflow=workflow, + bundle_dir=bundle_dir, + params=params, + scroll_delta=(dx, dy) if position_required[0] else None, + misses=misses, + ) + return self._predicate_holds( + stop_pred, + frame_png, + bundle_dir, + params, + workflow=workflow, + ) + + holds = readiness_holds(before_png) + if self._governed_asset_mutation is not None: + return self._governed_asset_mutation + if holds: + # Target already in view; nothing to scroll and no input sent. + # Record why, only for the next-target readiness check, so the + # outcome classifier can tell a correctly skipped scroll from an + # undelivered action (execution_profiles.classify_execution_outcome). + if implicit_target_readiness and result.delivery_attempted is not True: + result.scroll_target_ready_before = True + return None + + increment = math.hypot(dx, dy) + budget = SCROLL_BUDGET_FACTOR * increment + scrolled = 0.0 + while scrolled + increment <= budget: + # Readiness callbacks run after the outer step preflight. Bind + # every wheel edge to a new exact remote frame and repeat the + # context/identity checks after those callbacks. This also avoids + # reusing the one-shot lease consumed by the prior wheel edge. + if self._step_needs_consequential_revalidation(step, workflow): + ( + _scroll_resolution, + _scroll_region, + fresh_scroll_frame, + scroll_error, + ) = self._revalidate_consequential_actuation( + step, + None, + None, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir=run_dir, + ) + if scroll_error is not None: + return scroll_error + if readiness_holds(fresh_scroll_frame): + return None + remote_preflight_error = self._prepare_remote_scroll_input(step, result) + if remote_preflight_error is not None: + return remote_preflight_error + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: self.backend.scroll(dx, dy), + ) + scrolled += increment + frame = self._wait_for_scroll_transition_and_settle(before_png) + if not self.vision.pixels_changed(before_png, frame): + # The page did not move (its end): visibility is enough. + position_required[0] = False + before_png = frame + holds = readiness_holds(frame) + if self._governed_asset_mutation is not None: + return self._governed_asset_mutation + if holds: + return None + + following = ( + graph_ctx.following + if graph_ctx is not None + else ( + workflow.steps[step_index + 1] + if step_index + 1 < len(workflow.steps) + else None + ) + ) + if following is not None and following.action is ActionKind.SCROLL: + # The next SCROLL step continues the loop with its own budget. + return None + if ( + graph_ctx is None + and implicit_target_readiness + and next_step is not None + and following is next_step + and next_step.action in _SCROLL_HANDOFF_ACTIONS + and misses + and all(miss == "unresolved" for miss in misses) + and not step.risk_review_required + and not self._step_is_consequential(step, workflow) + ): + # The scroll did its bounded work; what failed is the next click's + # target, which never resolved on any frame (a relabelled or moved + # control). Halting here would name the scroll and hide the + # control that changed. Hand the miss to that click instead: it + # resolves on direct deterministic rungs only and, when they find + # nothing, halts on its own target before any input. A target + # that resolved but sat out of position or failed its identity + # check still halts here, as before. + self._scroll_missed_target = ( + next_step.id, + f"the preceding scroll '{step.id}' used {scrolled:.0f}px of its " + f"{budget:.0f}px budget and the target never came into view; " + "no deterministic rung found it, so no input was sent", + ) + return None + target_desc = ( + f"the anchor of step '{next_step.id}' ({next_step.intent})" + if next_step is not None + else self._describe_predicate(stop_pred) + ) + return ( + f"Step '{step.id}' ({step.intent}): closed-loop scroll exhausted " + f"its budget ({scrolled:.0f}px of {budget:.0f}px allowed, " + f"{SCROLL_BUDGET_FACTOR}x the recorded distance) without " + f"{target_desc} resolving — target never came into view; run aborted" + ) + + @timed_span("settle") + def _wait_for_scroll_transition_and_settle(self, baseline_png: bytes) -> bytes: + """Wait for a delivered scroll to change the frame, then settle it. + + A remote client can return two identical pre-action frames before the + wheel packet reaches the remote application. A generic settle poll can + therefore accept the old screen as stable. Require one visual + transition within the configured readiness window before applying the + normal settled-state check. At a real scroll boundary, return the last + unchanged frame after the bounded wait so the closed loop can continue + or refuse when its action budget is exhausted. + """ + + deadline = time.monotonic() + self.settle_readiness_timeout_s + frame = self.backend.screenshot() + while not self.vision.pixels_changed(baseline_png, frame): + if time.monotonic() >= deadline: + return frame + time.sleep(self.poll_interval_s) + frame = self.backend.screenshot() + return self.vision.wait_settled(self.backend) + + def _prepare_remote_scroll_input( + self, + step: Step, + result: StepResult, + ) -> Optional[str]: + """Bind one remote wheel edge to a fresh, one-use frame lease. + + Target-readiness probes can take longer than a remote backend's frame + lease. A wheel gesture has no coordinate to re-resolve, but it is + still real input into an opaque session. Acquire the remote backend's + exact-content lease after readiness checks and before the final + authorization, environment, and delivery checks. Do not extend the + allowed frame age or reuse the observation from the probe. + """ + + if not isinstance(self.backend, RemoteActuationBackend): + return None + try: + self.backend.acquire_actuation_frame() + except Exception as exc: # noqa: BLE001 - backend boundary must halt + if self.governed_authorization is not None: + result.safety_halt = True + detail = _scrub_phi(str(exc)) or type(exc).__name__ + return ( + "Remote scroll preflight HALTED before input for step " + f"'{step.id}' ({step.intent}): {detail}" + ) + return None diff --git a/packages/flow/openadapt_flow/runtime/replay/api_tier.py b/packages/flow/openadapt_flow/runtime/replay/api_tier.py new file mode 100644 index 000000000..648ab1c6b --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/api_tier.py @@ -0,0 +1,1261 @@ +# mypy: disable-error-code="empty-body" +"""The API tier and web-form spans: carry out a step through the application's API. + +Moved verbatim from ``openadapt_flow.runtime.replayer.Replayer`` by +scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1). +``_ApiTierHost`` declares, for mypy only, what this code reads from the +Replayer it is mixed into. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional # noqa: F401 +from urllib.parse import urlsplit + +from openadapt_flow.ir import ( + ActionDeliveryUncertainty, + ActionKind, + IdentityCheck, + IdentitySignalEvidence, + SafetyRefusalEvidence, + StepResult, +) +from openadapt_flow.qualification_faults import effect_verifier_input_sha256 +from openadapt_flow.runtime.authorization import ( + runtime_param_text, + runtime_params_for_gui, +) +from openadapt_flow.runtime.replay._shared import _WebFormApprovalView + +if TYPE_CHECKING: + from pathlib import Path + from typing import Mapping + + from openadapt_flow.backend import Backend + from openadapt_flow.ir import ApiBinding, ExecutionTargetKind, Step, Workflow + from openadapt_flow.runtime.authorization import ( + GovernedRunAuthorization, + RuntimeParamScalar, + ) + from openadapt_flow.runtime.effects import Effect, EffectState + + +class _ApiTierHost: + def _approval_gated(self) -> bool: ... + + _candidate_binding_refusal: Any + _capture_effect_pre_state: Any + + def _check_postconditions( + self, + step: Step, + frame_png: bytes, + bundle_dir: Path, + start_state: dict[str, Any], + result: Optional[StepResult] = None, + *, + workflow: Optional[Workflow] = None, + ) -> tuple[bool, bytes, list[str]]: ... + + _control_overlay_progress: Any + _defer_readback: Any + _delivery_authorization_refusal: Any + _emit_control_overlay_phase: Any + _keep_undo_prior_state: Any + _may_pipeline_readback: Any + _profile_effect_tier_refusal: Any + _require_qualification_environment_current: Any + _required_effect_pre_state_unreadable: Any + _resolve_effects: Any + + def _run_step(self, step: Step, **kwargs: Any) -> StepResult: ... + def _structural_state(self) -> dict[str, Any]: ... + + _verify_effects: Any + _web_form_approval: dict[int, Any] + _web_form_fallback_reason: Optional[str] + _web_form_form_state: Optional[tuple[str, str]] + _web_form_form_state_set: Any + _web_form_span_active: Any + api_actuator: Optional[Any] + api_steps: bool + api_steps_off_reason: Optional[str] + backend: Backend + deployment_surface: Optional[str] + governed_authorization: Optional[GovernedRunAuthorization] + use_structural: bool + vision: Any + web_form_actuator: Optional[Any] + + +if TYPE_CHECKING: + _Base = _ApiTierHost +else: + _Base = object + + +class ApiTier(_Base): + """The API tier and web-form spans: carry out a step through the application's API.""" + + def _web_form_span_at( + self, workflow: Workflow, step_index: int, *, durable: bool + ) -> Optional[tuple[int, Step, Optional[str]]]: + """The API step whose span starts at ``step_index``, and why it can't run. + + Returns ``(save_index, save_step, reason)``; ``reason`` is None when + the API step may run. None when no API step starts here. + """ + + from openadapt_flow.runtime.actuators.web_form import folded_checks_digest + + first = workflow.steps[step_index].id + for end_index in range(step_index, len(workflow.steps)): + save = workflow.steps[end_index] + binding = save.api_binding + if ( + binding is None + or binding.kind != "web_form" + or binding.web_form is None + or binding.web_form.replaces_steps[0] != first + ): + continue + ids = [s.id for s in workflow.steps[step_index : end_index + 1]] + if not self.api_steps: + reason = self.api_steps_off_reason or "API steps are off for this run" + elif surface_reason := self._web_form_surface_refusal( + workflow, step_index, end_index, binding + ): + reason = surface_reason + elif self._approval_gated() and not callable( + getattr(self._actuator_for(binding), "prepare", None) + ): + # Under an approval mode the person approves the exact request + # the API step sends. An actuator that can't prepare the + # request before sending it can't show it. + reason = ( + "approval mode needs the exact request before the save, and " + "this API actuator can't prepare it, so the GUI steps run" + ) + elif not binding.admitted(): + reason = "the API step is not admitted (openadapt-flow api admit)" + elif binding.web_form.other_writes_seen: + # Proposed before such spans were refused: the steps it would + # replace also wrote, and one request can't stand in for them. + reason = ( + "the steps the API step would replace also sent other writes " + f"({', '.join(binding.web_form.other_writes_seen)})" + ) + elif save.identity_armed and not binding.identity: + # The GUI save checks the right record on screen. The API step + # must check it too, so it needs a right-record contract. + reason = ( + "the save checks the right record on screen, and the API " + "step names no right-record contract (openadapt-flow api " + "admit --identity)" + ) + elif list(binding.effects) != list(save.effects): + reason = ( + "the save step's record check differs from the one admitted " + "with the API step (openadapt-flow api admit)" + ) + elif ids != list(binding.web_form.replaces_steps): + reason = "the program's steps no longer match the API step" + elif ( + folded_checks_digest(workflow.steps[step_index : end_index + 1]) + != binding.web_form.folded_checks_digest + ): + reason = ( + "the steps the API step replaces changed after it was " + "proposed (an action, value, target, check, guard or record " + "check), so the API step no longer stands for them " + "(openadapt-flow api propose)" + ) + elif self.governed_authorization is not None and ( + governed_reason := self._governed_web_form_refusal(workflow, save) + ): + reason = governed_reason + elif self._actuator_for(binding) is None: + reason = "there is no live browser session for the API step" + else: + reason = None + return end_index, save, reason + return None + + def _web_form_surface_refusal( + self, workflow: Workflow, start_index: int, end_index: int, binding: Any + ) -> Optional[str]: + """Why a multi-surface run can't use this API step, or None when it can. + + The API step is bound to its span's surface the way it is bound to the + page in a single-surface run: every step it replaces runs on that one + browser surface, none of them switches, opens or closes a surface, the + surface is the active one, and the request goes to that surface's + session on its declared origin. + """ + + router = getattr(self, "_surface_router", None) + if router is None: + return None + span = workflow.steps[start_index : end_index + 1] + surfaces = {s.surface or router.primary_id for s in span} + if len(surfaces) != 1: + return ( + "the steps the API step replaces run on more than one surface, " + "so the GUI steps run" + ) + if any( + s.action is ActionKind.SWITCH_SURFACE or s.opens or s.closes_surface + for s in span + ): + return ( + "the steps the API step replaces switch, open or close a " + "surface, so the GUI steps run" + ) + (surface_id,) = surfaces + if surface_id != router.active_id: + return "the API step's surface is not the active one, so the GUI steps run" + spec = router.specs[surface_id] + if spec.kind != "web": + return "the API step's surface is not a browser, so the GUI steps run" + actuator = self._actuator_for(binding) + if actuator is None: + return "there is no live browser session for the API step" + from openadapt_flow.runtime.surfaces import url_origin + + if spec.is_page: + expected = spec.origin + else: + seen = router.observations.get(surface_id) + expected = seen.origin if seen is not None else None + actual = url_origin( + getattr(getattr(actuator, "transport", None), "base_url", None) + ) + if expected is not None and actual != url_origin(expected): + return ( + "the API step's session is on another origin than its surface, " + "so the GUI steps run" + ) + return None + + def _governed_web_form_refusal( + self, workflow: Workflow, save: Step + ) -> Optional[str]: + """Why a governed run can't use this API step, or None when it can. + + The run's authorization must have admitted this exact API step. For a + managed (Cloud) dispatch that is Cloud's authorization, whose + ``api_steps`` are inside the dispatch binding digest Cloud signs. + + A qualification case runs the path its contract names for the save: + the API step when that path is ``api``, the GUI steps otherwise. A run + under a production qualification admission uses the API step only + when the qualified effect contract, which the signed admission binds, + names the save's API path; that contract exists only when the + qualification campaign ran the API path's cases. + """ + + authorization = self.governed_authorization + assert authorization is not None + if authorization.qualification_case_id is not None: + if authorization.qualification_case_action_paths.get(save.id) != "api": + return "a qualification case runs the GUI path it names" + elif authorization.production_qualification_admission_id is not None: + if not authorization.effect_requirements(save.id, "api"): + return ( + "the production qualification admission didn't qualify this " + "API step's path, so the qualified GUI path runs" + ) + return authorization.admitted_api_step(save, workflow) + + def _run_web_form_span( + self, + workflow: Workflow, + start_index: int, + end_index: int, + *, + params: Mapping[str, RuntimeParamScalar], + bundle_dir: Path, + run_dir: Path, + new_crops: dict[str, bytes], + ) -> Optional[list[StepResult]]: + """Run the API step for steps ``start_index..end_index``. + + Returns the results (the folded GUI steps, then the save) when the API + step took responsibility: it saved and the record check ran, or it + stopped after sending. Returns None when the save was not sent, with + the reason in ``self._web_form_fallback_reason``; the caller then runs + the GUI steps. + """ + + save = workflow.steps[end_index] + self._web_form_span_active = True + try: + result = self._run_step( + save, + workflow=workflow, + step_index=end_index, + params=params, + bundle_dir=bundle_dir, + run_dir=run_dir, + new_crops=new_crops, + ) + finally: + self._web_form_span_active = False + self._web_form_approval.clear() + if result.api_fallback is not None: + self._web_form_fallback_reason = result.api_fallback + return None + router = getattr(self, "_surface_router", None) + surface = router.active_id if router is not None else None + folded = [ + StepResult( + step_id=s.id, + intent=s.intent, + ok=True, + skipped=True, + actuation="api_folded", + surface=surface, + risk=s.risk, + risk_explanation=s.risk_explanation, + risk_review_required=s.risk_review_required, + delivery_attempted=False, + ) + for s in workflow.steps[start_index:end_index] + ] + return [*folded, result] + + def _lookup_capability_refusal( + self, + workflow: Workflow, + execution_target_kind: Optional[ExecutionTargetKind], + ) -> Optional[str]: + """Refuse a surface that offers fewer lookups than were qualified. + + A workflow qualified with structure lookups (DOM / accessibility) + must not run where only pixels exist, such as an RDP or Citrix window + viewed from outside the session: it passed qualification on evidence + this surface cannot produce. Checked before any action. + """ + + from openadapt_flow.lookup_capabilities import ( + DeploymentSurface, + declared_surface_problem, + lookup_capability_refusal, + qualified_lookup_of, + resolve_deployment_surface, + ) + + if qualified_lookup_of(workflow) is None: + return None + kind = execution_target_kind or workflow.surface + surface: DeploymentSurface + if declared_surface_problem(self.deployment_surface, kind) is not None: + # A declaration that contradicts the backend is read the + # conservative way: the remote session is viewed from outside. + surface = "remote_display" + else: + try: + surface = resolve_deployment_surface(self.deployment_surface, kind) + except ValueError: + surface = "remote_display" + structure_available = ( + surface != "remote_display" + and self.use_structural + and callable(getattr(self.backend, "locate_structural", None)) + ) + return lookup_capability_refusal( + workflow, surface=surface, structure_available=structure_available + ) + + @staticmethod + def _url_path_targets_param(url_template: str, param: str) -> bool: + """Require the identity placeholder as one whole URL path segment.""" + + token = "{" + param + "}" + return token in [ + segment for segment in urlsplit(url_template).path.split("/") if segment + ] + + @staticmethod + def _api_request_pointer_value(binding: ApiBinding, pointer: str) -> Any: + """Resolve one admitted target-bearing pointer without consulting headers.""" + + if pointer.startswith("/url/"): + return binding.url_template + if binding.kind == "web_form": + from openadapt_flow.runtime.actuators.web_form import ( + web_form_pointer_template, + ) + + return web_form_pointer_template(binding, pointer) + root_name, *tokens = pointer.lstrip("/").split("/") + if root_name == "body": + value: Any = binding.body_template + elif root_name == "query": + value = binding.query + else: + return None + for token in tokens: + decoded = token.replace("~1", "/").replace("~0", "~") + if isinstance(value, dict): + if decoded not in value: + return None + value = value[decoded] + elif isinstance(value, list): + try: + index = int(decoded) + except ValueError: + return None + if index < 0 or index >= len(value): + return None + value = value[index] + else: + return None + return value + + @staticmethod + def _api_request_pointer_field_path(pointer: str) -> tuple[str, ...]: + """Return the fully decoded target-field path after the request root.""" + + return tuple( + token.replace("~1", "/").replace("~0", "~") + for token in pointer.lstrip("/").split("/")[1:] + ) + + def _api_identity_refusal( + self, + step: Step, + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + effects: list["Effect"], + result: StepResult, + ) -> Optional[str]: + """Validate an exact qualified identity binding before an API write.""" + + authorization_requires = ( + self.governed_authorization is not None + and self.governed_authorization.requires_verified_identity(step.id) + ) + if step.api_binding is not None and step.api_binding.kind == "web_form": + # A form save whose GUI step checks the right record, or whose + # admission named a right-record parameter, is checked the same + # way in every run, governed or not (its durable checkpoint + # needs the evidence too). + authorization_requires = authorization_requires or bool( + step.identity_armed or step.api_binding.identity + ) + project = workflow.qualification + policy = project.identity_policies.get(step.id) if project is not None else None + if not authorization_requires and policy is None: + return None + + binding = step.api_binding + assert binding is not None + if not binding.identity: + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): the qualified action has no exact " + "api_binding.identity contract — refusing to send the request" + ) + + allowed_keys: Optional[set[str]] = None + required = 1 + if policy is not None and policy.enforcement.value == "signal_quorum": + allowed_keys = {signal.key.value for signal in policy.signals} + required = policy.quorum + + evidence: list[IdentitySignalEvidence] = [] + for identity in binding.identity: + if allowed_keys is not None and identity.key not in allowed_keys: + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): semantic signal {identity.key!r} is not " + "part of the qualified identity policy" + ) + raw_value = params.get(identity.param) + value = runtime_param_text(raw_value) if raw_value is not None else None + if value is None or value == "": + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): required identity parameter " + f"{identity.param!r} is missing" + ) + pointed_templates = [ + self._api_request_pointer_value(binding, pointer) + for pointer in identity.request_pointers + ] + exact_token = "{" + identity.param + "}" + effect_target_path = tuple(identity.effect_field.split(".")) + # A form field's name is the page's, not the record's (form_lname + # versus lname): for a form save the field is bound to the record + # check by sending the same parameter the check selects on. + names_must_match = binding.kind != "web_form" + pointers_are_bound = all( + template is not None + and ( + not names_must_match + or self._api_request_pointer_field_path(pointer) + == effect_target_path + ) + and ( + isinstance(template, str) + and self._url_path_targets_param(template, identity.param) + if pointer.startswith("/url/") + else isinstance(template, str) and template == exact_token + ) + for pointer, template in zip( + identity.request_pointers, pointed_templates, strict=True + ) + ) + if not pointers_are_bound: + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): identity parameter {identity.param!r} is " + "not bound at every declared target-bearing request pointer" + ) + from openadapt_flow.runtime.actuators.web_form import ( + effect_selector_rows, + ) + + effect_exprs = [ + row.get(identity.effect_field) + for effect in effects + for row in effect_selector_rows(effect) + ] + if not any( + expr is not None and expr.param == identity.param + for expr in effect_exprs + ): + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): effect selector {identity.effect_field!r} " + f"is not bound to identity parameter {identity.param!r}" + ) + evidence.append( + IdentitySignalEvidence( + signal=identity.key, + source="api_parameter", + verdict="verified", + evidence_class="api_request_effect_binding", + match="exact", + ) + ) + + if len(evidence) < required: + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): {len(evidence)}/{required} exact qualified " + "identity bindings verified — refusing to send the request" + ) + result.identity = IdentityCheck( + status="verified", + mode="signal_quorum", + coverage=1.0, + signal_evidence=evidence, + quorum_required=required, + quorum_verified=len(evidence), + ) + return None + + def _actuator_for(self, binding: Any) -> Optional[Any]: + """The actuator that runs ``binding``, or None when none can.""" + + if binding is None: + return None + if binding.kind != "web_form": + return self.api_actuator + if self.web_form_actuator is not None: + return self.web_form_actuator + # In a multi-surface run the request goes through the active + # surface's session, which on a resume may not be self.backend yet. + router = getattr(self, "_surface_router", None) + backend = router.active_backend if router is not None else self.backend + page = getattr(backend, "page", None) + if page is None: + return None + from openadapt_flow.runtime.actuators.web_form import ( + PlaywrightSessionTransport, + WebFormActuator, + ) + + try: + return WebFormActuator(PlaywrightSessionTransport(page)) + except Exception: # noqa: BLE001 - no live session: the GUI runs + return None + + def _try_api_tier( + self, + step: Step, + params: Mapping[str, RuntimeParamScalar], + result: StepResult, + *, + workflow: Workflow, + step_index: int, + bundle_dir: Path, + effect_verifier: Any, + ) -> bool: + """Perform ``step.api_binding``'s write via the API and confirm it. + + The TOP of the capability ladder (RFC section 4 ``api`` tier): a + deterministic, ``$0``, model-free write, gated by the SAME + ``EffectVerifier`` as a GUI write. For an API write the target + "identity" is the explicit API parameter -- stronger than a resolved + pixel band -- so no identity gate is weakened by skipping the GUI. + + Fail-safe ordering (the no-double-write contract): + + - refuse BEFORE any request when the write could not be confirmed (no + effect contract, or no verifier) -- an unverifiable consequential + write never proceeds; + - snapshot the system of record, then actuate ONCE; + - UNAVAILABLE (request never sent) -> use the binding's configured GUI + fallback or typed pre-delivery halt; nothing was written; + - HALT (request sent, outcome unknown / rejected) -> run the configured + postcondition and independent-effect checks without retry or GUI + fallback; resolve only if the complete contract proves the effect; + - ACTUATED (2xx) -> CONFIRM with the EffectVerifier; a non-CONFIRMED + verdict HALTs exactly as it would for a GUI write. + + Returns True when the API tier took responsibility for the step (the + result is final -- actuated+verified, HALTed, or a config-error HALT), + False only when the API tier is UNAVAILABLE and the binding permits the + caller to fall through to the GUI resolution ladder. + """ + binding = step.api_binding + assert binding is not None # guaranteed by the caller + api_actuator = self._actuator_for(binding) + assert api_actuator is not None # guaranteed by the caller + + # An API write MUST be confirmable against its own exact effect + # contract. ``step.effects`` belongs exclusively to the GUI fallback; + # substituting it here could confirm a different record or operation. + effects = binding.effects + if not effects: + result.effect_verified = False + result.effect_results.append( + "API binding declares no api_binding.effects contract to " + "confirm its write -- refusing an " + "unverifiable API write (fail-safe HALT)" + ) + result.ok = False + result.error = ( + f"Step '{step.id}' ({step.intent}) has an API binding but no " + "system-of-record effect to CONFIRM the write -- an API write " + "must be verifiable; refusing to actuate " + "(deployment/configuration error); run aborted" + ) + return True + identity_refusal = self._api_identity_refusal( + step, + params, + workflow, + effects, + result, + ) + if identity_refusal is not None: + result.ok = False + result.safety_halt = True + result.failure_category = "governed_refusal" + result.error = identity_refusal + return True + # Bind the effect contracts to this run before either the verifier + # strength check or its detector-input digest is evaluated. + effects = self._resolve_effects(effects, params) + if effect_verifier is None: + result.effect_verified = False + result.effect_results.append( + "API binding present but no EffectVerifier is configured to " + "confirm the write (fail-safe HALT)" + ) + result.ok = False + result.error = ( + f"Step '{step.id}' ({step.intent}) would actuate via the API " + "but no EffectVerifier is configured to confirm the write -- " + "refusing an unverifiable consequential write; run aborted" + ) + result.safety_refusal_evidence = SafetyRefusalEvidence( + stage="effect_verifier", + code="effect_verifier_missing", + detector_input_sha256=effect_verifier_input_sha256(None, effects), + ) + return True + + # The same parameter substitution that the ApiActuator applies to the + # URL/query/body now also applies to the effect checked below. + api_hash_start = len(result.effect_contract_hashes) + result.effect_contract_hashes.extend( + effect.contract_hash() for effect in effects + ) + refusal = self._profile_effect_tier_refusal( + workflow, + step, + "api", + effects, + effect_verifier, + ) + if refusal is not None: + result.effect_verified = False + result.safety_halt = True + result.failure_category = "governed_refusal" + result.effect_results.append(refusal) + result.error = refusal + result.safety_refusal_evidence = SafetyRefusalEvidence( + stage="effect_strength", + code="effect_strength_insufficient", + detector_input_sha256=effect_verifier_input_sha256( + effect_verifier, + effects, + ), + ) + return True + + # Snapshot the system of record BEFORE the write so the verifier counts + # only what THIS actuation wrote (delta / at-most-once / collateral + # loss), then actuate exactly once. + try: + before = self._capture_effect_pre_state(effect_verifier, effects) + except Exception as exc: # noqa: BLE001 - deployment verifier boundary + result.effect_verified = False + result.effect_results.append( + "[api] effect pre-state capture failed before request delivery " + f"({type(exc).__name__}); API actuation was not attempted" + ) + result.ok = False + result.error = ( + f"Step '{step.id}' ({step.intent}) could not capture the " + "system-of-record pre-state before API actuation; refusing to " + "send the request — run aborted" + ) + return True + binding_refusal = self._candidate_binding_refusal(before, effects) + if binding_refusal is not None: + result.effect_verified = False + result.effect_results.append(f"[api] {binding_refusal}") + result.ok = False + result.safety_halt = True + result.failure_category = "governed_refusal" + result.error = binding_refusal + return True + if self._required_effect_pre_state_unreadable(effect_verifier, before, effects): + result.effect_verified = False + result.effect_results.append( + "[api] effect pre-state is unreachable before request delivery" + ) + result.ok = False + result.error = ( + f"Step '{step.id}' ({step.intent}) could not read the " + "system-of-record pre-state before API actuation; refusing to " + "send the request — run aborted" + ) + return True + self._keep_undo_prior_state(workflow, step, before, effects, params) + refusal = self._profile_effect_tier_refusal( + workflow, + step, + "api", + effects, + effect_verifier, + ) + if refusal is not None: + result.effect_verified = False + result.safety_halt = True + result.failure_category = "governed_refusal" + result.effect_results.append(refusal) + result.error = refusal + result.safety_refusal_evidence = SafetyRefusalEvidence( + stage="effect_strength", + code="effect_strength_insufficient", + detector_input_sha256=effect_verifier_input_sha256( + effect_verifier, + effects, + ), + ) + return True + overlay_current, overlay_total = self._control_overlay_progress( + workflow, step_index, None + ) + self._emit_control_overlay_phase( + "executing", + current_step=overlay_current, + total_steps=overlay_total, + ) + approved_view = self._web_form_approval.pop(id(result), None) + send_prepared: Optional[Any] = None + again: Any = None + if approved_view is not None: + # Under an approval mode: prepare the request again now (fresh + # form, fresh pre-checks). The last-point gate compares its + # operation digest with the one a person approved; a request that + # can't be prepared again reads as an unreadable form. + again = self._web_form_approval_view(step, params) + self._web_form_form_state = ( + None if isinstance(again, str) else ("api_request", again.digest) + ) + self._web_form_form_state_set = True + try: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + finally: + self._web_form_form_state = None + self._web_form_form_state_set = False + if refusal is None and approved_view is not None: + if isinstance(again, str) or again.digest != approved_view.digest: + # Also refused when the gate let it through (for example a + # save that passed problems_only's checks without a person): + # only the exact request prepared before the gate is sent. + why = ( + again + if isinstance(again, str) + else "the request, the record or the record check differs" + ) + result.safety_halt = True + result.failure_category = "governed_refusal" + refusal = ( + f"Step '{step.id}' ({step.intent}): the API request changed " + f"after the approval gate ({why}); the save was not sent " + "and the GUI steps do not run in its place" + ) + else: + # Exactly the request whose digest was approved and re-checked. + send_prepared = again.prepared + if refusal is not None: + result.ok = False + result.error = refusal + return True + # Structural postconditions compare against the last observation before + # request delivery, not against state captured earlier during + # authorization or verifier setup. + start_state = self._structural_state() + # This transition is the explicit delivery boundary. Set it BEFORE the + # call because an exception/timeout may occur after the request left the + # process but before the actuator can return a receipt-like outcome. + result.delivery_attempted = True + try: + self._require_qualification_environment_current() + if send_prepared is not None: + # Exactly the request whose digest was approved and re-checked. + outcome = api_actuator.send(binding, send_prepared) + else: + outcome = api_actuator.actuate( + binding, + runtime_params_for_gui(params), + ) + except Exception as exc: # noqa: BLE001 - external actuator boundary + # The actuator contract is no-throw, but a deployment adapter can + # still violate it. The delivery boundary was crossed immediately + # before the call, so fail toward an attempted/unknown write and + # never fall through to the GUI. + result.actuation = "api" + result.effect_verified = False + result.effect_results.append( + "[api] actuator raised after delivery may have begun " + f"({type(exc).__name__}); outcome requires reconciliation" + ) + return self._verify_uncertain_api_delivery( + step, + result, + workflow=workflow, + step_index=step_index, + bundle_dir=bundle_dir, + start_state=start_state, + before=before, + effects=effects, + effect_verifier=effect_verifier, + cause_type=type(exc).__name__, + ) + + from openadapt_flow.runtime.actuators import ( + ActuationStatus, + ApiActuationResult, + ApiHaltKind, + ) + + # Only the validated model can prove the pre-delivery UNAVAILABLE + # state. A deployment-owned actuator may return any object after its + # call, including a duck-typed object that claims ``UNAVAILABLE``. + # Treat that as possible delivery: collect evidence, never fall + # through to the GUI, and never turn it into success. + if type(outcome) is not ApiActuationResult: + result.actuation = "api" + result.effect_results.append( + "[api] actuator returned an untyped result after delivery may " + f"have begun ({type(outcome).__name__}); treating it as a " + "protocol error that cannot resolve to success" + ) + return self._verify_uncertain_api_delivery( + step, + result, + workflow=workflow, + step_index=step_index, + bundle_dir=bundle_dir, + start_state=start_state, + before=before, + effects=effects, + effect_verifier=effect_verifier, + cause_type="ApiActuatorProtocolError", + allow_contract_resolution=False, + ) + try: + # Revalidate the return value because an adapter can mutate a + # Pydantic instance after construction. The runtime trusts the + # copied, validated receipt only. + outcome = ApiActuationResult.model_validate( + outcome.model_dump(mode="python") + ) + except Exception as exc: # noqa: BLE001 - deployment actuator boundary + result.actuation = "api" + result.effect_results.append( + "[api] actuator returned a malformed typed result after " + "delivery may have begun " + f"({type(exc).__name__}); treating it as a protocol error " + "that cannot resolve to success" + ) + return self._verify_uncertain_api_delivery( + step, + result, + workflow=workflow, + step_index=step_index, + bundle_dir=bundle_dir, + start_state=start_state, + before=before, + effects=effects, + effect_verifier=effect_verifier, + cause_type="ApiActuatorProtocolError", + allow_contract_resolution=False, + ) + + outcome_status = outcome.status + outcome_reason = outcome.reason + + if outcome_status is ActuationStatus.UNAVAILABLE: + # The request was NEVER sent -- nothing was written. Remove the + # unused API-path contracts before applying the binding's exact + # unavailability policy. A GUI fallback result must describe only + # the path responsible for delivery, never both alternatives. + result.delivery_attempted = False + del result.effect_contract_hashes[api_hash_start:] + result.effect_results.append(f"[api] {outcome_reason}") + if binding.on_unavailable == "halt": + self._set_api_unavailable_refusal( + step, + result, + reason=outcome_reason, + ) + return True + return False + + # From here the request WAS attempted -- this step is API-tier and is + # NEVER also GUI-written (the no-double-write contract). + result.actuation = "api" + + if outcome_status is ActuationStatus.HALT: + result.effect_results.append(f"[api] {outcome_reason}") + halt_kind = getattr(outcome, "halt_kind", None) + typed_halt = isinstance(halt_kind, ApiHaltKind) + if not typed_halt: + result.effect_results.append( + "[api] actuator returned an untyped HALT result; treating " + "the delivery as a protocol error that cannot resolve to " + "success" + ) + return self._verify_uncertain_api_delivery( + step, + result, + workflow=workflow, + step_index=step_index, + bundle_dir=bundle_dir, + start_state=start_state, + before=before, + effects=effects, + effect_verifier=effect_verifier, + cause_type=( + "ApiActuatorProtocolError" + if not typed_halt + else ( + "ApiResponseRejected" + if halt_kind is ApiHaltKind.RESPONSE_REJECTED + else "ApiDeliveryUncertain" + ) + ), + allow_contract_resolution=( + typed_halt and halt_kind is ApiHaltKind.DELIVERY_UNCERTAIN + ), + ) + + # ACTUATED (2xx): confirm the write against the system of record with + # the same EffectVerifier that gates a GUI write. A non-CONFIRMED + # verdict HALTs. The GUI resolve/act (and screen postconditions) are + # SKIPPED -- the record, not the screen, is the oracle for an API write. + result.effect_results.append(f"[api] actuated {outcome_reason}") + self._emit_control_overlay_phase( + "verifying", + current_step=overlay_current, + total_steps=overlay_total, + ) + if self._may_pipeline_readback(effect_verifier, step): + # Pipelined read-back (runtime/readback_pipeline.py), as for a GUI + # save: the same record check on the side thread. The item's + # report stays unsuccessful until it settles with the verdict. + self._defer_readback( + step, + before, + result, + effects=effects, + verifier=effect_verifier, + actuation="api", + ) + result.ok = True + result.error = None + return True + try: + error = self._verify_effects( + step, + before, + result, + effects=effects, + verifier=effect_verifier, + ) + except Exception as exc: # noqa: BLE001 - deployment verifier boundary + result.effect_verified = False + result.effect_results.append( + "[api] effect verification raised after API delivery " + f"({type(exc).__name__}); outcome requires reconciliation" + ) + error = ( + f"System-of-record verification for API step '{step.id}' " + f"({step.intent}) raised {type(exc).__name__} after delivery; " + "the write may have landed — run aborted" + ) + result.ok = error is None + result.error = error + return True + + def _verify_uncertain_api_delivery( + self, + step: Step, + result: StepResult, + *, + workflow: Workflow, + step_index: int, + bundle_dir: Path, + start_state: dict[str, Any], + before: EffectState, + effects: list["Effect"], + effect_verifier: Any, + cause_type: str, + allow_contract_resolution: bool = True, + ) -> bool: + """Resolve one possibly dispatched API write without another actuation. + + A response loss, rejected response, or non-conforming actuator + exception does not end evidence collection. The runtime checks the + configured postcondition and the independent system-of-record effect. + A response loss can resolve only when both contracts confirm. An + explicit response outside the binding's accepted contract always + remains halted, even when later evidence confirms a business effect. + The API request is never retried and the GUI fallback never runs. + """ + + result.delivery_uncertainty = ActionDeliveryUncertainty( + operation="api_request", + native=False, + observed_at=datetime.now(timezone.utc).isoformat(), + cause_type=cause_type, + ) + overlay_current, overlay_total = self._control_overlay_progress( + workflow, step_index, None + ) + self._emit_control_overlay_phase( + "verifying", + current_step=overlay_current, + total_steps=overlay_total, + ) + verification_errors: list[str] = [] + + if step.expect: + try: + after_png = self.vision.wait_settled(self.backend) + postconditions_ok, _last_frame, failed = self._check_postconditions( + step, + after_png, + bundle_dir, + start_state, + result, + workflow=workflow, + ) + result.postconditions_ok = postconditions_ok + if not postconditions_ok: + verification_errors.append( + "postcondition did not confirm: " + + ("; ".join(failed) or "unknown postcondition") + ) + except Exception as exc: # noqa: BLE001 - observation boundary + result.postconditions_ok = False + verification_errors.append( + f"postcondition observation failed ({type(exc).__name__})" + ) + else: + # An absent postcondition cannot resolve uncertain delivery. + result.postconditions_ok = None + verification_errors.append("no postcondition was declared") + + try: + effect_error = self._verify_effects( + step, + before, + result, + effects=effects, + verifier=effect_verifier, + ) + except Exception as exc: # noqa: BLE001 - deployment verifier boundary + result.effect_verified = False + result.effect_results.append( + "[api] effect verification raised after uncertain delivery " + f"({type(exc).__name__}); outcome requires reconciliation" + ) + effect_error = ( + f"independent effect verification failed ({type(exc).__name__})" + ) + if effect_error is not None: + verification_errors.append(effect_error) + + uncertainty = result.delivery_uncertainty + assert uncertainty is not None + uncertainty.verification_attempted = True + uncertainty.postconditions_confirmed = result.postconditions_ok is True + uncertainty.effects_confirmed = result.effect_verified is True + uncertainty.resolved_by_contract = ( + allow_contract_resolution + and uncertainty.postconditions_confirmed + and uncertainty.effects_confirmed + and not verification_errors + ) + result.ok = uncertainty.resolved_by_contract + if result.ok: + result.error = None + else: + result.safety_halt = True + result.failure_category = ( + "governed_refusal" + if self.governed_authorization is not None + else "safety_halt" + ) + detail = "; ".join(verification_errors) or "verification confirmed" + if allow_contract_resolution: + result.error = ( + f"API delivery for step '{step.id}' ({step.intent}) was " + "uncertain and was not retried; the complete postcondition " + "and independent effect contract did not confirm: " + f"{detail}" + ) + else: + result.error = ( + f"API delivery for step '{step.id}' ({step.intent}) " + "received a response outside the binding's accepted " + "contract and was not retried; later verification evidence " + f"was collected but cannot override that rejection: {detail}" + ) + return True + + @staticmethod + def _set_api_unavailable_refusal( + step: Step, + result: StepResult, + *, + reason: str, + ) -> None: + """Record an exact pre-delivery API-only refusal. + + The caller invokes this only when no request was sent. The typed + evidence distinguishes that state from an attempted or uncertain API + delivery and prevents any GUI fallback. + """ + + binding = step.api_binding + assert binding is not None + detector_input = json.dumps( + { + "binding": binding.model_dump(mode="json"), + "reason_sha256": hashlib.sha256(reason.encode("utf-8")).hexdigest(), + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + result.ok = False + result.delivery_attempted = False + result.safety_halt = True + result.failure_category = "governed_refusal" + result.effect_verified = False + result.error = ( + f"API-only step '{step.id}' ({step.intent}) is unavailable before " + "delivery; refusing GUI fallback — run aborted" + ) + result.safety_refusal_evidence = SafetyRefusalEvidence( + stage="api_admission", + code="api_path_unavailable", + detector_input_sha256=hashlib.sha256(detector_input).hexdigest(), + ) + + def _web_form_approval_view( + self, step: Step, params: Mapping[str, RuntimeParamScalar] + ) -> "Any | str": + """Prepare an API step's request for a person to approve. + + Returns the view (the prepared request, its operation digest, the + rows and image the person sees), or the reason it can't be prepared. + Only reads are sent. + """ + + from openadapt_flow.approval_mode.item_approval import ApiRequestRef + from openadapt_flow.runtime.actuators import ApiActuationResult + from openadapt_flow.runtime.actuators.web_form import ( + operation_digest, + operation_rows, + render_operation_png, + ) + + binding = step.api_binding + assert binding is not None + actuator = self._actuator_for(binding) + if actuator is None or not callable(getattr(actuator, "prepare", None)): + return "this API actuator can't prepare the request for approval" + gui_params = runtime_params_for_gui(params) + try: + prepared = actuator.prepare(binding, gui_params) + except Exception as exc: # noqa: BLE001 - only reads were sent + return f"preparing the request failed ({type(exc).__name__})" + if isinstance(prepared, ApiActuationResult): + return prepared.reason or "the API step is unavailable" + effects = self._resolve_effects(list(binding.effects), params) + # Several record checks usually select the same record (one per + # field): the person reads, and the digest binds, each pair once. + record = list( + dict.fromkeys( + (key, str(expr)) + for effect in effects + for key, expr in sorted(effect.match.items()) + ) + ) + digest = operation_digest( + prepared, + record_check=[effect.contract_hash() for effect in effects], + record=record, + ) + rows = operation_rows(binding, prepared, gui_params, record=record) + return _WebFormApprovalView( + prepared=prepared, + digest=digest, + rows=rows, + png=render_operation_png( + f"Approve this request: {step.intent or step.id}", rows + ), + ref=ApiRequestRef( + method=prepared.method.upper(), + path=prepared.path, + payload_sha256=prepared.payload_sha256(), + ), + ) diff --git a/packages/flow/openadapt_flow/runtime/replay/confirming.py b/packages/flow/openadapt_flow/runtime/replay/confirming.py new file mode 100644 index 000000000..58d26af03 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/confirming.py @@ -0,0 +1,1053 @@ +# mypy: disable-error-code="empty-body" +"""Confirming an action: typed input landed, the right option is selected, step guards hold. + +Moved verbatim from ``openadapt_flow.runtime.replayer.Replayer`` by +scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1). +``_ConfirmingHost`` declares, for mypy only, what this code reads from the +Replayer it is mixed into. +""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING, Any, Optional # noqa: F401 + +from openadapt_flow.ir import ActionKind +from openadapt_flow.runtime import identity as identity_mod +from openadapt_flow.runtime.authorization import runtime_params_for_gui +from openadapt_flow.runtime.program_predicates import evaluate_program_predicate +from openadapt_flow.runtime.replay._shared import ( + _PAGE_SCALE_BAND, + FIELD_REGION_PAD, + FIELD_REGION_SIZE, + MASKED_MIN_CONFIDENCE, + MASKED_NEW_TEXT_SLACK, + MASKED_REPEAT_FRACTION, + PC_STRUCTURAL_TEMPLATE_THRESHOLD, + PC_TEMPLATE_SEARCH_PAD, + PC_TEMPLATE_THRESHOLD, + _frame_viewport, + _uniform_page_scale, +) +from openadapt_flow.runtime.resolver import pad_region +from openadapt_flow.runtime.step_timing import timed_span + +if TYPE_CHECKING: + from pathlib import Path + from typing import Mapping + + from openadapt_flow.backend import Backend + from openadapt_flow.ir import ( + Point, + Predicate, + Region, + Resolution, + Step, + StepResult, + Workflow, + ) + from openadapt_flow.runtime.authorization import RuntimeParamScalar + + +class _ConfirmingHost: + _asset_bytes: Any + _cancel_guarded_keyboard: Any + _deliver_backend_call: Any + _delivery_authorization_refusal: Any + + @staticmethod + def _describe_postcondition(pc: Any) -> str: ... + @staticmethod + def _describe_predicate(pred: Predicate) -> str: ... + + _governed_asset_mutation: Optional[str] + _handle_interstitials: Any + _mapped_selection_readback_region: Any + _pc_page_scale: Any + _pc_scaled_match: Any + + def _poll_postconditions( + self, + step: Step, + frame_png: bytes, + bundle_dir: Path, + start_state: dict[str, Any], + ) -> tuple[bool, bytes]: ... + @staticmethod + def _postcondition_confirms_a_write( + step: Step, workflow: Optional[Workflow] + ) -> bool: ... + def _postcondition_passes( + self, + pc: Any, + frame_png: bytes, + bundle_dir: Path, + start_state: Optional[dict[str, Any]] = None, + ) -> bool: ... + def _profile_runtime_refusal(self, workflow: Workflow) -> Optional[str]: ... + + _require_qualification_environment_current: Any + + def _resolve_step( + self, + step: Step, + screen_png: bytes, + bundle_dir: Path, + workflow: Workflow, + *, + allow_grounder: bool = True, + allow_target_ocr: bool = True, + ) -> tuple[Optional[Resolution], Optional[Region], Optional[str]]: ... + + _revalidate_consequential_actuation: Any + _settle_seeded: Any + _step_needs_consequential_revalidation: Any + + def _wait_starting_state_settled( + self, before_png: bytes + ) -> tuple[bytes, Optional[str], Optional[bool]]: ... + + backend: Backend + poll_interval_s: float + state_verifier: Optional[Any] + vision: Any + + +if TYPE_CHECKING: + _Base = _ConfirmingHost +else: + _Base = object + + +class Confirming(_Base): + """Confirming an action: typed input landed, the right option is selected, step guards hold.""" + + def _apply_step_gates( + self, + step: Step, + before_png: bytes, + bundle_dir: Path, + params: Mapping[str, RuntimeParamScalar], + result: StepResult, + workflow: Optional[Workflow] = None, + ) -> tuple[bool, Optional[str], bytes]: + """Evaluate readiness, interstitials, guard, then wait_until before acting. + + Runs in state-dependency order, all model-free: + + 1. **readiness** (``require_settled``) - refuse to act on a frame that + never settled (still-loading / mid-transition); HALT gracefully. + 2. **interstitials** - a KNOWN reversible, non-consequential overlay on + the entry frame gets one audited dismissal plus declared clearance + verification, or HALTs before the workflow action. + 3. **guard** (precondition) on the entry frame - unmet HALTs (default) + or SKIPs (``on_unmet="skip"``). + 4. **wait_until** (readiness predicate) - polled, bounded, HALT on + timeout (never proceed-anyway). + + Returns ``(proceed, error, before_png)``: + + - ``(True, None, frame)`` - proceed to resolve/act (frame may have been + re-settled while polling a wait_until predicate). + - ``(False, None, frame)`` - the guard was unmet with ``on_unmet="skip"``: + the step is a no-op success (caller marks it ``skipped``). + - ``(False, error, frame)`` - HALT: an un-ready screen, a blocking / + undismissable interstitial, an unmet ``halt`` guard, or a + ``wait_until`` predicate that never held within its bound (fail-safe; + the run NEVER proceeds-anyway on an unmet readiness predicate). + + Model-free: predicates are evaluated by :meth:`_predicate_holds`. + """ + if workflow is not None: + profile_refusal = self._profile_runtime_refusal(workflow) + if profile_refusal is not None: + result.safety_halt = True + result.failure_category = "governed_refusal" + return ( + False, + f"{profile_refusal} at the actuation boundary — refusing " + "to act; run aborted", + before_png, + ) + # (1) Readiness: never act on a frame that never settled. + before_png, readiness_error, settled = self._wait_starting_state_settled( + before_png + ) + result.starting_state_settled = settled + if readiness_error is not None: + # Like an interstitial refusal, a not-ready starting state is a + # safety halt. A program on_exception edge must not convert the + # explicit refusal into a successful terminal outcome. + result.safety_halt = True + return False, readiness_error, before_png + + # (2) Interstitials: dismiss a KNOWN overlay, or HALT gracefully. + before_png, interstitial_error = self._handle_interstitials( + step, + before_png, + bundle_dir, + params, + result, + result.interstitial_actions, + workflow, + ) + if interstitial_error is not None: + # An interstitial refusal is a state-safety halt, not an ordinary + # workflow exception a program on_exception edge may turn into a + # successful outcome after an uncertain/blocked pre-action state. + result.safety_halt = True + return False, interstitial_error, before_png + + # Guard (precondition) on the entry frame. + guard_holds = step.guard is None or self._predicate_holds( + step.guard.predicate, + before_png, + bundle_dir, + params, + workflow=workflow, + ) + if self._governed_asset_mutation is not None: + return False, self._governed_asset_mutation, before_png + if not guard_holds: + assert step.guard is not None + if step.guard.on_unmet == "skip": + return False, None, before_png + return ( + False, + ( + f"Guard precondition for step '{step.id}' ({step.intent}) is " + f"unmet on the current screen " + f"({self._describe_predicate(step.guard.predicate)}) — " + "refusing to act; run aborted" + ), + before_png, + ) + + # wait_until (readiness). SCROLL consumes its own predicate as the stop + # condition of its closed loop (_act_scroll), so it is skipped here. + if step.wait_until is not None and step.action is not ActionKind.SCROLL: + pred = step.wait_until + deadline = time.monotonic() + pred.timeout_s + while True: + if self._predicate_holds( + pred, + before_png, + bundle_dir, + params, + workflow=workflow, + ): + return True, None, before_png + if time.monotonic() >= deadline: + return ( + False, + ( + f"wait_until predicate for step '{step.id}' " + f"({step.intent}) did not hold within " + f"{pred.timeout_s:.1f}s " + f"({self._describe_predicate(pred)}) — readiness never " + "reached; refusing to proceed-anyway; run aborted" + ), + before_png, + ) + time.sleep(self.poll_interval_s) + before_png = self.vision.wait_settled(self.backend) + + return True, None, before_png + + def _predicate_holds( + self, + pred: Predicate, + frame_png: bytes, + bundle_dir: Path, + params: Mapping[str, RuntimeParamScalar], + *, + workflow: Optional[Workflow] = None, + ) -> bool: + """Evaluate a Predicate against the current frame / run params. + + Deterministic and ZERO model calls (the $0 runtime guarantee): + ``anchor_resolves`` runs the resolution ladder with NO grounder, + ``text_present`` / ``text_absent`` use the tolerant OCR presence check, + ``param_equals`` is a string compare, and ``and`` / ``or`` / ``not`` + compose. An unknown kind fails safe (does not hold). + """ + return evaluate_program_predicate( + pred, + frame_png, + runtime_params_for_gui(params), + vision=self.vision, + viewport=_frame_viewport(frame_png), + asset_loader=lambda rel: self._asset_bytes( + bundle_dir, + rel, + workflow=workflow, + ), + ) + + def _field_region( + self, + field_point: Optional[Point], + structural_region: Optional[Region] = None, + ) -> Optional[Region]: + """Region to observe for typed input, or None for the whole frame.""" + vw, vh = self.backend.viewport + if structural_region is not None: + sx, sy, sw, sh = structural_region + x = max(0, sx - FIELD_REGION_PAD) + y = max(0, sy - FIELD_REGION_PAD) + right = min(vw, sx + sw + FIELD_REGION_PAD) + bottom = min(vh, sy + sh + FIELD_REGION_PAD) + if right > x and bottom > y: + return (x, y, right - x, bottom - y) + if field_point is None: + return None + w = min(FIELD_REGION_SIZE[0], vw) + h = min(FIELD_REGION_SIZE[1], vh) + x = min(max(0, field_point[0] - w // 2), max(0, vw - w)) + y = min(max(0, field_point[1] - h // 2), max(0, vh - h)) + return (x, y, w, h) + + def _ocr_squashed(self, png: bytes, region: Optional[Region]) -> str: + """Squashed OCR text of a (region of a) frame.""" + lines = self.vision.ocr(png, region=region) + return identity_mod.squash(" ".join(line.text for line in lines)) + + def _readable_chars(self, png: bytes, region: Optional[Region]) -> int: + """Confidently readable alphanumeric characters in a region. + + The masked-acceptance metric (see ``MASKED_NEW_TEXT_SLACK``): + password dots OCR as nothing, low-confidence noise, punctuation + runs, or — on some platform renderers — confident homogeneous + digit runs, while a dialog adds confident words. Counting only + alphanumeric characters of confident, non-homogeneous lines is + also invariant to OCR merging or splitting the same text into + different boxes between frames. + """ + total = 0 + for line in self.vision.ocr(png, region=region): + if getattr(line, "confidence", 1.0) < MASKED_MIN_CONFIDENCE: + continue + alnum = [ch for ch in line.text if ch.isalnum()] + if not alnum: + continue + most_common = max(alnum.count(ch) for ch in set(alnum)) + if len(alnum) >= 4 and most_common / len(alnum) >= MASKED_REPEAT_FRACTION: + continue # homogeneous glyph run: masked-dot misread + total += len(alnum) + return total + + def _text_value_at(self, field_point: Optional[Point]) -> Optional[str]: + """Read an exact editable value when the backend can expose one. + + ``None`` means the substrate cannot make this observation. Empty text + is returned as ``""`` and remains distinguishable from unavailable. + Exceptions fail closed into the visual verifier rather than escaping. + """ + reader = getattr( + self.backend, + "text_value_at" if field_point is not None else "focused_text_value", + None, + ) + if not callable(reader): + return None + try: + value = reader(*field_point) if field_point is not None else reader() + except Exception: + return None + return value if isinstance(value, str) else None + + def _typed_input_landed( + self, + text: str, + field_point: Optional[Point], + baseline_png: bytes, + *, + field_region: Optional[Region] = None, + baseline_field_value: Optional[str] = None, + allow_masked: bool = False, + ) -> tuple[bool, bool]: + """Did the just-typed ``text`` visibly land? + + Exact structural readback is authoritative when the backend exposes + it. Otherwise, for an OCR-able value (>= + ``identity.MIN_PARAM_CHARS`` squashed chars), a contiguous squashed + run of the value (retried at 2x resolution) must be readable in the + field region. Pixel-only acceptance is restricted to steps declared + ``secret``: masked password dots may render no readable text. A dialog + painting over the region or an unreadable ordinary field must never + count as "input landed". Values too short for OCR to arbitrate retain + the legacy diff fallback. + + Returns: + ``(landed, changed)`` — the verdict, and whether the field + region's pixels changed at all (the caller's retry decision: + retyping is only safe when nothing changed). + """ + after_png = self._settle_seeded() + region = self._field_region(field_point, field_region) + if baseline_field_value is not None: + expected = baseline_field_value + text + after_at_point = ( + self._text_value_at(field_point) if field_point is not None else None + ) + after_focused = self._text_value_at(None) + + # Focus-bound readback is the strongest confirmation. A full-page + # cover can hide the field from hit-testing while leaving that + # exact editable element focused, so focused readback also + # preserves the intended safe-halt-at-the-next-action behavior. + if after_focused == expected: + return True, expected != baseline_field_value + + # If hit-testing can read the value but focus moved elsewhere, an + # interstitial/dialog stole focus after delivery. That is an + # unknown state, not a success: the value could belong to an + # obscured/replaced control. + if after_at_point == expected: + return False, True + + observed = [ + value for value in (after_at_point, after_focused) if value is not None + ] + if observed and all(value == baseline_field_value for value in observed): + return False, False + if observed: + # A partial/different value is an unknown state and must halt + # without destructive select-all. + return False, True + # A structurally readable field disappearing after delivery means + # it was covered, replaced, or navigated away. Do not downgrade to + # OCR, which could read the intended text elsewhere on the screen. + return False, True + changed = self.vision.pixels_changed(baseline_png, after_png, region=region) + # Dropdown: typing selects the option whose label matches. When the + # focused at the field point now shows exactly the typed - # label (whitespace-squashed, case-insensitive), the input landed. - # Any other answer falls through to the visual check, unchanged. - if field_point is not None and not allow_masked: - reader = getattr(self.backend, "focused_select_label_at", None) - label: Optional[str] = None - if callable(reader): - try: - label = reader(int(field_point[0]), int(field_point[1])) - except Exception: - label = None - if ( - isinstance(label, str) - and identity_mod.squash(text) - and identity_mod.squash(label).casefold() - == identity_mod.squash(text).casefold() + try: + with ( + Image.open(io.BytesIO(before_png)) as a_img, + Image.open(io.BytesIO(fresh_png)) as b_img, ): - return True, True - needle = identity_mod.squash(text) - if len(needle) < identity_mod.MIN_PARAM_CHARS: - return changed, changed # too short for OCR to arbitrate - need = identity_mod.required_run(len(needle)) - after_hay = self._ocr_squashed(after_png, region) - if identity_mod.longest_run(needle, after_hay) >= need: - return True, changed - if region is not None: - upscaled = identity_mod.upscale_crop(after_png, region) - if upscaled is not None: - up_hay = self._ocr_squashed(upscaled, None) - if identity_mod.longest_run(needle, up_hay) >= need: - return True, changed - if not changed: - return False, False - # Pixels changed but the value is unreadable: masked rendering is - # the only acceptable explanation, and masked rendering adds no - # confidently readable alphanumeric text (dot glyphs OCR as - # nothing, noise, or punctuation — platform-dependent). Anything - # else (a dialog over the field, another widget's text) must fail - # the verdict. - if not allow_masked: - return False, changed - landed = ( - self._readable_chars(after_png, region) - <= self._readable_chars(baseline_png, region) + MASKED_NEW_TEXT_SLACK - ) - return landed, changed + a = np.asarray(a_img.convert("RGB"), dtype=np.int16) + b = np.asarray(b_img.convert("RGB"), dtype=np.int16) + except Exception: # noqa: BLE001 - an unreadable frame is a change + return ["frame_size"] + if a.shape != b.shape: + return ["frame_size"] + height, width = a.shape[:2] + changed: list[str] = [] + for name, (x, y, w, h) in regions.items(): + x0, y0 = max(0, int(x)), max(0, int(y)) + x1, y1 = min(width, int(x + w)), min(height, int(y + h)) + if x1 <= x0 or y1 <= y0: + continue + diff = np.abs(a[y0:y1, x0:x1] - b[y0:y1, x0:x1]).max(axis=2) + if int((diff > _PIXEL_RECHECK_LEVEL).sum()) >= _PIXEL_RECHECK_MIN_PIXELS: + changed.append(name) + return changed - @timed_span("postcondition") - def _verify_typed_input( + def _pixel_recheck( self, step: Step, - text: str, - field_point: Optional[Point], - baseline_png: bytes, - result: StepResult, - *, - workflow: Workflow, + resolution: Resolution, + checked_png: bytes, params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, bundle_dir: Path, + result: StepResult, run_dir: Path, - resolution: Optional[Resolution], - field_region: Optional[Region] = None, - baseline_field_value: Optional[str] = None, - allow_masked: bool = False, - allow_retry: bool = True, - ) -> Optional[str]: - """Verify a TYPE action landed; one guarded refocus-and-retype retry. - - The retry only fires when exact readback or pixels prove the first - attempt changed NOTHING in the field (keystrokes fell on a - non-rendering target): re-click - the field (when its point is known), select-all so a false-negative - first attempt is replaced rather than duplicated, retype. When the - region DID change but the value cannot be read (a dialog over the - field, an unexpected re-render), retyping is not safe — select-all - could destroy pre-existing field content and the re-click could - re-fire whatever now sits at that point — so the run halts - immediately with the accurate reason. Typed input that cannot be - confirmed must never be reported as success (VALIDATION.md 'focus - stolen' finding). + ) -> Union[Resolution, str]: + """Guard the gap between a pixel step's check and its click. + + Right before the click, take a fresh frame and compare the target, + identity and hover regions with the frame the step was checked on (the + hover frame when the step took a hover look). When one changed (a + dialog opened over the target, the list re-sorted), do not click: + re-resolve the target and re-run the identity check on the fresh + frame, with no input, and compare again on the next frame. After + :data:`_PIXEL_RECHECK_REOBSERVES` re-observations that still see + changes, or when the target or its identity cannot be confirmed again, + refuse the click. Structured (DOM / accessibility) targets are bound + by element tokens instead and are not rechecked here. + + Returns the resolution to click, or the refusal. """ - landed, changed = self._typed_input_landed( - text, - field_point, - baseline_png, - field_region=field_region, - baseline_field_value=baseline_field_value, - allow_masked=allow_masked, - ) - if landed: - result.input_verified = True - return None - if changed: - result.input_verified = False - return ( - f"Typed input could not be verified for step '{step.id}' " - f"({step.intent}): the field region changed but the typed " - "value is not readable there (something else rendered over " - "or instead of the input) — retyping is unsafe in this " - "state; run aborted" - ) - if not allow_retry: - result.input_verified = False - return ( - f"Typed input could not be verified for step '{step.id}' " - f"({step.intent}): the identity-bound browser field showed no " - "confirmed change; refusing an unleased refocus/retype retry" + if resolution.rung == "structural": + return resolution + baseline = self.__dict__.get("_hover_frames", {}).get(step.id) or checked_png + current = resolution + region = self._active_delivery_region + for attempt in range(_PIXEL_RECHECK_REOBSERVES + 1): + fresh = self.backend.screenshot() + changed = self._changed_regions( + baseline, fresh, self._check_regions(step, current, region) ) - result.input_retried = True - retry_resolution = resolution - retry_region = field_region - retry_frame = baseline_png - needs_revalidation = self._step_needs_consequential_revalidation(step, workflow) - if field_point is not None: - if needs_revalidation: - ( - retry_resolution, - retry_region, - retry_frame, - retry_error, - ) = self._revalidate_consequential_actuation( - step, - retry_resolution, - retry_region, - retry_frame, - params, - workflow, - bundle_dir, - result, - run_dir=run_dir, + if not changed: + if current is not resolution: + result.resolution = current + return current + what = ", ".join(changed) + if attempt == _PIXEL_RECHECK_REOBSERVES: + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="aborted") # type: ignore[arg-type] ) - if retry_error is not None: - result.input_verified = False - return retry_error - if retry_resolution is None: - result.input_verified = False - return ( - f"Typed input retry for step '{step.id}' ({step.intent}) " - "could not freshly resolve its field; run aborted" - ) - field_point = retry_resolution.point - refusal = self._delivery_authorization_refusal( - workflow, params, step, result - ) - if refusal is not None: - result.input_verified = False - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call( - result, - lambda: self.backend.click(*field_point), + break + fresh_resolution, fresh_region, error = self._resolve_step( + step, fresh, bundle_dir, workflow ) - # Replace, don't append: if the first attempt DID land but was - # not visible to the diff/OCR, retyping raw would double it. - if needs_revalidation: - ( - retry_resolution, - retry_region, - retry_frame, - retry_error, - ) = self._revalidate_consequential_actuation( + if error is None and fresh_resolution is not None: + error = self._identity_gate_error( step, - retry_resolution, - retry_region, - retry_frame, + fresh_resolution, + fresh, params, workflow, bundle_dir, result, - run_dir=run_dir, - arm_keyboard=True, + evidence_run_dir=run_dir, ) - if retry_error is not None: - result.input_verified = False - return retry_error - if retry_resolution is None or retry_resolution.point != field_point: - self._cancel_guarded_keyboard() - result.input_verified = False - return ( - f"Typed input retry for step '{step.id}' ({step.intent}) " - "resolved to a different field after refocus; run aborted" - ) - refusal = self._delivery_authorization_refusal( - workflow, params, step, result - ) - if refusal is not None: - result.input_verified = False - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call( - result, - lambda: self.backend.press("ControlOrMeta+a"), - ) - - # The refocus click and select-all key each consume an opaque remote - # lease. Acquire and validate one more exact frame for the retype edge. - if needs_revalidation: - ( - retry_resolution, - retry_region, - retry_baseline, - retry_error, - ) = self._revalidate_consequential_actuation( - step, - retry_resolution, - retry_region, - retry_frame, - params, - workflow, - bundle_dir, - result, - run_dir=run_dir, - arm_keyboard=True, - ) - if retry_error is not None: - result.input_verified = False - return retry_error - else: - retry_baseline = self.backend.screenshot() - if ( - needs_revalidation - and field_point is not None - and (retry_resolution is None or retry_resolution.point != field_point) - ): - self._cancel_guarded_keyboard() - result.input_verified = False - return ( - f"Typed input retry for step '{step.id}' ({step.intent}) " - "resolved to a different field before retyping; run aborted" - ) - refusal = self._delivery_authorization_refusal(workflow, params, step, result) - if refusal is not None: - result.input_verified = False - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call(result, lambda: self.backend.type_text(text)) - landed, _changed = self._typed_input_landed( - text, - field_point, - retry_baseline, - field_region=retry_region, - # Select-all means the retry's expected exact value is ``text`` - # regardless of what the field contained before replacement. - baseline_field_value="" if baseline_field_value is not None else None, - allow_masked=allow_masked, - ) - if landed: - result.input_verified = True - return None - result.input_verified = False - return ( - f"Typed input could not be verified for step '{step.id}' " - f"({step.intent}): the screen did not change where the text " - "should have appeared and OCR could not find the typed value, " - "after one refocus-and-retype retry — keystrokes likely fell on " - "a non-input target (focus lost); run aborted" - ) - - @timed_span("postcondition") - def _verify_selected_option( - self, - step: Step, - text: str, - after_png: bytes, - bundle_dir: Path, - workflow: Workflow, - result: StepResult, - ) -> Optional[str]: - """Require the option value on the final settled, re-resolved field. - - Unlike ordinary TYPE verification, selection never accepts pixel - change alone and never retries: the commit key may already have changed - the control. Structural readback must equal the intended option, or OCR - must recover one complete exactly normalized line inside only the final - live field region. The compiled region qualifies geometry but never - supplies runtime coordinates. This method does not settle again: its - caller passes the one final frame used by all post-action checks. - """ - - qualified_readback = step.selection_region - qualified_target = step.anchor.region if step.anchor is not None else None - if qualified_readback is None or qualified_target is None: - result.input_verified = False - return f"Option selection for step '{step.id}' has no field region" - resolution, live_region, resolve_error = self._resolve_step( - step, - after_png, - bundle_dir, - workflow, - # Final option verification is exact retained-evidence work. A - # model may propose a governed repair after a halt, but it cannot - # supply the target region that certifies the committed value. - allow_grounder=False, - ) - if ( - resolve_error is not None - or resolution is None - or not self._selection_region_compatible(qualified_target, live_region) - ): - result.input_verified = False - result.safety_halt = True - result.failure_category = "governed_refusal" - return ( - f"Selected option field could not be re-resolved with compatible " - f"geometry on the final settled frame for step '{step.id}' " - f"({step.intent}); the run halted" - ) - assert live_region is not None - live_readback = self._mapped_selection_readback_region( - qualified_target, - qualified_readback, - live_region, - after_png, - ) - if live_readback is None: - result.input_verified = False - result.safety_halt = True - result.failure_category = "governed_refusal" - return ( - f"Selected option readback region could not be mapped onto the " - f"final settled frame for step '{step.id}' ({step.intent}); the " - "run halted" - ) - exact = self._text_value_at(resolution.point) - if exact is not None: - verified = identity_mod.squash(exact) == identity_mod.squash(text) - else: - needle = identity_mod.squash(text) - verified = any( - getattr(line, "confidence", 1.0) >= 0.5 - and identity_mod.squash(line.text) == needle - for line in self.vision.ocr(after_png, region=live_readback) + if error is not None or fresh_resolution is None: + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="aborted") # type: ignore[arg-type] + ) + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}): the screen changed at the " + f"{what} region between the check and the click, and the " + "target could not be confirmed again on a fresh frame " + f"({error or 'target not found'}) — click not delivered" + ) + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="reobserved") # type: ignore[arg-type] ) - if not verified: - upscaled = identity_mod.upscale_crop(after_png, live_readback) - if upscaled is not None: - verified = any( - getattr(line, "confidence", 1.0) >= 0.5 - and identity_mod.squash(line.text) == needle - for line in self.vision.ocr(upscaled, region=None) - ) - result.input_verified = verified - if verified: - return None + current, region, baseline = fresh_resolution, fresh_region, fresh result.safety_halt = True - result.failure_category = "governed_refusal" - return ( - f"Selected option could not be verified for step '{step.id}' " - f"({step.intent}) inside its exact live re-resolved field region; the " - "commit may have selected a different value, so the run halted" - ) - - @staticmethod - def _selection_region_compatible( - qualified: Region, - live: Optional[Region], - ) -> bool: - """Check live field shape without treating recorded coordinates as truth. - - Remote drift may translate the field, so x/y intentionally do not - participate. A resolution rung that returns no region, flips the field - orientation, or changes either dimension by more than 25% cannot safely - inherit the recorded option-selection contract. - """ - - if live is None: - return False - _, _, qualified_w, qualified_h = qualified - _, _, live_w, live_h = live - if min(qualified_w, qualified_h, live_w, live_h) <= 0: - return False - if (qualified_w >= qualified_h) != (live_w >= live_h): - return False + result.failure_category = "safety_halt" return ( - 0.75 <= live_w / qualified_w <= 1.25 - and 0.75 <= live_h / qualified_h <= 1.25 + f"Step '{step.id}' ({step.intent}): the screen kept changing at the " + f"{what} region between the check and the click — click not " + "delivered; run aborted" ) - @staticmethod - def _selection_target_continuous( - qualified: Region, - focused: Point, - refreshed: Point, - ) -> bool: - """Require both select phases to name the same focused field. - - Reuse the selection contract's 25% size tolerance for point drift. - This permits small OCR/template jitter but refuses a second control. - """ - - _, _, qualified_w, qualified_h = qualified - if qualified_w <= 0 or qualified_h <= 0: - return False + def _copy_look_allowed(self) -> bool: + """The copy look is input: never in governed or qualification runs.""" return ( - abs(refreshed[0] - focused[0]) <= qualified_w * 0.25 - and abs(refreshed[1] - focused[1]) <= qualified_h * 0.25 + self.identity_copy is not None + and self._read_clipboard is not None + and self.governed_authorization is None + and self.managed_dispatch_binding is None + and self.production_qualification_guard is None + and self.qualification_campaign_guard is None + and self.qualification_fault_driver is None + and getattr(self, "approval_gate", None) is None ) - @staticmethod - def _mapped_selection_readback_region( - recorded_target: Region, - recorded_readback: Region, - live_target: Region, - frame_png: bytes, - ) -> Optional[Region]: - """Map the recorded readback band onto a fresh live target. - - Absolute recording coordinates never drive verification. The mapping - preserves the band's target-relative offsets while allowing the - already-qualified target geometry to translate or scale modestly, then - clamps it to the exact final frame. - """ - - from openadapt_flow.vision import imgops - - frame = imgops.decode_bgr(frame_png) - if frame is None: - return None - frame_h, frame_w = frame.shape[:2] - target_x, target_y, target_w, target_h = recorded_target - read_x, read_y, read_w, read_h = recorded_readback - live_x, live_y, live_w, live_h = live_target - if min(target_w, target_h, read_w, read_h, live_w, live_h) <= 0: - return None - scale_x = live_w / target_w - scale_y = live_h / target_h - mapped_x = round(live_x + (read_x - target_x) * scale_x) - mapped_y = round(live_y + (read_y - target_y) * scale_y) - mapped_w = max(1, round(read_w * scale_x)) - mapped_h = max(1, round(read_h * scale_y)) - left = max(0, mapped_x) - top = max(0, mapped_y) - right = min(frame_w, mapped_x + mapped_w) - bottom = min(frame_h, mapped_y + mapped_h) - if right <= left or bottom <= top: - return None - return (left, top, right - left, bottom - top) + def _copy_look_possible(self, identifier_region: Optional[Region]) -> bool: + return identifier_region is not None and self._copy_look_allowed() - # -- closed-loop scroll ------------------------------------------------------ + def _embedded_identity_params( + self, anchor: Anchor, workflow: Workflow + ) -> list[str]: + """Workflow parameters whose demonstrated value is in the identity band.""" + tmpl = anchor.identity_template + if tmpl is not None and tmpl.tokens: + return list(tmpl.param_token_indices) + if anchor.context_text: + return identity_mod.embedded_params(anchor.context_text, workflow.params) + return [] - @timed_span("resolve") - def _implicit_scroll_target_ready( + def _look_again( self, step: Step, - frame_png: bytes, - *, + first: IdentityCheck, + magnified: Optional[IdentityCheck], + read: Callable[[bytes], IdentityCheck], + params: Optional[Mapping[str, RuntimeParamScalar]], workflow: Workflow, - bundle_dir: Path, - params: Mapping[str, RuntimeParamScalar], - scroll_delta: Optional[tuple[int, int]] = None, - misses: Optional[list[str]] = None, - ) -> bool: - """Resolve an implicit scroll target and honor its armed identity. - - With ``scroll_delta``, the target must also be no further along the - scroll direction than where the demonstration had it (within - :data:`SCROLL_POSITION_SLACK_FRACTION` of the frame). A target that - is visible at the bottom edge but was clicked mid-screen in the - recording still needs the scroll: the step after the click was - recorded with that scroll position (for example, a section that - expands below the clicked header). - - A generic input crop can visually resolve in several places on a long - form. Declaring the target visible from that visual match alone makes - the preceding scroll a false no-op. Structural resolution stays - available, model grounding stays disabled, and an identity-armed target - is ready only when the same pre-action identity gate verifies it. - - When not ready, ``misses`` (if given) gets why: ``unresolved`` (no - direct deterministic rung found the target), ``offscreen`` (the page - reports the recorded element outside the viewport), ``position`` or - ``identity``. - """ + identifier_region: Optional[Region], + frame_png: bytes, + recorded_id_matches: Optional[Callable[[str], bool]] = None, + ) -> IdentityCheck: + """Extra looks before stopping on a look-alike identifier. - def miss(reason: str) -> bool: - if misses is not None: - misses.append(reason) - return False + Returns the verdict of :func:`identity_looks.decide`: ``verified`` + when every fresh frame agrees and the known-ID list or an exact copy + settles the record, the contradicting read when a look names another + record, or ``first`` carrying the looks. A known-ID list that may not + be used now, or a copy look that changed the screen, returns ``first`` + marked with a named halt. + """ + from openadapt_flow.runtime import known_ids as known_ids_mod - if step.anchor is None: - return miss("unresolved") - structural = ( - self.backend - if self.use_structural and hasattr(self.backend, "locate_structural") + admission = ( + self.known_ids + if isinstance(self.known_ids, known_ids_mod.KnownIdAdmission) else None ) - offscreen = getattr(structural, "structural_offscreen", None) - if ( - callable(offscreen) - and step.anchor.structural is not None - and offscreen(step.anchor.structural) is True - ): - # The page itself shows the unique recorded target exists below - # (or above) the viewport: not ready, keep scrolling. Stricter - # than letting a visual rung find a look-alike on screen, and it - # skips a full-frame OCR search per probe. The control exists, so - # this miss never hands the halt to the click (#1331). - return miss("offscreen") - template_png = self._asset_bytes( - bundle_dir, step.anchor.template, workflow=workflow - ) - try: - resolved = resolve( - step.anchor, - frame_png, - self.vision, - None, # scroll readiness must remain deterministic and model-free - step.intent, - template_png=template_png, - viewport=_frame_viewport(frame_png), - structural=structural, - ) - except OcrResolutionRefused: - # Readiness is a bounded, non-actuating probe. An ambiguous OCR - # candidate means that the target is not ready yet, so the scroll - # loop must continue. The later target action runs the full - # resolver again and still halts on the same ambiguity. - return miss("unresolved") - if resolved is None: - return miss("unresolved") - resolution, _matched_region = resolved - # Geometry estimates a point from surrounding landmarks; it does not - # prove that the target itself is visible. In particular, a fixed - # header can project the recorded point while the form row is still - # off-screen, turning the preceding scroll into a false no-op. Scroll - # readiness therefore requires direct deterministic target evidence. - # Model grounding is disabled above and future weaker rungs must opt in - # deliberately rather than silently acquiring readiness authority. - if resolution.rung not in _DIRECT_TARGET_RUNGS: - return miss("unresolved") - if scroll_delta is not None and not _scroll_target_positioned( - step.anchor.click_point, resolution.point, scroll_delta, frame_png - ): - return miss("position") - if not step.identity_armed: - return True - check = self._verify_identity( - step, resolution, frame_png, params, workflow, bundle_dir - ) - return check.status == "verified" or miss("identity") - - def _act_scroll( - self, - step: Step, - *, - workflow: Workflow, - step_index: int, - bundle_dir: Path, - run_dir: Path, - before_png: bytes, - params: Mapping[str, RuntimeParamScalar], - result: StepResult, - graph_ctx: Optional["_GraphStepContext"] = None, - ) -> Optional[str]: - """Execute a SCROLL step as a closed loop on a wait_until predicate. - - A recorded scroll's purpose is to bring the next target into view, so - the step scrolls by its recorded delta until a READINESS predicate holds - on a settled frame — not a fixed number of times. That predicate is the - step's own ``wait_until`` when set; otherwise it defaults to - ``ANCHOR_RESOLVES`` on the NEXT anchored step's anchor — today's closed - loop, now expressed as the first concrete ``wait_until`` predicate - rather than a special case (RFC §6). The step probes BEFORE scrolling - (a preceding SCROLL step may already have brought the target into view, - making this one a no-op) and stops as soon as the predicate holds. - - The loop is bounded: this step may scroll at most - ``SCROLL_BUDGET_FACTOR`` times its own recorded distance. On budget - exhaustion the step fails loudly — unless the immediately following - step is another SCROLL step, which inherits the loop (so a recorded - run of N scrolls shares a combined ~2.5x budget). - - Falls back to the fixed recorded delta (open-loop, one gesture) when no - readiness predicate exists (no ``wait_until`` and no later anchor) or - the recorded delta is zero. Predicate probes never call the grounder: - closed-loop scrolling must stay model-free. + known: Any = self.known_ids + if admission is not None: + halt = admission.check() + if halt is not None: + return first.model_copy(update={"known_id_halt": halt.reason}) + known = admission.known_ids + run_values = runtime_params_for_gui(params or {}) - Returns: - An error string on budget exhaustion (see above) or None. - """ - dx = step.scroll_dx or 0 - dy = step.scroll_dy or 0 - # The next anchored step (default scroll stop condition): linear mode - # scans forward in ``workflow.steps``; program mode uses the - # interpreter-supplied successor (the target of this state's - # unconditional transition chain). - next_step = ( - graph_ctx.next_anchored - if graph_ctx is not None - else self._next_anchored_step(workflow, step_index) - ) - # The scroll's readiness is a wait_until predicate: an explicit - # step.wait_until wins; otherwise the next anchor's ANCHOR_RESOLVES. - stop_pred = step.wait_until - implicit_target_readiness = stop_pred is None and next_step is not None - if stop_pred is None and next_step is not None: - stop_pred = Predicate( - kind=PredicateKind.ANCHOR_RESOLVES, - anchor=next_step.anchor, - intent=next_step.intent, - ) - if stop_pred is None or (dx == 0 and dy == 0): - remote_preflight_error = self._prepare_remote_scroll_input(step, result) - if remote_preflight_error is not None: - return remote_preflight_error - refusal = self._delivery_authorization_refusal( - workflow, params, step, result - ) - if refusal is not None: - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call( - result, - lambda: self.backend.scroll(dx, dy), - ) - return None + frames: list[tuple[IdentityCheck, Optional[str]]] = [] + for _ in range(looks_mod.FRAME_LOOKS): + time.sleep(self.poll_interval_s) + png = self.backend.screenshot() + frames.append((read(png), sha256_bytes(png))) + if frames[-1][0].status == "mismatch": + break # final; no further looks - # Until a wheel edge stops moving the page, readiness also requires - # the target at (or past) its demonstrated position. - position_required = [True] - # Why each readiness probe of the next target failed; they decide who - # owns a budget-exhaustion halt (see below). - misses: list[str] = [] - - def readiness_holds(frame_png: bytes) -> bool: - if implicit_target_readiness: - assert next_step is not None - return self._implicit_scroll_target_ready( - next_step, - frame_png, - workflow=workflow, - bundle_dir=bundle_dir, - params=params, - scroll_delta=(dx, dy) if position_required[0] else None, - misses=misses, - ) - return self._predicate_holds( - stop_pred, - frame_png, - bundle_dir, - params, - workflow=workflow, + frame_reads = [first, *(r for r, _ in frames)] + known_lift: Optional[IdentityCheck] = None + if known is not None and all( + looks_mod.is_glyph_abstain(r) for r in frame_reads + ): + lifted = known_ids_mod.resolve_lookalike_abstain( + frame_reads, + known, + run_values=run_values, + recorded_id_matches=recorded_id_matches, ) + if lifted is not None and lifted.known_id_evidence is not None: + update: dict[str, Any] = {"frames": len(frame_reads)} + if admission is not None: + update["confirmed_at"] = admission.confirmation.confirmed_at + update["export_sha256"] = admission.confirmation.sha256 + known_lift = lifted.model_copy( + update={ + "known_id_evidence": lifted.known_id_evidence.model_copy( + update=update + ) + } + ) - holds = readiness_holds(before_png) - if self._governed_asset_mutation is not None: - return self._governed_asset_mutation - if holds: - # Target already in view; nothing to scroll and no input sent. - # Record why, only for the next-target readiness check, so the - # outcome classifier can tell a correctly skipped scroll from an - # undelivered action (execution_profiles.classify_execution_outcome). - if implicit_target_readiness and result.delivery_attempted is not True: - result.scroll_target_ready_before = True - return None - - increment = math.hypot(dx, dy) - budget = SCROLL_BUDGET_FACTOR * increment - scrolled = 0.0 - while scrolled + increment <= budget: - # Readiness callbacks run after the outer step preflight. Bind - # every wheel edge to a new exact remote frame and repeat the - # context/identity checks after those callbacks. This also avoids - # reusing the one-shot lease consumed by the prior wheel edge. - if self._step_needs_consequential_revalidation(step, workflow): - ( - _scroll_resolution, - _scroll_region, - fresh_scroll_frame, - scroll_error, - ) = self._revalidate_consequential_actuation( - step, - None, - None, - before_png, - params, - workflow, - bundle_dir, - result, - run_dir=run_dir, + copied: Optional[looks_mod.ClipboardRead] = None + classes = looks_mod.lookalike_classes(first.observed) + bindings = self.__dict__.setdefault("_identity_look_bindings", {}) + if ( + identifier_region is not None + and self._copy_look_allowed() + and all(r.status != "mismatch" for r, _ in frames) + ): + assert self.identity_copy is not None + if self._identity_input_looks: + try: + cell_lines = self.vision.ocr(frame_png, region=identifier_region) + except Exception: # noqa: BLE001 - no box, no copy look + cell_lines = [] + cell = looks_mod.copy_region(cell_lines, identifier_region) + copied = ( + looks_mod.copy_identifier( + self.backend, + self._read_clipboard, + cell, + gesture=self.identity_copy.gesture, + copy_keys=self.identity_copy.copy_keys, + crop_region=identifier_region, + ) + if cell is not None + else looks_mod.ClipboardRead( + text=None, reason="no_single_identifier_box" + ) ) - if scroll_error is not None: - return scroll_error - if readiness_holds(fresh_scroll_frame): - return None - remote_preflight_error = self._prepare_remote_scroll_input(step, result) - if remote_preflight_error is not None: - return remote_preflight_error - refusal = self._delivery_authorization_refusal( - workflow, params, step, result - ) - if refusal is not None: - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call( - result, - lambda: self.backend.scroll(dx, dy), - ) - scrolled += increment - frame = self._wait_for_scroll_transition_and_settle(before_png) - if not self.vision.pixels_changed(before_png, frame): - # The page did not move (its end): visibility is enough. - position_required[0] = False - before_png = frame - holds = readiness_holds(frame) - if self._governed_asset_mutation is not None: - return self._governed_asset_mutation - if holds: - return None + if copied.changed_outside: + look = IdentityLook( + kind="clipboard", + outcome="unavailable", + reason="screen_changed_outside_cell", + frame_sha256=copied.frame_sha256, + ) + return first.model_copy( + update={ + "look_halt": "identity_look_changed_screen", + "looks": [look], + } + ) + crops = tuple( + c + for c in ( + looks_mod.crop_png(frame_png, identifier_region), + copied.pre_crop, + copied.post_crop, + ) + if c is not None + ) + bindings[step.id] = looks_mod.LookBinding( + step_id=step.id, classes=classes, read=copied, crops=crops + ) + else: + binding = bindings.get(step.id) + live_crop = looks_mod.crop_png(frame_png, identifier_region) + if binding is not None and binding.matches(step.id, classes, live_crop): + copied = binding.read - following = ( - graph_ctx.following - if graph_ctx is not None - else ( - workflow.steps[step_index + 1] - if step_index + 1 < len(workflow.steps) - else None - ) + known_list = None + if known is not None: + known_list = getattr(known, "known_ids", known) + decided = looks_mod.decide( + first, + frames=frames, + magnified=magnified, + contradicts=identity_read_contradicts, + known_ids=known_list, + known_ids_lift=known_lift, + clipboard=copied, + exact_expected=looks_mod.exact_expected_ids( + run_values, + self._embedded_identity_params(step.anchor, workflow) + if step.anchor is not None + else [], + ), ) - if following is not None and following.action is ActionKind.SCROLL: - # The next SCROLL step continues the loop with its own budget. - return None + binding = bindings.get(step.id) if ( - graph_ctx is None - and implicit_target_readiness - and next_step is not None - and following is next_step - and next_step.action in _SCROLL_HANDOFF_ACTIONS - and misses - and all(miss == "unresolved" for miss in misses) - and not step.risk_review_required - and not self._step_is_consequential(step, workflow) - ): - # The scroll did its bounded work; what failed is the next click's - # target, which never resolved on any frame (a relabelled or moved - # control). Halting here would name the scroll and hide the - # control that changed. Hand the miss to that click instead: it - # resolves on direct deterministic rungs only and, when they find - # nothing, halts on its own target before any input. A target - # that resolved but sat out of position or failed its identity - # check still halts here, as before. - self._scroll_missed_target = ( - next_step.id, - f"the preceding scroll '{step.id}' used {scrolled:.0f}px of its " - f"{budget:.0f}px budget and the target never came into view; " - "no deterministic rung found it, so no input was sent", + self._identity_input_looks + and binding is not None + and decided.status == "verified" + and any( + look.kind == "clipboard" and look.outcome == "resolves" + for look in decided.looks ) - return None - target_desc = ( - f"the anchor of step '{next_step.id}' ({next_step.intent})" - if next_step is not None - else self._describe_predicate(stop_pred) - ) - return ( - f"Step '{step.id}' ({step.intent}): closed-loop scroll exhausted " - f"its budget ({scrolled:.0f}px of {budget:.0f}px allowed, " - f"{SCROLL_BUDGET_FACTOR}x the recorded distance) without " - f"{target_desc} resolving — target never came into view; run aborted" - ) - - @timed_span("settle") - def _wait_for_scroll_transition_and_settle(self, baseline_png: bytes) -> bytes: - """Wait for a delivered scroll to change the frame, then settle it. - - A remote client can return two identical pre-action frames before the - wheel packet reaches the remote application. A generic settle poll can - therefore accept the old screen as stable. Require one visual - transition within the configured readiness window before applying the - normal settled-state check. At a real scroll boundary, return the last - unchanged frame after the bounded wait so the closed loop can continue - or refuse when its action budget is exhausted. - """ - - deadline = time.monotonic() + self.settle_readiness_timeout_s - frame = self.backend.screenshot() - while not self.vision.pixels_changed(baseline_png, frame): - if time.monotonic() >= deadline: - return frame - time.sleep(self.poll_interval_s) - frame = self.backend.screenshot() - return self.vision.wait_settled(self.backend) + ): + bindings[step.id] = dataclasses.replace(binding, verdict=decided) + return decided - def _prepare_remote_scroll_input( - self, - step: Step, - result: StepResult, - ) -> Optional[str]: - """Bind one remote wheel edge to a fresh, one-use frame lease. - - Target-readiness probes can take longer than a remote backend's frame - lease. A wheel gesture has no coordinate to re-resolve, but it is - still real input into an opaque session. Acquire the remote backend's - exact-content lease after readiness checks and before the final - authorization, environment, and delivery checks. Do not extend the - allowed frame age or reuse the observation from the probe. - """ + # -- typed-input verification ------------------------------------------------- - if not isinstance(self.backend, RemoteActuationBackend): - return None - try: - self.backend.acquire_actuation_frame() - except Exception as exc: # noqa: BLE001 - backend boundary must halt - if self.governed_authorization is not None: - result.safety_halt = True - detail = _scrub_phi(str(exc)) or type(exc).__name__ - return ( - "Remote scroll preflight HALTED before input for step " - f"'{step.id}' ({step.intent}): {detail}" - ) - return None + # -- closed-loop scroll ------------------------------------------------------ @staticmethod def _next_anchored_step(workflow: Workflow, step_index: int) -> Optional[Step]: @@ -16651,23 +10622,6 @@ def _next_anchored_step(workflow: Workflow, step_index: int) -> Optional[Step]: # -- postconditions -------------------------------------------------------- - def _structural_state(self) -> dict[str, Any]: - """Structural observations the backend can provide right now. - - Backends MAY expose ``url`` / ``page_title`` / ``page_count`` (see - ``openadapt_flow.backend.StructuralBackend``). Missing observations - are simply absent from the dict. - """ - state: dict[str, Any] = {} - for key in ("url", "page_title", "page_count"): - try: - value = getattr(self.backend, key, None) - except Exception: - value = None - if value is not None: - state[key] = value - return state - def _attended_transition_observation(self) -> Any: """Capture ephemeral browser structure for a signed pause baseline. @@ -16687,91 +10641,6 @@ def _attended_transition_observation(self) -> Any: page_count=page_count if isinstance(page_count, int) else None, ) - def _structural_changed( - self, key: str, start_state: dict[str, Any] - ) -> Optional[bool]: - """Whether structural observation ``key`` differs from step start. - - Returns None when the observation is unavailable on either side — - the caller treats that as an honestly-unverifiable pass (a bundle - recorded on a structural backend may replay on one that is not; - see docs/LIMITS.md). - """ - try: - current = getattr(self.backend, key, None) - except Exception: - current = None - start = start_state.get(key) - if current is None or start is None: - return None - return current != start - - @timed_span("postcondition") - def _check_postconditions( - self, - step: Step, - frame_png: bytes, - bundle_dir: Path, - start_state: dict[str, Any], - result: Optional[StepResult] = None, - *, - workflow: Optional[Workflow] = None, - ) -> tuple[bool, bytes, list[str]]: - """Poll postconditions until each passes or times out. - - Each postcondition is polled (fresh screenshots) up to its own - ``timeout_s``. If any fails, the screen is re-settled once and all - postconditions are re-checked a single time. If a VLM state-verifier - is configured, a render-drift-sensitive postcondition that still failed - gets one last drift-oracle pass (see :meth:`_drift_oracle_rescue`), - but never on a step that can leave a business effect behind (see - :meth:`_postcondition_confirms_a_write`). ``workflow`` lets a qualified - operator risk decision count; without it the classification is the - stricter step-only one. - - Returns: - (ok, last_frame, failed) — the frame the final verdict was based - on, plus human-readable descriptions of the postconditions that - failed the final check (empty when ok). - """ - self._pc_page_scale = _uniform_page_scale(step, result) - self._pc_scaled_match = False - try: - ok, frame_png = self._poll_postconditions( - step, frame_png, bundle_dir, start_state - ) - if ok: - self._note_page_scale(result) - return True, frame_png, [] - # One re-settle retry. - frame_png = self.vision.wait_settled(self.backend) - failed_pcs = [ - pc - for pc in step.expect - if not self._postcondition_passes( - pc, frame_png, bundle_dir, start_state - ) - ] - if not failed_pcs: - self._note_page_scale(result) - finally: - self._pc_page_scale = None - if ( - failed_pcs - and self.state_verifier is not None - and not self._postcondition_confirms_a_write(step, workflow) - ): - failed_pcs = self._drift_oracle_rescue(failed_pcs, frame_png, result) - failed = [self._describe_postcondition(pc) for pc in failed_pcs] - return not failed_pcs, frame_png, failed - - def _note_page_scale(self, result: Optional[StepResult]) -> None: - """Record on ``result`` the page scale a passing check needed.""" - - scale = getattr(self, "_pc_page_scale", None) - if result is not None and scale is not None and self._pc_scaled_match: - result.postcondition_page_scale = scale - @staticmethod def _postcondition_confirms_a_write( step: Step, workflow: Optional[Workflow] @@ -16801,63 +10670,6 @@ def _postcondition_confirms_a_write( return True return is_consequential(step, workflow) - def _drift_oracle_rescue( - self, - failed_pcs: list[Any], - frame_png: bytes, - result: Optional[StepResult], - ) -> list[Any]: - """Give each deterministically-failed, render-drift-sensitive - postcondition one confirmation pass through the VLM state-verifier. - - VETO-SAFE by construction: - - * The caller never invokes this on an irreversible or effect-bearing - step (see :meth:`_postcondition_confirms_a_write`). - * Only ``text_present`` and ``region_stable`` are eligible — the kinds - a theme/scale/font re-render can legitimately break. Structural - postconditions (``url_changed`` etc.) and ``text_absent`` are NEVER - rescued: those failures are real, not render drift. - * A postcondition is rescued ONLY on a confident ``"yes"``; ``"no"``, - ``"uncertain"``, and any appliance outage keep it failed (halt). - * Every call and every rescue is recorded on ``result`` for the report - (and counted as a model call), so a rescue is auditable, never silent. - - Returns the postconditions that remain failed after the pass. - """ - assert self.state_verifier is not None # guaranteed by the caller (line ~2618) - survivors: list[Any] = [] - for pc in failed_pcs: - expected = self._expected_state_text(pc) - if expected is None: # not a drift-rescuable kind - survivors.append(pc) - continue - if result is not None: - result.drift_oracle_calls += 1 - try: - confirmed = self.state_verifier.holds(frame_png, expected) - except Exception: - confirmed = False # fail-safe: any error keeps the halt - if confirmed: - if result is not None: - result.postcondition_drift_rescues.append( - self._describe_postcondition(pc) - ) - else: - survivors.append(pc) - return survivors - - @staticmethod - def _expected_state_text(pc: Any) -> Optional[str]: - """A natural-language expected-state for the drift-oracle, or None if - the postcondition kind is not render-drift-rescuable.""" - kind = pc.kind.value if hasattr(pc.kind, "value") else pc.kind - if kind == "text_present" and pc.text: - return f"the text {pc.text!r} is visible on the screen" - if kind == "region_stable": - return "the content recorded in the highlighted region is present" - return None - def _local_edit_held(self, pc: Any, start_state: dict[str, Any]) -> bool: """A compiler-classified local edit stayed local (fail closed). @@ -17049,80 +10861,6 @@ def _postcondition_passes( return distance <= pc.phash_tolerance return False - def _scaled_region_match( - self, frame_png: bytes, template_png: bytes, region: tuple[int, ...] - ) -> bool: - """The recorded content, at the page scale this step's click showed. - - A whole-page zoom (browser zoom, or an app's own scale) grows every - element and moves it away from the top-left corner by the same - factor, so the recorded region falls outside the search window and - its content outside the matcher's scale band. When this step's own - click landed at the recorded point times one uniform factor - (:func:`_uniform_page_scale`), look for the same template at that - factor (within one percent), at the recorded region times that - factor. Thresholds are the unscaled check's: a modal, a missing panel or another layout still - fails. - """ - - scale = getattr(self, "_pc_page_scale", None) - if scale is None: - return False - x, y, w, h = region - scaled = ( - int(round(x * scale)), - int(round(y * scale)), - int(round(w * scale)), - int(round(h * scale)), - ) - search = pad_region(scaled, PC_TEMPLATE_SEARCH_PAD, _frame_viewport(frame_png)) - matchers = [ - (self.vision.find_template, PC_TEMPLATE_THRESHOLD), - ( - getattr(self.vision, "find_structural_template", None), - PC_STRUCTURAL_TEMPLATE_THRESHOLD, - ), - ] - for matcher, threshold in matchers: - if not callable(matcher): - continue - try: - match = matcher( - frame_png, - template_png, - search_region=search, - scales=tuple(round(scale * f, 4) for f in _PAGE_SCALE_BAND), - threshold=threshold, - ) - except TypeError: # a vision stand-in with no scale band - return False - if match is not None: - self._pc_scaled_match = True - return True - return False - - def _dom_text_visible(self, text: str) -> bool: - """True only when the backend proves ``text`` is visibly rendered. - - Optional backend seam ``visible_text_present`` (browser DOM). Any - other answer, a missing seam, or an error is "not proven" -> False, - so the caller keeps its visual check. - """ - reader = getattr(self.backend, "visible_text_present", None) - if not callable(reader): - return False - try: - return reader(text) is True - except Exception: - return False - - def _postcondition_template(self, pc: Any, bundle_dir: Path) -> Optional[bytes]: - """Bytes of a REGION_STABLE postcondition's template crop, if any.""" - rel = getattr(pc, "template", None) - if not rel: - return None - return self._asset_bytes(bundle_dir, rel) - def _raise_on_governed_asset_mutation(self) -> None: """Make predicate-only mutation a non-catchable program safety halt.""" if self._governed_asset_mutation is not None: diff --git a/packages/flow/tests/test_multi_surface_fail_closed.py b/packages/flow/tests/test_multi_surface_fail_closed.py index a32f74b4d..4f0d93763 100644 --- a/packages/flow/tests/test_multi_surface_fail_closed.py +++ b/packages/flow/tests/test_multi_surface_fail_closed.py @@ -280,9 +280,9 @@ def _dialog_error(seen: list[dict[str, Any]], dialog: DialogResponse) -> str: def test_a_dialog_mismatch_error_quotes_no_dialog_text(monkeypatch) -> None: - import openadapt_flow.runtime.replayer as replayer_mod + import openadapt_flow.runtime.replay.acting as acting_mod # reads _DIALOG_WAIT_S - monkeypatch.setattr(replayer_mod, "_DIALOG_WAIT_S", 0.0) + monkeypatch.setattr(acting_mod, "_DIALOG_WAIT_S", 0.0) live = "Discharge patient Jane Roe (MRN 991)?" dialog = DialogResponse(response="accept", message="Discharge this patient?") error = _dialog_error([{"message": live, "matched": False}], dialog) @@ -294,9 +294,9 @@ def test_a_dialog_mismatch_error_quotes_no_dialog_text(monkeypatch) -> None: def test_a_dialog_pattern_is_armed_as_a_pattern(monkeypatch) -> None: - import openadapt_flow.runtime.replayer as replayer_mod + import openadapt_flow.runtime.replay.acting as acting_mod # reads _DIALOG_WAIT_S - monkeypatch.setattr(replayer_mod, "_DIALOG_WAIT_S", 0.0) + monkeypatch.setattr(acting_mod, "_DIALOG_WAIT_S", 0.0) dialog = DialogResponse(response="accept", message_pattern=r"Delete\ record\ \d+\?") replayer = Replayer.__new__(Replayer) backend = _DialogBackend([{"message": "Delete record 7?", "matched": True}]) diff --git a/packages/flow/tests/test_replayer.py b/packages/flow/tests/test_replayer.py index 5030a74eb..061186714 100644 --- a/packages/flow/tests/test_replayer.py +++ b/packages/flow/tests/test_replayer.py @@ -2240,10 +2240,15 @@ def scripted_resolve(*args, **kwargs): raise outcome return outcome - monkeypatch.setattr( + # The scroll's readiness probe resolves in replay.acting and the click in + # the replayer; interstitial handling (replay.settling) would resolve too. + # All draw from the same scripted outcomes. + for target in ( "openadapt_flow.runtime.replayer.resolve", - scripted_resolve, - ) + "openadapt_flow.runtime.replay.acting.resolve", + "openadapt_flow.runtime.replay.settling.resolve", + ): + monkeypatch.setattr(target, scripted_resolve) backend = FakeBackend() workflow = Workflow(name="wf", steps=[scroll_step(), click_step()]) From 0b9124c15961365ab09730fb32ae21eb8000e754 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 00:54:15 -0400 Subject: [PATCH 9/9] ci(flow): strict mypy and code owners cover runtime/core and runtime/replay The moved replayer code keeps the strict type check and safety-path review it had in replayer.py. Co-Authored-By: Claude Opus 5.5 --- .github/CODEOWNERS | 1 + .github/workflows/packages.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 98c45c70d..e42b700d2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,7 @@ /packages/flow/openadapt_flow/qualification.py @abrichr /packages/flow/openadapt_flow/runtime/replayer.py @abrichr /packages/flow/openadapt_flow/runtime/core/ @abrichr +/packages/flow/openadapt_flow/runtime/replay/ @abrichr /packages/flow/.importlinter @abrichr /packages/flow/openadapt_flow/runtime/resolver.py @abrichr /packages/flow/openadapt_flow/submit_binding.py @abrichr diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 6f0f29108..6a1fc1f59 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -309,8 +309,8 @@ jobs: openadapt_flow/runtime/identity_template.py \ openadapt_flow/runtime/identity_vlm.py \ openadapt_flow/runtime/replayer.py \ - openadapt_flow/runtime/core/__init__.py \ - openadapt_flow/runtime/core/actions.py \ + openadapt_flow/runtime/core \ + openadapt_flow/runtime/replay \ openadapt_flow/runtime/resolver.py \ openadapt_flow/submit_binding.py \ openadapt_flow/runtime/effects/effect.py \