diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4390a4020..e42b700d2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,6 +12,9 @@ /packages/flow/openadapt_flow/policies/ @abrichr /packages/flow/openadapt_flow/qualification.py @abrichr /packages/flow/openadapt_flow/runtime/replayer.py @abrichr +/packages/flow/openadapt_flow/runtime/core/ @abrichr +/packages/flow/openadapt_flow/runtime/replay/ @abrichr +/packages/flow/.importlinter @abrichr /packages/flow/openadapt_flow/runtime/resolver.py @abrichr /packages/flow/openadapt_flow/submit_binding.py @abrichr /packages/flow/openadapt_flow/validation/ @abrichr diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 20556c037..6a1fc1f59 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -288,6 +288,11 @@ jobs: run: ruff check openadapt_flow - name: Ruff format check run: ruff format --check openadapt_flow tests + - name: Import boundaries (runtime.core imports no governance) + # Contracts in packages/flow/.importlinter; DECOMPOSITION.md, section 2. + run: | + pip install 'import-linter==2.15' + lint-imports - name: Mypy run: mypy - name: Mypy (strict, safety path only) @@ -304,6 +309,8 @@ jobs: openadapt_flow/runtime/identity_template.py \ openadapt_flow/runtime/identity_vlm.py \ openadapt_flow/runtime/replayer.py \ + openadapt_flow/runtime/core \ + openadapt_flow/runtime/replay \ openadapt_flow/runtime/resolver.py \ openadapt_flow/submit_binding.py \ openadapt_flow/runtime/effects/effect.py \ diff --git a/packages/flow/.importlinter b/packages/flow/.importlinter new file mode 100644 index 000000000..68c3c179d --- /dev/null +++ b/packages/flow/.importlinter @@ -0,0 +1,122 @@ +# Import boundaries for openadapt-flow, checked by `lint-imports` in CI +# (.github/workflows/packages.yml, flow core lint). See +# docs/design/DECOMPOSITION.md, section 2. +# +# runtime.core carries out steps; governance (authorization, approvals, +# qualification, profiles, runners, the replayer that wires them) is passed in +# as gates. tests/test_import_boundaries.py keeps the forbidden list complete. + +[importlinter] +root_package = openadapt_flow +include_external_packages = False + +[importlinter:contract:runtime-core-direct] +name = runtime.core imports no governance module +type = forbidden +source_modules = + openadapt_flow.runtime.core +forbidden_modules = + openadapt_flow.runner + openadapt_flow.hosted + openadapt_flow.approval_mode + openadapt_flow.jit + openadapt_flow.runtime.replayer + openadapt_flow.production_qualification + openadapt_flow.qualification + openadapt_flow.qualification_admission + openadapt_flow.qualification_admission_v2 + openadapt_flow.qualification_admission_v4 + openadapt_flow.qualification_campaign_authority + openadapt_flow.qualification_campaign_permit + openadapt_flow.qualification_dev_signer + openadapt_flow.qualification_drills + openadapt_flow.qualification_environment + openadapt_flow.qualification_faults + openadapt_flow.qualification_identity_evidence + openadapt_flow.qualification_oracle_gate + openadapt_flow.qualification_proposal + openadapt_flow.qualification_web_session + openadapt_flow.qualified_business_decisions + openadapt_flow.qualify +allow_indirect_imports = True + +[importlinter:contract:runtime-core-indirect] +# The same boundary through every chain of imports. Today the schema and +# several shared modules import governance, so every module runtime.core +# depends on reaches it. Each edge below is one of those imports. A new import +# of governance by any module runtime.core depends on fails; a new route to an +# edge already listed does not, so keep the list short. Cutting an edge removes +# its line; an empty list makes the boundary complete. +name = runtime.core reaches no governance module except through listed edges +type = forbidden +source_modules = + openadapt_flow.runtime.core +forbidden_modules = + openadapt_flow.runner + openadapt_flow.hosted + openadapt_flow.approval_mode + openadapt_flow.jit + openadapt_flow.runtime.replayer + openadapt_flow.production_qualification + openadapt_flow.qualification + openadapt_flow.qualification_admission + openadapt_flow.qualification_admission_v2 + openadapt_flow.qualification_admission_v4 + openadapt_flow.qualification_campaign_authority + openadapt_flow.qualification_campaign_permit + openadapt_flow.qualification_dev_signer + openadapt_flow.qualification_drills + openadapt_flow.qualification_environment + openadapt_flow.qualification_faults + openadapt_flow.qualification_identity_evidence + openadapt_flow.qualification_oracle_gate + openadapt_flow.qualification_proposal + openadapt_flow.qualification_web_session + openadapt_flow.qualified_business_decisions + openadapt_flow.qualify +ignore_imports = + openadapt_flow.admit_pipeline -> openadapt_flow.approval_mode.approver + openadapt_flow.admit_pipeline -> openadapt_flow.approval_mode.signed_policy + openadapt_flow.admit_pipeline -> openadapt_flow.jit.lifecycle + openadapt_flow.admit_pipeline -> openadapt_flow.qualification_proposal + openadapt_flow.admitted_composition -> openadapt_flow.qualification_admission + openadapt_flow.deployment -> openadapt_flow.qualification_environment + openadapt_flow.execution_profiles -> openadapt_flow.qualification + openadapt_flow.execution_profiles -> openadapt_flow.qualification_identity_evidence + openadapt_flow.fleet.fingerprint -> openadapt_flow.qualification_environment + openadapt_flow.fleet.signing -> openadapt_flow.qualification_admission_v2 + openadapt_flow.fleet.status -> openadapt_flow.qualification_admission_v2 + openadapt_flow.ir -> openadapt_flow.approval_mode.model + openadapt_flow.ir -> openadapt_flow.qualification + openadapt_flow.ir -> openadapt_flow.qualification_faults + openadapt_flow.policy -> openadapt_flow.qualification + openadapt_flow.repair.candidate -> openadapt_flow.qualification + openadapt_flow.repair.invariants -> openadapt_flow.qualification + openadapt_flow.repair.lifecycle -> openadapt_flow.jit.authority + openadapt_flow.repair.readmission -> openadapt_flow.jit.lifecycle + openadapt_flow.repair.readmission -> openadapt_flow.jit.signature + openadapt_flow.repair.readmission -> openadapt_flow.qualify.campaign + openadapt_flow.repair.registration -> openadapt_flow.qualification + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.approver + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.form_state + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.gate + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.item_approval + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.model + openadapt_flow.repair.self_repair -> openadapt_flow.approval_mode.post + openadapt_flow.repair.self_repair -> openadapt_flow.jit.adapters + openadapt_flow.repair.self_repair -> openadapt_flow.jit.keys + openadapt_flow.repair.self_repair -> openadapt_flow.jit.lifecycle + openadapt_flow.repair.self_repair -> openadapt_flow.jit.signature + openadapt_flow.repair.standard_change -> openadapt_flow.jit.authority + openadapt_flow.run_gate -> openadapt_flow.qualification + openadapt_flow.run_gate -> openadapt_flow.qualification_faults + openadapt_flow.runtime.authorization -> openadapt_flow.qualification + openadapt_flow.runtime.authorization -> openadapt_flow.qualification_admission + openadapt_flow.runtime.durable.attended -> openadapt_flow.qualification + openadapt_flow.runtime.durable.business_decision -> openadapt_flow.qualification + openadapt_flow.runtime.durable.controller -> openadapt_flow.approval_mode.model + openadapt_flow.runtime.durable.resume -> openadapt_flow.runtime.replayer + openadapt_flow.runtime_validation -> openadapt_flow.hosted + openadapt_flow.sections -> openadapt_flow.qualification + openadapt_flow.teach_scope -> openadapt_flow.jit.adapters + openadapt_flow.terminal_verification_v2 -> openadapt_flow.qualification_admission_v2 diff --git a/packages/flow/docs/design/DECOMPOSITION.md b/packages/flow/docs/design/DECOMPOSITION.md index b5e5f6d9e..604efd6df 100644 --- a/packages/flow/docs/design/DECOMPOSITION.md +++ b/packages/flow/docs/design/DECOMPOSITION.md @@ -23,6 +23,7 @@ repeat the measurements. 7. [Coordination with work in flight](#7-coordination-with-work-in-flight) 8. [How the numbers were measured](#8-how-the-numbers-were-measured) 9. [Review decisions](#9-review-decisions) +10. [Duplicates to collapse](#10-duplicates-to-collapse) ## 1. What replayer.py is today @@ -299,7 +300,37 @@ state, and the production lifecycle work that changes them often. object. Reason: the report schema is a public contract, so its construction should be in one place. -### 2.4 Deliberately not done +### 2.4 The `runtime.core` boundary + +`openadapt_flow/runtime/core/` holds the code that carries out a step through +a backend and nothing that decides whether it may. Governance reaches it only +as gates the caller passes in: `ActionContext.authorize` runs delivery +authorization and the qualification-environment check at the last point +before input, in the order the replayer always ran them. + +Two import-linter contracts in `packages/flow/.importlinter`, run by +`lint-imports` in the flow lint job, keep it that way. Neither may import +`runner`, `hosted`, `approval_mode`, `jit`, the replayer or any +`qualification*` module: + +- **Direct.** No module in `runtime.core` imports a governance module. This + holds with no exceptions. +- **Indirect.** The same through every chain of imports, except 45 listed + edges. Measured on `origin/main` at `ecc162e12`: a module that imports only + `ir`, `backend` and the resolver reaches 249 of the package's 562 modules, + 44 of them governance. The routes run through the schema (`ir` imports + `qualification`, `qualification_faults` and `approval_mode.model`), the + effect verifier (`effects.effect` → `authorization` → `execution_profiles` + → `durable` → the replayer → `runner`), and the policy, repair, fleet and + durable modules. A new governance import in any module the core depends on + fails the contract. Each cut removes a line; PR 5 removes the `ir` edges. + +`tests/test_import_boundaries.py` keeps the forbidden list complete as +`qualification_*` modules are added, and proves every `ActionKind` is carried +out in exactly one place: `runtime.core.ACTION_EXECUTORS` (KEY, HOTKEY and +WAIT so far) or the replayer's list of kinds not yet moved. + +### 2.5 Deliberately not done - **Resolution rungs as registered strategies.** `resolver.resolve()` is already separate, and the rung order is part of the safety argument @@ -326,6 +357,8 @@ how the pull request proves it. | Approval gates need a signed human answer | Approval tests; no change to approval code paths outside the moves (and none before #1486 merges) | | Same side effects in the same order | Golden runs record every backend call (method and arguments) and compare the trace | | Import paths keep working | `replayer.py` re-exports every moved name; a test imports every name importable from the module on `main` before the first move | +| Contract hashes over source bytes unchanged | Nothing moves out of `runtime/resolver.py` or `runtime/program_predicates.py`: the program-predicate evaluator contract hashes their source bytes (and the vision module's) | +| Command identities unchanged | The CLI split keeps `_SEALED_READY_COMMANDS` matching, because it keys on each command function's `module:name` | | Strict typing and code-owner review keep covering the moved code | Same pull request updates the strict mypy list and CODEOWNERS to cover `runtime/replay/` | ### 3.1 Golden runs @@ -370,19 +403,20 @@ mix.py:7: error: The erased type of self "host.Replayer" is not a supertype of its class "mix.IdentityMixin" [misc] ``` -Instead, each mixin declares what it needs from the host as a `Protocol` that -only mypy sees: +Instead, each mixin declares what it needs from the host in a host class +that only mypy sees as its base: ```python -class _IdentityHost(Protocol): +# mypy: disable-error-code="empty-body" +class _IdentityChecksHost: backend: Backend - vision: Vision - identity_vlm: IdentityVlm | None + vision: Any + identity_vlm: Optional[Any] pixel_verify_enabled: bool - def _asset_bytes(self, ...) -> bytes: ... + def _asset_bytes(self, bundle_dir: Path, rel: str, *, workflow: Optional[Workflow] = None) -> Optional[bytes]: ... if TYPE_CHECKING: - _Base = _IdentityHost + _Base = _IdentityChecksHost else: _Base = object @@ -391,14 +425,37 @@ class IdentityChecks(_Base): ... ``` -This was checked with mypy 1.11.2 using the strict flags from CI. The mixin -type-checks; at run time the class is a plain `type` whose method resolution -order is `Replayer`, the mixins, `object` (no `Protocol` metaclass); and when -`Replayer` assigns an attribute a type that no longer matches the declaration, -mypy reports an error. Drift is caught, not silent. - -The host protocols are also the first deliverable of the design: each one -lists exactly what its concern reads, which is the input for Stage B. +The host is a plain class, not a `Protocol`. A first version used +`Protocol` and failed in two ways under mypy 1.11 and 2.4: + +- A protocol attribute declared without a value is abstract, so a mixin + method that assigns it (`self._render_scale_inner = None`) re-infers its + type as `None`, and the replayer's own declaration then conflicts. +- Protocol methods with empty bodies are implicitly abstract. When one mixin + stubs a method that a later mixin defines, the stub comes first in the + method resolution order and mypy reports the replayer as abstract. + +A plain host class avoids both. Its stubs' empty bodies are allowed by a +per-module `empty-body` exemption, which cannot hide anything in the moved +code because moved methods are never empty. The attribute types are the +replayer's own (class-level annotations, `__init__` annotations, or the +annotation of the parameter an attribute is assigned from), with `Any` only +where the replayer states no type. + +This was checked with mypy 2.4 using the strict flags from CI, on the real +moves in PR 2: the mixins type-check; at run time the class is a plain +`type` whose method resolution order is `Replayer`, the mixins, `object`; +and when `Replayer` assigns an attribute a type that no longer matches a +host declaration, mypy reports an error. Drift is caught, not silent. + +`scripts/extract_mixin.py` generates the mixin module, its imports and its +host class. Each host lists exactly what its concern reads, which is the +input for Stage B. + +A concern goes into `runtime.core` only when its host reads no governance +state (approval, authorization, qualification, durable or profile +attributes) and its module imports no governance module, even for types. +Everything else goes into `runtime/replay/`. Stage A also reduces merge conflicts. In the October merge-conflict review, three API-step pull requests (#1370, #1371, #1372) each conflicted on @@ -656,3 +713,31 @@ Decided in review on 2026-10-24: 4. **Resolution rungs:** no second ladder is planned, so `resolver.resolve()` stays one ladder. Checking identity against the system of record is a check source (S5), not a ladder. +5. **Core boundary:** an import-linter contract keeps `runtime.core` free of + governance imports (2.4), and the first executors move behind it at the + start of PR 2. Governance becomes gates around the core. +6. **Landing order:** #1510 (PR 1) lands last among the in-flight pull + requests, with budgets reset to `main` at merge time and the committed + golden baselines; the ratchet then becomes blocking. + +## 10. Duplicates to collapse + +Measured on `origin/main` at `ecc162e12` by reading and `grep`. "Users" counts +non-test files that reference the symbol. Each family names the canonical +implementation and the pull request in section 4.3 that collapses it. + +| Family | What exists | Canonical | PR | +|---|---|---|---| +| Check-strength scales | 8 ladders for one idea: `CheckLevel` (types, 0–5, higher is stronger, already maps the others), `VerificationTier` (1–4, lower is stronger, 46 users, on the wire), `EffectVerificationTier`, `EffectStrengthV1`, `OracleTier`, `ReadBackTier` (defined in types and hand-copied in `approval_mode/approver.py:237`), six copies of the `independent_readback \| screen_readback \| unchecked` literal, two identical `ReadGrade` literals | `CheckLevel`; `VerificationTier` stays as a wire alias | 5 | +| Run outcomes | `TransactionOutcome` (10 states), `ExecutionOutcome` and the envelope literal (5), `QualificationOutcome` (the same 5 in lower case), `ProductionExecutionOutcome`, `ExecuteTerminalOutcomeV1`, `PlainResultV1`, a parallel plain vocabulary in `desktop/outcomes.py`, free strings in the agent runner; `RunReport` carries `success`, `execution_outcome`, `outcome_envelope` and `transaction_outcome` at once | `TransactionOutcome` for the terminal state; `PlainResultV1` for people | 3 (report), 5 (schema) | +| Admission formats | Qualification admission v1 (enforced today), v2, v4; local-dev admission; JIT `admission.json`; execute admission; admit-pipeline pending and decision records; product-release admission | The v4 envelope, with dev, JIT and execute as issuer or mode variants | 5, after a cross-repository contract review | +| Certify paths | Three certify commands repeat the same policy-source resolution; Desktop `certify_bundle` evaluates qualification twice; JIT, repair readmission and the admit pipeline each sign their own records | `certify_project` and one `resolve_policy` helper | 4 | +| Signing and canonical JSON | About 45 canonical-JSON helpers. They are not byte-identical: some use `ensure_ascii=True` (`runner_capability.py:177`, `jit/lifecycle.py:192` by default), most `False`. `flow/openadapt_flow/agent/seal.py` copies `types/openadapt_types/seal.py`. Six key stores | `openadapt_types.seal.canonical_json` with an explicit `ensure_ascii`; signed formats keep a versioned helper so existing signatures still verify | 3 (agent seal copy first), 5 | +| Pagination | One pager (`runtime/list_source.py`), used only by `list_guard.py` | Nothing to collapse | — | +| Census | Not on `main`; on its feature branch a positional-id check exists in Python twice and once in JavaScript | One implementation with a generated JavaScript mirror | After that branch merges | +| Runtimes | One step interpreter (`Replayer`). Four callers start it as a subprocess, five through `deployment.build_replayer`, and about ten construct `Replayer(...)` directly with their own arguments, which can skip governance wiring | `build_replayer` for every in-process caller | 3 | + +Cheapest first: the agent seal copy, the shared literals, the copied outcome +string sets, `resolve_policy`, routing direct `Replayer(...)` callers through +`build_replayer`. Strength scales, outcomes and admissions change persisted or +signed formats and come last, each with a versioned mapping. diff --git a/packages/flow/openadapt_flow/runtime/core/__init__.py b/packages/flow/openadapt_flow/runtime/core/__init__.py new file mode 100644 index 000000000..9aaf99927 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/core/__init__.py @@ -0,0 +1,20 @@ +"""The governance-free core of step execution. + +Code here decides HOW a step is carried out through a backend. It never +decides WHETHER it may be: authorization, approvals, qualification, profiles +and durable state are gates the caller passes in (see ``ActionContext``), so +the core can be tested against a backend alone and reused by any runner. + +An import-linter contract (``packages/flow/.importlinter``) enforces the +boundary: nothing in this package may import ``runner``, ``hosted``, +``qualification*``, ``approval_mode``, ``jit`` or the replayer. See +docs/design/DECOMPOSITION.md, section 2. +""" + +from openadapt_flow.runtime.core.actions import ( + ACTION_EXECUTORS, + ActionContext, + ActionExecutor, +) + +__all__ = ["ACTION_EXECUTORS", "ActionContext", "ActionExecutor"] diff --git a/packages/flow/openadapt_flow/runtime/core/actions.py b/packages/flow/openadapt_flow/runtime/core/actions.py new file mode 100644 index 000000000..017ccdd2f --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/core/actions.py @@ -0,0 +1,117 @@ +"""Action executors: carry out one step's action through the backend. + +One executor per ``ActionKind``. An executor returns an error string when it +performed no action (or only part of one), and None when the action was +delivered. Every input it delivers goes through two callables from the +caller, in this order: + +1. ``authorize()``: every governance check that must pass at the last point + before input (delivery authorization, the qualification environment). It + returns a refusal string, or None to proceed, and may raise. +2. ``deliver(result, call)``: the caller's delivery bookkeeping around the + backend call (attempt reporting, settle seeds, acknowledgements). + +Kinds not listed in ``ACTION_EXECUTORS`` are still carried out by +``Replayer._act_without_dialog``; they move here one at a time +(docs/design/DECOMPOSITION.md, seam S4). +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any, Optional, Protocol, TypeVar, cast + +from openadapt_flow.backend import ( + GuardedKeyboardActionBackend, + RemoteActuationBackend, +) +from openadapt_flow.ir import ActionKind, Step, StepResult + +_T = TypeVar("_T") + + +class Deliver(Protocol): + def __call__(self, result: StepResult, call: Callable[[], _T]) -> _T: ... + + +@dataclass(frozen=True) +class ActionContext: + """What an executor needs: the step, the backend and the caller's gates.""" + + step: Step + result: StepResult + backend: Any + # The settled frame the step's checks ran on; guarded input binds to it. + before_png: bytes + # Governance at the last point before input: a refusal, or None. + authorize: Callable[[], Optional[str]] + deliver: Deliver + # Whether keyboard input must stay bound to the verified focus. Called + # only when an executor needs it, in the order the original code did. + requires_atomic_keyboard: Callable[[], bool] + + +ActionExecutor = Callable[[ActionContext], Optional[str]] + + +def execute_key(ctx: ActionContext) -> Optional[str]: + """KEY and HOTKEY: press one key or one modifier combination.""" + step, result = ctx.step, ctx.result + if not step.key: + return ( + f"Step '{step.id}' ({step.intent}) is a " + f"{step.action.value} step with no key" + ) + key = ( + "+".join([*step.modifiers, step.key]) + if step.action is ActionKind.HOTKEY + else step.key + ) + requires_atomic_keyboard = ctx.requires_atomic_keyboard() + guarded_key = ( + requires_atomic_keyboard + and not isinstance(ctx.backend, RemoteActuationBackend) + and isinstance(ctx.backend, GuardedKeyboardActionBackend) + ) + if guarded_key: + refusal = ctx.authorize() + if refusal is not None: + return refusal + result.delivery_receipt = ctx.deliver( + result, + lambda: cast(GuardedKeyboardActionBackend, ctx.backend).press_guarded( + key, + expected_frame_sha256=hashlib.sha256(ctx.before_png).hexdigest(), + ), + ) + result.actuation = "guarded_keyboard" + elif requires_atomic_keyboard and not isinstance( + ctx.backend, RemoteActuationBackend + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) lost its guarded keyboard " + "capability before delivery — refusing unguarded key input; " + "run aborted" + ) + else: + refusal = ctx.authorize() + if refusal is not None: + return refusal + ctx.deliver(result, lambda: ctx.backend.press(key)) + return None + + +def execute_wait(ctx: ActionContext) -> Optional[str]: + """WAIT: no input. The settle after every action does the waiting.""" + return None + + +ACTION_EXECUTORS: Mapping[ActionKind, ActionExecutor] = { + ActionKind.KEY: execute_key, + ActionKind.HOTKEY: execute_key, + ActionKind.WAIT: execute_wait, +} diff --git a/packages/flow/openadapt_flow/runtime/core/render.py b/packages/flow/openadapt_flow/runtime/core/render.py new file mode 100644 index 000000000..1f904e3c8 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/core/render.py @@ -0,0 +1,283 @@ +# mypy: disable-error-code="empty-body" +"""Render scale and layout drift: keep recorded templates comparable to the live screen. + +Moved verbatim from ``openadapt_flow.runtime.replayer.Replayer`` by +scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1). +``_RenderScaleAndLayoutDriftHost`` declares, for mypy only, what this code reads from the +Replayer it is mixed into. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional, cast # noqa: F401 + +from openadapt_flow.backend import Backend + +if TYPE_CHECKING: + from typing import Any, Callable + + from openadapt_flow.ir import ExecutionTargetKind, Step, Workflow + from openadapt_flow.runtime.surfaces import SurfaceRouter + + +class _RenderScaleAndLayoutDriftHost: + def _active_surface_binding( + self, workflow: Workflow + ) -> tuple[Optional[ExecutionTargetKind], Optional[str]]: ... + def _asset_bytes( + self, bundle_dir: Path, rel: str, *, workflow: Optional[Workflow] = None + ) -> Optional[bytes]: ... + + _layout_fingerprints: Optional[dict[str, Any]] + _layout_target_kind: Optional[str] + _render_scale_attempted: bool + _render_scale_evidence: Optional[dict[str, Any]] + _render_scale_inner: Optional[Backend] + _surface_router: Optional[SurfaceRouter] + backend: Backend + layout_drift: Optional[bool] + layout_drift_advisor: Optional[Callable[[Any], Optional[str]]] + layout_drift_tolerances: Optional[Any] + render_scale: Optional[bool] + vision: Any + + +if TYPE_CHECKING: + _Base = _RenderScaleAndLayoutDriftHost +else: + _Base = object + + +class RenderScaleAndLayoutDrift(_Base): + """Render scale and layout drift: keep recorded templates comparable to the live screen.""" + + def _layout_drift_on(self, workflow: Workflow) -> bool: + if self.layout_drift is not None: + return self.layout_drift + kind: Optional[str] + if self._surface_router is not None: + # Multi-surface run: follow the surface the step acts on, so an + # RDP segment of a web-first bundle keeps the check. + kind, _ = self._active_surface_binding(workflow) + else: + kind = self._layout_target_kind or workflow.surface + if kind in ("rdp", "citrix"): + return True + backend = self._render_scale_inner or self.backend + return type(backend).__name__ == "FreeRDPBackend" + + def _live_session_profile(self) -> Optional[str]: + """Digest of the pinned remote-session profile, when there is one.""" + transport = getattr(self.backend, "transport", None) + profile = getattr(getattr(transport, "profile", None), "sha256", None) + return profile if isinstance(profile, str) else None + + def _session_profile_changed(self, recorded: Any) -> bool: + live = self._live_session_profile() + return bool( + recorded.profile_sha256 and live and recorded.profile_sha256 != live + ) + + def _reset_render_scale(self) -> None: + """Return to the caller's backend and forget the last estimate.""" + if self._render_scale_inner is not None: + release = getattr(self.backend, "release_frame_sources", None) + if callable(release): + release() + self.backend = self._render_scale_inner + self._render_scale_inner = None + self._render_scale_attempted = False + self._render_scale_evidence = None + + def _load_layout_fingerprints( + self, bundle_dir: Path, workflow: Workflow + ) -> dict[str, Any]: + from openadapt_flow.runtime import layout_drift as drift_mod + + if self._layout_fingerprints is None: + self._layout_fingerprints = drift_mod.load_bundle_fingerprints( + self._asset_bytes( + Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow + ) + ) + return self._layout_fingerprints + + def _maybe_scale_render( + self, + step: Step, + frame_png: bytes, + bundle_dir: Optional[Path], + workflow: Workflow, + ) -> bytes: + """Estimate the session's display scale once; return the frame to use. + + Runs at the first step that has a recorded fingerprint. A frame that + matches the fingerprint, or gives no confident estimate, is returned + unchanged and the layout-drift check judges it as before. Otherwise + the backend is replaced by a + :class:`~openadapt_flow.runtime.render_scale.RenderScaledBackend` and + the frame is returned in the recorded geometry, where the same + layout-drift check must then pass. + """ + if self._render_scale_attempted or bundle_dir is None: + return frame_png + enabled = ( + self.render_scale + if self.render_scale is not None + else self._layout_drift_on(workflow) + ) + if not enabled or self._surface_router is not None: + return frame_png + from openadapt_flow.runtime import layout_drift as drift_mod + from openadapt_flow.runtime import render_scale as scale_mod + + recorded = self._load_layout_fingerprints(Path(bundle_dir), workflow).get( + step.id + ) + if recorded is None: + return frame_png + self._render_scale_attempted = True + try: + import io + + from PIL import Image + + with Image.open(io.BytesIO(frame_png)) as img: + viewport = (int(img.width), int(img.height)) + except Exception: # noqa: BLE001 - undecodable: leave it to the drift check + return frame_png + if self._session_profile_changed(recorded): + # A pinned remote-session profile (scale, DPI, font smoothing) is + # a deployment decision; a different profile still halts below. + self._render_scale_evidence = { + "status": "no_estimate", + "reason": "the pinned remote session profile changed", + } + return frame_png + lines = self.vision.ocr(frame_png) + live = drift_mod.fingerprint(lines, viewport) + verdict = drift_mod.compare( + recorded, + live, + self.layout_drift_tolerances or drift_mod.Tolerances(), + ) + if not verdict.drifted: + self._render_scale_evidence = {"status": "recorded_geometry"} + return frame_png + salt, hashed = drift_mod.load_scale_landmarks( + self._asset_bytes( + Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow + ) + ) + landmarks = scale_mod.hashed_landmarks( + hashed.get(step.id, []) + ) + scale_mod.anchor_landmarks(workflow.steps) + pairs = scale_mod.match_landmarks(landmarks, lines, salt_hex=salt) + transform, reason = scale_mod.estimate_render_transform( + pairs, + recorded_viewport=recorded.viewport, + live_viewport=viewport, + ) + if transform is None: + self._render_scale_evidence = { + "status": "no_estimate", + "reason": reason, + "drift": list(verdict.reasons), + } + return frame_png + scaled = scale_mod.render_scaled_backend(self.backend, transform) + self._render_scale_inner = self.backend + self.backend = cast(Backend, scaled) + self._render_scale_evidence = {"status": "normalized", **transform.evidence()} + # The layout-drift check below now judges the normalized frame, so a + # font or layout change that is not a pure rescale still stops here. + return scaled.normalize(frame_png) + + def _layout_drift_error( + self, + step: Step, + frame_png: bytes, + bundle_dir: Optional[Path], + workflow: Workflow, + ) -> Optional[str]: + """Halt message when the frame drifted from the recorded fingerprint. + + Steps without a recorded fingerprint (older bundles, non-click steps) + are not checked. Model-free: the advisor, when configured, is called + only after drift is found and never changes the verdict. + """ + if bundle_dir is None or not self._layout_drift_on(workflow): + return None + from openadapt_flow.runtime import layout_drift as drift_mod + + recorded = self._load_layout_fingerprints(Path(bundle_dir), workflow).get( + step.id + ) + if recorded is None: + return None + live_profile = self._live_session_profile() + try: + import io + + from PIL import Image + + with Image.open(io.BytesIO(frame_png)) as img: + viewport = (int(img.width), int(img.height)) + except Exception: # noqa: BLE001 - an undecodable frame is drift + viewport = (0, 0) + lines = self.vision.ocr(frame_png) + tolerances = self.layout_drift_tolerances or drift_mod.Tolerances() + if self._render_scale_inner is not None and not self._session_profile_changed( + recorded + ): + # A normalized frame is judged label by label: the recorded label + # set says where each label sat and how large it was, so OCR box + # jitter on other lines cannot pass for a font change. + from openadapt_flow.runtime import render_scale as scale_mod + + salt, hashed = drift_mod.load_scale_landmarks( + self._asset_bytes( + Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow + ) + ) + details = ( + scale_mod.check_normalized_frame( + hashed.get(step.id, []), + lines, + salt_hex=salt, + text_height_ratio=tolerances.text_height_ratio, + landmark_px=tolerances.landmark_px, + ) + if salt is not None + else None + ) + if details: + return ( + f"Layout drift before step '{step.id}' ({step.intent}) on " + f"the display-scaled session: {'; '.join(details)} — the " + "screen no longer matches the recording; refusing to act." + ) + if details is not None: + return None + live = drift_mod.fingerprint( + lines, + viewport, + landmarks=list(recorded.landmarks), + profile_sha256=(live_profile if isinstance(live_profile, str) else None), + ) + try: + drift_mod.check_layout( + recorded, + live, + tolerances=tolerances, + advisor=self.layout_drift_advisor, + ) + except drift_mod.LayoutDriftHalt as halt: + advice = f" Advice: {halt.verdict.advice}" if halt.verdict.advice else "" + return ( + f"Layout drift before step '{step.id}' ({step.intent}): " + f"{drift_mod.describe(halt.verdict)} — the screen no longer " + f"matches the recording; refusing to act.{advice}" + ) + return None diff --git a/packages/flow/openadapt_flow/runtime/replay/__init__.py b/packages/flow/openadapt_flow/runtime/replay/__init__.py new file mode 100644 index 000000000..d82a6e701 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/__init__.py @@ -0,0 +1 @@ +"""Replayer concerns that involve governance, one module each (docs/design/DECOMPOSITION.md, section 4).""" diff --git a/packages/flow/openadapt_flow/runtime/replay/_shared.py b/packages/flow/openadapt_flow/runtime/replay/_shared.py new file mode 100644 index 000000000..8a446483b --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/_shared.py @@ -0,0 +1,527 @@ +"""Constants, helpers and small types the replayer's concerns share. + +Moved verbatim from ``openadapt_flow.runtime.replayer`` by +scripts/extract_mixin.py; the source module re-exports every name. +""" + +from __future__ import annotations + +import re +import struct +from dataclasses import dataclass +from typing import Any, Mapping, Optional, TypeVar + +from openadapt_flow.ir import ( + ActionKind, + LoopIterationBinding, + RunReport, + StateKind, + Step, + Workflow, +) +from openadapt_flow.network_scope import EndpointScope, endpoint_scope +from openadapt_flow.runtime.authorization import RuntimeParamScalar +from openadapt_flow.runtime.durable.program_checkpoint import ( + GraphFrame, + ProgramCheckpoint, +) + +_DeliveryResultT = TypeVar("_DeliveryResultT") +_MAX_FRESH_ACTUATION_REACQUISITIONS = 2 +# Check-to-click guard on the pixel path (``Replayer._pixel_recheck``): how +# often a changed target or identity region is re-observed before the click +# is refused, and how much a pixel must change to count (remote encoders +# shift colours by a few levels; a dialog or a new row changes far more). +_PIXEL_RECHECK_REOBSERVES = 2 +_PIXEL_RECHECK_LEVEL = 24 +_PIXEL_RECHECK_MIN_PIXELS = 4 +# How long a step that raises a browser dialog waits for it to open. +_DIALOG_WAIT_S = 5.0 + +# How long past its step's timeout a held record check may run under an +# approval mode before the held report decides the item (a person). +READBACK_DEADLINE_MARGIN_S = 120.0 +# Private module capability. The public ``run`` API cannot mint a durable +# continuation. ``runtime.durable.resume`` imports this object only after it +# has validated the pause, approval, bundle, checkpoint, and live state. +_DURABLE_RESUME_AUTHORITY = object() + + +# REGION_STABLE template check: how far the expected content may shift from +# the recorded region (real apps re-layout by a few pixels between runs), +# and the minimum template-match score to accept it. +PC_TEMPLATE_SEARCH_PAD = 80 + + +_FIELD_CHECK_LIMIT = 32 +_FIELD_VALUE_LIMIT = 256 + + +def _field_text(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (str, int, float)): + return str(value)[:_FIELD_VALUE_LIMIT] + return None + + +def effect_field_checks(effect: Any, verdict: Any, tier: Any) -> list[Any]: + """The field comparisons one effect verdict made, for the local report. + + A ``field_equals`` effect compares one field. A record effect compares + each literal ``match`` value against the first matched record. Values + that aren't plain text or numbers are left out rather than guessed. + """ + + from openadapt_flow.ir import EffectFieldCheck + + tier_value = int(tier) if tier is not None else None + checks: list[EffectFieldCheck] = [] + field = getattr(effect, "field", None) + expected_value = getattr(verdict, "expected_value", None) + if field and expected_value is not None: + checks.append( + EffectFieldCheck( + field=str(field)[:_FIELD_VALUE_LIMIT], + expected=_field_text(expected_value), + observed=_field_text(getattr(verdict, "observed_value", None)), + matched=bool(getattr(verdict, "confirmed", False)), + verification_tier=tier_value, + ) + ) + return checks + records = getattr(verdict, "matched_records", None) or [] + record = records[0] if records and isinstance(records[0], dict) else None + for name, want in (getattr(effect, "match", None) or {}).items(): + expected = _field_text(want) + if expected is None: + continue + observed = _field_text(record.get(name)) if record is not None else None + checks.append( + EffectFieldCheck( + field=str(name)[:_FIELD_VALUE_LIMIT], + expected=expected, + observed=observed, + matched=observed == expected, + verification_tier=tier_value, + ) + ) + if len(checks) >= _FIELD_CHECK_LIMIT: + break + return checks + + +#: A uniform page scale is inferred only from a click point at least this far +#: from both edges at the origin, only when both axes agree to within this +#: fraction, and only inside these bounds (and not within 5% of 1). +_PAGE_SCALE_MIN_COORD = 40 +_PAGE_SCALE_AXIS_AGREEMENT = 0.02 +_PAGE_SCALE_BOUNDS = (0.5, 2.0) +#: One pixel of click-point noise moves the estimate by a few tenths of a +#: percent, enough to drop a wide template below threshold; the check tries +#: this tight band around it. +_PAGE_SCALE_BAND = (0.99, 0.995, 1.0, 1.005, 1.01) + + +def _uniform_page_scale(step: Any, result: Any) -> Optional[float]: + """The page scale this step's click showed, or ``None``. + + The recorded click point and the point the ladder resolved this run, + divided axis by axis. A whole-page zoom multiplies both coordinates by + the same factor; a control that merely moved almost never does, so the + two axes must agree. + """ + + resolution = getattr(result, "resolution", None) + anchor = getattr(step, "anchor", None) + if resolution is None or anchor is None: + return None + try: + rx, ry = (float(v) for v in anchor.click_point) + lx, ly = (float(v) for v in resolution.point) + except (TypeError, ValueError): + return None + if min(rx, ry) < _PAGE_SCALE_MIN_COORD: + return None + sx, sy = lx / rx, ly / ry + if abs(sx - sy) > _PAGE_SCALE_AXIS_AGREEMENT * max(sx, sy): + return None + scale = (sx + sy) / 2 + low, high = _PAGE_SCALE_BOUNDS + if not low <= scale <= high or abs(scale - 1.0) < 0.05: + return None + return round(scale, 3) + + +def _frame_viewport(frame_png: bytes) -> tuple[int, int]: + """Return the viewport of THIS frame, read from the frame itself. + + ``self.backend.viewport`` is a live read. Pairing it with an + already-captured frame lets a resize or move between capture and use + reinterpret that frame under geometry it never had, which can place a + coordinate where the operator never demonstrated one. A PNG carries its own + dimensions in the IHDR chunk, so the frame can always answer for itself. + """ + + if len(frame_png) < 24 or not frame_png.startswith(b"\x89PNG\r\n\x1a\n"): + raise ValueError("frame viewport requires valid PNG bytes") + width, height = struct.unpack(">II", frame_png[16:24]) + if width <= 0 or height <= 0: + raise ValueError("frame viewport must be positive") + return int(width), int(height) + + +PC_TEMPLATE_THRESHOLD = 0.9 +# REGION_STABLE asserts recorded structure, not palette. The ordinary +# grayscale template matcher remains the first (stricter) check; this edge-map +# threshold is a color-invariant fallback. Exact v1.16.1 MockMed evidence +# measured correct theme+parameter substitution at 0.86 and a true blocking +# modal at 0.24, leaving a deliberate margin on both sides. +PC_STRUCTURAL_TEMPLATE_THRESHOLD = 0.8 + +# Typed-input verification: size of the "field region" diffed/OCRed around +# the focusing click point after a TYPE action. Generous on purpose — the +# typed text renders at the field's own left edge / first line, not at the +# click point — and clamped to the viewport. When no focusing click is +# known (keyboard-only focus moves, e.g. Tab between fields), the whole +# frame is used instead. +FIELD_REGION_SIZE = (640, 240) +FIELD_REGION_PAD = 16 + +# Typed-input verification, secret-only diff acceptance: when a SECRET typed +# value is OCR-able but OCR cannot find it, a pixel change alone is accepted +# only if the field region gained no other READABLE text — masked fields render +# dots, which OCR reads as nothing, low-confidence noise, or punctuation +# runs, while a dialog rendering over the region adds confident words. +# "Readable" therefore counts alphanumeric characters of lines OCR is +# confident about (>= MASKED_MIN_CONFIDENCE), EXCLUDING homogeneous glyph +# runs: a dot row can misread as a confident digit run (measured on the +# Linux renderer: 17 bullets -> '0000000000006' at 0.81 confidence), and +# such runs are >= MASKED_REPEAT_FRACTION one repeated character — +# no real dialog sentence is. Alnum counts are also invariant to +# segmentation differences between frames. The gain may not exceed +# MASKED_NEW_TEXT_SLACK. +MASKED_NEW_TEXT_SLACK = 3 +MASKED_MIN_CONFIDENCE = 0.6 +MASKED_REPEAT_FRACTION = 0.66 + +# Closed-loop scroll: a SCROLL step keeps scrolling by its recorded delta +# until the NEXT anchored step's anchor resolves on a settled frame, bounded +# by this multiple of the step's own recorded scroll distance. Consecutive +# SCROLL steps hand the loop to each other (each probes first and no-ops +# once the anchor is in view), so a run of N recorded scrolls has a combined +# budget of ~2.5x the total recorded distance. +SCROLL_BUDGET_FACTOR = 2.5 + +# A secret TYPE step's value is never stored in the bundle; it is read at +# replay from this environment variable (the param name upper-cased, with +# non-alphanumeric characters mapped to '_'). See ir.Step.secret. +SECRET_ENV_PREFIX = "OPENADAPT_FLOW_SECRET_" + + +def secret_env_var(param: str) -> str: + """Environment variable a secret parameter's value is read from.""" + key = "".join(ch if ch.isalnum() else "_" for ch in param).upper() + return f"{SECRET_ENV_PREFIX}{key}" + + +# Bounds for the Phase-2 program interpreter -- deterministic termination +# guarantees. A worklist loop is bounded per-loop (LoopSpec.max_iterations); the +# graph walk itself is bounded by a total step budget (so an authored cycle of +# always-true transitions can never spin forever) and subflow/loop nesting is +# depth-bounded. +PROGRAM_MAX_STEPS = 100_000 +PROGRAM_MAX_DEPTH = 64 + + +class _GraphStepContext: + """Per-action-state context the program interpreter feeds ``_run_step``. + + Supplies the two pieces the linear loop derives from list position but a + graph lacks: the previously EXECUTED action (for the click-to-focus TYPE + heuristic) and the SCROLL closed-loop successors (the next anchored action + state and the immediately following state). All optional; a None field means + "no such successor" (the scroll loop then falls back to its open-loop + gesture, exactly as at the end of a linear workflow). + """ + + __slots__ = ( + "prev_action", + "next_anchored", + "following", + "has_exception_handler", + ) + + def __init__( + self, + prev_action: Optional[ActionKind] = None, + next_anchored: Optional[Step] = None, + following: Optional[Step] = None, + has_exception_handler: bool = False, + ) -> None: + self.prev_action = prev_action + self.next_anchored = next_anchored + self.following = following + self.has_exception_handler = has_exception_handler + + +def _pipeline_safe_verifier(verifier: Any) -> bool: + """Whether a read-back may run on a side thread while the next item runs. + + Only verifiers that read the system of record over the network and never + touch the screen: REST, FHIR, GraphQL and SQL. Anything that binds to a + backend (on-screen read-backs, and the redacting and candidate wrappers, + which forward ``bind_backend``) or any other kind verifies inline. + """ + + if ( + verifier is None + or hasattr(verifier, "bind_backend") + or getattr(verifier, "backend", None) is not None + or getattr(verifier, "_backend", None) is not None + ): + return False + safe: list[type] = [] + for module, name in ( + ("openadapt_flow.runtime.effects.rest", "RestRecordVerifier"), + ("openadapt_flow.runtime.effects.fhir", "FhirEffectVerifier"), + ("openadapt_flow.runtime.effects.graphql", "GraphQLRecordVerifier"), + ("openadapt_flow.runtime.effects.sql", "SqlRecordVerifier"), + ): + try: + import importlib + + safe.append(getattr(importlib.import_module(module), name)) + except Exception: # noqa: BLE001 - an unavailable kind is not safe + continue + return isinstance(verifier, tuple(safe)) + + +class _ProgramHalt(Exception): + """Internal signal: an unrecoverable state was reached (an unhandled action + failure, a dead-end with no matching transition, a bound exceeded, or a + ``halt`` / ``escalate`` terminal). Carries the outcome + reason the run + report records. Caught only by ``_interpret_program`` -- never escapes the + Replayer.""" + + def __init__(self, outcome: str, reason: str, *, safety: bool = False) -> None: + super().__init__(reason) + self.outcome = outcome + self.reason = reason + self.safety = safety + # Populated at an ACTION-state failure before graph frames unwind. + # Empty means the halt is not eligible for a generic attended + # Continue/Skip transition. + self.program_frames: list[GraphFrame] = [] + self.program_params: dict[str, RuntimeParamScalar] = {} + self.program_history_hash: str = "" + + +def effect_endpoint_scope(verifier: Any) -> EndpointScope: + """Where an effect verifier reads its system of record. + + A verifier that names a ``base_url`` (REST, FHIR) is ``loopback`` when that + URL is on this computer and ``external`` otherwise. A verifier without one + is ``unknown``, which keeps a network substrate counted as egress. + """ + base_url = getattr(verifier, "base_url", None) + return endpoint_scope(base_url) if isinstance(base_url, str) else "unknown" + + +def _record_loop_iteration( + report: RunReport, + loop_state_id: str, + row_index: int, + row: Mapping[str, RuntimeParamScalar], +) -> None: + """Record the row values one loop iteration binds, before it runs. + + ``report.params`` is the run's base scope. A row field overrides it for + that iteration, so the audit report needs each row's own values to state + what the run typed. + """ + report.loop_iterations.append( + LoopIterationBinding( + loop_state_id=loop_state_id, + row_index=row_index, + params=dict(row), + ) + ) + + +def _loop_iterations_from_checkpoints( + checkpoints: list[ProgramCheckpoint], +) -> list[LoopIterationBinding]: + """Rebuild the loop rows that earlier legs of a resumed run bound. + + A resumed report carries the earlier legs' results, so it also needs the + row values those results typed. Each checkpoint's frame stack names the + loop row it ran in. A row is listed once, at its first checkpoint, keyed by + its position in the frame stack so a nested loop's rows under different + outer rows stay distinct. + """ + seen: set[tuple[tuple[str, str, int], ...]] = set() + bindings: list[LoopIterationBinding] = [] + for checkpoint in checkpoints: + path: tuple[tuple[str, str, int], ...] = () + for frame in checkpoint.frames: + cursor = frame.loop + if cursor is None: + path = (*path, (frame.graph_id, "", -1)) + continue + path = (*path, (frame.graph_id, cursor.loop_state_id, cursor.row_index)) + if path in seen or not 0 <= cursor.row_index < len(cursor.rows): + continue + seen.add(path) + bindings.append( + LoopIterationBinding( + loop_state_id=cursor.loop_state_id, + row_index=cursor.row_index, + params=dict(cursor.rows[cursor.row_index]), + ) + ) + return bindings + + +def _all_workflow_steps(workflow: Workflow): + """Yield every ``Step`` in a workflow for whole-bundle audits. + + Linear bundle: the ``steps`` list. Program bundle: every ``action`` state's + step across the top-level program graph AND all subflows (a loop body's + steps included). Order is stable but not execution order -- used only for + static coverage counting, not for running. + """ + if workflow.program is None: + yield from workflow.steps + return + graphs = [workflow.program, *workflow.subflows.values()] + for graph in graphs: + for state in graph.states.values(): + if state.kind is StateKind.ACTION and state.step is not None: + yield state.step + + +def _effect_read_wording(tier: Any) -> tuple[str, str]: + """Name an effect check in a halt message by its evidence tier. + + A separate-session record-page read (tier 2) and a screen read-back + (tiers 3 and 4) are not system-of-record reads, so their halt messages + must not call them that. Untyped legacy verifiers keep the old wording. + """ + + from openadapt_flow.verification import VerificationTier + + if tier is VerificationTier.INDEPENDENT_SESSION: + return "Effect verification", "record page read in a separate session" + if tier in ( + VerificationTier.PERSISTED_STATE_REACQUISITION, + VerificationTier.IMMEDIATE_SCREEN, + ): + return "Effect verification", "screen read-back" + return "System-of-record effect verification", "system of record" + + +class EgressNotPermitted(RuntimeError): + """Raised when an egress-capable model component (a grounder / identity-VLM + / state-verifier that could send a screenshot off the box) is wired without + the operator's explicit ``allow_model_grounding`` opt-in (PHI audit REM-3). + """ + + +#: How far past its demonstrated position (as a fraction of the frame's extent +#: along the scroll axis) a scroll target may still be before the closed-loop +#: scroll counts it as ready. +SCROLL_POSITION_SLACK_FRACTION = 0.25 + +#: Rungs that show the target itself on screen: what closed-loop scroll +#: readiness accepts, and all a step accepts after its scroll missed it. +_DIRECT_TARGET_RUNGS = frozenset({"structural", "template", "template_global", "ocr"}) + +#: Steps that own the halt when the closed-loop scroll before them never found +#: their target (see ``Replayer._act_scroll``). +_SCROLL_HANDOFF_ACTIONS = frozenset( + {ActionKind.CLICK, ActionKind.DOUBLE_CLICK, ActionKind.RIGHT_CLICK} +) + + +def _scroll_target_positioned( + recorded: tuple[int, int], + resolved: tuple[int, int], + scroll_delta: tuple[int, int], + frame_png: bytes, +) -> bool: + """Is the resolved target no further along the scroll than recorded?""" + + dx, dy = scroll_delta + w, h = _frame_viewport(frame_png) + if dy: + slack = SCROLL_POSITION_SLACK_FRACTION * h + ahead = (resolved[1] - recorded[1]) if dy > 0 else (recorded[1] - resolved[1]) + if ahead > slack: + return False + if dx: + slack = SCROLL_POSITION_SLACK_FRACTION * w + ahead = (resolved[0] - recorded[0]) if dx > 0 else (recorded[0] - resolved[0]) + if ahead > slack: + return False + return True + + +def _identity_tokens(text: Any) -> set[str]: + return {t for t in re.split(r"[^0-9a-z]+", str(text or "").lower()) if t} + + +def identity_read_contradicts(check: Any) -> bool: + """A ``mismatch`` read that saw text the expected identity doesn't have. + + "mismatch" also covers a read that is merely incomplete (columns cut + off). Only a read that shows a token the expected record lacks (another + MRN, a different spelling or date) is evidence of a different record. + """ + + if getattr(check, "status", None) != "mismatch": + return False + expected = _identity_tokens(getattr(check, "expected", None)) + observed = _identity_tokens(getattr(check, "observed", None)) + if not expected or not observed: + return True # nothing to compare: keep the refusal + return not observed <= expected + + +def better_identity_read(first: Any, retry: Any) -> Any: + """The identity read to keep from a first read and its upscaled re-read. + + A read that contradicts the expected record (:func:`identity_read_contradicts`) + is final: a second read of the same screen must never turn it into + ``verified``. A mismatch that is only incomplete can be rescued by a + re-read that verifies. Otherwise ``verified`` beats the "cannot certify" + outcomes, then a mismatch, then ``abstain``, then a blank ``unreadable``; + a tie goes to the read that covered more of the expected identity. + """ + + if identity_read_contradicts(first): + return first + if identity_read_contradicts(retry): + return retry + rank = {"unreadable": 0, "abstain": 1, "mismatch": 2, "verified": 3} + if (rank[retry.status], retry.coverage) > (rank[first.status], first.coverage): + return retry + return first + + +#: Server answers kept per step (a save makes one or a few write requests). +_MAX_SERVER_RESPONSES = 64 + + +@dataclass(frozen=True) +class _WebFormApprovalView: + """An API step's prepared request, as a person is asked to approve it.""" + + prepared: Any + #: :func:`~openadapt_flow.runtime.actuators.web_form.operation_digest`. + digest: str + rows: list[tuple[str, str]] + png: bytes + ref: Any diff --git a/packages/flow/openadapt_flow/runtime/replay/acting.py b/packages/flow/openadapt_flow/runtime/replay/acting.py new file mode 100644 index 000000000..494a9f623 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/acting.py @@ -0,0 +1,1682 @@ +# mypy: disable-error-code="empty-body" +"""Acting by kind: deliver clicks, drags, typing, selections, scrolls and dialog answers. + +Moved verbatim from ``openadapt_flow.runtime.replayer.Replayer`` by +scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1). +``_ActingHost`` declares, for mypy only, what this code reads from the +Replayer it is mixed into. +""" + +from __future__ import annotations + +import hashlib +import math +import os +import time +from typing import TYPE_CHECKING, Any, Callable, Optional, cast # noqa: F401 + +from openadapt_flow.backend import ( + GuardedCoordinateActionBackend, + GuardedDragActionBackend, + GuardedKeyboardActionBackend, + GuardedRemotePointerActionBackend, + GuardedSelectOptionBackend, + RemoteActuationBackend, + RichPointerActionBackend, + SelectOptionBackend, +) +from openadapt_flow.ir import ActionKind, Point, Predicate, PredicateKind, Region +from openadapt_flow.privacy import scrub_text as _scrub_phi +from openadapt_flow.runtime.authorization import runtime_param_text +from openadapt_flow.runtime.core import ACTION_EXECUTORS, ActionContext +from openadapt_flow.runtime.replay._shared import ( + _DIALOG_WAIT_S, + _DIRECT_TARGET_RUNGS, + _SCROLL_HANDOFF_ACTIONS, + SCROLL_BUDGET_FACTOR, + _frame_viewport, + _scroll_target_positioned, + secret_env_var, +) +from openadapt_flow.runtime.resolver import resolve +from openadapt_flow.runtime.step_timing import timed_span +from openadapt_flow.vision.ocr import OcrResolutionRefused + +if TYPE_CHECKING: + from pathlib import Path + from typing import Mapping, Union + + from openadapt_flow.backend import Backend + from openadapt_flow.ir import ( + Anchor, + IdentityCheck, + Resolution, + Step, + StepResult, + Workflow, + ) + from openadapt_flow.qualification_faults import QualificationFaultDriver + from openadapt_flow.runtime.authorization import ( + GovernedRunAuthorization, + RuntimeParamScalar, + ) + from openadapt_flow.runtime.replay._shared import _GraphStepContext + + +class _ActingHost: + _asset_bytes: Any + + def _authorize_delivery( + self, + workflow: Workflow, + params: Mapping[str, RuntimeParamScalar], + step: Step, + result: StepResult, + ) -> Optional[str]: ... + + _cancel_guarded_coordinate: Any + _cancel_guarded_keyboard: Any + _deliver_backend_call: Any + _delivery_authorization_refusal: Any + + @staticmethod + def _describe_predicate(pred: Predicate) -> str: ... + + _field_reads: dict[str, Any] + _governed_asset_mutation: Optional[str] + _last_click_point: Optional[Point] + _last_click_region: Optional[Region] + + @staticmethod + def _next_anchored_step(workflow: Workflow, step_index: int) -> Optional[Step]: ... + def _pixel_recheck( + self, + step: Step, + resolution: Resolution, + checked_png: bytes, + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + bundle_dir: Path, + result: StepResult, + run_dir: Path, + ) -> Union[Resolution, str]: ... + def _predicate_holds( + self, + pred: Predicate, + frame_png: bytes, + bundle_dir: Path, + params: Mapping[str, RuntimeParamScalar], + *, + workflow: Optional[Workflow] = None, + ) -> bool: ... + + _read_clipboard: Optional[Any] + + def _read_gate_refusal(self, step: Step, result: StepResult) -> Optional[str]: ... + + _require_qualification_environment_current: Any + _requires_atomic_identity_keyboard: Any + _requires_atomic_identity_pointer: Any + + def _resolve_drag_end( + self, step: Step, screen_png: bytes, bundle_dir: Path, workflow: Workflow + ) -> tuple[Optional[Resolution], Optional[Region], Optional[str]]: ... + def _retain_visual_resolution_evidence( + self, + *, + workflow: Workflow, + anchor: Anchor, + resolution: Resolution, + matched_region: Region, + frame_png: bytes, + bundle_dir: Path, + run_dir: Path, + allow_target_ocr: bool = True, + ) -> Resolution: ... + + _revalidate_consequential_actuation: Any + _run_id: str + _scroll_missed_target: Optional[tuple[str, str]] + _selection_contract_error: Any + + @staticmethod + def _selection_region_compatible( + qualified: Region, live: Optional[Region] + ) -> bool: ... + @staticmethod + def _selection_target_continuous( + qualified: Region, focused: Point, refreshed: Point + ) -> bool: ... + + _step_is_consequential: Any + _step_needs_consequential_revalidation: Any + _step_requires_remote_guarded_actuation: Any + + def _text_value_at(self, field_point: Optional[Point]) -> Optional[str]: ... + + _translate_recorded_region: Any + _typed_remote_receipt_required: Any + + def _verify_identity( + self, + step: Step, + resolution: Resolution, + before_png: bytes, + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + bundle_dir: Optional[Path] = None, + ) -> IdentityCheck: ... + def _verify_typed_input( + self, + step: Step, + text: str, + field_point: Optional[Point], + baseline_png: bytes, + result: StepResult, + *, + workflow: Workflow, + params: Mapping[str, RuntimeParamScalar], + bundle_dir: Path, + run_dir: Path, + resolution: Optional[Resolution], + field_region: Optional[Region] = None, + baseline_field_value: Optional[str] = None, + allow_masked: bool = False, + allow_retry: bool = True, + ) -> Optional[str]: ... + + backend: Backend + governed_authorization: Optional[GovernedRunAuthorization] + poll_interval_s: float + qualification_fault_driver: Optional[QualificationFaultDriver] + settle_readiness_timeout_s: float + use_structural: bool + vision: Any + + +if TYPE_CHECKING: + _Base = _ActingHost +else: + _Base = object + + +class Acting(_Base): + """Acting by kind: deliver clicks, drags, typing, selections, scrolls and dialog answers.""" + + def _act_read( + self, + step: Step, + resolution: Optional[Resolution], + *, + workflow: Workflow, + params: Mapping[str, RuntimeParamScalar], + before_png: bytes, + result: StepResult, + ) -> Optional[str]: + """READ: read one field into ``result.field_read``. Zero model calls: + OCR and the clipboard are deterministic channels.""" + + from openadapt_flow.records.read import read_field + from openadapt_flow.records.schema import SealRef + + spec = step.read + assert spec is not None + region = self._translate_recorded_region(step, resolution, spec.region) + clipboard = self._read_clipboard + if spec.proof.double_read and clipboard is None: + result.failure_category = "runtime_failure" + return ( + f"Step '{step.id}' ({step.intent}) asks for a double read but " + "this run has no clipboard channel" + ) + if clipboard is not None: + # Focusing and copying a field is input delivery, so it passes the + # same authorization gate as any other input. + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + result.delivery_attempted = True + seal = SealRef(run_id=self._run_id) if spec.proof.seal else None + try: + field_read = read_field( + spec.field, + self.backend, + region, + proof=spec.proof, + clipboard=clipboard, + copy_keys=tuple(spec.copy_keys), + ocr=self.vision.ocr, + step_id=step.id, + seal=seal, + frame_png=before_png, + ) + except Exception as exc: # noqa: BLE001 - a read failure is a halt + result.failure_category = "runtime_failure" + return f"Step '{step.id}' ({step.intent}) could not read: {exc}" + result.field_read = field_read + result.actuation = "read" + self._field_reads[spec.field.name] = field_read + if field_read.needs_review and spec.on_review == "halt": + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) read a value that needs " + f"review ({field_read.review_reason}). Halting for a person" + ) + return None + + @timed_span("actuate") + def _act( + self, + step: Step, + resolution: Optional[Resolution], + params: Mapping[str, RuntimeParamScalar], + **kwargs: Any, + ) -> Optional[str]: + """Perform the step's action. + + A click recorded as closing its own popup (``Step.closes_surface``) + arms the page backend's one-shot expectation first + (:class:`~openadapt_flow.backend.PageCloseExpectationBackend`), so a + "page closed" report from that click counts as delivery once the page + is confirmed closed. Every other step leaves such a report uncertain. + """ + + expect = ( + getattr(self.backend, "expect_page_close", None) + if step.closes_surface + else None + ) + if not callable(expect): + return self._act_answering_dialog(step, resolution, params, **kwargs) + backend = self.backend + expect() + try: + return self._act_answering_dialog(step, resolution, params, **kwargs) + finally: + cancel = getattr(backend, "cancel_page_close", None) + if callable(cancel): + cancel() + + def _act_answering_dialog( + self, + step: Step, + resolution: Optional[Resolution], + params: Mapping[str, RuntimeParamScalar], + **kwargs: Any, + ) -> Optional[str]: + """Perform the step's action, answering the dialog it raises, if any. + + A step with a recorded ``dialog`` arms a one-shot answer on the + browser backend before its input edge. The dialog's message is its + identity: a different message is dismissed, never accepted, and the + step halts; so does a dialog that never appears. A backend that cannot + see browser dialogs refuses the step before any input + (docs/MULTI_SURFACE.md, "Dialogs"). Errors name a code + (``dialog_missing``, ``dialog_mismatch``), never the dialog's text: + the error lands in report.json and REPORT.md, and a live dialog can + show application data. + """ + + dialog = step.dialog + if dialog is None: + return self._act_without_dialog(step, resolution, params, **kwargs) + arm = getattr(self.backend, "arm_dialog", None) + take = getattr(self.backend, "take_dialogs", None) + if not callable(arm) or not callable(take): + return ( + f"Step '{step.id}' ({step.intent}) answers a browser dialog, but " + "the active backend cannot see browser dialogs; run aborted" + ) + self._arm_dialog(arm, dialog) + try: + error = self._act_without_dialog(step, resolution, params, **kwargs) + finally: + seen = take(0.0) + if error is not None: + return error + if not seen: + seen = self._wait_for_dialog(take, dialog) + if not seen: + return ( + f"Step '{step.id}' ({step.intent}): the recorded browser dialog " + "did not appear (dialog_missing); run aborted" + ) + mismatched = [entry for entry in seen if not entry.get("matched")] + if mismatched: + return ( + f"Step '{step.id}' ({step.intent}): a browser dialog did not " + "show the recorded message (dialog_mismatch); it was " + "dismissed, not answered; run aborted" + ) + return None + + @staticmethod + def _arm_dialog(arm: Callable[..., Any], dialog: Any) -> None: + if dialog.message_pattern is not None: + arm(dialog.response, None, message_pattern=dialog.message_pattern) + else: + arm(dialog.response, dialog.message) + + @timed_span("postcondition") + def _wait_for_dialog(self, take: Callable[..., Any], dialog: Any) -> list[Any]: + """Wait briefly for an armed dialog that opens after the input edge.""" + + arm = getattr(self.backend, "arm_dialog", None) + if callable(arm): + self._arm_dialog(arm, dialog) + seen = take(_DIALOG_WAIT_S) + return list(seen) if isinstance(seen, list) else [] + + def _act_without_dialog( + self, + step: Step, + resolution: Optional[Resolution], + params: Mapping[str, RuntimeParamScalar], + *, + workflow: Workflow, + step_index: int, + bundle_dir: Path, + before_png: bytes, + result: StepResult, + run_dir: Path, + graph_ctx: Optional["_GraphStepContext"] = None, + ) -> Optional[str]: + """Perform the step's action through the backend. + + Returns: + An error string (no action performed / partial) or None. + """ + if step.read_gate is not None: + refusal = self._read_gate_refusal(step, result) + if refusal is not None: + return refusal + if step.action is ActionKind.READ: + return self._act_read( + step, + resolution, + workflow=workflow, + params=params, + before_png=before_png, + result=result, + ) + if step.action in (ActionKind.CLICK, ActionKind.DOUBLE_CLICK): + assert resolution is not None # guaranteed by _resolve_step + x, y = resolution.point + native_act = getattr(self.backend, "act_structural", None) + requires_atomic_identity = self._requires_atomic_identity_pointer( + step, workflow + ) + if ( + resolution.rung == "structural" + and resolution.structural_handle is not None + and step.anchor is not None + and step.anchor.structural is not None + and callable(native_act) + ): + if ( + requires_atomic_identity + and resolution.structural_handle.target_fingerprint is None + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "identity-gated click, but its structural target has no " + "fresh actuation fingerprint — refusing raw coordinate " + "delivery; run aborted" + ) + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + structural_locator = step.anchor.structural + structural_handle = resolution.structural_handle + structural_act = cast(Callable[..., Any], native_act) + delivery_receipt = self._deliver_backend_call( + result, + lambda: structural_act( + structural_locator, + structural_handle, + double=step.action is ActionKind.DOUBLE_CLICK, + ), + ) + result.delivery_receipt = delivery_receipt + result.actuation = "uia" if delivery_receipt.native else "dom" + else: + remote_consequential = self._step_requires_remote_guarded_actuation( + step, workflow + ) + typed_remote = isinstance( + self.backend, + GuardedRemotePointerActionBackend, + ) + if ( + remote_consequential + and not typed_remote + and self._typed_remote_receipt_required() + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "remote click, but this backend cannot bind its exact " + "fresh frame and target to delivery; run aborted" + ) + if remote_consequential: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + if typed_remote: + remote_pointer = cast( + GuardedRemotePointerActionBackend, self.backend + ) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: remote_pointer.click_guarded( + x, + y, + expected_frame_sha256=hashlib.sha256( + before_png + ).hexdigest(), + double=step.action is ActionKind.DOUBLE_CLICK, + ), + ) + result.actuation = "remote_guarded" + else: + # See ``_typed_remote_receipt_required``: the exact + # fresh frame is already bound by the backend's own + # one-shot lease, armed immediately above by + # ``_revalidate_consequential_actuation``. + self._deliver_backend_call( + result, + lambda: self.backend.click( + x, + y, + double=step.action is ActionKind.DOUBLE_CLICK, + ), + ) + elif requires_atomic_identity: + if isinstance(self.backend, GuardedCoordinateActionBackend): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + coordinate_backend = cast( + GuardedCoordinateActionBackend, self.backend + ) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: coordinate_backend.act_guarded_coordinate( + x, + y, + expected_frame_sha256=hashlib.sha256( + before_png + ).hexdigest(), + double=step.action is ActionKind.DOUBLE_CLICK, + ), + ) + result.actuation = "guarded_coordinate" + else: + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "identity-gated click, but this backend cannot bind " + "identity verification to the same actuation operation " + "— refusing raw coordinate delivery; run aborted" + ) + else: + rechecked = self._pixel_recheck( + step, + resolution, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir, + ) + if isinstance(rechecked, str): + return rechecked + resolution = rechecked + x, y = resolution.point + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: self.backend.click( + x, + y, + double=step.action is ActionKind.DOUBLE_CLICK, + ), + ) + self._last_click_point = (x, y) + # Only a STRUCTURAL handle reports the element's own bounds. A + # visual rung's ``matched_region`` is where the anchor TEMPLATE + # matched, which is not the field: narrowing a following TYPE's + # verification window to it hides the typed value and false-halts + # a healthy run. + self._last_click_region = ( + resolution.structural_handle.region + if resolution.rung == "structural" + and resolution.structural_handle is not None + else None + ) + return None + + if step.action is ActionKind.RIGHT_CLICK: + assert resolution is not None # guaranteed by _resolve_step + x, y = resolution.point + requires_atomic_identity = self._requires_atomic_identity_pointer( + step, workflow + ) + remote_consequential = self._step_requires_remote_guarded_actuation( + step, workflow + ) + if remote_consequential and not isinstance( + self.backend, + GuardedRemotePointerActionBackend, + ): + if self._typed_remote_receipt_required(): + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "remote right click, but this backend cannot bind its " + "exact fresh frame and target to delivery; run aborted" + ) + # See ``_typed_remote_receipt_required``: the exact fresh frame + # is already bound by the backend's own one-shot lease, and its + # first input edge consumes it. + if not isinstance(self.backend, RichPointerActionBackend): + return ( + f"Step '{step.id}' ({step.intent}) requires a right " + "click, but this backend has no bounded right-click " + "operation" + ) + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: cast(RichPointerActionBackend, self.backend).right_click( + x, y + ), + ) + return None + if remote_consequential: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + remote_pointer = cast(GuardedRemotePointerActionBackend, self.backend) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: remote_pointer.right_click_guarded( + x, + y, + expected_frame_sha256=hashlib.sha256(before_png).hexdigest(), + ), + ) + result.actuation = "remote_guarded" + elif requires_atomic_identity: + if isinstance(self.backend, GuardedCoordinateActionBackend): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + coordinate_backend = cast( + GuardedCoordinateActionBackend, self.backend + ) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: coordinate_backend.act_guarded_coordinate( + x, + y, + expected_frame_sha256=hashlib.sha256( + before_png + ).hexdigest(), + button="right", + ), + ) + result.actuation = "guarded_coordinate" + else: + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "identity-gated right click, but this backend cannot bind " + "identity verification to delivery; run aborted" + ) + elif isinstance(self.backend, RichPointerActionBackend): + rechecked = self._pixel_recheck( + step, + resolution, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir, + ) + if isinstance(rechecked, str): + return rechecked + x, y = rechecked.point + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: cast(RichPointerActionBackend, self.backend).right_click( + x, y + ), + ) + else: + return ( + f"Step '{step.id}' ({step.intent}) requires a right click, " + "but this backend does not implement it" + ) + return None + + if step.action is ActionKind.DRAG: + assert resolution is not None # guaranteed by _resolve_step + drag_end, drag_end_region, drag_error = self._resolve_drag_end( + step, + before_png, + bundle_dir, + workflow, + ) + if ( + drag_end is not None + and drag_end_region is not None + and step.drag_end_anchor is not None + ): + drag_end = self._retain_visual_resolution_evidence( + workflow=workflow, + anchor=step.drag_end_anchor, + resolution=drag_end, + matched_region=drag_end_region, + frame_png=before_png, + bundle_dir=bundle_dir, + run_dir=run_dir, + ) + result.drag_end_resolution = drag_end + if drag_error is not None: + self._cancel_guarded_coordinate() + return drag_error + assert drag_end is not None + x, y = resolution.point + end_x, end_y = drag_end.point + requires_atomic_identity = self._requires_atomic_identity_pointer( + step, workflow + ) + structural_drag = getattr(self.backend, "drag_structural_guarded", None) + if ( + resolution.rung == "structural" + and drag_end.rung == "structural" + and resolution.structural_handle is not None + and drag_end.structural_handle is not None + and step.anchor is not None + and step.anchor.structural is not None + and step.drag_end_anchor is not None + and step.drag_end_anchor.structural is not None + and callable(structural_drag) + ): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + source_locator = step.anchor.structural + source_handle = resolution.structural_handle + destination_locator = step.drag_end_anchor.structural + destination_handle = drag_end.structural_handle + structural_drag_act = cast(Callable[..., Any], structural_drag) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: structural_drag_act( + source_locator, + source_handle, + destination_locator, + destination_handle, + ), + ) + result.actuation = "dom" + elif self._step_requires_remote_guarded_actuation(step, workflow) and ( + isinstance(self.backend, GuardedRemotePointerActionBackend) + or self._typed_remote_receipt_required() + ): + if not isinstance( + self.backend, + GuardedRemotePointerActionBackend, + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "remote drag, but this backend cannot bind both exact " + "fresh-frame endpoints to delivery; run aborted" + ) + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + remote_pointer = cast(GuardedRemotePointerActionBackend, self.backend) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: remote_pointer.drag_guarded( + x, + y, + end_x, + end_y, + expected_frame_sha256=hashlib.sha256(before_png).hexdigest(), + ), + ) + result.actuation = "remote_guarded" + elif requires_atomic_identity and not isinstance( + self.backend, RemoteActuationBackend + ): + if not isinstance(self.backend, GuardedDragActionBackend): + self._cancel_guarded_coordinate() + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "drag, but this backend cannot bind both freshly " + "resolved endpoints to delivery; run aborted" + ) + # Pixel-only source and destination evidence were resolved + # against ``before_png``. Drop any unused structural tokens, + # then bind the source coordinate and require that exact frame + # immediately before mouse-down. Fully structural browser + # drags take the stronger two-element lease above. + cancel_structural = getattr( + self.backend, + "cancel_pending_structural_guards", + None, + ) + if callable(cancel_structural): + cancel_structural() + if not isinstance(self.backend, GuardedCoordinateActionBackend): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "drag, but this backend cannot bind its freshly " + "resolved source to delivery; run aborted" + ) + try: + self.backend.arm_guarded_coordinate(x, y) + except Exception as exc: # noqa: BLE001 - backend boundary + result.safety_halt = True + result.failure_category = "safety_halt" + detail = _scrub_phi(str(exc)) or type(exc).__name__ + return ( + f"Step '{step.id}' ({step.intent}) could not bind its " + f"freshly resolved drag source: {detail}; run aborted" + ) + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + drag_backend = cast(GuardedDragActionBackend, self.backend) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: drag_backend.drag_guarded( + x, + y, + end_x, + end_y, + expected_frame_sha256=hashlib.sha256(before_png).hexdigest(), + ), + ) + result.actuation = "guarded_coordinate" + elif isinstance(self.backend, RichPointerActionBackend): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: cast(RichPointerActionBackend, self.backend).drag( + x, y, end_x, end_y + ), + ) + else: + return ( + f"Step '{step.id}' ({step.intent}) requires a drag, but " + "this backend does not implement it" + ) + return None + + if step.action in (ActionKind.TYPE, ActionKind.SELECT_OPTION): + if step.secret: + # Secret value is never in the bundle/params: inject it from + # the environment, failing fast with an actionable message. + env_var = getattr(self, "secret_env_overrides", {}).get( + step.param or "" + ) or secret_env_var(step.param or "") + text = os.environ.get(env_var, "") + if not step.param: + return ( + f"Step '{step.id}' ({step.intent}) is marked secret " + "but names no parameter" + ) + if not text: + return ( + f"Step '{step.id}' ({step.intent}) requires secret " + f"parameter '{step.param}', but the environment " + f"variable {env_var} is not set — export it with the " + "secret value and re-run" + ) + elif step.param is not None: + if step.param not in params: + return ( + f"Step '{step.id}' ({step.intent}) requires parameter " + f"'{step.param}' but it was not provided" + ) + text = runtime_param_text(params[step.param]) + elif step.text is not None: + text = step.text + else: + return ( + f"Step '{step.id}' ({step.intent}) is a TYPE step with " + "neither text nor param" + ) + identity_atomic_keyboard = self._requires_atomic_identity_keyboard( + step, workflow + ) + requires_atomic_keyboard = ( + identity_atomic_keyboard or self.qualification_fault_driver is not None + ) + if ( + requires_atomic_keyboard + and not isinstance(self.backend, RemoteActuationBackend) + and not isinstance(self.backend, GuardedKeyboardActionBackend) + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a qualification or " + "identity-gated keyboard action, but this backend cannot " + "bind the verified focus and execution context to the same " + "input operation — refusing unguarded keyboard delivery; " + "run aborted" + ) + if step.selection_commit_key is not None: + selection_contract_error = self._selection_contract_error( + step, workflow + ) + if selection_contract_error is not None: + result.safety_halt = True + result.failure_category = "safety_halt" + return selection_contract_error + # The field point: this step's own focusing click (anchored + # TYPE), or the immediately preceding step's click point (the + # recorder's click-to-focus-then-type pattern). When focus was + # moved some other way (Tab between fields), there is no known + # field point — verification diffs the whole frame and the + # retry cannot re-click. + field_point: Optional[Point] = None + field_region: Optional[Region] = None + if resolution is not None: + # Anchored TYPE: click to focus the field first. + x, y = resolution.point + native_act = getattr(self.backend, "act_structural", None) + requires_atomic_identity = self._requires_atomic_identity_pointer( + step, workflow + ) + if ( + resolution.rung == "structural" + and resolution.structural_handle is not None + and step.anchor is not None + and step.anchor.structural is not None + and callable(native_act) + ): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + structural_locator = step.anchor.structural + structural_handle = resolution.structural_handle + structural_act = cast(Callable[..., Any], native_act) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: structural_act( + structural_locator, + structural_handle, + ), + ) + result.actuation = "uia" + else: + if ( + requires_atomic_identity + and not isinstance(self.backend, RemoteActuationBackend) + and isinstance(self.backend, GuardedCoordinateActionBackend) + ): + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + coordinate_backend = cast( + GuardedCoordinateActionBackend, self.backend + ) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: coordinate_backend.act_guarded_coordinate( + x, + y, + expected_frame_sha256=hashlib.sha256( + before_png + ).hexdigest(), + ), + ) + result.actuation = "guarded_coordinate" + elif requires_atomic_identity and not isinstance( + self.backend, RemoteActuationBackend + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) is a consequential " + "identity-gated focusing click, but this backend " + "cannot bind target and context verification to the " + "same coordinate operation — refusing raw coordinate " + "delivery; run aborted" + ) + else: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, lambda: self.backend.click(x, y) + ) + field_point = (x, y) + # See ``_last_click_region``: a visual rung's matched region is + # the anchor template's live position, not the field's bounds. + # SELECT_OPTION does not need it here — it always revalidates + # below and consumes that pass's fresh matched region. + field_region = ( + resolution.structural_handle.region + if resolution.rung == "structural" + and resolution.structural_handle is not None + else None + ) + # Fresh baseline AFTER the focusing click so its own focus ring + # never counts as "input landed". A consequential remote TYPE + # is a second input gesture after that click, so reacquire and + # re-run target/identity before typing; the focusing click + # consumed the first one-shot remote lease. + if self._step_needs_consequential_revalidation(step, workflow): + focused_field_point = field_point + ( + refreshed, + refreshed_region, + before_png, + remote_error, + ) = self._revalidate_consequential_actuation( + step, + resolution, + field_region, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir=run_dir, + arm_keyboard=True, + ) + if remote_error is not None: + return remote_error + if refreshed is not None: + if ( + step.action is ActionKind.SELECT_OPTION + and focused_field_point is not None + and step.anchor is not None + and not self._selection_target_continuous( + step.anchor.region, + focused_field_point, + refreshed.point, + ) + ): + self._cancel_guarded_keyboard() + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) re-resolved " + "to a different field after focus; refusing " + "option selection" + ) + assert step.anchor is not None + if refreshed_region is not None: + refreshed = self._retain_visual_resolution_evidence( + workflow=workflow, + anchor=step.anchor, + resolution=refreshed, + matched_region=refreshed_region, + frame_png=before_png, + bundle_dir=bundle_dir, + run_dir=run_dir, + allow_target_ocr=False, + ) + resolution = refreshed + result.resolution = refreshed + field_point = refreshed.point + # A visual match region is the retained template crop. It + # is not the editable control's bounds. Treating that crop + # as the typed-value readback region can exclude text that + # renders at the left edge of a wide field. Keep exact + # structural bounds when the backend supplies them; + # otherwise use the point-centred visual readback window. + if step.action is ActionKind.SELECT_OPTION: + # Composite selection binds its type-ahead and commit + # to the exact live resolved selection region. + field_region = refreshed_region + else: + field_region = ( + refreshed.structural_handle.region + if ( + refreshed is not None + and refreshed.rung == "structural" + and refreshed.structural_handle is not None + ) + else None + ) + else: + before_png = self.backend.screenshot() + elif self._prev_was_click(workflow, step_index, graph_ctx): + field_point = self._last_click_point + field_region = self._last_click_region + baseline_field_value = self._text_value_at(field_point) + if baseline_field_value is None: + baseline_field_value = self._text_value_at(None) + if step.selection_commit_key is not None: + selection_commit_key = step.selection_commit_key + assert step.selection_region is not None + assert isinstance(self.backend, SelectOptionBackend) + assert step.anchor is not None + if not self._selection_region_compatible( + step.anchor.region, field_region + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) re-resolved to " + "incompatible live field geometry; refusing option " + "selection" + ) + assert field_region is not None + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + if isinstance(self.backend, GuardedSelectOptionBackend): + assert field_point is not None + guarded_select = cast(GuardedSelectOptionBackend, self.backend) + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: guarded_select.select_option_guarded( + text, + selection_commit_key, + target_point=field_point, + expected_frame_sha256=hashlib.sha256( + before_png + ).hexdigest(), + ), + ) + result.actuation = ( + "remote_guarded" + if isinstance(self.backend, RemoteActuationBackend) + else "guarded_keyboard" + ) + else: + select_backend = cast(SelectOptionBackend, self.backend) + self._deliver_backend_call( + result, + lambda: select_backend.select_option( + text, selection_commit_key + ), + ) + return None + if ( + self.qualification_fault_driver is not None + and not identity_atomic_keyboard + and isinstance(self.backend, GuardedKeyboardActionBackend) + ): + if field_point is None: + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) has no focused field " + "point for exact qualification delivery; run aborted" + ) + self._require_qualification_environment_current() + cast(GuardedKeyboardActionBackend, self.backend).arm_guarded_keyboard( + *field_point + ) + guarded_type = ( + requires_atomic_keyboard + and not isinstance(self.backend, RemoteActuationBackend) + and isinstance(self.backend, GuardedKeyboardActionBackend) + ) + if guarded_type: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + result.delivery_receipt = self._deliver_backend_call( + result, + lambda: cast( + GuardedKeyboardActionBackend, self.backend + ).type_text_guarded( + text, + expected_frame_sha256=hashlib.sha256(before_png).hexdigest(), + ), + ) + result.actuation = "guarded_keyboard" + elif requires_atomic_keyboard and not isinstance( + self.backend, RemoteActuationBackend + ): + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}) lost its guarded keyboard " + "capability before delivery — refusing unguarded text input; " + "run aborted" + ) + else: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call(result, lambda: self.backend.type_text(text)) + if not text: + return None # nothing typed, nothing to verify + return self._verify_typed_input( + step, + text, + field_point, + before_png, + result, + workflow=workflow, + params=params, + bundle_dir=bundle_dir, + run_dir=run_dir, + resolution=resolution, + field_region=field_region, + baseline_field_value=baseline_field_value, + allow_masked=step.secret, + allow_retry=not guarded_type, + ) + + executor = ACTION_EXECUTORS.get(step.action) + if executor is not None: + # KEY, HOTKEY and WAIT run in the governance-free core; the gates + # it must pass before input come from here. + return executor( + ActionContext( + step=step, + result=result, + backend=self.backend, + before_png=before_png, + authorize=lambda: self._authorize_delivery( + workflow, params, step, result + ), + deliver=self._deliver_backend_call, + requires_atomic_keyboard=lambda: ( + self._requires_atomic_identity_keyboard(step, workflow) + ), + ) + ) + + if step.action is ActionKind.SCROLL: + return self._act_scroll( + step, + workflow=workflow, + step_index=step_index, + bundle_dir=bundle_dir, + run_dir=run_dir, + before_png=before_png, + params=params, + result=result, + graph_ctx=graph_ctx, + ) + + return f"Step '{step.id}' has unsupported action {step.action!r}" + + @staticmethod + def _prev_was_click( + workflow: Workflow, + step_index: int, + graph_ctx: Optional["_GraphStepContext"], + ) -> bool: + """Whether the step executed just before this TYPE step was a click. + + Linear mode reads ``workflow.steps[step_index - 1]`` exactly as before; + program mode reads the interpreter-supplied ``graph_ctx.prev_action`` + (the previously EXECUTED action state -- the graph has no positional + predecessor). The click-to-focus-then-type heuristic is thereby + identical on both paths. + """ + if graph_ctx is not None: + return graph_ctx.prev_action in ( + ActionKind.CLICK, + ActionKind.DOUBLE_CLICK, + ) + return step_index > 0 and workflow.steps[step_index - 1].action in ( + ActionKind.CLICK, + ActionKind.DOUBLE_CLICK, + ) + + @timed_span("resolve") + def _implicit_scroll_target_ready( + self, + step: Step, + frame_png: bytes, + *, + workflow: Workflow, + bundle_dir: Path, + params: Mapping[str, RuntimeParamScalar], + scroll_delta: Optional[tuple[int, int]] = None, + misses: Optional[list[str]] = None, + ) -> bool: + """Resolve an implicit scroll target and honor its armed identity. + + With ``scroll_delta``, the target must also be no further along the + scroll direction than where the demonstration had it (within + :data:`SCROLL_POSITION_SLACK_FRACTION` of the frame). A target that + is visible at the bottom edge but was clicked mid-screen in the + recording still needs the scroll: the step after the click was + recorded with that scroll position (for example, a section that + expands below the clicked header). + + A generic input crop can visually resolve in several places on a long + form. Declaring the target visible from that visual match alone makes + the preceding scroll a false no-op. Structural resolution stays + available, model grounding stays disabled, and an identity-armed target + is ready only when the same pre-action identity gate verifies it. + + When not ready, ``misses`` (if given) gets why: ``unresolved`` (no + direct deterministic rung found the target), ``offscreen`` (the page + reports the recorded element outside the viewport), ``position`` or + ``identity``. + """ + + def miss(reason: str) -> bool: + if misses is not None: + misses.append(reason) + return False + + if step.anchor is None: + return miss("unresolved") + structural = ( + self.backend + if self.use_structural and hasattr(self.backend, "locate_structural") + else None + ) + offscreen = getattr(structural, "structural_offscreen", None) + if ( + callable(offscreen) + and step.anchor.structural is not None + and offscreen(step.anchor.structural) is True + ): + # The page itself shows the unique recorded target exists below + # (or above) the viewport: not ready, keep scrolling. Stricter + # than letting a visual rung find a look-alike on screen, and it + # skips a full-frame OCR search per probe. The control exists, so + # this miss never hands the halt to the click (#1331). + return miss("offscreen") + template_png = self._asset_bytes( + bundle_dir, step.anchor.template, workflow=workflow + ) + try: + resolved = resolve( + step.anchor, + frame_png, + self.vision, + None, # scroll readiness must remain deterministic and model-free + step.intent, + template_png=template_png, + viewport=_frame_viewport(frame_png), + structural=structural, + ) + except OcrResolutionRefused: + # Readiness is a bounded, non-actuating probe. An ambiguous OCR + # candidate means that the target is not ready yet, so the scroll + # loop must continue. The later target action runs the full + # resolver again and still halts on the same ambiguity. + return miss("unresolved") + if resolved is None: + return miss("unresolved") + resolution, _matched_region = resolved + # Geometry estimates a point from surrounding landmarks; it does not + # prove that the target itself is visible. In particular, a fixed + # header can project the recorded point while the form row is still + # off-screen, turning the preceding scroll into a false no-op. Scroll + # readiness therefore requires direct deterministic target evidence. + # Model grounding is disabled above and future weaker rungs must opt in + # deliberately rather than silently acquiring readiness authority. + if resolution.rung not in _DIRECT_TARGET_RUNGS: + return miss("unresolved") + if scroll_delta is not None and not _scroll_target_positioned( + step.anchor.click_point, resolution.point, scroll_delta, frame_png + ): + return miss("position") + if not step.identity_armed: + return True + check = self._verify_identity( + step, resolution, frame_png, params, workflow, bundle_dir + ) + return check.status == "verified" or miss("identity") + + def _act_scroll( + self, + step: Step, + *, + workflow: Workflow, + step_index: int, + bundle_dir: Path, + run_dir: Path, + before_png: bytes, + params: Mapping[str, RuntimeParamScalar], + result: StepResult, + graph_ctx: Optional["_GraphStepContext"] = None, + ) -> Optional[str]: + """Execute a SCROLL step as a closed loop on a wait_until predicate. + + A recorded scroll's purpose is to bring the next target into view, so + the step scrolls by its recorded delta until a READINESS predicate holds + on a settled frame — not a fixed number of times. That predicate is the + step's own ``wait_until`` when set; otherwise it defaults to + ``ANCHOR_RESOLVES`` on the NEXT anchored step's anchor — today's closed + loop, now expressed as the first concrete ``wait_until`` predicate + rather than a special case (RFC §6). The step probes BEFORE scrolling + (a preceding SCROLL step may already have brought the target into view, + making this one a no-op) and stops as soon as the predicate holds. + + The loop is bounded: this step may scroll at most + ``SCROLL_BUDGET_FACTOR`` times its own recorded distance. On budget + exhaustion the step fails loudly — unless the immediately following + step is another SCROLL step, which inherits the loop (so a recorded + run of N scrolls shares a combined ~2.5x budget). + + Falls back to the fixed recorded delta (open-loop, one gesture) when no + readiness predicate exists (no ``wait_until`` and no later anchor) or + the recorded delta is zero. Predicate probes never call the grounder: + closed-loop scrolling must stay model-free. + + Returns: + An error string on budget exhaustion (see above) or None. + """ + dx = step.scroll_dx or 0 + dy = step.scroll_dy or 0 + # The next anchored step (default scroll stop condition): linear mode + # scans forward in ``workflow.steps``; program mode uses the + # interpreter-supplied successor (the target of this state's + # unconditional transition chain). + next_step = ( + graph_ctx.next_anchored + if graph_ctx is not None + else self._next_anchored_step(workflow, step_index) + ) + # The scroll's readiness is a wait_until predicate: an explicit + # step.wait_until wins; otherwise the next anchor's ANCHOR_RESOLVES. + stop_pred = step.wait_until + implicit_target_readiness = stop_pred is None and next_step is not None + if stop_pred is None and next_step is not None: + stop_pred = Predicate( + kind=PredicateKind.ANCHOR_RESOLVES, + anchor=next_step.anchor, + intent=next_step.intent, + ) + if stop_pred is None or (dx == 0 and dy == 0): + remote_preflight_error = self._prepare_remote_scroll_input(step, result) + if remote_preflight_error is not None: + return remote_preflight_error + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: self.backend.scroll(dx, dy), + ) + return None + + # Until a wheel edge stops moving the page, readiness also requires + # the target at (or past) its demonstrated position. + position_required = [True] + # Why each readiness probe of the next target failed; they decide who + # owns a budget-exhaustion halt (see below). + misses: list[str] = [] + + def readiness_holds(frame_png: bytes) -> bool: + if implicit_target_readiness: + assert next_step is not None + return self._implicit_scroll_target_ready( + next_step, + frame_png, + workflow=workflow, + bundle_dir=bundle_dir, + params=params, + scroll_delta=(dx, dy) if position_required[0] else None, + misses=misses, + ) + return self._predicate_holds( + stop_pred, + frame_png, + bundle_dir, + params, + workflow=workflow, + ) + + holds = readiness_holds(before_png) + if self._governed_asset_mutation is not None: + return self._governed_asset_mutation + if holds: + # Target already in view; nothing to scroll and no input sent. + # Record why, only for the next-target readiness check, so the + # outcome classifier can tell a correctly skipped scroll from an + # undelivered action (execution_profiles.classify_execution_outcome). + if implicit_target_readiness and result.delivery_attempted is not True: + result.scroll_target_ready_before = True + return None + + increment = math.hypot(dx, dy) + budget = SCROLL_BUDGET_FACTOR * increment + scrolled = 0.0 + while scrolled + increment <= budget: + # Readiness callbacks run after the outer step preflight. Bind + # every wheel edge to a new exact remote frame and repeat the + # context/identity checks after those callbacks. This also avoids + # reusing the one-shot lease consumed by the prior wheel edge. + if self._step_needs_consequential_revalidation(step, workflow): + ( + _scroll_resolution, + _scroll_region, + fresh_scroll_frame, + scroll_error, + ) = self._revalidate_consequential_actuation( + step, + None, + None, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir=run_dir, + ) + if scroll_error is not None: + return scroll_error + if readiness_holds(fresh_scroll_frame): + return None + remote_preflight_error = self._prepare_remote_scroll_input(step, result) + if remote_preflight_error is not None: + return remote_preflight_error + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + if refusal is not None: + return refusal + self._require_qualification_environment_current() + self._deliver_backend_call( + result, + lambda: self.backend.scroll(dx, dy), + ) + scrolled += increment + frame = self._wait_for_scroll_transition_and_settle(before_png) + if not self.vision.pixels_changed(before_png, frame): + # The page did not move (its end): visibility is enough. + position_required[0] = False + before_png = frame + holds = readiness_holds(frame) + if self._governed_asset_mutation is not None: + return self._governed_asset_mutation + if holds: + return None + + following = ( + graph_ctx.following + if graph_ctx is not None + else ( + workflow.steps[step_index + 1] + if step_index + 1 < len(workflow.steps) + else None + ) + ) + if following is not None and following.action is ActionKind.SCROLL: + # The next SCROLL step continues the loop with its own budget. + return None + if ( + graph_ctx is None + and implicit_target_readiness + and next_step is not None + and following is next_step + and next_step.action in _SCROLL_HANDOFF_ACTIONS + and misses + and all(miss == "unresolved" for miss in misses) + and not step.risk_review_required + and not self._step_is_consequential(step, workflow) + ): + # The scroll did its bounded work; what failed is the next click's + # target, which never resolved on any frame (a relabelled or moved + # control). Halting here would name the scroll and hide the + # control that changed. Hand the miss to that click instead: it + # resolves on direct deterministic rungs only and, when they find + # nothing, halts on its own target before any input. A target + # that resolved but sat out of position or failed its identity + # check still halts here, as before. + self._scroll_missed_target = ( + next_step.id, + f"the preceding scroll '{step.id}' used {scrolled:.0f}px of its " + f"{budget:.0f}px budget and the target never came into view; " + "no deterministic rung found it, so no input was sent", + ) + return None + target_desc = ( + f"the anchor of step '{next_step.id}' ({next_step.intent})" + if next_step is not None + else self._describe_predicate(stop_pred) + ) + return ( + f"Step '{step.id}' ({step.intent}): closed-loop scroll exhausted " + f"its budget ({scrolled:.0f}px of {budget:.0f}px allowed, " + f"{SCROLL_BUDGET_FACTOR}x the recorded distance) without " + f"{target_desc} resolving — target never came into view; run aborted" + ) + + @timed_span("settle") + def _wait_for_scroll_transition_and_settle(self, baseline_png: bytes) -> bytes: + """Wait for a delivered scroll to change the frame, then settle it. + + A remote client can return two identical pre-action frames before the + wheel packet reaches the remote application. A generic settle poll can + therefore accept the old screen as stable. Require one visual + transition within the configured readiness window before applying the + normal settled-state check. At a real scroll boundary, return the last + unchanged frame after the bounded wait so the closed loop can continue + or refuse when its action budget is exhausted. + """ + + deadline = time.monotonic() + self.settle_readiness_timeout_s + frame = self.backend.screenshot() + while not self.vision.pixels_changed(baseline_png, frame): + if time.monotonic() >= deadline: + return frame + time.sleep(self.poll_interval_s) + frame = self.backend.screenshot() + return self.vision.wait_settled(self.backend) + + def _prepare_remote_scroll_input( + self, + step: Step, + result: StepResult, + ) -> Optional[str]: + """Bind one remote wheel edge to a fresh, one-use frame lease. + + Target-readiness probes can take longer than a remote backend's frame + lease. A wheel gesture has no coordinate to re-resolve, but it is + still real input into an opaque session. Acquire the remote backend's + exact-content lease after readiness checks and before the final + authorization, environment, and delivery checks. Do not extend the + allowed frame age or reuse the observation from the probe. + """ + + if not isinstance(self.backend, RemoteActuationBackend): + return None + try: + self.backend.acquire_actuation_frame() + except Exception as exc: # noqa: BLE001 - backend boundary must halt + if self.governed_authorization is not None: + result.safety_halt = True + detail = _scrub_phi(str(exc)) or type(exc).__name__ + return ( + "Remote scroll preflight HALTED before input for step " + f"'{step.id}' ({step.intent}): {detail}" + ) + return None diff --git a/packages/flow/openadapt_flow/runtime/replay/api_tier.py b/packages/flow/openadapt_flow/runtime/replay/api_tier.py new file mode 100644 index 000000000..648ab1c6b --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/api_tier.py @@ -0,0 +1,1261 @@ +# mypy: disable-error-code="empty-body" +"""The API tier and web-form spans: carry out a step through the application's API. + +Moved verbatim from ``openadapt_flow.runtime.replayer.Replayer`` by +scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1). +``_ApiTierHost`` declares, for mypy only, what this code reads from the +Replayer it is mixed into. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional # noqa: F401 +from urllib.parse import urlsplit + +from openadapt_flow.ir import ( + ActionDeliveryUncertainty, + ActionKind, + IdentityCheck, + IdentitySignalEvidence, + SafetyRefusalEvidence, + StepResult, +) +from openadapt_flow.qualification_faults import effect_verifier_input_sha256 +from openadapt_flow.runtime.authorization import ( + runtime_param_text, + runtime_params_for_gui, +) +from openadapt_flow.runtime.replay._shared import _WebFormApprovalView + +if TYPE_CHECKING: + from pathlib import Path + from typing import Mapping + + from openadapt_flow.backend import Backend + from openadapt_flow.ir import ApiBinding, ExecutionTargetKind, Step, Workflow + from openadapt_flow.runtime.authorization import ( + GovernedRunAuthorization, + RuntimeParamScalar, + ) + from openadapt_flow.runtime.effects import Effect, EffectState + + +class _ApiTierHost: + def _approval_gated(self) -> bool: ... + + _candidate_binding_refusal: Any + _capture_effect_pre_state: Any + + def _check_postconditions( + self, + step: Step, + frame_png: bytes, + bundle_dir: Path, + start_state: dict[str, Any], + result: Optional[StepResult] = None, + *, + workflow: Optional[Workflow] = None, + ) -> tuple[bool, bytes, list[str]]: ... + + _control_overlay_progress: Any + _defer_readback: Any + _delivery_authorization_refusal: Any + _emit_control_overlay_phase: Any + _keep_undo_prior_state: Any + _may_pipeline_readback: Any + _profile_effect_tier_refusal: Any + _require_qualification_environment_current: Any + _required_effect_pre_state_unreadable: Any + _resolve_effects: Any + + def _run_step(self, step: Step, **kwargs: Any) -> StepResult: ... + def _structural_state(self) -> dict[str, Any]: ... + + _verify_effects: Any + _web_form_approval: dict[int, Any] + _web_form_fallback_reason: Optional[str] + _web_form_form_state: Optional[tuple[str, str]] + _web_form_form_state_set: Any + _web_form_span_active: Any + api_actuator: Optional[Any] + api_steps: bool + api_steps_off_reason: Optional[str] + backend: Backend + deployment_surface: Optional[str] + governed_authorization: Optional[GovernedRunAuthorization] + use_structural: bool + vision: Any + web_form_actuator: Optional[Any] + + +if TYPE_CHECKING: + _Base = _ApiTierHost +else: + _Base = object + + +class ApiTier(_Base): + """The API tier and web-form spans: carry out a step through the application's API.""" + + def _web_form_span_at( + self, workflow: Workflow, step_index: int, *, durable: bool + ) -> Optional[tuple[int, Step, Optional[str]]]: + """The API step whose span starts at ``step_index``, and why it can't run. + + Returns ``(save_index, save_step, reason)``; ``reason`` is None when + the API step may run. None when no API step starts here. + """ + + from openadapt_flow.runtime.actuators.web_form import folded_checks_digest + + first = workflow.steps[step_index].id + for end_index in range(step_index, len(workflow.steps)): + save = workflow.steps[end_index] + binding = save.api_binding + if ( + binding is None + or binding.kind != "web_form" + or binding.web_form is None + or binding.web_form.replaces_steps[0] != first + ): + continue + ids = [s.id for s in workflow.steps[step_index : end_index + 1]] + if not self.api_steps: + reason = self.api_steps_off_reason or "API steps are off for this run" + elif surface_reason := self._web_form_surface_refusal( + workflow, step_index, end_index, binding + ): + reason = surface_reason + elif self._approval_gated() and not callable( + getattr(self._actuator_for(binding), "prepare", None) + ): + # Under an approval mode the person approves the exact request + # the API step sends. An actuator that can't prepare the + # request before sending it can't show it. + reason = ( + "approval mode needs the exact request before the save, and " + "this API actuator can't prepare it, so the GUI steps run" + ) + elif not binding.admitted(): + reason = "the API step is not admitted (openadapt-flow api admit)" + elif binding.web_form.other_writes_seen: + # Proposed before such spans were refused: the steps it would + # replace also wrote, and one request can't stand in for them. + reason = ( + "the steps the API step would replace also sent other writes " + f"({', '.join(binding.web_form.other_writes_seen)})" + ) + elif save.identity_armed and not binding.identity: + # The GUI save checks the right record on screen. The API step + # must check it too, so it needs a right-record contract. + reason = ( + "the save checks the right record on screen, and the API " + "step names no right-record contract (openadapt-flow api " + "admit --identity)" + ) + elif list(binding.effects) != list(save.effects): + reason = ( + "the save step's record check differs from the one admitted " + "with the API step (openadapt-flow api admit)" + ) + elif ids != list(binding.web_form.replaces_steps): + reason = "the program's steps no longer match the API step" + elif ( + folded_checks_digest(workflow.steps[step_index : end_index + 1]) + != binding.web_form.folded_checks_digest + ): + reason = ( + "the steps the API step replaces changed after it was " + "proposed (an action, value, target, check, guard or record " + "check), so the API step no longer stands for them " + "(openadapt-flow api propose)" + ) + elif self.governed_authorization is not None and ( + governed_reason := self._governed_web_form_refusal(workflow, save) + ): + reason = governed_reason + elif self._actuator_for(binding) is None: + reason = "there is no live browser session for the API step" + else: + reason = None + return end_index, save, reason + return None + + def _web_form_surface_refusal( + self, workflow: Workflow, start_index: int, end_index: int, binding: Any + ) -> Optional[str]: + """Why a multi-surface run can't use this API step, or None when it can. + + The API step is bound to its span's surface the way it is bound to the + page in a single-surface run: every step it replaces runs on that one + browser surface, none of them switches, opens or closes a surface, the + surface is the active one, and the request goes to that surface's + session on its declared origin. + """ + + router = getattr(self, "_surface_router", None) + if router is None: + return None + span = workflow.steps[start_index : end_index + 1] + surfaces = {s.surface or router.primary_id for s in span} + if len(surfaces) != 1: + return ( + "the steps the API step replaces run on more than one surface, " + "so the GUI steps run" + ) + if any( + s.action is ActionKind.SWITCH_SURFACE or s.opens or s.closes_surface + for s in span + ): + return ( + "the steps the API step replaces switch, open or close a " + "surface, so the GUI steps run" + ) + (surface_id,) = surfaces + if surface_id != router.active_id: + return "the API step's surface is not the active one, so the GUI steps run" + spec = router.specs[surface_id] + if spec.kind != "web": + return "the API step's surface is not a browser, so the GUI steps run" + actuator = self._actuator_for(binding) + if actuator is None: + return "there is no live browser session for the API step" + from openadapt_flow.runtime.surfaces import url_origin + + if spec.is_page: + expected = spec.origin + else: + seen = router.observations.get(surface_id) + expected = seen.origin if seen is not None else None + actual = url_origin( + getattr(getattr(actuator, "transport", None), "base_url", None) + ) + if expected is not None and actual != url_origin(expected): + return ( + "the API step's session is on another origin than its surface, " + "so the GUI steps run" + ) + return None + + def _governed_web_form_refusal( + self, workflow: Workflow, save: Step + ) -> Optional[str]: + """Why a governed run can't use this API step, or None when it can. + + The run's authorization must have admitted this exact API step. For a + managed (Cloud) dispatch that is Cloud's authorization, whose + ``api_steps`` are inside the dispatch binding digest Cloud signs. + + A qualification case runs the path its contract names for the save: + the API step when that path is ``api``, the GUI steps otherwise. A run + under a production qualification admission uses the API step only + when the qualified effect contract, which the signed admission binds, + names the save's API path; that contract exists only when the + qualification campaign ran the API path's cases. + """ + + authorization = self.governed_authorization + assert authorization is not None + if authorization.qualification_case_id is not None: + if authorization.qualification_case_action_paths.get(save.id) != "api": + return "a qualification case runs the GUI path it names" + elif authorization.production_qualification_admission_id is not None: + if not authorization.effect_requirements(save.id, "api"): + return ( + "the production qualification admission didn't qualify this " + "API step's path, so the qualified GUI path runs" + ) + return authorization.admitted_api_step(save, workflow) + + def _run_web_form_span( + self, + workflow: Workflow, + start_index: int, + end_index: int, + *, + params: Mapping[str, RuntimeParamScalar], + bundle_dir: Path, + run_dir: Path, + new_crops: dict[str, bytes], + ) -> Optional[list[StepResult]]: + """Run the API step for steps ``start_index..end_index``. + + Returns the results (the folded GUI steps, then the save) when the API + step took responsibility: it saved and the record check ran, or it + stopped after sending. Returns None when the save was not sent, with + the reason in ``self._web_form_fallback_reason``; the caller then runs + the GUI steps. + """ + + save = workflow.steps[end_index] + self._web_form_span_active = True + try: + result = self._run_step( + save, + workflow=workflow, + step_index=end_index, + params=params, + bundle_dir=bundle_dir, + run_dir=run_dir, + new_crops=new_crops, + ) + finally: + self._web_form_span_active = False + self._web_form_approval.clear() + if result.api_fallback is not None: + self._web_form_fallback_reason = result.api_fallback + return None + router = getattr(self, "_surface_router", None) + surface = router.active_id if router is not None else None + folded = [ + StepResult( + step_id=s.id, + intent=s.intent, + ok=True, + skipped=True, + actuation="api_folded", + surface=surface, + risk=s.risk, + risk_explanation=s.risk_explanation, + risk_review_required=s.risk_review_required, + delivery_attempted=False, + ) + for s in workflow.steps[start_index:end_index] + ] + return [*folded, result] + + def _lookup_capability_refusal( + self, + workflow: Workflow, + execution_target_kind: Optional[ExecutionTargetKind], + ) -> Optional[str]: + """Refuse a surface that offers fewer lookups than were qualified. + + A workflow qualified with structure lookups (DOM / accessibility) + must not run where only pixels exist, such as an RDP or Citrix window + viewed from outside the session: it passed qualification on evidence + this surface cannot produce. Checked before any action. + """ + + from openadapt_flow.lookup_capabilities import ( + DeploymentSurface, + declared_surface_problem, + lookup_capability_refusal, + qualified_lookup_of, + resolve_deployment_surface, + ) + + if qualified_lookup_of(workflow) is None: + return None + kind = execution_target_kind or workflow.surface + surface: DeploymentSurface + if declared_surface_problem(self.deployment_surface, kind) is not None: + # A declaration that contradicts the backend is read the + # conservative way: the remote session is viewed from outside. + surface = "remote_display" + else: + try: + surface = resolve_deployment_surface(self.deployment_surface, kind) + except ValueError: + surface = "remote_display" + structure_available = ( + surface != "remote_display" + and self.use_structural + and callable(getattr(self.backend, "locate_structural", None)) + ) + return lookup_capability_refusal( + workflow, surface=surface, structure_available=structure_available + ) + + @staticmethod + def _url_path_targets_param(url_template: str, param: str) -> bool: + """Require the identity placeholder as one whole URL path segment.""" + + token = "{" + param + "}" + return token in [ + segment for segment in urlsplit(url_template).path.split("/") if segment + ] + + @staticmethod + def _api_request_pointer_value(binding: ApiBinding, pointer: str) -> Any: + """Resolve one admitted target-bearing pointer without consulting headers.""" + + if pointer.startswith("/url/"): + return binding.url_template + if binding.kind == "web_form": + from openadapt_flow.runtime.actuators.web_form import ( + web_form_pointer_template, + ) + + return web_form_pointer_template(binding, pointer) + root_name, *tokens = pointer.lstrip("/").split("/") + if root_name == "body": + value: Any = binding.body_template + elif root_name == "query": + value = binding.query + else: + return None + for token in tokens: + decoded = token.replace("~1", "/").replace("~0", "~") + if isinstance(value, dict): + if decoded not in value: + return None + value = value[decoded] + elif isinstance(value, list): + try: + index = int(decoded) + except ValueError: + return None + if index < 0 or index >= len(value): + return None + value = value[index] + else: + return None + return value + + @staticmethod + def _api_request_pointer_field_path(pointer: str) -> tuple[str, ...]: + """Return the fully decoded target-field path after the request root.""" + + return tuple( + token.replace("~1", "/").replace("~0", "~") + for token in pointer.lstrip("/").split("/")[1:] + ) + + def _api_identity_refusal( + self, + step: Step, + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + effects: list["Effect"], + result: StepResult, + ) -> Optional[str]: + """Validate an exact qualified identity binding before an API write.""" + + authorization_requires = ( + self.governed_authorization is not None + and self.governed_authorization.requires_verified_identity(step.id) + ) + if step.api_binding is not None and step.api_binding.kind == "web_form": + # A form save whose GUI step checks the right record, or whose + # admission named a right-record parameter, is checked the same + # way in every run, governed or not (its durable checkpoint + # needs the evidence too). + authorization_requires = authorization_requires or bool( + step.identity_armed or step.api_binding.identity + ) + project = workflow.qualification + policy = project.identity_policies.get(step.id) if project is not None else None + if not authorization_requires and policy is None: + return None + + binding = step.api_binding + assert binding is not None + if not binding.identity: + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): the qualified action has no exact " + "api_binding.identity contract — refusing to send the request" + ) + + allowed_keys: Optional[set[str]] = None + required = 1 + if policy is not None and policy.enforcement.value == "signal_quorum": + allowed_keys = {signal.key.value for signal in policy.signals} + required = policy.quorum + + evidence: list[IdentitySignalEvidence] = [] + for identity in binding.identity: + if allowed_keys is not None and identity.key not in allowed_keys: + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): semantic signal {identity.key!r} is not " + "part of the qualified identity policy" + ) + raw_value = params.get(identity.param) + value = runtime_param_text(raw_value) if raw_value is not None else None + if value is None or value == "": + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): required identity parameter " + f"{identity.param!r} is missing" + ) + pointed_templates = [ + self._api_request_pointer_value(binding, pointer) + for pointer in identity.request_pointers + ] + exact_token = "{" + identity.param + "}" + effect_target_path = tuple(identity.effect_field.split(".")) + # A form field's name is the page's, not the record's (form_lname + # versus lname): for a form save the field is bound to the record + # check by sending the same parameter the check selects on. + names_must_match = binding.kind != "web_form" + pointers_are_bound = all( + template is not None + and ( + not names_must_match + or self._api_request_pointer_field_path(pointer) + == effect_target_path + ) + and ( + isinstance(template, str) + and self._url_path_targets_param(template, identity.param) + if pointer.startswith("/url/") + else isinstance(template, str) and template == exact_token + ) + for pointer, template in zip( + identity.request_pointers, pointed_templates, strict=True + ) + ) + if not pointers_are_bound: + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): identity parameter {identity.param!r} is " + "not bound at every declared target-bearing request pointer" + ) + from openadapt_flow.runtime.actuators.web_form import ( + effect_selector_rows, + ) + + effect_exprs = [ + row.get(identity.effect_field) + for effect in effects + for row in effect_selector_rows(effect) + ] + if not any( + expr is not None and expr.param == identity.param + for expr in effect_exprs + ): + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): effect selector {identity.effect_field!r} " + f"is not bound to identity parameter {identity.param!r}" + ) + evidence.append( + IdentitySignalEvidence( + signal=identity.key, + source="api_parameter", + verdict="verified", + evidence_class="api_request_effect_binding", + match="exact", + ) + ) + + if len(evidence) < required: + return ( + f"API identity verification HALTED step '{step.id}' " + f"({step.intent}): {len(evidence)}/{required} exact qualified " + "identity bindings verified — refusing to send the request" + ) + result.identity = IdentityCheck( + status="verified", + mode="signal_quorum", + coverage=1.0, + signal_evidence=evidence, + quorum_required=required, + quorum_verified=len(evidence), + ) + return None + + def _actuator_for(self, binding: Any) -> Optional[Any]: + """The actuator that runs ``binding``, or None when none can.""" + + if binding is None: + return None + if binding.kind != "web_form": + return self.api_actuator + if self.web_form_actuator is not None: + return self.web_form_actuator + # In a multi-surface run the request goes through the active + # surface's session, which on a resume may not be self.backend yet. + router = getattr(self, "_surface_router", None) + backend = router.active_backend if router is not None else self.backend + page = getattr(backend, "page", None) + if page is None: + return None + from openadapt_flow.runtime.actuators.web_form import ( + PlaywrightSessionTransport, + WebFormActuator, + ) + + try: + return WebFormActuator(PlaywrightSessionTransport(page)) + except Exception: # noqa: BLE001 - no live session: the GUI runs + return None + + def _try_api_tier( + self, + step: Step, + params: Mapping[str, RuntimeParamScalar], + result: StepResult, + *, + workflow: Workflow, + step_index: int, + bundle_dir: Path, + effect_verifier: Any, + ) -> bool: + """Perform ``step.api_binding``'s write via the API and confirm it. + + The TOP of the capability ladder (RFC section 4 ``api`` tier): a + deterministic, ``$0``, model-free write, gated by the SAME + ``EffectVerifier`` as a GUI write. For an API write the target + "identity" is the explicit API parameter -- stronger than a resolved + pixel band -- so no identity gate is weakened by skipping the GUI. + + Fail-safe ordering (the no-double-write contract): + + - refuse BEFORE any request when the write could not be confirmed (no + effect contract, or no verifier) -- an unverifiable consequential + write never proceeds; + - snapshot the system of record, then actuate ONCE; + - UNAVAILABLE (request never sent) -> use the binding's configured GUI + fallback or typed pre-delivery halt; nothing was written; + - HALT (request sent, outcome unknown / rejected) -> run the configured + postcondition and independent-effect checks without retry or GUI + fallback; resolve only if the complete contract proves the effect; + - ACTUATED (2xx) -> CONFIRM with the EffectVerifier; a non-CONFIRMED + verdict HALTs exactly as it would for a GUI write. + + Returns True when the API tier took responsibility for the step (the + result is final -- actuated+verified, HALTed, or a config-error HALT), + False only when the API tier is UNAVAILABLE and the binding permits the + caller to fall through to the GUI resolution ladder. + """ + binding = step.api_binding + assert binding is not None # guaranteed by the caller + api_actuator = self._actuator_for(binding) + assert api_actuator is not None # guaranteed by the caller + + # An API write MUST be confirmable against its own exact effect + # contract. ``step.effects`` belongs exclusively to the GUI fallback; + # substituting it here could confirm a different record or operation. + effects = binding.effects + if not effects: + result.effect_verified = False + result.effect_results.append( + "API binding declares no api_binding.effects contract to " + "confirm its write -- refusing an " + "unverifiable API write (fail-safe HALT)" + ) + result.ok = False + result.error = ( + f"Step '{step.id}' ({step.intent}) has an API binding but no " + "system-of-record effect to CONFIRM the write -- an API write " + "must be verifiable; refusing to actuate " + "(deployment/configuration error); run aborted" + ) + return True + identity_refusal = self._api_identity_refusal( + step, + params, + workflow, + effects, + result, + ) + if identity_refusal is not None: + result.ok = False + result.safety_halt = True + result.failure_category = "governed_refusal" + result.error = identity_refusal + return True + # Bind the effect contracts to this run before either the verifier + # strength check or its detector-input digest is evaluated. + effects = self._resolve_effects(effects, params) + if effect_verifier is None: + result.effect_verified = False + result.effect_results.append( + "API binding present but no EffectVerifier is configured to " + "confirm the write (fail-safe HALT)" + ) + result.ok = False + result.error = ( + f"Step '{step.id}' ({step.intent}) would actuate via the API " + "but no EffectVerifier is configured to confirm the write -- " + "refusing an unverifiable consequential write; run aborted" + ) + result.safety_refusal_evidence = SafetyRefusalEvidence( + stage="effect_verifier", + code="effect_verifier_missing", + detector_input_sha256=effect_verifier_input_sha256(None, effects), + ) + return True + + # The same parameter substitution that the ApiActuator applies to the + # URL/query/body now also applies to the effect checked below. + api_hash_start = len(result.effect_contract_hashes) + result.effect_contract_hashes.extend( + effect.contract_hash() for effect in effects + ) + refusal = self._profile_effect_tier_refusal( + workflow, + step, + "api", + effects, + effect_verifier, + ) + if refusal is not None: + result.effect_verified = False + result.safety_halt = True + result.failure_category = "governed_refusal" + result.effect_results.append(refusal) + result.error = refusal + result.safety_refusal_evidence = SafetyRefusalEvidence( + stage="effect_strength", + code="effect_strength_insufficient", + detector_input_sha256=effect_verifier_input_sha256( + effect_verifier, + effects, + ), + ) + return True + + # Snapshot the system of record BEFORE the write so the verifier counts + # only what THIS actuation wrote (delta / at-most-once / collateral + # loss), then actuate exactly once. + try: + before = self._capture_effect_pre_state(effect_verifier, effects) + except Exception as exc: # noqa: BLE001 - deployment verifier boundary + result.effect_verified = False + result.effect_results.append( + "[api] effect pre-state capture failed before request delivery " + f"({type(exc).__name__}); API actuation was not attempted" + ) + result.ok = False + result.error = ( + f"Step '{step.id}' ({step.intent}) could not capture the " + "system-of-record pre-state before API actuation; refusing to " + "send the request — run aborted" + ) + return True + binding_refusal = self._candidate_binding_refusal(before, effects) + if binding_refusal is not None: + result.effect_verified = False + result.effect_results.append(f"[api] {binding_refusal}") + result.ok = False + result.safety_halt = True + result.failure_category = "governed_refusal" + result.error = binding_refusal + return True + if self._required_effect_pre_state_unreadable(effect_verifier, before, effects): + result.effect_verified = False + result.effect_results.append( + "[api] effect pre-state is unreachable before request delivery" + ) + result.ok = False + result.error = ( + f"Step '{step.id}' ({step.intent}) could not read the " + "system-of-record pre-state before API actuation; refusing to " + "send the request — run aborted" + ) + return True + self._keep_undo_prior_state(workflow, step, before, effects, params) + refusal = self._profile_effect_tier_refusal( + workflow, + step, + "api", + effects, + effect_verifier, + ) + if refusal is not None: + result.effect_verified = False + result.safety_halt = True + result.failure_category = "governed_refusal" + result.effect_results.append(refusal) + result.error = refusal + result.safety_refusal_evidence = SafetyRefusalEvidence( + stage="effect_strength", + code="effect_strength_insufficient", + detector_input_sha256=effect_verifier_input_sha256( + effect_verifier, + effects, + ), + ) + return True + overlay_current, overlay_total = self._control_overlay_progress( + workflow, step_index, None + ) + self._emit_control_overlay_phase( + "executing", + current_step=overlay_current, + total_steps=overlay_total, + ) + approved_view = self._web_form_approval.pop(id(result), None) + send_prepared: Optional[Any] = None + again: Any = None + if approved_view is not None: + # Under an approval mode: prepare the request again now (fresh + # form, fresh pre-checks). The last-point gate compares its + # operation digest with the one a person approved; a request that + # can't be prepared again reads as an unreadable form. + again = self._web_form_approval_view(step, params) + self._web_form_form_state = ( + None if isinstance(again, str) else ("api_request", again.digest) + ) + self._web_form_form_state_set = True + try: + refusal = self._delivery_authorization_refusal( + workflow, params, step, result + ) + finally: + self._web_form_form_state = None + self._web_form_form_state_set = False + if refusal is None and approved_view is not None: + if isinstance(again, str) or again.digest != approved_view.digest: + # Also refused when the gate let it through (for example a + # save that passed problems_only's checks without a person): + # only the exact request prepared before the gate is sent. + why = ( + again + if isinstance(again, str) + else "the request, the record or the record check differs" + ) + result.safety_halt = True + result.failure_category = "governed_refusal" + refusal = ( + f"Step '{step.id}' ({step.intent}): the API request changed " + f"after the approval gate ({why}); the save was not sent " + "and the GUI steps do not run in its place" + ) + else: + # Exactly the request whose digest was approved and re-checked. + send_prepared = again.prepared + if refusal is not None: + result.ok = False + result.error = refusal + return True + # Structural postconditions compare against the last observation before + # request delivery, not against state captured earlier during + # authorization or verifier setup. + start_state = self._structural_state() + # This transition is the explicit delivery boundary. Set it BEFORE the + # call because an exception/timeout may occur after the request left the + # process but before the actuator can return a receipt-like outcome. + result.delivery_attempted = True + try: + self._require_qualification_environment_current() + if send_prepared is not None: + # Exactly the request whose digest was approved and re-checked. + outcome = api_actuator.send(binding, send_prepared) + else: + outcome = api_actuator.actuate( + binding, + runtime_params_for_gui(params), + ) + except Exception as exc: # noqa: BLE001 - external actuator boundary + # The actuator contract is no-throw, but a deployment adapter can + # still violate it. The delivery boundary was crossed immediately + # before the call, so fail toward an attempted/unknown write and + # never fall through to the GUI. + result.actuation = "api" + result.effect_verified = False + result.effect_results.append( + "[api] actuator raised after delivery may have begun " + f"({type(exc).__name__}); outcome requires reconciliation" + ) + return self._verify_uncertain_api_delivery( + step, + result, + workflow=workflow, + step_index=step_index, + bundle_dir=bundle_dir, + start_state=start_state, + before=before, + effects=effects, + effect_verifier=effect_verifier, + cause_type=type(exc).__name__, + ) + + from openadapt_flow.runtime.actuators import ( + ActuationStatus, + ApiActuationResult, + ApiHaltKind, + ) + + # Only the validated model can prove the pre-delivery UNAVAILABLE + # state. A deployment-owned actuator may return any object after its + # call, including a duck-typed object that claims ``UNAVAILABLE``. + # Treat that as possible delivery: collect evidence, never fall + # through to the GUI, and never turn it into success. + if type(outcome) is not ApiActuationResult: + result.actuation = "api" + result.effect_results.append( + "[api] actuator returned an untyped result after delivery may " + f"have begun ({type(outcome).__name__}); treating it as a " + "protocol error that cannot resolve to success" + ) + return self._verify_uncertain_api_delivery( + step, + result, + workflow=workflow, + step_index=step_index, + bundle_dir=bundle_dir, + start_state=start_state, + before=before, + effects=effects, + effect_verifier=effect_verifier, + cause_type="ApiActuatorProtocolError", + allow_contract_resolution=False, + ) + try: + # Revalidate the return value because an adapter can mutate a + # Pydantic instance after construction. The runtime trusts the + # copied, validated receipt only. + outcome = ApiActuationResult.model_validate( + outcome.model_dump(mode="python") + ) + except Exception as exc: # noqa: BLE001 - deployment actuator boundary + result.actuation = "api" + result.effect_results.append( + "[api] actuator returned a malformed typed result after " + "delivery may have begun " + f"({type(exc).__name__}); treating it as a protocol error " + "that cannot resolve to success" + ) + return self._verify_uncertain_api_delivery( + step, + result, + workflow=workflow, + step_index=step_index, + bundle_dir=bundle_dir, + start_state=start_state, + before=before, + effects=effects, + effect_verifier=effect_verifier, + cause_type="ApiActuatorProtocolError", + allow_contract_resolution=False, + ) + + outcome_status = outcome.status + outcome_reason = outcome.reason + + if outcome_status is ActuationStatus.UNAVAILABLE: + # The request was NEVER sent -- nothing was written. Remove the + # unused API-path contracts before applying the binding's exact + # unavailability policy. A GUI fallback result must describe only + # the path responsible for delivery, never both alternatives. + result.delivery_attempted = False + del result.effect_contract_hashes[api_hash_start:] + result.effect_results.append(f"[api] {outcome_reason}") + if binding.on_unavailable == "halt": + self._set_api_unavailable_refusal( + step, + result, + reason=outcome_reason, + ) + return True + return False + + # From here the request WAS attempted -- this step is API-tier and is + # NEVER also GUI-written (the no-double-write contract). + result.actuation = "api" + + if outcome_status is ActuationStatus.HALT: + result.effect_results.append(f"[api] {outcome_reason}") + halt_kind = getattr(outcome, "halt_kind", None) + typed_halt = isinstance(halt_kind, ApiHaltKind) + if not typed_halt: + result.effect_results.append( + "[api] actuator returned an untyped HALT result; treating " + "the delivery as a protocol error that cannot resolve to " + "success" + ) + return self._verify_uncertain_api_delivery( + step, + result, + workflow=workflow, + step_index=step_index, + bundle_dir=bundle_dir, + start_state=start_state, + before=before, + effects=effects, + effect_verifier=effect_verifier, + cause_type=( + "ApiActuatorProtocolError" + if not typed_halt + else ( + "ApiResponseRejected" + if halt_kind is ApiHaltKind.RESPONSE_REJECTED + else "ApiDeliveryUncertain" + ) + ), + allow_contract_resolution=( + typed_halt and halt_kind is ApiHaltKind.DELIVERY_UNCERTAIN + ), + ) + + # ACTUATED (2xx): confirm the write against the system of record with + # the same EffectVerifier that gates a GUI write. A non-CONFIRMED + # verdict HALTs. The GUI resolve/act (and screen postconditions) are + # SKIPPED -- the record, not the screen, is the oracle for an API write. + result.effect_results.append(f"[api] actuated {outcome_reason}") + self._emit_control_overlay_phase( + "verifying", + current_step=overlay_current, + total_steps=overlay_total, + ) + if self._may_pipeline_readback(effect_verifier, step): + # Pipelined read-back (runtime/readback_pipeline.py), as for a GUI + # save: the same record check on the side thread. The item's + # report stays unsuccessful until it settles with the verdict. + self._defer_readback( + step, + before, + result, + effects=effects, + verifier=effect_verifier, + actuation="api", + ) + result.ok = True + result.error = None + return True + try: + error = self._verify_effects( + step, + before, + result, + effects=effects, + verifier=effect_verifier, + ) + except Exception as exc: # noqa: BLE001 - deployment verifier boundary + result.effect_verified = False + result.effect_results.append( + "[api] effect verification raised after API delivery " + f"({type(exc).__name__}); outcome requires reconciliation" + ) + error = ( + f"System-of-record verification for API step '{step.id}' " + f"({step.intent}) raised {type(exc).__name__} after delivery; " + "the write may have landed — run aborted" + ) + result.ok = error is None + result.error = error + return True + + def _verify_uncertain_api_delivery( + self, + step: Step, + result: StepResult, + *, + workflow: Workflow, + step_index: int, + bundle_dir: Path, + start_state: dict[str, Any], + before: EffectState, + effects: list["Effect"], + effect_verifier: Any, + cause_type: str, + allow_contract_resolution: bool = True, + ) -> bool: + """Resolve one possibly dispatched API write without another actuation. + + A response loss, rejected response, or non-conforming actuator + exception does not end evidence collection. The runtime checks the + configured postcondition and the independent system-of-record effect. + A response loss can resolve only when both contracts confirm. An + explicit response outside the binding's accepted contract always + remains halted, even when later evidence confirms a business effect. + The API request is never retried and the GUI fallback never runs. + """ + + result.delivery_uncertainty = ActionDeliveryUncertainty( + operation="api_request", + native=False, + observed_at=datetime.now(timezone.utc).isoformat(), + cause_type=cause_type, + ) + overlay_current, overlay_total = self._control_overlay_progress( + workflow, step_index, None + ) + self._emit_control_overlay_phase( + "verifying", + current_step=overlay_current, + total_steps=overlay_total, + ) + verification_errors: list[str] = [] + + if step.expect: + try: + after_png = self.vision.wait_settled(self.backend) + postconditions_ok, _last_frame, failed = self._check_postconditions( + step, + after_png, + bundle_dir, + start_state, + result, + workflow=workflow, + ) + result.postconditions_ok = postconditions_ok + if not postconditions_ok: + verification_errors.append( + "postcondition did not confirm: " + + ("; ".join(failed) or "unknown postcondition") + ) + except Exception as exc: # noqa: BLE001 - observation boundary + result.postconditions_ok = False + verification_errors.append( + f"postcondition observation failed ({type(exc).__name__})" + ) + else: + # An absent postcondition cannot resolve uncertain delivery. + result.postconditions_ok = None + verification_errors.append("no postcondition was declared") + + try: + effect_error = self._verify_effects( + step, + before, + result, + effects=effects, + verifier=effect_verifier, + ) + except Exception as exc: # noqa: BLE001 - deployment verifier boundary + result.effect_verified = False + result.effect_results.append( + "[api] effect verification raised after uncertain delivery " + f"({type(exc).__name__}); outcome requires reconciliation" + ) + effect_error = ( + f"independent effect verification failed ({type(exc).__name__})" + ) + if effect_error is not None: + verification_errors.append(effect_error) + + uncertainty = result.delivery_uncertainty + assert uncertainty is not None + uncertainty.verification_attempted = True + uncertainty.postconditions_confirmed = result.postconditions_ok is True + uncertainty.effects_confirmed = result.effect_verified is True + uncertainty.resolved_by_contract = ( + allow_contract_resolution + and uncertainty.postconditions_confirmed + and uncertainty.effects_confirmed + and not verification_errors + ) + result.ok = uncertainty.resolved_by_contract + if result.ok: + result.error = None + else: + result.safety_halt = True + result.failure_category = ( + "governed_refusal" + if self.governed_authorization is not None + else "safety_halt" + ) + detail = "; ".join(verification_errors) or "verification confirmed" + if allow_contract_resolution: + result.error = ( + f"API delivery for step '{step.id}' ({step.intent}) was " + "uncertain and was not retried; the complete postcondition " + "and independent effect contract did not confirm: " + f"{detail}" + ) + else: + result.error = ( + f"API delivery for step '{step.id}' ({step.intent}) " + "received a response outside the binding's accepted " + "contract and was not retried; later verification evidence " + f"was collected but cannot override that rejection: {detail}" + ) + return True + + @staticmethod + def _set_api_unavailable_refusal( + step: Step, + result: StepResult, + *, + reason: str, + ) -> None: + """Record an exact pre-delivery API-only refusal. + + The caller invokes this only when no request was sent. The typed + evidence distinguishes that state from an attempted or uncertain API + delivery and prevents any GUI fallback. + """ + + binding = step.api_binding + assert binding is not None + detector_input = json.dumps( + { + "binding": binding.model_dump(mode="json"), + "reason_sha256": hashlib.sha256(reason.encode("utf-8")).hexdigest(), + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + result.ok = False + result.delivery_attempted = False + result.safety_halt = True + result.failure_category = "governed_refusal" + result.effect_verified = False + result.error = ( + f"API-only step '{step.id}' ({step.intent}) is unavailable before " + "delivery; refusing GUI fallback — run aborted" + ) + result.safety_refusal_evidence = SafetyRefusalEvidence( + stage="api_admission", + code="api_path_unavailable", + detector_input_sha256=hashlib.sha256(detector_input).hexdigest(), + ) + + def _web_form_approval_view( + self, step: Step, params: Mapping[str, RuntimeParamScalar] + ) -> "Any | str": + """Prepare an API step's request for a person to approve. + + Returns the view (the prepared request, its operation digest, the + rows and image the person sees), or the reason it can't be prepared. + Only reads are sent. + """ + + from openadapt_flow.approval_mode.item_approval import ApiRequestRef + from openadapt_flow.runtime.actuators import ApiActuationResult + from openadapt_flow.runtime.actuators.web_form import ( + operation_digest, + operation_rows, + render_operation_png, + ) + + binding = step.api_binding + assert binding is not None + actuator = self._actuator_for(binding) + if actuator is None or not callable(getattr(actuator, "prepare", None)): + return "this API actuator can't prepare the request for approval" + gui_params = runtime_params_for_gui(params) + try: + prepared = actuator.prepare(binding, gui_params) + except Exception as exc: # noqa: BLE001 - only reads were sent + return f"preparing the request failed ({type(exc).__name__})" + if isinstance(prepared, ApiActuationResult): + return prepared.reason or "the API step is unavailable" + effects = self._resolve_effects(list(binding.effects), params) + # Several record checks usually select the same record (one per + # field): the person reads, and the digest binds, each pair once. + record = list( + dict.fromkeys( + (key, str(expr)) + for effect in effects + for key, expr in sorted(effect.match.items()) + ) + ) + digest = operation_digest( + prepared, + record_check=[effect.contract_hash() for effect in effects], + record=record, + ) + rows = operation_rows(binding, prepared, gui_params, record=record) + return _WebFormApprovalView( + prepared=prepared, + digest=digest, + rows=rows, + png=render_operation_png( + f"Approve this request: {step.intent or step.id}", rows + ), + ref=ApiRequestRef( + method=prepared.method.upper(), + path=prepared.path, + payload_sha256=prepared.payload_sha256(), + ), + ) diff --git a/packages/flow/openadapt_flow/runtime/replay/confirming.py b/packages/flow/openadapt_flow/runtime/replay/confirming.py new file mode 100644 index 000000000..58d26af03 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/replay/confirming.py @@ -0,0 +1,1053 @@ +# mypy: disable-error-code="empty-body" +"""Confirming an action: typed input landed, the right option is selected, step guards hold. + +Moved verbatim from ``openadapt_flow.runtime.replayer.Replayer`` by +scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1). +``_ConfirmingHost`` declares, for mypy only, what this code reads from the +Replayer it is mixed into. +""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING, Any, Optional # noqa: F401 + +from openadapt_flow.ir import ActionKind +from openadapt_flow.runtime import identity as identity_mod +from openadapt_flow.runtime.authorization import runtime_params_for_gui +from openadapt_flow.runtime.program_predicates import evaluate_program_predicate +from openadapt_flow.runtime.replay._shared import ( + _PAGE_SCALE_BAND, + FIELD_REGION_PAD, + FIELD_REGION_SIZE, + MASKED_MIN_CONFIDENCE, + MASKED_NEW_TEXT_SLACK, + MASKED_REPEAT_FRACTION, + PC_STRUCTURAL_TEMPLATE_THRESHOLD, + PC_TEMPLATE_SEARCH_PAD, + PC_TEMPLATE_THRESHOLD, + _frame_viewport, + _uniform_page_scale, +) +from openadapt_flow.runtime.resolver import pad_region +from openadapt_flow.runtime.step_timing import timed_span + +if TYPE_CHECKING: + from pathlib import Path + from typing import Mapping + + from openadapt_flow.backend import Backend + from openadapt_flow.ir import ( + Point, + Predicate, + Region, + Resolution, + Step, + StepResult, + Workflow, + ) + from openadapt_flow.runtime.authorization import RuntimeParamScalar + + +class _ConfirmingHost: + _asset_bytes: Any + _cancel_guarded_keyboard: Any + _deliver_backend_call: Any + _delivery_authorization_refusal: Any + + @staticmethod + def _describe_postcondition(pc: Any) -> str: ... + @staticmethod + def _describe_predicate(pred: Predicate) -> str: ... + + _governed_asset_mutation: Optional[str] + _handle_interstitials: Any + _mapped_selection_readback_region: Any + _pc_page_scale: Any + _pc_scaled_match: Any + + def _poll_postconditions( + self, + step: Step, + frame_png: bytes, + bundle_dir: Path, + start_state: dict[str, Any], + ) -> tuple[bool, bytes]: ... + @staticmethod + def _postcondition_confirms_a_write( + step: Step, workflow: Optional[Workflow] + ) -> bool: ... + def _postcondition_passes( + self, + pc: Any, + frame_png: bytes, + bundle_dir: Path, + start_state: Optional[dict[str, Any]] = None, + ) -> bool: ... + def _profile_runtime_refusal(self, workflow: Workflow) -> Optional[str]: ... + + _require_qualification_environment_current: Any + + def _resolve_step( + self, + step: Step, + screen_png: bytes, + bundle_dir: Path, + workflow: Workflow, + *, + allow_grounder: bool = True, + allow_target_ocr: bool = True, + ) -> tuple[Optional[Resolution], Optional[Region], Optional[str]]: ... + + _revalidate_consequential_actuation: Any + _settle_seeded: Any + _step_needs_consequential_revalidation: Any + + def _wait_starting_state_settled( + self, before_png: bytes + ) -> tuple[bytes, Optional[str], Optional[bool]]: ... + + backend: Backend + poll_interval_s: float + state_verifier: Optional[Any] + vision: Any + + +if TYPE_CHECKING: + _Base = _ConfirmingHost +else: + _Base = object + + +class Confirming(_Base): + """Confirming an action: typed input landed, the right option is selected, step guards hold.""" + + def _apply_step_gates( + self, + step: Step, + before_png: bytes, + bundle_dir: Path, + params: Mapping[str, RuntimeParamScalar], + result: StepResult, + workflow: Optional[Workflow] = None, + ) -> tuple[bool, Optional[str], bytes]: + """Evaluate readiness, interstitials, guard, then wait_until before acting. + + Runs in state-dependency order, all model-free: + + 1. **readiness** (``require_settled``) - refuse to act on a frame that + never settled (still-loading / mid-transition); HALT gracefully. + 2. **interstitials** - a KNOWN reversible, non-consequential overlay on + the entry frame gets one audited dismissal plus declared clearance + verification, or HALTs before the workflow action. + 3. **guard** (precondition) on the entry frame - unmet HALTs (default) + or SKIPs (``on_unmet="skip"``). + 4. **wait_until** (readiness predicate) - polled, bounded, HALT on + timeout (never proceed-anyway). + + Returns ``(proceed, error, before_png)``: + + - ``(True, None, frame)`` - proceed to resolve/act (frame may have been + re-settled while polling a wait_until predicate). + - ``(False, None, frame)`` - the guard was unmet with ``on_unmet="skip"``: + the step is a no-op success (caller marks it ``skipped``). + - ``(False, error, frame)`` - HALT: an un-ready screen, a blocking / + undismissable interstitial, an unmet ``halt`` guard, or a + ``wait_until`` predicate that never held within its bound (fail-safe; + the run NEVER proceeds-anyway on an unmet readiness predicate). + + Model-free: predicates are evaluated by :meth:`_predicate_holds`. + """ + if workflow is not None: + profile_refusal = self._profile_runtime_refusal(workflow) + if profile_refusal is not None: + result.safety_halt = True + result.failure_category = "governed_refusal" + return ( + False, + f"{profile_refusal} at the actuation boundary — refusing " + "to act; run aborted", + before_png, + ) + # (1) Readiness: never act on a frame that never settled. + before_png, readiness_error, settled = self._wait_starting_state_settled( + before_png + ) + result.starting_state_settled = settled + if readiness_error is not None: + # Like an interstitial refusal, a not-ready starting state is a + # safety halt. A program on_exception edge must not convert the + # explicit refusal into a successful terminal outcome. + result.safety_halt = True + return False, readiness_error, before_png + + # (2) Interstitials: dismiss a KNOWN overlay, or HALT gracefully. + before_png, interstitial_error = self._handle_interstitials( + step, + before_png, + bundle_dir, + params, + result, + result.interstitial_actions, + workflow, + ) + if interstitial_error is not None: + # An interstitial refusal is a state-safety halt, not an ordinary + # workflow exception a program on_exception edge may turn into a + # successful outcome after an uncertain/blocked pre-action state. + result.safety_halt = True + return False, interstitial_error, before_png + + # Guard (precondition) on the entry frame. + guard_holds = step.guard is None or self._predicate_holds( + step.guard.predicate, + before_png, + bundle_dir, + params, + workflow=workflow, + ) + if self._governed_asset_mutation is not None: + return False, self._governed_asset_mutation, before_png + if not guard_holds: + assert step.guard is not None + if step.guard.on_unmet == "skip": + return False, None, before_png + return ( + False, + ( + f"Guard precondition for step '{step.id}' ({step.intent}) is " + f"unmet on the current screen " + f"({self._describe_predicate(step.guard.predicate)}) — " + "refusing to act; run aborted" + ), + before_png, + ) + + # wait_until (readiness). SCROLL consumes its own predicate as the stop + # condition of its closed loop (_act_scroll), so it is skipped here. + if step.wait_until is not None and step.action is not ActionKind.SCROLL: + pred = step.wait_until + deadline = time.monotonic() + pred.timeout_s + while True: + if self._predicate_holds( + pred, + before_png, + bundle_dir, + params, + workflow=workflow, + ): + return True, None, before_png + if time.monotonic() >= deadline: + return ( + False, + ( + f"wait_until predicate for step '{step.id}' " + f"({step.intent}) did not hold within " + f"{pred.timeout_s:.1f}s " + f"({self._describe_predicate(pred)}) — readiness never " + "reached; refusing to proceed-anyway; run aborted" + ), + before_png, + ) + time.sleep(self.poll_interval_s) + before_png = self.vision.wait_settled(self.backend) + + return True, None, before_png + + def _predicate_holds( + self, + pred: Predicate, + frame_png: bytes, + bundle_dir: Path, + params: Mapping[str, RuntimeParamScalar], + *, + workflow: Optional[Workflow] = None, + ) -> bool: + """Evaluate a Predicate against the current frame / run params. + + Deterministic and ZERO model calls (the $0 runtime guarantee): + ``anchor_resolves`` runs the resolution ladder with NO grounder, + ``text_present`` / ``text_absent`` use the tolerant OCR presence check, + ``param_equals`` is a string compare, and ``and`` / ``or`` / ``not`` + compose. An unknown kind fails safe (does not hold). + """ + return evaluate_program_predicate( + pred, + frame_png, + runtime_params_for_gui(params), + vision=self.vision, + viewport=_frame_viewport(frame_png), + asset_loader=lambda rel: self._asset_bytes( + bundle_dir, + rel, + workflow=workflow, + ), + ) + + def _field_region( + self, + field_point: Optional[Point], + structural_region: Optional[Region] = None, + ) -> Optional[Region]: + """Region to observe for typed input, or None for the whole frame.""" + vw, vh = self.backend.viewport + if structural_region is not None: + sx, sy, sw, sh = structural_region + x = max(0, sx - FIELD_REGION_PAD) + y = max(0, sy - FIELD_REGION_PAD) + right = min(vw, sx + sw + FIELD_REGION_PAD) + bottom = min(vh, sy + sh + FIELD_REGION_PAD) + if right > x and bottom > y: + return (x, y, right - x, bottom - y) + if field_point is None: + return None + w = min(FIELD_REGION_SIZE[0], vw) + h = min(FIELD_REGION_SIZE[1], vh) + x = min(max(0, field_point[0] - w // 2), max(0, vw - w)) + y = min(max(0, field_point[1] - h // 2), max(0, vh - h)) + return (x, y, w, h) + + def _ocr_squashed(self, png: bytes, region: Optional[Region]) -> str: + """Squashed OCR text of a (region of a) frame.""" + lines = self.vision.ocr(png, region=region) + return identity_mod.squash(" ".join(line.text for line in lines)) + + def _readable_chars(self, png: bytes, region: Optional[Region]) -> int: + """Confidently readable alphanumeric characters in a region. + + The masked-acceptance metric (see ``MASKED_NEW_TEXT_SLACK``): + password dots OCR as nothing, low-confidence noise, punctuation + runs, or — on some platform renderers — confident homogeneous + digit runs, while a dialog adds confident words. Counting only + alphanumeric characters of confident, non-homogeneous lines is + also invariant to OCR merging or splitting the same text into + different boxes between frames. + """ + total = 0 + for line in self.vision.ocr(png, region=region): + if getattr(line, "confidence", 1.0) < MASKED_MIN_CONFIDENCE: + continue + alnum = [ch for ch in line.text if ch.isalnum()] + if not alnum: + continue + most_common = max(alnum.count(ch) for ch in set(alnum)) + if len(alnum) >= 4 and most_common / len(alnum) >= MASKED_REPEAT_FRACTION: + continue # homogeneous glyph run: masked-dot misread + total += len(alnum) + return total + + def _text_value_at(self, field_point: Optional[Point]) -> Optional[str]: + """Read an exact editable value when the backend can expose one. + + ``None`` means the substrate cannot make this observation. Empty text + is returned as ``""`` and remains distinguishable from unavailable. + Exceptions fail closed into the visual verifier rather than escaping. + """ + reader = getattr( + self.backend, + "text_value_at" if field_point is not None else "focused_text_value", + None, + ) + if not callable(reader): + return None + try: + value = reader(*field_point) if field_point is not None else reader() + except Exception: + return None + return value if isinstance(value, str) else None + + def _typed_input_landed( + self, + text: str, + field_point: Optional[Point], + baseline_png: bytes, + *, + field_region: Optional[Region] = None, + baseline_field_value: Optional[str] = None, + allow_masked: bool = False, + ) -> tuple[bool, bool]: + """Did the just-typed ``text`` visibly land? + + Exact structural readback is authoritative when the backend exposes + it. Otherwise, for an OCR-able value (>= + ``identity.MIN_PARAM_CHARS`` squashed chars), a contiguous squashed + run of the value (retried at 2x resolution) must be readable in the + field region. Pixel-only acceptance is restricted to steps declared + ``secret``: masked password dots may render no readable text. A dialog + painting over the region or an unreadable ordinary field must never + count as "input landed". Values too short for OCR to arbitrate retain + the legacy diff fallback. + + Returns: + ``(landed, changed)`` — the verdict, and whether the field + region's pixels changed at all (the caller's retry decision: + retyping is only safe when nothing changed). + """ + after_png = self._settle_seeded() + region = self._field_region(field_point, field_region) + if baseline_field_value is not None: + expected = baseline_field_value + text + after_at_point = ( + self._text_value_at(field_point) if field_point is not None else None + ) + after_focused = self._text_value_at(None) + + # Focus-bound readback is the strongest confirmation. A full-page + # cover can hide the field from hit-testing while leaving that + # exact editable element focused, so focused readback also + # preserves the intended safe-halt-at-the-next-action behavior. + if after_focused == expected: + return True, expected != baseline_field_value + + # If hit-testing can read the value but focus moved elsewhere, an + # interstitial/dialog stole focus after delivery. That is an + # unknown state, not a success: the value could belong to an + # obscured/replaced control. + if after_at_point == expected: + return False, True + + observed = [ + value for value in (after_at_point, after_focused) if value is not None + ] + if observed and all(value == baseline_field_value for value in observed): + return False, False + if observed: + # A partial/different value is an unknown state and must halt + # without destructive select-all. + return False, True + # A structurally readable field disappearing after delivery means + # it was covered, replaced, or navigated away. Do not downgrade to + # OCR, which could read the intended text elsewhere on the screen. + return False, True + changed = self.vision.pixels_changed(baseline_png, after_png, region=region) + # Dropdown: typing selects the option whose label matches. When the + # focused at the field point now shows exactly the typed - # label (whitespace-squashed, case-insensitive), the input landed. - # Any other answer falls through to the visual check, unchanged. - if field_point is not None and not allow_masked: - reader = getattr(self.backend, "focused_select_label_at", None) - label: Optional[str] = None - if callable(reader): - try: - label = reader(int(field_point[0]), int(field_point[1])) - except Exception: - label = None - if ( - isinstance(label, str) - and identity_mod.squash(text) - and identity_mod.squash(label).casefold() - == identity_mod.squash(text).casefold() - ): - return True, True - needle = identity_mod.squash(text) - if len(needle) < identity_mod.MIN_PARAM_CHARS: - return changed, changed # too short for OCR to arbitrate - need = identity_mod.required_run(len(needle)) - after_hay = self._ocr_squashed(after_png, region) - if identity_mod.longest_run(needle, after_hay) >= need: - return True, changed - if region is not None: - upscaled = identity_mod.upscale_crop(after_png, region) - if upscaled is not None: - up_hay = self._ocr_squashed(upscaled, None) - if identity_mod.longest_run(needle, up_hay) >= need: - return True, changed - if not changed: - return False, False - # Pixels changed but the value is unreadable: masked rendering is - # the only acceptable explanation, and masked rendering adds no - # confidently readable alphanumeric text (dot glyphs OCR as - # nothing, noise, or punctuation — platform-dependent). Anything - # else (a dialog over the field, another widget's text) must fail - # the verdict. - if not allow_masked: - return False, changed - landed = ( - self._readable_chars(after_png, region) - <= self._readable_chars(baseline_png, region) + MASKED_NEW_TEXT_SLACK + looked = self._look_again( + step, + first, + None, + read, + params, + workflow, + identifier_region, + frame_png, + matches, ) - return landed, changed + if ( + looked.status == "verified" + or looked.known_id_halt is not None + or looked.look_halt is not None + ): + return looked + return check.model_copy(update={"looks": list(looked.looks)}) - @timed_span("postcondition") - def _verify_typed_input( - self, - step: Step, - text: str, - field_point: Optional[Point], - baseline_png: bytes, - result: StepResult, - *, - workflow: Workflow, - params: Mapping[str, RuntimeParamScalar], - bundle_dir: Path, - run_dir: Path, - resolution: Optional[Resolution], - field_region: Optional[Region] = None, - baseline_field_value: Optional[str] = None, - allow_masked: bool = False, - allow_retry: bool = True, - ) -> Optional[str]: - """Verify a TYPE action landed; one guarded refocus-and-retype retry. - - The retry only fires when exact readback or pixels prove the first - attempt changed NOTHING in the field (keystrokes fell on a - non-rendering target): re-click - the field (when its point is known), select-all so a false-negative - first attempt is replaced rather than duplicated, retype. When the - region DID change but the value cannot be read (a dialog over the - field, an unexpected re-render), retyping is not safe — select-all - could destroy pre-existing field content and the re-click could - re-fire whatever now sits at that point — so the run halts - immediately with the accurate reason. Typed input that cannot be - confirmed must never be reported as success (VALIDATION.md 'focus - stolen' finding). + def _check_regions( + self, step: Step, resolution: Resolution, matched_region: Optional[Region] + ) -> dict[str, Region]: + """The regions a pixel step was checked on, at ``resolution``. + + ``target``: where the target matched (or a box of the recorded + target's size at the point). ``identity``: the identifier region, or + the identity band's row, when the step has identity evidence. + ``hover``: the recorded hover response, when there is one. """ - landed, changed = self._typed_input_landed( - text, - field_point, - baseline_png, - field_region=field_region, - baseline_field_value=baseline_field_value, - allow_masked=allow_masked, - ) - if landed: - result.input_verified = True - return None - if changed: - result.input_verified = False - return ( - f"Typed input could not be verified for step '{step.id}' " - f"({step.intent}): the field region changed but the typed " - "value is not readable there (something else rendered over " - "or instead of the input) — retyping is unsafe in this " - "state; run aborted" - ) - if not allow_retry: - result.input_verified = False - return ( - f"Typed input could not be verified for step '{step.id}' " - f"({step.intent}): the identity-bound browser field showed no " - "confirmed change; refusing an unleased refocus/retype retry" - ) - result.input_retried = True - retry_resolution = resolution - retry_region = field_region - retry_frame = baseline_png - needs_revalidation = self._step_needs_consequential_revalidation(step, workflow) - if field_point is not None: - if needs_revalidation: - ( - retry_resolution, - retry_region, - retry_frame, - retry_error, - ) = self._revalidate_consequential_actuation( - step, - retry_resolution, - retry_region, - retry_frame, - params, - workflow, - bundle_dir, - result, - run_dir=run_dir, + anchor = step.anchor + px, py = int(resolution.point[0]), int(resolution.point[1]) + regions: dict[str, Region] = {} + if matched_region is not None: + regions["target"] = matched_region + elif anchor is not None: + w, h = int(anchor.region[2]), int(anchor.region[3]) + regions["target"] = (px - w // 2, py - h // 2, w, h) + else: + regions["target"] = (px - 8, py - 8, 16, 16) + if anchor is not None: + if anchor.identifier_region is not None: + rx, ry, rw, rh = anchor.identifier_region + regions["identity"] = ( + rx + px - anchor.click_point[0], + ry + py - anchor.click_point[1], + rw, + rh, ) - if retry_error is not None: - result.input_verified = False - return retry_error - if retry_resolution is None: - result.input_verified = False - return ( - f"Typed input retry for step '{step.id}' ({step.intent}) " - "could not freshly resolve its field; run aborted" - ) - field_point = retry_resolution.point - refusal = self._delivery_authorization_refusal( - workflow, params, step, result - ) - if refusal is not None: - result.input_verified = False - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call( - result, - lambda: self.backend.click(*field_point), - ) - # Replace, don't append: if the first attempt DID land but was - # not visible to the diff/OCR, retyping raw would double it. - if needs_revalidation: - ( - retry_resolution, - retry_region, - retry_frame, - retry_error, - ) = self._revalidate_consequential_actuation( - step, - retry_resolution, - retry_region, - retry_frame, - params, - workflow, - bundle_dir, - result, - run_dir=run_dir, - arm_keyboard=True, + elif anchor.context_text or anchor.identity_template: + h = int(anchor.region[3]) + regions["identity"] = ( + 0, + py - h // 2 - 4, + int(self.backend.viewport[0]), + h + 8, ) - if retry_error is not None: - result.input_verified = False - return retry_error - if retry_resolution is None or retry_resolution.point != field_point: - self._cancel_guarded_keyboard() - result.input_verified = False - return ( - f"Typed input retry for step '{step.id}' ({step.intent}) " - "resolved to a different field after refocus; run aborted" - ) - refusal = self._delivery_authorization_refusal( - workflow, params, step, result - ) - if refusal is not None: - result.input_verified = False - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call( - result, - lambda: self.backend.press("ControlOrMeta+a"), - ) + if step.hover is not None and step.hover.region is not None: + dx, dy, w, h = step.hover.region + regions["hover"] = (px + dx, py + dy, w, h) + return regions - # The refocus click and select-all key each consume an opaque remote - # lease. Acquire and validate one more exact frame for the retype edge. - if needs_revalidation: - ( - retry_resolution, - retry_region, - retry_baseline, - retry_error, - ) = self._revalidate_consequential_actuation( - step, - retry_resolution, - retry_region, - retry_frame, - params, - workflow, - bundle_dir, - result, - run_dir=run_dir, - arm_keyboard=True, - ) - if retry_error is not None: - result.input_verified = False - return retry_error - else: - retry_baseline = self.backend.screenshot() - if ( - needs_revalidation - and field_point is not None - and (retry_resolution is None or retry_resolution.point != field_point) - ): - self._cancel_guarded_keyboard() - result.input_verified = False - return ( - f"Typed input retry for step '{step.id}' ({step.intent}) " - "resolved to a different field before retyping; run aborted" - ) - refusal = self._delivery_authorization_refusal(workflow, params, step, result) - if refusal is not None: - result.input_verified = False - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call(result, lambda: self.backend.type_text(text)) - landed, _changed = self._typed_input_landed( - text, - field_point, - retry_baseline, - field_region=retry_region, - # Select-all means the retry's expected exact value is ``text`` - # regardless of what the field contained before replacement. - baseline_field_value="" if baseline_field_value is not None else None, - allow_masked=allow_masked, - ) - if landed: - result.input_verified = True - return None - result.input_verified = False - return ( - f"Typed input could not be verified for step '{step.id}' " - f"({step.intent}): the screen did not change where the text " - "should have appeared and OCR could not find the typed value, " - "after one refocus-and-retype retry — keystrokes likely fell on " - "a non-input target (focus lost); run aborted" - ) + @staticmethod + def _changed_regions( + before_png: bytes, fresh_png: bytes, regions: Mapping[str, Region] + ) -> list[str]: + """Names of ``regions`` whose pixels differ between two frames.""" + import numpy as np + from PIL import Image - @timed_span("postcondition") - def _verify_selected_option( + try: + with ( + Image.open(io.BytesIO(before_png)) as a_img, + Image.open(io.BytesIO(fresh_png)) as b_img, + ): + a = np.asarray(a_img.convert("RGB"), dtype=np.int16) + b = np.asarray(b_img.convert("RGB"), dtype=np.int16) + except Exception: # noqa: BLE001 - an unreadable frame is a change + return ["frame_size"] + if a.shape != b.shape: + return ["frame_size"] + height, width = a.shape[:2] + changed: list[str] = [] + for name, (x, y, w, h) in regions.items(): + x0, y0 = max(0, int(x)), max(0, int(y)) + x1, y1 = min(width, int(x + w)), min(height, int(y + h)) + if x1 <= x0 or y1 <= y0: + continue + diff = np.abs(a[y0:y1, x0:x1] - b[y0:y1, x0:x1]).max(axis=2) + if int((diff > _PIXEL_RECHECK_LEVEL).sum()) >= _PIXEL_RECHECK_MIN_PIXELS: + changed.append(name) + return changed + + def _pixel_recheck( self, step: Step, - text: str, - after_png: bytes, - bundle_dir: Path, + resolution: Resolution, + checked_png: bytes, + params: Mapping[str, RuntimeParamScalar], workflow: Workflow, + bundle_dir: Path, result: StepResult, - ) -> Optional[str]: - """Require the option value on the final settled, re-resolved field. - - Unlike ordinary TYPE verification, selection never accepts pixel - change alone and never retries: the commit key may already have changed - the control. Structural readback must equal the intended option, or OCR - must recover one complete exactly normalized line inside only the final - live field region. The compiled region qualifies geometry but never - supplies runtime coordinates. This method does not settle again: its - caller passes the one final frame used by all post-action checks. - """ + run_dir: Path, + ) -> Union[Resolution, str]: + """Guard the gap between a pixel step's check and its click. - qualified_readback = step.selection_region - qualified_target = step.anchor.region if step.anchor is not None else None - if qualified_readback is None or qualified_target is None: - result.input_verified = False - return f"Option selection for step '{step.id}' has no field region" - resolution, live_region, resolve_error = self._resolve_step( - step, - after_png, - bundle_dir, - workflow, - # Final option verification is exact retained-evidence work. A - # model may propose a governed repair after a halt, but it cannot - # supply the target region that certifies the committed value. - allow_grounder=False, - ) - if ( - resolve_error is not None - or resolution is None - or not self._selection_region_compatible(qualified_target, live_region) - ): - result.input_verified = False - result.safety_halt = True - result.failure_category = "governed_refusal" - return ( - f"Selected option field could not be re-resolved with compatible " - f"geometry on the final settled frame for step '{step.id}' " - f"({step.intent}); the run halted" + Right before the click, take a fresh frame and compare the target, + identity and hover regions with the frame the step was checked on (the + hover frame when the step took a hover look). When one changed (a + dialog opened over the target, the list re-sorted), do not click: + re-resolve the target and re-run the identity check on the fresh + frame, with no input, and compare again on the next frame. After + :data:`_PIXEL_RECHECK_REOBSERVES` re-observations that still see + changes, or when the target or its identity cannot be confirmed again, + refuse the click. Structured (DOM / accessibility) targets are bound + by element tokens instead and are not rechecked here. + + Returns the resolution to click, or the refusal. + """ + if resolution.rung == "structural": + return resolution + baseline = self.__dict__.get("_hover_frames", {}).get(step.id) or checked_png + current = resolution + region = self._active_delivery_region + for attempt in range(_PIXEL_RECHECK_REOBSERVES + 1): + fresh = self.backend.screenshot() + changed = self._changed_regions( + baseline, fresh, self._check_regions(step, current, region) ) - assert live_region is not None - live_readback = self._mapped_selection_readback_region( - qualified_target, - qualified_readback, - live_region, - after_png, - ) - if live_readback is None: - result.input_verified = False - result.safety_halt = True - result.failure_category = "governed_refusal" - return ( - f"Selected option readback region could not be mapped onto the " - f"final settled frame for step '{step.id}' ({step.intent}); the " - "run halted" + if not changed: + if current is not resolution: + result.resolution = current + return current + what = ", ".join(changed) + if attempt == _PIXEL_RECHECK_REOBSERVES: + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="aborted") # type: ignore[arg-type] + ) + break + fresh_resolution, fresh_region, error = self._resolve_step( + step, fresh, bundle_dir, workflow ) - exact = self._text_value_at(resolution.point) - if exact is not None: - verified = identity_mod.squash(exact) == identity_mod.squash(text) - else: - needle = identity_mod.squash(text) - verified = any( - getattr(line, "confidence", 1.0) >= 0.5 - and identity_mod.squash(line.text) == needle - for line in self.vision.ocr(after_png, region=live_readback) + if error is None and fresh_resolution is not None: + error = self._identity_gate_error( + step, + fresh_resolution, + fresh, + params, + workflow, + bundle_dir, + result, + evidence_run_dir=run_dir, + ) + if error is not None or fresh_resolution is None: + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="aborted") # type: ignore[arg-type] + ) + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}): the screen changed at the " + f"{what} region between the check and the click, and the " + "target could not be confirmed again on a fresh frame " + f"({error or 'target not found'}) — click not delivered" + ) + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="reobserved") # type: ignore[arg-type] ) - if not verified: - upscaled = identity_mod.upscale_crop(after_png, live_readback) - if upscaled is not None: - verified = any( - getattr(line, "confidence", 1.0) >= 0.5 - and identity_mod.squash(line.text) == needle - for line in self.vision.ocr(upscaled, region=None) - ) - result.input_verified = verified - if verified: - return None + current, region, baseline = fresh_resolution, fresh_region, fresh result.safety_halt = True - result.failure_category = "governed_refusal" - return ( - f"Selected option could not be verified for step '{step.id}' " - f"({step.intent}) inside its exact live re-resolved field region; the " - "commit may have selected a different value, so the run halted" - ) - - @staticmethod - def _selection_region_compatible( - qualified: Region, - live: Optional[Region], - ) -> bool: - """Check live field shape without treating recorded coordinates as truth. - - Remote drift may translate the field, so x/y intentionally do not - participate. A resolution rung that returns no region, flips the field - orientation, or changes either dimension by more than 25% cannot safely - inherit the recorded option-selection contract. - """ - - if live is None: - return False - _, _, qualified_w, qualified_h = qualified - _, _, live_w, live_h = live - if min(qualified_w, qualified_h, live_w, live_h) <= 0: - return False - if (qualified_w >= qualified_h) != (live_w >= live_h): - return False + result.failure_category = "safety_halt" return ( - 0.75 <= live_w / qualified_w <= 1.25 - and 0.75 <= live_h / qualified_h <= 1.25 + f"Step '{step.id}' ({step.intent}): the screen kept changing at the " + f"{what} region between the check and the click — click not " + "delivered; run aborted" ) - @staticmethod - def _selection_target_continuous( - qualified: Region, - focused: Point, - refreshed: Point, - ) -> bool: - """Require both select phases to name the same focused field. - - Reuse the selection contract's 25% size tolerance for point drift. - This permits small OCR/template jitter but refuses a second control. - """ - - _, _, qualified_w, qualified_h = qualified - if qualified_w <= 0 or qualified_h <= 0: - return False + def _copy_look_allowed(self) -> bool: + """The copy look is input: never in governed or qualification runs.""" return ( - abs(refreshed[0] - focused[0]) <= qualified_w * 0.25 - and abs(refreshed[1] - focused[1]) <= qualified_h * 0.25 + self.identity_copy is not None + and self._read_clipboard is not None + and self.governed_authorization is None + and self.managed_dispatch_binding is None + and self.production_qualification_guard is None + and self.qualification_campaign_guard is None + and self.qualification_fault_driver is None + and getattr(self, "approval_gate", None) is None ) - @staticmethod - def _mapped_selection_readback_region( - recorded_target: Region, - recorded_readback: Region, - live_target: Region, - frame_png: bytes, - ) -> Optional[Region]: - """Map the recorded readback band onto a fresh live target. - - Absolute recording coordinates never drive verification. The mapping - preserves the band's target-relative offsets while allowing the - already-qualified target geometry to translate or scale modestly, then - clamps it to the exact final frame. - """ - - from openadapt_flow.vision import imgops - - frame = imgops.decode_bgr(frame_png) - if frame is None: - return None - frame_h, frame_w = frame.shape[:2] - target_x, target_y, target_w, target_h = recorded_target - read_x, read_y, read_w, read_h = recorded_readback - live_x, live_y, live_w, live_h = live_target - if min(target_w, target_h, read_w, read_h, live_w, live_h) <= 0: - return None - scale_x = live_w / target_w - scale_y = live_h / target_h - mapped_x = round(live_x + (read_x - target_x) * scale_x) - mapped_y = round(live_y + (read_y - target_y) * scale_y) - mapped_w = max(1, round(read_w * scale_x)) - mapped_h = max(1, round(read_h * scale_y)) - left = max(0, mapped_x) - top = max(0, mapped_y) - right = min(frame_w, mapped_x + mapped_w) - bottom = min(frame_h, mapped_y + mapped_h) - if right <= left or bottom <= top: - return None - return (left, top, right - left, bottom - top) + def _copy_look_possible(self, identifier_region: Optional[Region]) -> bool: + return identifier_region is not None and self._copy_look_allowed() - # -- closed-loop scroll ------------------------------------------------------ + def _embedded_identity_params( + self, anchor: Anchor, workflow: Workflow + ) -> list[str]: + """Workflow parameters whose demonstrated value is in the identity band.""" + tmpl = anchor.identity_template + if tmpl is not None and tmpl.tokens: + return list(tmpl.param_token_indices) + if anchor.context_text: + return identity_mod.embedded_params(anchor.context_text, workflow.params) + return [] - @timed_span("resolve") - def _implicit_scroll_target_ready( + def _look_again( self, step: Step, - frame_png: bytes, - *, + first: IdentityCheck, + magnified: Optional[IdentityCheck], + read: Callable[[bytes], IdentityCheck], + params: Optional[Mapping[str, RuntimeParamScalar]], workflow: Workflow, - bundle_dir: Path, - params: Mapping[str, RuntimeParamScalar], - scroll_delta: Optional[tuple[int, int]] = None, - misses: Optional[list[str]] = None, - ) -> bool: - """Resolve an implicit scroll target and honor its armed identity. - - With ``scroll_delta``, the target must also be no further along the - scroll direction than where the demonstration had it (within - :data:`SCROLL_POSITION_SLACK_FRACTION` of the frame). A target that - is visible at the bottom edge but was clicked mid-screen in the - recording still needs the scroll: the step after the click was - recorded with that scroll position (for example, a section that - expands below the clicked header). - - A generic input crop can visually resolve in several places on a long - form. Declaring the target visible from that visual match alone makes - the preceding scroll a false no-op. Structural resolution stays - available, model grounding stays disabled, and an identity-armed target - is ready only when the same pre-action identity gate verifies it. - - When not ready, ``misses`` (if given) gets why: ``unresolved`` (no - direct deterministic rung found the target), ``offscreen`` (the page - reports the recorded element outside the viewport), ``position`` or - ``identity``. - """ + identifier_region: Optional[Region], + frame_png: bytes, + recorded_id_matches: Optional[Callable[[str], bool]] = None, + ) -> IdentityCheck: + """Extra looks before stopping on a look-alike identifier. - def miss(reason: str) -> bool: - if misses is not None: - misses.append(reason) - return False + Returns the verdict of :func:`identity_looks.decide`: ``verified`` + when every fresh frame agrees and the known-ID list or an exact copy + settles the record, the contradicting read when a look names another + record, or ``first`` carrying the looks. A known-ID list that may not + be used now, or a copy look that changed the screen, returns ``first`` + marked with a named halt. + """ + from openadapt_flow.runtime import known_ids as known_ids_mod - if step.anchor is None: - return miss("unresolved") - structural = ( - self.backend - if self.use_structural and hasattr(self.backend, "locate_structural") + admission = ( + self.known_ids + if isinstance(self.known_ids, known_ids_mod.KnownIdAdmission) else None ) - offscreen = getattr(structural, "structural_offscreen", None) - if ( - callable(offscreen) - and step.anchor.structural is not None - and offscreen(step.anchor.structural) is True - ): - # The page itself shows the unique recorded target exists below - # (or above) the viewport: not ready, keep scrolling. Stricter - # than letting a visual rung find a look-alike on screen, and it - # skips a full-frame OCR search per probe. The control exists, so - # this miss never hands the halt to the click (#1331). - return miss("offscreen") - template_png = self._asset_bytes( - bundle_dir, step.anchor.template, workflow=workflow - ) - try: - resolved = resolve( - step.anchor, - frame_png, - self.vision, - None, # scroll readiness must remain deterministic and model-free - step.intent, - template_png=template_png, - viewport=_frame_viewport(frame_png), - structural=structural, - ) - except OcrResolutionRefused: - # Readiness is a bounded, non-actuating probe. An ambiguous OCR - # candidate means that the target is not ready yet, so the scroll - # loop must continue. The later target action runs the full - # resolver again and still halts on the same ambiguity. - return miss("unresolved") - if resolved is None: - return miss("unresolved") - resolution, _matched_region = resolved - # Geometry estimates a point from surrounding landmarks; it does not - # prove that the target itself is visible. In particular, a fixed - # header can project the recorded point while the form row is still - # off-screen, turning the preceding scroll into a false no-op. Scroll - # readiness therefore requires direct deterministic target evidence. - # Model grounding is disabled above and future weaker rungs must opt in - # deliberately rather than silently acquiring readiness authority. - if resolution.rung not in _DIRECT_TARGET_RUNGS: - return miss("unresolved") - if scroll_delta is not None and not _scroll_target_positioned( - step.anchor.click_point, resolution.point, scroll_delta, frame_png - ): - return miss("position") - if not step.identity_armed: - return True - check = self._verify_identity( - step, resolution, frame_png, params, workflow, bundle_dir - ) - return check.status == "verified" or miss("identity") + known: Any = self.known_ids + if admission is not None: + halt = admission.check() + if halt is not None: + return first.model_copy(update={"known_id_halt": halt.reason}) + known = admission.known_ids + run_values = runtime_params_for_gui(params or {}) - def _act_scroll( - self, - step: Step, - *, - workflow: Workflow, - step_index: int, - bundle_dir: Path, - run_dir: Path, - before_png: bytes, - params: Mapping[str, RuntimeParamScalar], - result: StepResult, - graph_ctx: Optional["_GraphStepContext"] = None, - ) -> Optional[str]: - """Execute a SCROLL step as a closed loop on a wait_until predicate. - - A recorded scroll's purpose is to bring the next target into view, so - the step scrolls by its recorded delta until a READINESS predicate holds - on a settled frame — not a fixed number of times. That predicate is the - step's own ``wait_until`` when set; otherwise it defaults to - ``ANCHOR_RESOLVES`` on the NEXT anchored step's anchor — today's closed - loop, now expressed as the first concrete ``wait_until`` predicate - rather than a special case (RFC §6). The step probes BEFORE scrolling - (a preceding SCROLL step may already have brought the target into view, - making this one a no-op) and stops as soon as the predicate holds. - - The loop is bounded: this step may scroll at most - ``SCROLL_BUDGET_FACTOR`` times its own recorded distance. On budget - exhaustion the step fails loudly — unless the immediately following - step is another SCROLL step, which inherits the loop (so a recorded - run of N scrolls shares a combined ~2.5x budget). - - Falls back to the fixed recorded delta (open-loop, one gesture) when no - readiness predicate exists (no ``wait_until`` and no later anchor) or - the recorded delta is zero. Predicate probes never call the grounder: - closed-loop scrolling must stay model-free. + frames: list[tuple[IdentityCheck, Optional[str]]] = [] + for _ in range(looks_mod.FRAME_LOOKS): + time.sleep(self.poll_interval_s) + png = self.backend.screenshot() + frames.append((read(png), sha256_bytes(png))) + if frames[-1][0].status == "mismatch": + break # final; no further looks - Returns: - An error string on budget exhaustion (see above) or None. - """ - dx = step.scroll_dx or 0 - dy = step.scroll_dy or 0 - # The next anchored step (default scroll stop condition): linear mode - # scans forward in ``workflow.steps``; program mode uses the - # interpreter-supplied successor (the target of this state's - # unconditional transition chain). - next_step = ( - graph_ctx.next_anchored - if graph_ctx is not None - else self._next_anchored_step(workflow, step_index) - ) - # The scroll's readiness is a wait_until predicate: an explicit - # step.wait_until wins; otherwise the next anchor's ANCHOR_RESOLVES. - stop_pred = step.wait_until - implicit_target_readiness = stop_pred is None and next_step is not None - if stop_pred is None and next_step is not None: - stop_pred = Predicate( - kind=PredicateKind.ANCHOR_RESOLVES, - anchor=next_step.anchor, - intent=next_step.intent, - ) - if stop_pred is None or (dx == 0 and dy == 0): - remote_preflight_error = self._prepare_remote_scroll_input(step, result) - if remote_preflight_error is not None: - return remote_preflight_error - refusal = self._delivery_authorization_refusal( - workflow, params, step, result - ) - if refusal is not None: - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call( - result, - lambda: self.backend.scroll(dx, dy), + frame_reads = [first, *(r for r, _ in frames)] + known_lift: Optional[IdentityCheck] = None + if known is not None and all( + looks_mod.is_glyph_abstain(r) for r in frame_reads + ): + lifted = known_ids_mod.resolve_lookalike_abstain( + frame_reads, + known, + run_values=run_values, + recorded_id_matches=recorded_id_matches, ) - return None + if lifted is not None and lifted.known_id_evidence is not None: + update: dict[str, Any] = {"frames": len(frame_reads)} + if admission is not None: + update["confirmed_at"] = admission.confirmation.confirmed_at + update["export_sha256"] = admission.confirmation.sha256 + known_lift = lifted.model_copy( + update={ + "known_id_evidence": lifted.known_id_evidence.model_copy( + update=update + ) + } + ) - # Until a wheel edge stops moving the page, readiness also requires - # the target at (or past) its demonstrated position. - position_required = [True] - # Why each readiness probe of the next target failed; they decide who - # owns a budget-exhaustion halt (see below). - misses: list[str] = [] - - def readiness_holds(frame_png: bytes) -> bool: - if implicit_target_readiness: - assert next_step is not None - return self._implicit_scroll_target_ready( - next_step, - frame_png, - workflow=workflow, - bundle_dir=bundle_dir, - params=params, - scroll_delta=(dx, dy) if position_required[0] else None, - misses=misses, + copied: Optional[looks_mod.ClipboardRead] = None + classes = looks_mod.lookalike_classes(first.observed) + bindings = self.__dict__.setdefault("_identity_look_bindings", {}) + if ( + identifier_region is not None + and self._copy_look_allowed() + and all(r.status != "mismatch" for r, _ in frames) + ): + assert self.identity_copy is not None + if self._identity_input_looks: + try: + cell_lines = self.vision.ocr(frame_png, region=identifier_region) + except Exception: # noqa: BLE001 - no box, no copy look + cell_lines = [] + cell = looks_mod.copy_region(cell_lines, identifier_region) + copied = ( + looks_mod.copy_identifier( + self.backend, + self._read_clipboard, + cell, + gesture=self.identity_copy.gesture, + copy_keys=self.identity_copy.copy_keys, + crop_region=identifier_region, + ) + if cell is not None + else looks_mod.ClipboardRead( + text=None, reason="no_single_identifier_box" + ) + ) + if copied.changed_outside: + look = IdentityLook( + kind="clipboard", + outcome="unavailable", + reason="screen_changed_outside_cell", + frame_sha256=copied.frame_sha256, + ) + return first.model_copy( + update={ + "look_halt": "identity_look_changed_screen", + "looks": [look], + } + ) + crops = tuple( + c + for c in ( + looks_mod.crop_png(frame_png, identifier_region), + copied.pre_crop, + copied.post_crop, + ) + if c is not None ) - return self._predicate_holds( - stop_pred, - frame_png, - bundle_dir, - params, - workflow=workflow, - ) - - holds = readiness_holds(before_png) - if self._governed_asset_mutation is not None: - return self._governed_asset_mutation - if holds: - # Target already in view; nothing to scroll and no input sent. - # Record why, only for the next-target readiness check, so the - # outcome classifier can tell a correctly skipped scroll from an - # undelivered action (execution_profiles.classify_execution_outcome). - if implicit_target_readiness and result.delivery_attempted is not True: - result.scroll_target_ready_before = True - return None - - increment = math.hypot(dx, dy) - budget = SCROLL_BUDGET_FACTOR * increment - scrolled = 0.0 - while scrolled + increment <= budget: - # Readiness callbacks run after the outer step preflight. Bind - # every wheel edge to a new exact remote frame and repeat the - # context/identity checks after those callbacks. This also avoids - # reusing the one-shot lease consumed by the prior wheel edge. - if self._step_needs_consequential_revalidation(step, workflow): - ( - _scroll_resolution, - _scroll_region, - fresh_scroll_frame, - scroll_error, - ) = self._revalidate_consequential_actuation( - step, - None, - None, - before_png, - params, - workflow, - bundle_dir, - result, - run_dir=run_dir, + bindings[step.id] = looks_mod.LookBinding( + step_id=step.id, classes=classes, read=copied, crops=crops ) - if scroll_error is not None: - return scroll_error - if readiness_holds(fresh_scroll_frame): - return None - remote_preflight_error = self._prepare_remote_scroll_input(step, result) - if remote_preflight_error is not None: - return remote_preflight_error - refusal = self._delivery_authorization_refusal( - workflow, params, step, result - ) - if refusal is not None: - return refusal - self._require_qualification_environment_current() - self._deliver_backend_call( - result, - lambda: self.backend.scroll(dx, dy), - ) - scrolled += increment - frame = self._wait_for_scroll_transition_and_settle(before_png) - if not self.vision.pixels_changed(before_png, frame): - # The page did not move (its end): visibility is enough. - position_required[0] = False - before_png = frame - holds = readiness_holds(frame) - if self._governed_asset_mutation is not None: - return self._governed_asset_mutation - if holds: - return None + else: + binding = bindings.get(step.id) + live_crop = looks_mod.crop_png(frame_png, identifier_region) + if binding is not None and binding.matches(step.id, classes, live_crop): + copied = binding.read - following = ( - graph_ctx.following - if graph_ctx is not None - else ( - workflow.steps[step_index + 1] - if step_index + 1 < len(workflow.steps) - else None - ) + known_list = None + if known is not None: + known_list = getattr(known, "known_ids", known) + decided = looks_mod.decide( + first, + frames=frames, + magnified=magnified, + contradicts=identity_read_contradicts, + known_ids=known_list, + known_ids_lift=known_lift, + clipboard=copied, + exact_expected=looks_mod.exact_expected_ids( + run_values, + self._embedded_identity_params(step.anchor, workflow) + if step.anchor is not None + else [], + ), ) - if following is not None and following.action is ActionKind.SCROLL: - # The next SCROLL step continues the loop with its own budget. - return None + binding = bindings.get(step.id) if ( - graph_ctx is None - and implicit_target_readiness - and next_step is not None - and following is next_step - and next_step.action in _SCROLL_HANDOFF_ACTIONS - and misses - and all(miss == "unresolved" for miss in misses) - and not step.risk_review_required - and not self._step_is_consequential(step, workflow) - ): - # The scroll did its bounded work; what failed is the next click's - # target, which never resolved on any frame (a relabelled or moved - # control). Halting here would name the scroll and hide the - # control that changed. Hand the miss to that click instead: it - # resolves on direct deterministic rungs only and, when they find - # nothing, halts on its own target before any input. A target - # that resolved but sat out of position or failed its identity - # check still halts here, as before. - self._scroll_missed_target = ( - next_step.id, - f"the preceding scroll '{step.id}' used {scrolled:.0f}px of its " - f"{budget:.0f}px budget and the target never came into view; " - "no deterministic rung found it, so no input was sent", + self._identity_input_looks + and binding is not None + and decided.status == "verified" + and any( + look.kind == "clipboard" and look.outcome == "resolves" + for look in decided.looks ) - return None - target_desc = ( - f"the anchor of step '{next_step.id}' ({next_step.intent})" - if next_step is not None - else self._describe_predicate(stop_pred) - ) - return ( - f"Step '{step.id}' ({step.intent}): closed-loop scroll exhausted " - f"its budget ({scrolled:.0f}px of {budget:.0f}px allowed, " - f"{SCROLL_BUDGET_FACTOR}x the recorded distance) without " - f"{target_desc} resolving — target never came into view; run aborted" - ) - - @timed_span("settle") - def _wait_for_scroll_transition_and_settle(self, baseline_png: bytes) -> bytes: - """Wait for a delivered scroll to change the frame, then settle it. - - A remote client can return two identical pre-action frames before the - wheel packet reaches the remote application. A generic settle poll can - therefore accept the old screen as stable. Require one visual - transition within the configured readiness window before applying the - normal settled-state check. At a real scroll boundary, return the last - unchanged frame after the bounded wait so the closed loop can continue - or refuse when its action budget is exhausted. - """ - - deadline = time.monotonic() + self.settle_readiness_timeout_s - frame = self.backend.screenshot() - while not self.vision.pixels_changed(baseline_png, frame): - if time.monotonic() >= deadline: - return frame - time.sleep(self.poll_interval_s) - frame = self.backend.screenshot() - return self.vision.wait_settled(self.backend) + ): + bindings[step.id] = dataclasses.replace(binding, verdict=decided) + return decided - def _prepare_remote_scroll_input( - self, - step: Step, - result: StepResult, - ) -> Optional[str]: - """Bind one remote wheel edge to a fresh, one-use frame lease. - - Target-readiness probes can take longer than a remote backend's frame - lease. A wheel gesture has no coordinate to re-resolve, but it is - still real input into an opaque session. Acquire the remote backend's - exact-content lease after readiness checks and before the final - authorization, environment, and delivery checks. Do not extend the - allowed frame age or reuse the observation from the probe. - """ + # -- typed-input verification ------------------------------------------------- - if not isinstance(self.backend, RemoteActuationBackend): - return None - try: - self.backend.acquire_actuation_frame() - except Exception as exc: # noqa: BLE001 - backend boundary must halt - if self.governed_authorization is not None: - result.safety_halt = True - detail = _scrub_phi(str(exc)) or type(exc).__name__ - return ( - "Remote scroll preflight HALTED before input for step " - f"'{step.id}' ({step.intent}): {detail}" - ) - return None + # -- closed-loop scroll ------------------------------------------------------ @staticmethod def _next_anchored_step(workflow: Workflow, step_index: int) -> Optional[Step]: @@ -17436,23 +10622,6 @@ def _next_anchored_step(workflow: Workflow, step_index: int) -> Optional[Step]: # -- postconditions -------------------------------------------------------- - def _structural_state(self) -> dict[str, Any]: - """Structural observations the backend can provide right now. - - Backends MAY expose ``url`` / ``page_title`` / ``page_count`` (see - ``openadapt_flow.backend.StructuralBackend``). Missing observations - are simply absent from the dict. - """ - state: dict[str, Any] = {} - for key in ("url", "page_title", "page_count"): - try: - value = getattr(self.backend, key, None) - except Exception: - value = None - if value is not None: - state[key] = value - return state - def _attended_transition_observation(self) -> Any: """Capture ephemeral browser structure for a signed pause baseline. @@ -17472,91 +10641,6 @@ def _attended_transition_observation(self) -> Any: page_count=page_count if isinstance(page_count, int) else None, ) - def _structural_changed( - self, key: str, start_state: dict[str, Any] - ) -> Optional[bool]: - """Whether structural observation ``key`` differs from step start. - - Returns None when the observation is unavailable on either side — - the caller treats that as an honestly-unverifiable pass (a bundle - recorded on a structural backend may replay on one that is not; - see docs/LIMITS.md). - """ - try: - current = getattr(self.backend, key, None) - except Exception: - current = None - start = start_state.get(key) - if current is None or start is None: - return None - return current != start - - @timed_span("postcondition") - def _check_postconditions( - self, - step: Step, - frame_png: bytes, - bundle_dir: Path, - start_state: dict[str, Any], - result: Optional[StepResult] = None, - *, - workflow: Optional[Workflow] = None, - ) -> tuple[bool, bytes, list[str]]: - """Poll postconditions until each passes or times out. - - Each postcondition is polled (fresh screenshots) up to its own - ``timeout_s``. If any fails, the screen is re-settled once and all - postconditions are re-checked a single time. If a VLM state-verifier - is configured, a render-drift-sensitive postcondition that still failed - gets one last drift-oracle pass (see :meth:`_drift_oracle_rescue`), - but never on a step that can leave a business effect behind (see - :meth:`_postcondition_confirms_a_write`). ``workflow`` lets a qualified - operator risk decision count; without it the classification is the - stricter step-only one. - - Returns: - (ok, last_frame, failed) — the frame the final verdict was based - on, plus human-readable descriptions of the postconditions that - failed the final check (empty when ok). - """ - self._pc_page_scale = _uniform_page_scale(step, result) - self._pc_scaled_match = False - try: - ok, frame_png = self._poll_postconditions( - step, frame_png, bundle_dir, start_state - ) - if ok: - self._note_page_scale(result) - return True, frame_png, [] - # One re-settle retry. - frame_png = self.vision.wait_settled(self.backend) - failed_pcs = [ - pc - for pc in step.expect - if not self._postcondition_passes( - pc, frame_png, bundle_dir, start_state - ) - ] - if not failed_pcs: - self._note_page_scale(result) - finally: - self._pc_page_scale = None - if ( - failed_pcs - and self.state_verifier is not None - and not self._postcondition_confirms_a_write(step, workflow) - ): - failed_pcs = self._drift_oracle_rescue(failed_pcs, frame_png, result) - failed = [self._describe_postcondition(pc) for pc in failed_pcs] - return not failed_pcs, frame_png, failed - - def _note_page_scale(self, result: Optional[StepResult]) -> None: - """Record on ``result`` the page scale a passing check needed.""" - - scale = getattr(self, "_pc_page_scale", None) - if result is not None and scale is not None and self._pc_scaled_match: - result.postcondition_page_scale = scale - @staticmethod def _postcondition_confirms_a_write( step: Step, workflow: Optional[Workflow] @@ -17586,63 +10670,6 @@ def _postcondition_confirms_a_write( return True return is_consequential(step, workflow) - def _drift_oracle_rescue( - self, - failed_pcs: list[Any], - frame_png: bytes, - result: Optional[StepResult], - ) -> list[Any]: - """Give each deterministically-failed, render-drift-sensitive - postcondition one confirmation pass through the VLM state-verifier. - - VETO-SAFE by construction: - - * The caller never invokes this on an irreversible or effect-bearing - step (see :meth:`_postcondition_confirms_a_write`). - * Only ``text_present`` and ``region_stable`` are eligible — the kinds - a theme/scale/font re-render can legitimately break. Structural - postconditions (``url_changed`` etc.) and ``text_absent`` are NEVER - rescued: those failures are real, not render drift. - * A postcondition is rescued ONLY on a confident ``"yes"``; ``"no"``, - ``"uncertain"``, and any appliance outage keep it failed (halt). - * Every call and every rescue is recorded on ``result`` for the report - (and counted as a model call), so a rescue is auditable, never silent. - - Returns the postconditions that remain failed after the pass. - """ - assert self.state_verifier is not None # guaranteed by the caller (line ~2618) - survivors: list[Any] = [] - for pc in failed_pcs: - expected = self._expected_state_text(pc) - if expected is None: # not a drift-rescuable kind - survivors.append(pc) - continue - if result is not None: - result.drift_oracle_calls += 1 - try: - confirmed = self.state_verifier.holds(frame_png, expected) - except Exception: - confirmed = False # fail-safe: any error keeps the halt - if confirmed: - if result is not None: - result.postcondition_drift_rescues.append( - self._describe_postcondition(pc) - ) - else: - survivors.append(pc) - return survivors - - @staticmethod - def _expected_state_text(pc: Any) -> Optional[str]: - """A natural-language expected-state for the drift-oracle, or None if - the postcondition kind is not render-drift-rescuable.""" - kind = pc.kind.value if hasattr(pc.kind, "value") else pc.kind - if kind == "text_present" and pc.text: - return f"the text {pc.text!r} is visible on the screen" - if kind == "region_stable": - return "the content recorded in the highlighted region is present" - return None - def _local_edit_held(self, pc: Any, start_state: dict[str, Any]) -> bool: """A compiler-classified local edit stayed local (fail closed). @@ -17834,344 +10861,6 @@ def _postcondition_passes( return distance <= pc.phash_tolerance return False - def _scaled_region_match( - self, frame_png: bytes, template_png: bytes, region: tuple[int, ...] - ) -> bool: - """The recorded content, at the page scale this step's click showed. - - A whole-page zoom (browser zoom, or an app's own scale) grows every - element and moves it away from the top-left corner by the same - factor, so the recorded region falls outside the search window and - its content outside the matcher's scale band. When this step's own - click landed at the recorded point times one uniform factor - (:func:`_uniform_page_scale`), look for the same template at that - factor (within one percent), at the recorded region times that - factor. Thresholds are the unscaled check's: a modal, a missing panel or another layout still - fails. - """ - - scale = getattr(self, "_pc_page_scale", None) - if scale is None: - return False - x, y, w, h = region - scaled = ( - int(round(x * scale)), - int(round(y * scale)), - int(round(w * scale)), - int(round(h * scale)), - ) - search = pad_region(scaled, PC_TEMPLATE_SEARCH_PAD, _frame_viewport(frame_png)) - matchers = [ - (self.vision.find_template, PC_TEMPLATE_THRESHOLD), - ( - getattr(self.vision, "find_structural_template", None), - PC_STRUCTURAL_TEMPLATE_THRESHOLD, - ), - ] - for matcher, threshold in matchers: - if not callable(matcher): - continue - try: - match = matcher( - frame_png, - template_png, - search_region=search, - scales=tuple(round(scale * f, 4) for f in _PAGE_SCALE_BAND), - threshold=threshold, - ) - except TypeError: # a vision stand-in with no scale band - return False - if match is not None: - self._pc_scaled_match = True - return True - return False - - def _dom_text_visible(self, text: str) -> bool: - """True only when the backend proves ``text`` is visibly rendered. - - Optional backend seam ``visible_text_present`` (browser DOM). Any - other answer, a missing seam, or an error is "not proven" -> False, - so the caller keeps its visual check. - """ - reader = getattr(self.backend, "visible_text_present", None) - if not callable(reader): - return False - try: - return reader(text) is True - except Exception: - return False - - def _postcondition_template(self, pc: Any, bundle_dir: Path) -> Optional[bytes]: - """Bytes of a REGION_STABLE postcondition's template crop, if any.""" - rel = getattr(pc, "template", None) - if not rel: - return None - return self._asset_bytes(bundle_dir, rel) - - def _layout_drift_on(self, workflow: Workflow) -> bool: - if self.layout_drift is not None: - return self.layout_drift - kind: Optional[str] - if self._surface_router is not None: - # Multi-surface run: follow the surface the step acts on, so an - # RDP segment of a web-first bundle keeps the check. - kind, _ = self._active_surface_binding(workflow) - else: - kind = self._layout_target_kind or workflow.surface - if kind in ("rdp", "citrix"): - return True - backend = self._render_scale_inner or self.backend - return type(backend).__name__ == "FreeRDPBackend" - - def _live_session_profile(self) -> Optional[str]: - """Digest of the pinned remote-session profile, when there is one.""" - transport = getattr(self.backend, "transport", None) - profile = getattr(getattr(transport, "profile", None), "sha256", None) - return profile if isinstance(profile, str) else None - - def _session_profile_changed(self, recorded: Any) -> bool: - live = self._live_session_profile() - return bool( - recorded.profile_sha256 and live and recorded.profile_sha256 != live - ) - - def _reset_render_scale(self) -> None: - """Return to the caller's backend and forget the last estimate.""" - if self._render_scale_inner is not None: - release = getattr(self.backend, "release_frame_sources", None) - if callable(release): - release() - self.backend = self._render_scale_inner - self._render_scale_inner = None - self._render_scale_attempted = False - self._render_scale_evidence = None - - def _load_layout_fingerprints( - self, bundle_dir: Path, workflow: Workflow - ) -> dict[str, Any]: - from openadapt_flow.runtime import layout_drift as drift_mod - - if self._layout_fingerprints is None: - self._layout_fingerprints = drift_mod.load_bundle_fingerprints( - self._asset_bytes( - Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow - ) - ) - return self._layout_fingerprints - - def _maybe_scale_render( - self, - step: Step, - frame_png: bytes, - bundle_dir: Optional[Path], - workflow: Workflow, - ) -> bytes: - """Estimate the session's display scale once; return the frame to use. - - Runs at the first step that has a recorded fingerprint. A frame that - matches the fingerprint, or gives no confident estimate, is returned - unchanged and the layout-drift check judges it as before. Otherwise - the backend is replaced by a - :class:`~openadapt_flow.runtime.render_scale.RenderScaledBackend` and - the frame is returned in the recorded geometry, where the same - layout-drift check must then pass. - """ - if self._render_scale_attempted or bundle_dir is None: - return frame_png - enabled = ( - self.render_scale - if self.render_scale is not None - else self._layout_drift_on(workflow) - ) - if not enabled or self._surface_router is not None: - return frame_png - from openadapt_flow.runtime import layout_drift as drift_mod - from openadapt_flow.runtime import render_scale as scale_mod - - recorded = self._load_layout_fingerprints(Path(bundle_dir), workflow).get( - step.id - ) - if recorded is None: - return frame_png - self._render_scale_attempted = True - try: - import io - - from PIL import Image - - with Image.open(io.BytesIO(frame_png)) as img: - viewport = (int(img.width), int(img.height)) - except Exception: # noqa: BLE001 - undecodable: leave it to the drift check - return frame_png - if self._session_profile_changed(recorded): - # A pinned remote-session profile (scale, DPI, font smoothing) is - # a deployment decision; a different profile still halts below. - self._render_scale_evidence = { - "status": "no_estimate", - "reason": "the pinned remote session profile changed", - } - return frame_png - lines = self.vision.ocr(frame_png) - live = drift_mod.fingerprint(lines, viewport) - verdict = drift_mod.compare( - recorded, - live, - self.layout_drift_tolerances or drift_mod.Tolerances(), - ) - if not verdict.drifted: - self._render_scale_evidence = {"status": "recorded_geometry"} - return frame_png - salt, hashed = drift_mod.load_scale_landmarks( - self._asset_bytes( - Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow - ) - ) - landmarks = scale_mod.hashed_landmarks( - hashed.get(step.id, []) - ) + scale_mod.anchor_landmarks(workflow.steps) - pairs = scale_mod.match_landmarks(landmarks, lines, salt_hex=salt) - transform, reason = scale_mod.estimate_render_transform( - pairs, - recorded_viewport=recorded.viewport, - live_viewport=viewport, - ) - if transform is None: - self._render_scale_evidence = { - "status": "no_estimate", - "reason": reason, - "drift": list(verdict.reasons), - } - return frame_png - scaled = scale_mod.render_scaled_backend(self.backend, transform) - self._render_scale_inner = self.backend - self.backend = cast(Backend, scaled) - self._render_scale_evidence = {"status": "normalized", **transform.evidence()} - # The layout-drift check below now judges the normalized frame, so a - # font or layout change that is not a pure rescale still stops here. - return scaled.normalize(frame_png) - - def _layout_drift_error( - self, - step: Step, - frame_png: bytes, - bundle_dir: Optional[Path], - workflow: Workflow, - ) -> Optional[str]: - """Halt message when the frame drifted from the recorded fingerprint. - - Steps without a recorded fingerprint (older bundles, non-click steps) - are not checked. Model-free: the advisor, when configured, is called - only after drift is found and never changes the verdict. - """ - if bundle_dir is None or not self._layout_drift_on(workflow): - return None - from openadapt_flow.runtime import layout_drift as drift_mod - - recorded = self._load_layout_fingerprints(Path(bundle_dir), workflow).get( - step.id - ) - if recorded is None: - return None - live_profile = self._live_session_profile() - try: - import io - - from PIL import Image - - with Image.open(io.BytesIO(frame_png)) as img: - viewport = (int(img.width), int(img.height)) - except Exception: # noqa: BLE001 - an undecodable frame is drift - viewport = (0, 0) - lines = self.vision.ocr(frame_png) - tolerances = self.layout_drift_tolerances or drift_mod.Tolerances() - if self._render_scale_inner is not None and not self._session_profile_changed( - recorded - ): - # A normalized frame is judged label by label: the recorded label - # set says where each label sat and how large it was, so OCR box - # jitter on other lines cannot pass for a font change. - from openadapt_flow.runtime import render_scale as scale_mod - - salt, hashed = drift_mod.load_scale_landmarks( - self._asset_bytes( - Path(bundle_dir), drift_mod.FINGERPRINT_ASSET, workflow=workflow - ) - ) - details = ( - scale_mod.check_normalized_frame( - hashed.get(step.id, []), - lines, - salt_hex=salt, - text_height_ratio=tolerances.text_height_ratio, - landmark_px=tolerances.landmark_px, - ) - if salt is not None - else None - ) - if details: - return ( - f"Layout drift before step '{step.id}' ({step.intent}) on " - f"the display-scaled session: {'; '.join(details)} — the " - "screen no longer matches the recording; refusing to act." - ) - if details is not None: - return None - live = drift_mod.fingerprint( - lines, - viewport, - landmarks=list(recorded.landmarks), - profile_sha256=(live_profile if isinstance(live_profile, str) else None), - ) - try: - drift_mod.check_layout( - recorded, - live, - tolerances=tolerances, - advisor=self.layout_drift_advisor, - ) - except drift_mod.LayoutDriftHalt as halt: - advice = f" Advice: {halt.verdict.advice}" if halt.verdict.advice else "" - return ( - f"Layout drift before step '{step.id}' ({step.intent}): " - f"{drift_mod.describe(halt.verdict)} — the screen no longer " - f"matches the recording; refusing to act.{advice}" - ) - return None - - def _asset_bytes( - self, - bundle_dir: Path, - rel: str, - *, - workflow: Optional[Workflow] = None, - ) -> Optional[bytes]: - """Return the verified run snapshot for governed asset consumers.""" - if not rel: - # No crop: a record-selecting click rebound to another record - # (runtime.target_param) carries none. - return None - if self.governed_authorization is not None: - data = self._governed_asset_snapshot.get(rel) - expected = self._governed_asset_hashes.get(rel) - if self._governed_plaintext_assets and expected is not None: - path = Path(bundle_dir) / rel - try: - current = path.read_bytes() - except OSError: - current = b"" - if hashlib.sha256(current).hexdigest() != expected: - self._governed_asset_mutation = ( - f"governed sealed asset {rel!r} changed after admission; " - "refusing to act; run aborted" - ) - return data - if workflow is not None and workflow.encrypted: - decrypted = workflow.decrypted_template(rel) - if decrypted is not None: - return decrypted - path = Path(bundle_dir) / rel - return path.read_bytes() if path.is_file() else None - def _raise_on_governed_asset_mutation(self) -> None: """Make predicate-only mutation a non-catchable program safety halt.""" if self._governed_asset_mutation is not None: @@ -18179,51 +10868,6 @@ def _raise_on_governed_asset_mutation(self) -> None: # -- healing --------------------------------------------------------------- - def _heal_step( - self, - step: Step, - resolution: Resolution, - matched_region: Region, - frame_png: bytes, - workflow: Workflow, - run_dir: Path, - new_crops: dict[str, bytes], - ): - """Build, govern, and persist a repair candidate. - - A heal is a governed PATCH, not a silent bundle swap: the raw event is - wrapped in a reviewable :class:`~openadapt_flow.runtime.healing.HealPatch` - and run through the regression gate (identity + effect + risk) before - it may touch the workflow. A patch that would weaken the step's - identity band -- the reviewed context-drop bug -- is QUARANTINED - (persisted under ``run_dir/heals//patch.json`` for review) - and NOT applied; the caller halts the run. A gate-passing local repair - can update the in-memory workflow. A gate-passing repair discovered - under exact authorization remains a promotable candidate, because the - running program cannot change after admission. - """ - event, crop_png = heal_mod.build_heal_event( - step, resolution, matched_region, frame_png, self.vision - ) - outcome = healing_mod.govern_heal(step, event, run_dir=run_dir) - if outcome.promoted: - if self.governed_authorization is None: - heal_mod.apply_heal(workflow, event) - source = self._execution_workflow_source - if source is not None and source is not workflow: - heal_mod.apply_heal(source, event) - self._accept_healed_anchor_in_workflow_snapshot(workflow, event) - new_crops[step.id] = crop_png - else: - # Exact authorization binds the admitted workflow semantics. - # Keep a gate-passing repair as a reviewable candidate for a - # later qualified bundle. Never change the program that is - # currently executing under that authorization. - outcome.patch.status = "promotable" - healing_mod.persist_patch(outcome.patch, run_dir) - heal_mod.persist_heal(event, crop_png, frame_png, run_dir) - return outcome - def _accept_healed_anchor_in_workflow_snapshot( self, workflow: Workflow, @@ -18266,24 +10910,6 @@ def _accept_healed_anchor_in_workflow_snapshot( # -- io ---------------------------------------------------------------------- - @staticmethod - def _new_step_evidence_key(run_dir: Path, step_id: str) -> str: - """Allocate one stable, collision-free file key for an action occurrence.""" - - safe = re.sub(r"[^A-Za-z0-9._-]+", "_", step_id).strip("._-") or "step" - if safe != step_id: - safe = f"{safe}_{hashlib.sha256(step_id.encode()).hexdigest()[:8]}" - steps_dir = Path(run_dir) / "steps" - candidate = safe - occurrence = 1 - while any( - _sealed.exists(steps_dir / f"{candidate}_{suffix}.png") - for suffix in ("before", "after", "attended-before", "attended-after") - ): - occurrence += 1 - candidate = f"{safe}__{occurrence:04d}" - return candidate - @staticmethod @timed_span("artifact_write") def _save_step_png(run_dir: Path, step_id: str, suffix: str, png: bytes) -> str: diff --git a/packages/flow/tests/test_core_key_actions.py b/packages/flow/tests/test_core_key_actions.py new file mode 100644 index 000000000..43ca053c9 --- /dev/null +++ b/packages/flow/tests/test_core_key_actions.py @@ -0,0 +1,125 @@ +"""KEY, HOTKEY and WAIT steps on the real browser. + +Characterization tests for the first action executors moved into +``openadapt_flow.runtime.core`` (docs/design/DECOMPOSITION.md, PR 2). They +pin what the replayer does today, through the real ``PlaywrightBackend``, so +the move can be shown to change nothing. The guarded-keyboard path is pinned +by ``test_playwright_atomic_actuation.py`` and the refusal of raw input on a +consequential identity step by ``test_replayer.py``. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from openadapt_flow.ir import ActionKind, Step, Workflow +from openadapt_flow.runtime.replayer import Replayer + +sync_api = pytest.importorskip("playwright.sync_api") + +_PAGE = """ + + +
+ +
+ + +""" + + +@pytest.fixture(scope="module") +def browser() -> Iterator[object]: + with sync_api.sync_playwright() as playwright: + chromium = playwright.chromium.launch(headless=True) + yield chromium + chromium.close() + + +def _run(browser: object, tmp_path: Path, *steps: Step) -> tuple[object, list[str]]: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + page = browser.new_page(viewport={"width": 640, "height": 320}) + try: + page.set_content(_PAGE) + page.locator("#field").focus() + bundle = tmp_path / "bundle" + bundle.mkdir() + report = Replayer(PlaywrightBackend(page), use_structural=False).run( + Workflow(name="keys", steps=list(steps)), + bundle_dir=bundle, + run_dir=tmp_path / "run", + ) + events = page.evaluate("window.events") + finally: + page.close() + return report, events + + +def test_key_step_presses_the_key_and_the_form_submits(browser, tmp_path) -> None: + report, events = _run( + browser, + tmp_path, + Step(id="k1", intent="submit", action=ActionKind.KEY, key="Enter"), + ) + + assert report.success is True + assert events == ["keydown:Enter", "submit"] + result = report.results[0] + assert result.delivery_attempted is True + assert result.actuation is None # an unguarded press records no actuation kind + assert result.delivery_receipt is None + + +def test_hotkey_step_joins_modifiers_and_key(browser, tmp_path) -> None: + report, events = _run( + browser, + tmp_path, + Step( + id="h1", + intent="select all", + action=ActionKind.HOTKEY, + key="a", + modifiers=["Shift"], + ), + ) + + assert report.success is True + # Playwright presses the lowercase key with Shift held. + assert events == ["keydown:Shift+Shift", "keydown:Shift+a"] + + +def test_key_step_without_a_key_fails_without_input(browser, tmp_path) -> None: + report, events = _run( + browser, + tmp_path, + Step(id="k0", intent="press nothing", action=ActionKind.KEY, key=None), + ) + + assert report.success is False + assert events == [] + result = report.results[0] + assert result.error == "Step 'k0' (press nothing) is a key step with no key" + assert result.delivery_attempted is False + + +def test_wait_step_sends_no_input(browser, tmp_path) -> None: + report, events = _run( + browser, + tmp_path, + Step(id="w1", intent="let it settle", action=ActionKind.WAIT), + ) + + assert report.success is True + assert events == [] + assert report.results[0].delivery_attempted is False diff --git a/packages/flow/tests/test_import_boundaries.py b/packages/flow/tests/test_import_boundaries.py new file mode 100644 index 000000000..097010795 --- /dev/null +++ b/packages/flow/tests/test_import_boundaries.py @@ -0,0 +1,67 @@ +"""The runtime.core boundary stays complete as the package grows. + +``lint-imports`` (CI, flow core lint) checks the contracts in +``.importlinter``. These tests keep the contracts themselves honest: every +governance module is on the forbidden list, and every action kind is carried +out by exactly one place. +""" + +from __future__ import annotations + +import configparser +import re +from pathlib import Path + +from openadapt_flow.ir import ActionKind +from openadapt_flow.runtime.core import ACTION_EXECUTORS + +FLOW = Path(__file__).resolve().parents[1] +PACKAGE = FLOW / "openadapt_flow" + +# Kinds the replayer still carries out itself (_act_without_dialog and the +# surface switch). Each moves into runtime.core with its own executor. +NOT_YET_IN_CORE = { + ActionKind.CLICK, + ActionKind.DOUBLE_CLICK, + ActionKind.RIGHT_CLICK, + ActionKind.DRAG, + ActionKind.TYPE, + ActionKind.SELECT_OPTION, + ActionKind.SCROLL, + ActionKind.READ, + ActionKind.SWITCH_SURFACE, +} + + +def _contract(name: str) -> configparser.SectionProxy: + config = configparser.ConfigParser() + config.read(FLOW / ".importlinter") + return config[f"importlinter:contract:{name}"] + + +def _modules(value: str) -> set[str]: + return {line.strip() for line in value.splitlines() if line.strip()} + + +def test_every_qualification_module_is_forbidden_to_the_core() -> None: + governance = { + f"openadapt_flow.{path.stem if path.is_file() else path.name}" + for path in PACKAGE.iterdir() + if re.match(r"(production_)?qualif", path.name) + and (path.suffix == ".py" or (path / "__init__.py").is_file()) + } + for name in ("runtime-core-direct", "runtime-core-indirect"): + forbidden = _modules(_contract(name)["forbidden_modules"]) + assert governance - forbidden == set(), name + + +def test_both_contracts_forbid_the_same_modules() -> None: + direct = _modules(_contract("runtime-core-direct")["forbidden_modules"]) + indirect = _modules(_contract("runtime-core-indirect")["forbidden_modules"]) + assert direct == indirect + + +def test_every_action_kind_has_exactly_one_executor() -> None: + in_core = set(ACTION_EXECUTORS) + assert in_core & NOT_YET_IN_CORE == set() + assert in_core | NOT_YET_IN_CORE == set(ActionKind) diff --git a/packages/flow/tests/test_multi_surface_fail_closed.py b/packages/flow/tests/test_multi_surface_fail_closed.py index a32f74b4d..4f0d93763 100644 --- a/packages/flow/tests/test_multi_surface_fail_closed.py +++ b/packages/flow/tests/test_multi_surface_fail_closed.py @@ -280,9 +280,9 @@ def _dialog_error(seen: list[dict[str, Any]], dialog: DialogResponse) -> str: def test_a_dialog_mismatch_error_quotes_no_dialog_text(monkeypatch) -> None: - import openadapt_flow.runtime.replayer as replayer_mod + import openadapt_flow.runtime.replay.acting as acting_mod # reads _DIALOG_WAIT_S - monkeypatch.setattr(replayer_mod, "_DIALOG_WAIT_S", 0.0) + monkeypatch.setattr(acting_mod, "_DIALOG_WAIT_S", 0.0) live = "Discharge patient Jane Roe (MRN 991)?" dialog = DialogResponse(response="accept", message="Discharge this patient?") error = _dialog_error([{"message": live, "matched": False}], dialog) @@ -294,9 +294,9 @@ def test_a_dialog_mismatch_error_quotes_no_dialog_text(monkeypatch) -> None: def test_a_dialog_pattern_is_armed_as_a_pattern(monkeypatch) -> None: - import openadapt_flow.runtime.replayer as replayer_mod + import openadapt_flow.runtime.replay.acting as acting_mod # reads _DIALOG_WAIT_S - monkeypatch.setattr(replayer_mod, "_DIALOG_WAIT_S", 0.0) + monkeypatch.setattr(acting_mod, "_DIALOG_WAIT_S", 0.0) dialog = DialogResponse(response="accept", message_pattern=r"Delete\ record\ \d+\?") replayer = Replayer.__new__(Replayer) backend = _DialogBackend([{"message": "Delete record 7?", "matched": True}]) diff --git a/packages/flow/tests/test_replayer.py b/packages/flow/tests/test_replayer.py index 5030a74eb..061186714 100644 --- a/packages/flow/tests/test_replayer.py +++ b/packages/flow/tests/test_replayer.py @@ -2240,10 +2240,15 @@ def scripted_resolve(*args, **kwargs): raise outcome return outcome - monkeypatch.setattr( + # The scroll's readiness probe resolves in replay.acting and the click in + # the replayer; interstitial handling (replay.settling) would resolve too. + # All draw from the same scripted outcomes. + for target in ( "openadapt_flow.runtime.replayer.resolve", - scripted_resolve, - ) + "openadapt_flow.runtime.replay.acting.resolve", + "openadapt_flow.runtime.replay.settling.resolve", + ): + monkeypatch.setattr(target, scripted_resolve) backend = FakeBackend() workflow = Workflow(name="wf", steps=[scroll_step(), click_step()]) diff --git a/scripts/extract_mixin.py b/scripts/extract_mixin.py new file mode 100644 index 000000000..2ee7184d1 --- /dev/null +++ b/scripts/extract_mixin.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python3 +"""Move methods of a large class into a mixin module, without editing them. + +The tool behind the move commits of the decomposition plan +(packages/flow/docs/design/DECOMPOSITION.md, section 4.1). Given a source +module, a class and a list of its methods, it: + +1. cuts each method out of the class verbatim (decorators and the comment + lines directly above it included); +2. writes them into a new mixin class in the target module, importing every + module-level name the bodies use from where the source module got it; +3. generates a host class that declares, for mypy only, exactly the + attributes and other methods the moved code reads from ``self``, with + the source class's own annotations (``Any`` when the source has none). + It is a plain class, not a ``Protocol``: mypy treats a protocol attribute + without a value as abstract and re-infers its type from the mixin's own + assignments, and implicitly abstract protocol methods could shadow the + real ones earlier in the method resolution order. The stubs' empty + bodies are allowed by a per-module ``empty-body`` exemption; +4. makes the source class inherit from the mixin. + +``--module-defs`` moves module-level functions and constants instead and +re-exports them from the source module, so their import paths keep working. + +The result must pass ``scripts/verify_moved_code.py``: identical ASTs for +everything moved, and only imports, re-exports and protocol stubs added. +Run ``ruff check --fix`` and ``ruff format`` on the touched files afterwards. + +Usage: + python scripts/extract_mixin.py SOURCE --class Replayer --target MODULE_PATH + --mixin NAME --methods m1,m2,... + python scripts/extract_mixin.py SOURCE --module-defs --target MODULE_PATH + --names n1,n2,... +""" + +from __future__ import annotations + +import argparse +import ast +import builtins +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +BUILTINS = set(dir(builtins)) + + +@dataclass +class Imports: + """Where each module-level name of the source module comes from.""" + + runtime: dict[str, str] # name -> import statement + type_checking: dict[str, str] + defined: dict[str, ast.stmt] # module-level defs in the source + + @classmethod + def of(cls, tree: ast.Module) -> Imports: + runtime: dict[str, str] = {} + type_checking: dict[str, str] = {} + defined: dict[str, ast.stmt] = {} + + def record(node: ast.stmt, into: dict[str, str]) -> None: + if isinstance(node, ast.ImportFrom): + for alias in node.names: + name = alias.asname or alias.name + text = f"from {'.' * node.level}{node.module or ''} import {alias.name}" + into[name] = text + (f" as {alias.asname}" if alias.asname else "") + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.asname: + into[alias.asname] = f"import {alias.name} as {alias.asname}" + else: + into[alias.name.split(".")[0]] = f"import {alias.name}" + + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + record(node, runtime) + elif ( + isinstance(node, ast.If) + and isinstance(node.test, ast.Name) + and node.test.id == "TYPE_CHECKING" + ): + for sub in node.body: + record(sub, type_checking) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + defined[node.name] = node + elif isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + defined[target.id] = node + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + defined[node.target.id] = node + return cls(runtime, type_checking, defined) + + +def names_in(nodes: list[ast.AST]) -> set[str]: + found: set[str] = set() + # Docstrings mention names in prose; they are not references. + docstrings = { + id(sub.value) + for node in nodes + for sub in ast.walk(node) + if isinstance(sub, ast.Expr) and isinstance(sub.value, ast.Constant) + } + for node in nodes: + for sub in ast.walk(node): + if id(sub) in docstrings: + continue + if isinstance(sub, ast.Name): + found.add(sub.id) + elif isinstance(sub, ast.Attribute): + base = sub + while isinstance(base, ast.Attribute): + base = base.value + if isinstance(base, ast.Name): + found.add(base.id) + elif isinstance(sub, ast.Constant) and isinstance(sub.value, str): + # String annotations ("Foo", "Optional[Bar]") name types too. + for word in re.findall(r"[A-Za-z_][A-Za-z_0-9]*", sub.value): + found.add(word) + return found + + +def annotation_names(nodes: list[ast.AST]) -> set[str]: + """Names used only in annotations of the given functions.""" + found: set[str] = set() + for node in nodes: + for sub in ast.walk(node): + if isinstance(sub, ast.arg) and sub.annotation is not None: + found |= names_in([sub.annotation]) + if isinstance(sub, (ast.FunctionDef, ast.AsyncFunctionDef)) and sub.returns: + found |= names_in([sub.returns]) + if isinstance(sub, ast.AnnAssign): + found |= names_in([sub.annotation]) + return found + + +def span(lines: list[str], node: ast.AST) -> tuple[int, int]: + """0-based [start, end) of a definition with decorators and comments above.""" + start = min([node.lineno] + [d.lineno for d in getattr(node, "decorator_list", [])]) - 1 + while start > 0 and lines[start - 1].lstrip().startswith("#"): + start -= 1 + end = node.end_lineno # type: ignore[attr-defined] + return start, end + + +def runtime_names(nodes: list[ast.AST]) -> set[str]: + """Names the bodies need at run time (annotations are strings here). + + Names a body imports for itself are left out: importing them at module + level as well would let ``ruff --fix`` delete the body's own import, + which is an edit, not a move. + """ + found: set[str] = set() + local: set[str] = set() + for node in nodes: + for sub in ast.walk(node): + if isinstance(sub, (ast.Import, ast.ImportFrom)): + for alias in sub.names: + local.add((alias.asname or alias.name).split(".")[0]) + for node in nodes: + assert isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + for deco in node.decorator_list: + found |= names_in([deco]) + for default in node.args.defaults + [d for d in node.args.kw_defaults if d]: + found |= names_in([default]) + for stmt in node.body: + for sub in ast.walk(stmt): + if isinstance(sub, ast.Name): + found.add(sub.id) + return found - local + + +def import_lines(names: set[str], imports: Imports, shared: dict[str, str]) -> tuple[list[str], list[str]]: + """Import statements for ``names``; second list is names nobody provides.""" + out, missing = [], [] + for name in sorted(names): + if name in BUILTINS or name == "self": + continue + if name in shared: + out.append(shared[name]) + elif name in imports.runtime: + out.append(imports.runtime[name]) + elif name in imports.type_checking: + out.append(imports.type_checking[name]) + elif name in imports.defined: + missing.append(name) + return sorted(set(out)), missing + + +def class_annotations(cls: ast.ClassDef) -> dict[str, str]: + """Attribute annotations the class states: class-level, then ``__init__``.""" + found: dict[str, str] = {} + for node in cls.body: + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + found[node.target.id] = ast.unparse(node.annotation) + init = next( + (n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "__init__"), + None, + ) + if init is None: + return found + params = { + a.arg: ast.unparse(a.annotation) + for a in init.args.args + init.args.kwonlyargs + if a.annotation is not None + } + for sub in ast.walk(init): + target = None + if isinstance(sub, ast.AnnAssign): + target, ann = sub.target, ast.unparse(sub.annotation) + elif isinstance(sub, ast.Assign) and len(sub.targets) == 1: + target = sub.targets[0] + value = sub.value + ann = params.get(value.id) if isinstance(value, ast.Name) else None + else: + continue + if ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "self" + and target.attr not in found + and ann + ): + found[target.attr] = ann + return found + + +def stub(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str: + copy = ast.parse(ast.unparse(node)).body[0] + assert isinstance(copy, (ast.FunctionDef, ast.AsyncFunctionDef)) + copy.body = [ast.Expr(ast.Constant(Ellipsis))] + # Only the decorators that change the signature belong on a stub; others + # (timing spans, caches) wrap behavior the stub does not have. + copy.decorator_list = [ + d + for d in copy.decorator_list + if isinstance(d, ast.Name) and d.id in ("staticmethod", "classmethod", "property") + ] + return ast.unparse(copy) + + +def extract_methods(args: argparse.Namespace) -> None: + source_path = Path(args.source) + text = source_path.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + tree = ast.parse(text) + imports = Imports.of(tree) + cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == args.cls) + methods = { + n.name: n for n in cls.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + wanted = [m for m in args.methods.split(",") if m] + unknown = [m for m in wanted if m not in methods] + if unknown: + sys.exit(f"not methods of {args.cls}: {unknown}") + moved = [methods[m] for m in wanted] + shared = shared_imports(args.shared, imports) + + # Runtime imports for the bodies. + runtime, missing = import_lines(runtime_names(moved) - set(wanted), imports, shared) + if args.cls in runtime_names(moved): + sys.exit(f"{args.cls} is referenced by name in the moved bodies; leave those methods") + if missing: + sys.exit(f"module-level names not yet in --shared: {missing}") + + # The host protocol: what the moved code reads from self. + used: set[str] = set() + for node in moved: + for sub in ast.walk(node): + if ( + isinstance(sub, ast.Attribute) + and isinstance(sub.value, ast.Name) + and sub.value.id == "self" + and not sub.attr.startswith("__") + ): + used.add(sub.attr) + used -= set(wanted) + annotations = class_annotations(cls) + host_lines: list[str] = [] + stub_nodes: list[ast.AST] = [] + for name in sorted(used): + if name in methods: + host_lines.append(_indent(stub(methods[name]))) + stub_nodes.append(methods[name]) + else: + host_lines.append(f" {name}: {annotations.get(name, 'Any')}") + ann_text = [ast.parse(a, mode="eval").body for a in annotations.values()] + type_names = ( + annotation_names(moved) + | annotation_names(stub_nodes) + | names_in([n for n in ann_text if any(f": {ast.unparse(n)}" in h for h in host_lines)]) + ) + type_lines, type_missing = import_lines(type_names - set(wanted), imports, shared) + type_only = sorted(set(type_lines) - set(runtime)) + if type_missing: + type_only += [f"from {args.source_module} import {n}" for n in sorted(type_missing)] + + host = f"_{args.mixin}Host" + body = "".join("".join(lines[a:b]) + "\n" for a, b in (span(lines, n) for n in moved)) + module = [ + '# mypy: disable-error-code="empty-body"\n', + f'"""{args.doc}\n\nMoved verbatim from ``{args.source_module}.{args.cls}`` by\n' + "scripts/extract_mixin.py (docs/design/DECOMPOSITION.md, section 4.1).\n" + f'``{host}`` declares, for mypy only, what this code reads from the\n' + f'{args.cls} it is mixed into.\n"""\n\n', + "from __future__ import annotations\n\n", + "from typing import TYPE_CHECKING, Any # noqa: F401\n", + *[line + "\n" for line in runtime], + "\nif TYPE_CHECKING:\n", + *([f" {line}\n" for line in type_only] or [" pass\n"]), + f"\n\nclass {host}:\n", + "\n".join(host_lines) + "\n" if host_lines else " pass\n", + "\n\nif TYPE_CHECKING:\n", + f" _Base = {host}\nelse:\n _Base = object\n\n\n", + f"class {args.mixin}(_Base):\n", + f' """{args.doc.splitlines()[0]}"""\n\n', + body, + ] + target = Path(args.target) + target.parent.mkdir(parents=True, exist_ok=True) + init = target.parent / "__init__.py" + if not init.exists(): + init.write_text(f'"""{args.package_doc}"""\n', encoding="utf-8") + target.write_text("".join(module), encoding="utf-8") + + # Cut the methods out of the source class, bottom up. + for a, b in sorted((span(lines, n) for n in moved), reverse=True): + del lines[a:b] + while a < len(lines) and a > 0 and not lines[a].strip() and not lines[a - 1].strip(): + del lines[a] + out = "".join(lines) + mixin_module = _module_of(target) + header = re.search(rf"^class {args.cls}(\((?P[^)]*)\))?:", out, re.M) + assert header + bases = [b.strip() for b in (header.group("bases") or "").split(",") if b.strip()] + bases.append(args.mixin) + out = out[: header.start()] + f"class {args.cls}({', '.join(bases)}):" + out[header.end() :] + out = _add_import(out, f"from {mixin_module} import {args.mixin}") + source_path.write_text(out, encoding="utf-8") + print(f"moved {len(moved)} methods into {target}") + + +def extract_module_defs(args: argparse.Namespace) -> None: + source_path = Path(args.source) + text = source_path.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + tree = ast.parse(text) + imports = Imports.of(tree) + wanted = [n for n in args.names.split(",") if n] + nodes = [] + for name in wanted: + node = imports.defined.get(name) + if node is None: + sys.exit(f"not a module-level definition: {name}") + if node not in nodes: + nodes.append(node) + names = set() + for node in nodes: + names |= names_in([node]) + provided = {n for node in nodes for n in _defined_names(node)} + local = { + (alias.asname or alias.name).split(".")[0] + for node in nodes + for sub in ast.walk(node) + if isinstance(sub, (ast.Import, ast.ImportFrom)) + for alias in sub.names + } + runtime, missing = import_lines(names - provided - local, imports, {}) + if missing: + sys.exit(f"these definitions use other module-level names; move them too: {missing}") + # Adjacent definitions (only blank or comment lines between them) move as + # one block, so comments that cover several constants stay with them. + spans = merge_adjacent(lines, sorted(span(lines, n) for n in nodes)) + body = "\n\n\n".join("".join(lines[a:b]).strip("\n") for a, b in spans) + target = Path(args.target) + target.parent.mkdir(parents=True, exist_ok=True) + init = target.parent / "__init__.py" + if not init.exists(): + init.write_text(f'"""{args.package_doc}"""\n', encoding="utf-8") + target.write_text( + f'"""{args.doc}\n\nMoved verbatim from ``{args.source_module}`` by\n' + 'scripts/extract_mixin.py; the source module re-exports every name.\n"""\n\n' + "from __future__ import annotations\n\n" + + "".join(line + "\n" for line in runtime) + + "\n\n" + + body + + "\n", + encoding="utf-8", + ) + for a, b in reversed(spans): + del lines[a:b] + out = "".join(lines) + exported = sorted(provided) + out = _add_import( + out, + f"from {_module_of(target)} import ( # noqa: F401 re-exported\n" + + "".join(f" {n},\n" for n in exported) + + ")", + ) + source_path.write_text(out, encoding="utf-8") + print(f"moved {len(exported)} names into {target}") + + +def merge_adjacent(lines: list[str], spans: list[tuple[int, int]]) -> list[tuple[int, int]]: + merged: list[tuple[int, int]] = [] + for a, b in spans: + if merged: + gap = lines[merged[-1][1] : a] + if all(not g.strip() or g.lstrip().startswith("#") for g in gap): + merged[-1] = (merged[-1][0], b) + continue + merged.append((a, b)) + return merged + + +def _defined_names(node: ast.stmt) -> list[str]: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return [node.name] + if isinstance(node, ast.Assign): + return [t.id for t in node.targets if isinstance(t, ast.Name)] + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + return [node.target.id] + return [] + + +def shared_imports(paths: list[str] | None, imports: Imports) -> dict[str, str]: + """Names other moved modules now provide: name -> import statement.""" + found: dict[str, str] = {} + for path in paths or []: + tree = ast.parse(Path(path).read_text(encoding="utf-8")) + module = _module_of(Path(path)) + for node in tree.body: + for name in _defined_names(node): + found[name] = f"from {module} import {name}" + return found + + +def _module_of(path: Path) -> str: + parts = path.with_suffix("").parts + start = next(i for i, p in enumerate(parts) if p.startswith("openadapt")) + return ".".join(parts[start:]) + + +def _indent(code: str) -> str: + return "".join(" " + line if line.strip() else line for line in code.splitlines(True)) + + +def _add_import(source: str, statement: str) -> str: + """Insert an import after the last top-level import of the module.""" + tree = ast.parse(source) + last = max( + n.end_lineno for n in tree.body if isinstance(n, (ast.Import, ast.ImportFrom)) + ) + lines = source.splitlines(keepends=True) + lines.insert(last, statement + "\n") + return "".join(lines) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("source") + parser.add_argument("--class", dest="cls", default="Replayer") + parser.add_argument("--source-module", default="openadapt_flow.runtime.replayer") + parser.add_argument("--target", required=True) + parser.add_argument("--mixin") + parser.add_argument("--methods", default="") + parser.add_argument("--module-defs", action="store_true") + parser.add_argument("--names", default="") + parser.add_argument("--shared", action="append") + parser.add_argument("--doc", default="Moved replayer code.") + parser.add_argument("--package-doc", default="Replayer concerns, one module each.") + args = parser.parse_args(argv) + if args.module_defs: + extract_module_defs(args) + else: + extract_methods(args) + + +if __name__ == "__main__": + main()