Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitleaksignore
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,6 @@ e8ec96725c4451aa64cb97230c18d04b55d98503:packages/flow/tests/e2e/test_mcp_teach_
# test_network_capture_redact_json.py; the next commit builds them at run time.
52502adf407793c211fb3c0ac5aba37556206d8c:packages/flow/tests/test_network_capture_redact_json.py:generic-api-key:17
52502adf407793c211fb3c0ac5aba37556206d8c:packages/flow/tests/test_network_capture_redact_json.py:generic-api-key:146
# PR #1543: `arg=self._fill_value_key` names a per-backend page key (a uuid made at
# runtime), not a credential. The line no longer exists after 81d00f5a4.
37605dd5fd04a1ef8517d3ec078ed88348c0a8f4:packages/flow/openadapt_flow/backends/playwright_backend.py:generic-api-key:1807
42 changes: 42 additions & 0 deletions packages/flow/docs/LIMITS.md
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,48 @@ The binding ties the click to the entry the run made. It doesn't check that
the entry is right: the values typed come from the run's parameters, and only
a record check can confirm what the application saved.

A labelled save on a create-new form (Trac's **Create ticket**, NetBox's
**Create**) has the same problem: no existing record is on screen next to it,
so it compiled with no identity check, and a policy that refuses unarmed saves
refused it. Its record is the one the run is typing into the form. When the
recording shows the save is the submit button of the one form that owns every
field typed since the last save, the compiler gives it a **form identity**
(`Anchor.form_identity`). Before the click, a run requires (identity mode
`form`):

- the button is still that form's submit button, and the form owns exactly the
fields this run typed into, each holding exactly what the run typed;
- the form submits to the same origin and path with the same method, and would
send the same set of field names and types, hidden ones included, so an edit
form that looks like the create form, or an address, method or record id a
script changed or added, halts;
- every field the run did not type into (a hidden parent or record id, a
select left on its default, a checked box) holds its demonstrated value,
stored as a salted digest. Hidden tokens are not compared: ones named like a
token (`csrf`, `token`, `nonce` and the like), whose value changes with every
page load, and ones that changed while the demonstration was typed. A
program that should create under a different parent on each run therefore
halts until that parent is checked another way.

During the click, a submit from any other form, or from the right form without
its save button (`requestSubmit()` from a click handler), is cancelled and
halts the run. A script that submits with `form.submit()` or `fetch` sends no
submit event, so the guard doesn't see it.

A backend that can't read the page leaves the identity unreadable, which halts
too. Each field must hold exactly what the run typed, so a controlled input
that reformats the value halts: one that reformats on every keystroke stops at
the typed-value read-back, and one that reformats when it loses focus (a
phone-number mask, for example) stops at the form check. A field that a React
keyed remount replaces with a new element, even with the same value, also
halts: the new element isn't one the run typed into. A submit button outside
the `<form>` element counts when its `form=` attribute names that form. One
with no form owner at all (a React portal that submits from script) gets no
form identity, so its save stays unarmed.

The form identity ties the click to the entry the run made in the demonstrated
form. It doesn't prove the app saved it correctly: only a record check does.

For consequential or entity-sensitive workflows, require a policy that refuses
unarmed actions, review the inferred risk and tags, and test identity against
real near-match and same-name records. Pure-pixel substrates should expect more
Expand Down
233 changes: 231 additions & 2 deletions packages/flow/openadapt_flow/backends/playwright_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from openadapt_flow.backend import ActionDeliveryUncertain, StructuralResolutionRefused
from openadapt_flow.ir import (
ActionDeliveryReceipt,
FormIdentity,
StructuralHandle,
StructuralLocator,
SubmitBinding,
Expand All @@ -42,7 +43,10 @@
from openadapt_flow.submit_binding import (
OBSERVE_SUBMIT_BINDING_JS,
SUBMIT_BINDING_FIELD_SELECTOR,
form_values_from_page,
form_values_record,
observation_from_page,
verify_form_identity,
verify_submit_binding,
)

Expand Down Expand Up @@ -456,7 +460,9 @@ class _FramePoint:
+ OBSERVE_SUBMIT_BINDING_JS
+ r""";
const fills = window[args.fillKey];
return observe(control, Array.isArray(fills) ? fills : []);
return observe(
control, Array.isArray(fills) ? fills : [], window[args.fillKey + '_hidden']
);
}"""
)

Expand All @@ -474,10 +480,123 @@ class _FramePoint:
Object.defineProperty(window, key, {value: fills, configurable: true});
}
if (fills.indexOf(el) < 0) fills.push(el);
// Each hidden control's value as typing starts, so a create-new form's
// fingerprint can tell a token that rotates while the form is filled in.
let hidden = window[key + '_hidden'];
if (!(hidden instanceof WeakMap)) {
hidden = new WeakMap();
Object.defineProperty(
window, key + '_hidden', {value: hidden, configurable: true});
}
for (const input of el.ownerDocument.querySelectorAll('input[type="hidden"]')) {
if (!hidden.has(input)) hidden.set(input, String(input.value));
}
return true;
}"""
)

# Remember what a field holds once the run has finished typing into it, and
# whether that is exactly what the run typed, for a create-new form's identity
# check (openadapt_flow.ir.FormIdentity). A select holds what was typed when
# its chosen option's label is the typed text. A field that reformatted the
# typed text (a controlled input, a mask) does not. The WeakMap lives in the
# field's own document under a per-backend random key and is only compared in
# the page; no value is returned to Python.
_SNAPSHOT_FILLED_VALUE_JS = r"""(el, args) => {
let values = window[args.key];
if (!(values instanceof WeakMap)) {
values = new WeakMap();
Object.defineProperty(window, args.key, {value: values, configurable: true});
}
const clean = (s) => String(s || '').replace(/\s+/g, ' ').trim();
const value = ('value' in el) ? String(el.value) : String(el.innerText || '');
let typed;
if (el.localName === 'select') {
const option = el.options[el.selectedIndex];
typed = Boolean(option) && clean(option.text) === clean(args.typed);
} else if ('value' in el) {
typed = value === String(args.typed);
} else {
typed = clean(value) === clean(args.typed);
}
values.set(el, {value: value, typed: typed});
return true;
}"""

# A create-new form's identity at a point: the submit-binding observation of
# the control there against the fields this run typed into, and how many of
# those fields in the control's form no longer hold what the run left in them
# (or were never snapshotted). Counts only: no value leaves the page.
_FORM_IDENTITY_AT_JS = (
r"""(args) => {
const hit = document.elementFromPoint(args.x, args.y);
if (!hit) return null;
const control = hit.closest("""
+ json.dumps(GUARDED_ACTIONABLE_SELECTOR)
+ r""");
if (!control) return null;
const observe = """
+ OBSERVE_SUBMIT_BINDING_JS
+ r""";
const fills = Array.isArray(window[args.fillKey]) ? window[args.fillKey] : [];
const out = observe(control, fills, window[args.fillKey + '_hidden']);
const values = window[args.valueKey];
const form = control.form || control.closest('form');
let changed = 0;
for (const field of fills) {
if (!field || !field.isConnected || !form) continue;
const owner = field.form || field.closest('form');
if (owner !== form) continue;
const now = ('value' in field) ? String(field.value)
: String(field.innerText || '');
const left = (values instanceof WeakMap) ? values.get(field) : null;
if (!left || !left.typed || left.value !== now) changed += 1;
}
return {observation: out.observation, changed: changed};
}"""
)

# Before a create-new form's save: cancel any submit during the click that is
# not the checked form submitted by the checked button (a click handler that
# submits another form, or the form without its button), and remember why.
# One shot: it removes itself at the first submit or after 30 s.
_ARM_FORM_SUBMIT_GUARD_JS = (
r"""(args) => {
const hit = document.elementFromPoint(args.x, args.y);
const control = hit ? hit.closest("""
+ json.dumps(GUARDED_ACTIONABLE_SELECTOR)
+ r""") : null;
const form = control ? (control.form || control.closest('form')) : null;
if (!form) return false;
const previous = window[args.key];
if (previous && previous.remove) previous.remove();
const state = {refused: null};
const handler = (event) => {
state.remove();
if (event.target === form && event.submitter === control) return;
event.preventDefault();
event.stopImmediatePropagation();
state.refused = event.target === form
? 'the form was submitted by something other than its save button'
: 'the click submitted a different form';
};
state.remove = () => window.removeEventListener('submit', handler, true);
window.addEventListener('submit', handler, true);
setTimeout(state.remove, 30000);
Object.defineProperty(
window, args.key, {value: state, configurable: true, writable: true});
return true;
}"""
)

_FORM_SUBMIT_GUARD_RESULT_JS = r"""(key) => {
const state = window[key];
if (!state) return null;
if (state.remove) state.remove();
delete window[key];
return state.refused || null;
}"""

# Page-structure identity of a FORM FIELD that sits in no record row.
#
# A data-entry form (OpenEMR's new-patient page, most CRUD screens) lays its
Expand Down Expand Up @@ -1303,6 +1422,9 @@ def __init__(
# Fields this backend typed into, kept page-side per document (element
# references only) for submit bindings. A new backend starts empty.
self._fill_store_key = f"__oaflow_filled_{uuid.uuid4().hex}"
self._fill_value_key = f"__oaflow_filled_values_{uuid.uuid4().hex}"
self._form_guard_key = f"__oaflow_form_guard_{uuid.uuid4().hex}"
self._form_guard_scope: Any = None
self._structural_tokens: dict[str, _StructuralGuard] = {}
self._guarded_coordinate: Optional[_CoordinateGuard] = None
# One-shot expectation that the next click closes this page surface
Expand Down Expand Up @@ -1940,6 +2062,110 @@ def _register_focused_field(self) -> None:
_REGISTER_FILLED_FIELD_JS, arg=self._fill_store_key, with_arg=True
)

def _snapshot_focused_field(self, typed: str) -> None:
"""Remember what the focused field holds now that ``typed`` is in,
and whether that is exactly ``typed``.

A create-new form's identity check compares each filled field with
this snapshot before the save. A field with no snapshot, or one that
did not end up holding exactly what was typed, counts as changed, so
a failure here makes that check refuse: the safe direction.
"""

self._focused_element_call(
_SNAPSHOT_FILLED_VALUE_JS,
arg={"key": self._fill_value_key, "typed": typed},
with_arg=True,
)

def form_identity_at(self, x: int, y: int, identity: FormIdentity) -> Optional[str]:
"""Check a create-new form's identity for the control at a point.

Returns None when the control is still the submit button of the
demonstrated form (same address, method and set of fields), the form
owns exactly the fields this run typed into, each holding exactly what
the run typed, and every other field holds its demonstrated value.
Then it arms a one-shot guard that cancels any other submit during the
click (:meth:`form_submit_guard_refusal` reports it). Otherwise the
reason, in plain words, including when the page can't be read.
"""

self.cancel_form_submit_guard()
try:
point = self._frame_point(int(x), int(y))
if point is None:
return "the resolved point has no readable page"
raw = point.scope.evaluate(
_FORM_IDENTITY_AT_JS,
{
"x": point.x,
"y": point.y,
"fillKey": self._fill_store_key,
"valueKey": self._fill_value_key,
},
)
if not isinstance(raw, dict):
return "no control at the resolved point"
observation = observation_from_page(raw.get("observation"))
values = form_values_from_page(raw.get("observation"))
changed = int(raw.get("changed", -1))
except Exception:
return "could not read the form at the resolved point"
if changed < 0:
return "could not read the form's field values"
reason = verify_form_identity(
identity, observation, changed_fields=changed, values=values
)
if reason is not None:
return reason
try:
armed = point.scope.evaluate(
_ARM_FORM_SUBMIT_GUARD_JS,
{"x": point.x, "y": point.y, "key": self._form_guard_key},
)
except Exception:
armed = False
if armed is not True:
return "could not guard the form's submission"
self._form_guard_scope = point.scope
return None

def form_submit_guard_refusal(self) -> Optional[str]:
"""After the click: why the guard cancelled a submit, or None.

Disarms the guard. A page that navigated away (the checked form's own
submit) has nothing to report.
"""

scope, self._form_guard_scope = self._form_guard_scope, None
if scope is None:
return None
try:
refused = scope.evaluate(_FORM_SUBMIT_GUARD_RESULT_JS, self._form_guard_key)
except Exception:
return None
return refused if isinstance(refused, str) else None

def cancel_form_submit_guard(self) -> None:
"""Disarm a guard whose click never happened."""

self.form_submit_guard_refusal()

def form_values_at(self, x: int, y: int) -> Optional[dict[str, Any]]:
"""RECORD-TIME: the values of the form's fields the demonstration did
not type into, for the control at a point (see
:func:`~openadapt_flow.submit_binding.form_values_record`), or None.
Plaintext for the recording only; the compiler keeps salted digests.
Never raises.
"""

try:
raw = self._observe_submit_binding(x, y)
values = form_values_from_page(raw.get("observation")) if raw else None
except Exception:
return None
return form_values_record(values) if values is not None else None

def _focused_element_call(
self, js: str, *, arg: Any = None, with_arg: bool = False
) -> Any:
Expand Down Expand Up @@ -3151,13 +3377,15 @@ def type_text_guarded(
"""Type through the pre-identity focused-element lease."""

self._register_focused_field()
return self._act_guarded_keyboard(
receipt = self._act_guarded_keyboard(
expected_frame_sha256=expected_frame_sha256,
operation="guarded_dom_type",
# Keys go through the leased element itself, never the page
# keyboard, so delivery stays bound to the checked target.
deliver=lambda locator: locator.press_sequentially(text, timeout=1000),
)
self._snapshot_focused_field(text)
return receipt

def _handle_screenshot_frame_lifecycle(self, _frame: Any = None) -> None:
"""Advance the irreversible frame-tree generation."""
Expand Down Expand Up @@ -3732,6 +3960,7 @@ def type_text(self, text: str) -> None:
self._assert_qualification_environment_current()
self._register_focused_field()
self._deliver_text(text, lambda: self.page.keyboard.type(text))
self._snapshot_focused_field(text)

def press(self, key: str) -> None:
"""Press a key or chord, e.g. ``'Enter'`` or ``'Meta+a'``."""
Expand Down
Loading
Loading