diff --git a/packages/flow/benchmark/fhir_medplum/compose.yml b/packages/flow/benchmark/fhir_medplum/compose.yml new file mode 100644 index 000000000..ae8ceff93 --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/compose.yml @@ -0,0 +1,81 @@ +# Medplum (Apache-2.0), self-hosted for the second-installation benchmark. +# Synthetic data only. Every image is pinned by digest, every published port +# binds to loopback, and Postgres and Redis publish nothing. Secrets come from +# the fixture's generated state (benchmark/fhir_medplum/fixture.py). +services: + postgres: + image: ${MEDPLUM_PG_IMAGE} + environment: + POSTGRES_USER: medplum + POSTGRES_PASSWORD: ${MEDPLUM_DB_PASSWORD} + command: + - postgres + - -c + - statement_timeout=60000 + - -c + - default_transaction_isolation=REPEATABLE READ + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U medplum"] + interval: 5s + timeout: 5s + retries: 30 + + redis: + image: ${MEDPLUM_REDIS_IMAGE} + command: ["redis-server", "--requirepass", "${MEDPLUM_REDIS_PASSWORD}"] + healthcheck: + test: ["CMD", "redis-cli", "-a", "${MEDPLUM_REDIS_PASSWORD}", "ping"] + interval: 5s + timeout: 5s + retries: 30 + + medplum-server: + image: ${MEDPLUM_SERVER_IMAGE} + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + command: ["env"] + ports: + - "127.0.0.1:${MEDPLUM_SERVER_PORT}:8103" + environment: + MEDPLUM_PORT: 8103 + MEDPLUM_BASE_URL: "http://localhost:${MEDPLUM_SERVER_PORT}/" + MEDPLUM_APP_BASE_URL: "http://localhost:${MEDPLUM_APP_PORT}/" + MEDPLUM_STORAGE_BASE_URL: "http://localhost:${MEDPLUM_SERVER_PORT}/storage/" + MEDPLUM_DATABASE_HOST: postgres + MEDPLUM_DATABASE_PORT: 5432 + MEDPLUM_DATABASE_DBNAME: medplum + MEDPLUM_DATABASE_USERNAME: medplum + MEDPLUM_DATABASE_PASSWORD: ${MEDPLUM_DB_PASSWORD} + MEDPLUM_REDIS_HOST: redis + MEDPLUM_REDIS_PORT: 6379 + MEDPLUM_REDIS_PASSWORD: ${MEDPLUM_REDIS_PASSWORD} + MEDPLUM_BINARY_STORAGE: "file:./binary/" + MEDPLUM_SUPPORT_EMAIL: '"Medplum" ' + MEDPLUM_MAX_JSON_SIZE: 1mb + MEDPLUM_MAX_BATCH_SIZE: 50mb + MEDPLUM_VM_CONTEXT_BOTS_ENABLED: "false" + MEDPLUM_ALLOWED_ORIGINS: "http://localhost:${MEDPLUM_APP_PORT}" + MEDPLUM_INTROSPECTION_ENABLED: "true" + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://localhost:8103/healthcheck').then(r => r.ok || process.exit(1)).catch(() => process.exit(1))"] + interval: 10s + timeout: 10s + retries: 60 + + medplum-app: + image: ${MEDPLUM_APP_IMAGE} + depends_on: + medplum-server: + condition: service_healthy + ports: + - "127.0.0.1:${MEDPLUM_APP_PORT}:3000" + environment: + MEDPLUM_BASE_URL: "http://localhost:${MEDPLUM_SERVER_PORT}/" + +volumes: + pgdata: diff --git a/packages/flow/benchmark/fhir_medplum/demo.py b/packages/flow/benchmark/fhir_medplum/demo.py new file mode 100644 index 000000000..5dd2a022e --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/demo.py @@ -0,0 +1,241 @@ +"""Teach Medplum's new-patient form: a scripted demonstration, then compile. + + PYTHONPATH=. python -m benchmark.fhir_medplum.demo record --out DIR [--values FILE] + PYTHONPATH=. python -m benchmark.fhir_medplum.demo compile --recording DIR --bundle DIR + +Signing in and opening the blank form are setup, not part of the +demonstration (``qualify_hooks.open_session``). The demonstration fills the +form and clicks Create. Every action goes through the ``Recorder``, so the +frames and events are what a person's demonstration would leave; record time +uses Playwright locators only to find where to click. + +Three choices the form forces: + +- Gender is an autocomplete over a ValueSet: the clerk types the code + (``female``), then ArrowDown and Enter take the exact match. Enter alone + submits the whole form. +- Telecom's system is a native select: the clerk types the option + (``phone``, ``email``) into it. +- Controls with nothing readable beside them are reached by keyboard (see + ``demonstrate``). +- Birth Date is a native date input; the backend sets an ISO date into it. + +Synthetic values only. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +#: The demonstration's patient (synthetic). Replays bring their own values. +DEFAULT_VALUES = { + "fname": "Rowan", + "lname": "Achterberg", + "DOB": "1979-11-23", + "sex": "female", + "street": "41 Quarry Lane", + "city": "Worcester", + "state_label": "Massachusetts", + "postal_code": "01609", + "phone_home": "508-555-0147", + "email": "rowan.achterberg@example.invalid", +} + +SETTLE = { + "settle_timeout_s": 10.0, + "settle_stable_frames": 3, + "settle_interval_s": 0.3, +} + +#: Targets are kept below the sticky header and banner, above the fold. +SAFE_TOP, SAFE_BOTTOM, ANCHOR_Y = 170, 800, 420 + + +def _center(recorder: Any, page: Any, locator: Any) -> tuple[int, int]: + """Bring ``locator`` into the safe band with recorded wheel scrolls.""" + + last = None + for _ in range(6): + box = locator.bounding_box() + if box is None: + raise RuntimeError("target is not on the page") + mid = box["y"] + box["height"] / 2 + if SAFE_TOP <= mid <= SAFE_BOTTOM: + return int(box["x"] + box["width"] / 2), int(mid) + if last is not None and abs(mid - last) < 1 and SAFE_TOP <= mid <= 890: + # The page can't scroll further (the Create button is last). + return int(box["x"] + box["width"] / 2), int(mid) + last = mid + recorder.scroll(0, int(mid - ANCHOR_Y)) + page.wait_for_timeout(400) + raise RuntimeError("could not bring the target into view") + + +def _click(recorder: Any, page: Any, locator: Any) -> None: + recorder.click(*_center(recorder, page, locator)) + page.wait_for_timeout(250) + + +_FOCUSED = ( + "() => { const e = document.activeElement; return [e.tagName," + " e.getAttribute('name') || '', e.getAttribute('data-testid') || ''," + " e.getAttribute('placeholder') || '', (e.innerText || '').trim().slice(0, 30)]; }" +) + + +def _keys( + recorder: Any, page: Any, key: str, times: int, expect: dict[str, str] +) -> None: + """Move the focus by keyboard, then check where it landed (not recorded).""" + + for _ in range(times): + recorder.press(key) + page.wait_for_timeout(120) + tag, name, testid, placeholder, text = page.evaluate(_FOCUSED) + got = { + "tag": tag, + "name": name, + "testid": testid, + "placeholder": placeholder, + "text": text, + } + wrong = {k: (v, got[k]) for k, v in expect.items() if got[k] != v} + if wrong: + raise RuntimeError(f"focus landed elsewhere after {times} x {key}: {wrong}") + + +def _keys_until( + recorder: Any, page: Any, key: str, expect: dict[str, str], most: int = 4 +) -> int: + """Press ``key`` until the focus matches ``expect``; the presses are recorded.""" + + for n in range(1, most + 1): + recorder.press(key) + page.wait_for_timeout(120) + tag, name, testid, placeholder, text = page.evaluate(_FOCUSED) + got = { + "tag": tag, + "name": name, + "testid": testid, + "placeholder": placeholder, + "text": text, + } + if all(got.get(k) == v for k, v in expect.items()): + return n + raise RuntimeError(f"focus never reached {expect} in {most} x {key}") + + +def demonstrate(recorder: Any, page: Any, values: dict[str, str]) -> None: + """Fill the new-patient form once and click Create. + + Medplum's generic resource form gives several controls nothing to read + beside them: the "Add Name" button, the Telecom selects and value fields, + Gender and Birth Date. A click on them can't carry an identity check, and + clinical-write refuses an unarmed click. So the demonstration reaches + them by keyboard from a click that does carry one, as a keyboard user + would; every keystroke is recorded, and record time checks the focus. + """ + + # The first control has nothing beside it, so the demonstration tabs + # from the top of the page to "Add Name" (Space presses a button). + _keys_until(recorder, page, "Tab", {"tag": "BUTTON", "text": "Add Name"}, most=15) + recorder.press("Space") + page.wait_for_timeout(300) + _click(recorder, page, page.get_by_placeholder("Given")) + recorder.type_text(values["fname"], param="fname") + _click(recorder, page, page.get_by_placeholder("Family")) + recorder.type_text(values["lname"], param="lname") + + # Each Telecom row: system, use, value, remove, then the Add button. + for system, param in (("phone", "phone_home"), ("email", "email")): + _click(recorder, page, page.get_by_role("button", name="Add Telecom")) + _keys(recorder, page, "Shift+Tab", 4, {"tag": "SELECT", "testid": "system"}) + recorder.type_text(system) + _keys(recorder, page, "Tab", 2, {"tag": "INPUT", "placeholder": "Value"}) + recorder.type_text(values[param], param=param) + + # Birth Date: the row's remove button, Add Telecom, Gender, Birth Date. + # Choosing a gender re-renders the field and drops the focus, so the date + # goes first and Gender last. + _keys(recorder, page, "Tab", 4, {"tag": "INPUT", "name": "birthDate"}) + recorder.type_text(values["DOB"], param="DOB") + _keys_until(recorder, page, "Shift+Tab", {"tag": "INPUT", "name": "gender"}) + recorder.type_text(values["sex"], param="sex") + page.wait_for_timeout(1200) + recorder.press("ArrowDown") + recorder.press("Enter") + page.wait_for_timeout(300) + + _click(recorder, page, page.get_by_role("button", name="Add Address")) + _click(recorder, page, page.get_by_placeholder("Line 1")) + recorder.type_text(values["street"], param="street") + _click(recorder, page, page.get_by_placeholder("City")) + recorder.type_text(values["city"], param="city") + _click(recorder, page, page.get_by_placeholder("State")) + recorder.type_text(values["state_label"], param="state_label") + _click(recorder, page, page.get_by_placeholder("Postal Code")) + recorder.type_text(values["postal_code"], param="postal_code") + + _click(recorder, page, page.get_by_role("button", name="Create")) + page.wait_for_url(lambda url: "/Patient/new" not in url, timeout=30_000) + page.wait_for_timeout(1500) + + +def cmd_record(args: argparse.Namespace) -> int: + from types import SimpleNamespace + + from benchmark.fhir_medplum import qualify_hooks + from openadapt_flow.recorder import Recorder + + out = Path(args.out) + if out.exists(): + raise SystemExit(f"recording path already exists: {out}") + values = dict(DEFAULT_VALUES) + if args.values: + values.update(json.loads(Path(args.values).read_text())) + config = SimpleNamespace(app=SimpleNamespace(viewport=(1280, 900))) + backend, close = qualify_hooks.open_session(config) + try: + recorder = Recorder(backend, out, app_url=backend.page.url, **SETTLE) + demonstrate(recorder, backend.page, values) + print("recording:", recorder.finish()) + print("saved:", backend.page.url) + finally: + close() + return 0 + + +def cmd_compile(args: argparse.Namespace) -> int: + from openadapt_flow.compiler import compile_recording + + bundle = Path(args.bundle) + if bundle.exists(): + raise SystemExit(f"bundle path already exists: {bundle}") + workflow = compile_recording( + Path(args.recording), bundle, name="medplum-new-patient" + ) + print("bundle:", bundle, "steps:", len(workflow.steps)) + print("params:", sorted(workflow.params) if workflow.params else []) + return 0 + + +def main(argv: Any = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="cmd", required=True) + p = sub.add_parser("record") + p.add_argument("--out", required=True) + p.add_argument("--values", default=None, help="JSON file of field values") + p.set_defaults(func=cmd_record) + p = sub.add_parser("compile") + p.add_argument("--recording", required=True) + p.add_argument("--bundle", required=True) + p.set_defaults(func=cmd_compile) + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/fhir_medplum/environment.lock.json b/packages/flow/benchmark/fhir_medplum/environment.lock.json new file mode 100644 index 000000000..d751a14e8 --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/environment.lock.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "generated_for": "OpenAdapt second-installation benchmark: Medplum (FHIR R4) with synthetic patients", + "license": "Apache-2.0 (Medplum); images pulled at run time, never stored in this repository", + "services": { + "server": "medplum/medplum-server:5.2.8@sha256:d70ee91bcf15aa5302c91ad3875f96a6f547f01c2e24c3499d9203f084321c0a", + "app": "medplum/medplum-app:5.2.8@sha256:4a4a4d411e28260e83d8f2802ea24b57156aa53096f5013813969af48b8a886d", + "postgres": "postgres:16@sha256:9e2cf6287f25c3151f4c9421ccce2b17f3e60061c111a29165e0d2cdd9ff0a97", + "redis": "redis:7@sha256:44960b3100c48159a66d28c18b0588c193492e25f7551d60e51dd464fc3eca5b" + }, + "platforms": "every image publishes linux/arm64 and linux/amd64", + "project": "openadapt-si-medplum" +} diff --git a/packages/flow/benchmark/fhir_medplum/fixture.py b/packages/flow/benchmark/fhir_medplum/fixture.py new file mode 100644 index 000000000..cc6b632f7 --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/fixture.py @@ -0,0 +1,334 @@ +"""A local Medplum (FHIR R4) server and web app with synthetic patients. + +The second FHIR system for the second-installation benchmark +(benchmark/second_installation). Medplum is Apache-2.0; its images are pulled +by digest at run time (environment.lock.json) and nothing of it is stored here. + +- The web app (``ui_url``) is what a clerk uses: it creates a Patient through + Medplum's resource form. +- The FHIR API (``fhir_url``) is the record channel: the run's record check and + duplicate check read it with a client-credentials token, as a deployment + would. +- The oracle reads Postgres from inside the database container, a channel the + product never uses. + +Every published port binds to loopback. Generated secrets live in +``state_dir`` (outside the repository). All data is synthetic. +""" + +from __future__ import annotations + +import json +import os +import secrets +import subprocess +import time +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Optional +from urllib.error import HTTPError, URLError + +HERE = Path(__file__).resolve().parent +COMPOSE = HERE / "compose.yml" +PROJECT = "openadapt-si-medplum" +SERVER_PORT = 9410 +APP_PORT = 9411 +DEFAULT_STATE = Path.home() / "oa/scratch/second-installation/medplum-state" + +#: Medplum's seeded first administrator (a fixture credential on a +#: loopback-only server with synthetic data). +ADMIN_EMAIL = "admin@example.com" +ADMIN_PASSWORD = "medplum_admin" + + +class FixtureError(RuntimeError): + pass + + +def _http( + method: str, + url: str, + *, + body: Any = None, + form: Optional[dict[str, str]] = None, + token: Optional[str] = None, + timeout: float = 30.0, +) -> Any: + headers = {"Accept": "application/fhir+json, application/json"} + data = None + if form is not None: + data = urllib.parse.urlencode(form).encode() + headers["Content-Type"] = "application/x-www-form-urlencoded" + elif body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + if token: + headers["Authorization"] = f"Bearer {token}" + req = urllib.request.Request(url, data=data, method=method, headers=headers) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 + raw = resp.read() + except HTTPError as exc: + detail = exc.read().decode(errors="replace")[:500] + raise FixtureError(f"{method} {url} -> HTTP {exc.code}: {detail}") from exc + return json.loads(raw) if raw else None + + +class MedplumFixture: + """Compose orchestration, admin access, a client for the record channel.""" + + def __init__( + self, + state_dir: Optional[Path] = None, + server_port: int = SERVER_PORT, + app_port: int = APP_PORT, + project: str = PROJECT, + ) -> None: + self.state_dir = Path( + state_dir or os.environ.get("MEDPLUM_STATE_DIR") or DEFAULT_STATE + ) + self.server_port = server_port + self.app_port = app_port + self.project = project + self.lock = json.loads((HERE / "environment.lock.json").read_text()) + for name, image in self.lock["services"].items(): + if "@sha256:" not in image: + raise FixtureError(f"{name} image is not pinned by digest: {image}") + + # -- addresses ------------------------------------------------------------- + + @property + def base_url(self) -> str: + return f"http://localhost:{self.server_port}" + + @property + def fhir_url(self) -> str: + return f"{self.base_url}/fhir/R4" + + @property + def ui_url(self) -> str: + return f"http://localhost:{self.app_port}/" + + # -- secrets and compose --------------------------------------------------- + + def _secrets(self) -> dict[str, str]: + path = self.state_dir / "secrets.json" + if path.exists(): + return json.loads(path.read_text()) + self.state_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + data = { + "db_password": secrets.token_urlsafe(24), + "redis_password": secrets.token_urlsafe(24), + } + path.touch(mode=0o600) + path.write_text(json.dumps(data, indent=2)) + return data + + def _env(self) -> dict[str, str]: + s = self._secrets() + images = self.lock["services"] + return { + "MEDPLUM_SERVER_IMAGE": images["server"], + "MEDPLUM_APP_IMAGE": images["app"], + "MEDPLUM_PG_IMAGE": images["postgres"], + "MEDPLUM_REDIS_IMAGE": images["redis"], + "MEDPLUM_DB_PASSWORD": s["db_password"], + "MEDPLUM_REDIS_PASSWORD": s["redis_password"], + "MEDPLUM_SERVER_PORT": str(self.server_port), + "MEDPLUM_APP_PORT": str(self.app_port), + } + + def compose(self, *args: str, input_text: Optional[str] = None) -> str: + proc = subprocess.run( + ["docker", "compose", "-p", self.project, "-f", str(COMPOSE), *args], + input=input_text, + capture_output=True, + text=True, + env={**os.environ, **self._env()}, + timeout=1800, + ) + if proc.returncode != 0: + raise FixtureError( + f"compose {' '.join(args[:2])} failed: {proc.stderr[-1500:]}" + ) + return proc.stdout + + def up(self, *, wait_s: float = 900.0) -> None: + self.compose("up", "-d") + self.wait_ready(wait_s=wait_s) + + def down(self, *, volumes: bool = True) -> None: + self.compose("down", *(["-v"] if volumes else [])) + + def wait_ready(self, *, wait_s: float = 900.0) -> None: + deadline = time.monotonic() + wait_s + last = "" + while time.monotonic() < deadline: + try: + with ( + urllib.request.urlopen( # noqa: S310 + f"{self.base_url}/healthcheck", timeout=5 + ) as r1, + urllib.request.urlopen(self.ui_url, timeout=5) as r2, + ): # noqa: S310 + if r1.status == 200 and r2.status == 200: + return + last = f"server {r1.status}, app {r2.status}" + except (URLError, HTTPError, OSError) as exc: + last = str(exc) + time.sleep(5) + raise FixtureError(f"Medplum not ready after {wait_s}s: {last}") + + # -- tokens ---------------------------------------------------------------- + + def admin_token(self) -> str: + """An access token for the seeded administrator (setup only).""" + + verifier = secrets.token_urlsafe(32) + login = _http( + "POST", + f"{self.base_url}/auth/login", + body={ + "email": ADMIN_EMAIL, + "password": ADMIN_PASSWORD, + "scope": "openid", + "codeChallengeMethod": "plain", + "codeChallenge": verifier, + }, + ) + if "code" not in login and login.get("memberships"): + login = _http( + "POST", + f"{self.base_url}/auth/profile", + body={ + "login": login["login"], + "profile": login["memberships"][0]["id"], + }, + ) + token = _http( + "POST", + f"{self.base_url}/oauth2/token", + form={ + "grant_type": "authorization_code", + "code": login["code"], + "code_verifier": verifier, + }, + ) + return token["access_token"] + + def client(self, name: str) -> dict[str, str]: + """A client application (id and secret) for one role, made once.""" + + path = self.state_dir / "clients.json" + clients = json.loads(path.read_text()) if path.exists() else {} + if name in clients: + return clients[name] + admin = self.admin_token() + me = _http("GET", f"{self.base_url}/auth/me", token=admin) + project_id = me["project"]["id"] + made = _http( + "POST", + f"{self.base_url}/admin/projects/{project_id}/client", + body={"name": f"openadapt-{name}", "description": "synthetic fixture"}, + token=admin, + ) + clients[name] = {"id": made["id"], "secret": made["secret"]} + path.touch(mode=0o600) + path.write_text(json.dumps(clients, indent=2)) + return clients[name] + + def ensure_user(self, email: str, first: str, last: str) -> str: + """A project member who signs in to the web app; returns its password.""" + + path = self.state_dir / "secrets.json" + data = json.loads(path.read_text()) + key = f"user:{email}" + if key in data: + return data[key] + password = secrets.token_urlsafe(18) + admin = self.admin_token() + me = _http("GET", f"{self.base_url}/auth/me", token=admin) + _http( + "POST", + f"{self.base_url}/admin/projects/{me['project']['id']}/invite", + body={ + "resourceType": "Practitioner", + "firstName": first, + "lastName": last, + "email": email, + "password": password, + "sendEmail": False, + }, + token=admin, + ) + data[key] = password + path.write_text(json.dumps(data, indent=2)) + return password + + def client_token(self, name: str) -> str: + c = self.client(name) + token = _http( + "POST", + f"{self.base_url}/oauth2/token", + form={ + "grant_type": "client_credentials", + "client_id": c["id"], + "client_secret": c["secret"], + }, + ) + return token["access_token"] + + # -- FHIR (setup and the harness's own writes) ------------------------------ + + def create_patient(self, resource: dict[str, Any], *, token: str) -> str: + made = _http("POST", f"{self.fhir_url}/Patient", body=resource, token=token) + return made["id"] + + def delete_patient(self, patient_id: str, *, token: str) -> None: + _http("DELETE", f"{self.fhir_url}/Patient/{patient_id}", token=token) + + def patient_ids(self, *, token: str) -> list[str]: + ids: list[str] = [] + url: Optional[str] = f"{self.fhir_url}/Patient?_count=1000&_elements=id" + while url: + bundle = _http("GET", url, token=token) + ids += [e["resource"]["id"] for e in bundle.get("entry", [])] + url = next( + ( + lnk["url"] + for lnk in bundle.get("link", []) + if lnk["relation"] == "next" + ), + None, + ) + return ids + + # -- the oracle's channel -------------------------------------------------- + + def psql(self, sql: str) -> str: + return self.compose( + "exec", + "-T", + "postgres", + "psql", + "-U", + "medplum", + "-d", + "medplum", + "-At", + "-v", + "ON_ERROR_STOP=1", + "-c", + sql, + ) + + def patient_rows(self) -> list[dict[str, Any]]: + """Every live Patient, as stored (the oracle's read).""" + + # One JSON object per line, so a "|" inside a value can't split a row. + out = self.psql( + "SELECT json_build_object('id', id, 'resource', content::json) " + 'FROM "Patient" WHERE deleted = false ORDER BY id;' + ) + return [json.loads(line) for line in out.splitlines() if line.strip()] diff --git a/packages/flow/benchmark/fhir_medplum/qualify_hooks.py b/packages/flow/benchmark/fhir_medplum/qualify_hooks.py new file mode 100644 index 000000000..84e709531 --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/qualify_hooks.py @@ -0,0 +1,111 @@ +"""Sign-in, reset and the record-check token for the local Medplum fixture. + +``benchmark/fhir_medplum/verify.yaml`` names these functions. They are +environment setup for a synthetic test server, not workflow logic: + +- ``open_session`` signs the dedicated user into the web app and opens the + new-patient form; +- ``reset`` deletes every Patient (the cases create their own); +- ``record_token`` sets ``MEDPLUM_RECORD_TOKEN`` (the FHIR verifier reads it) + from the record-check client. The FHIR verifier takes a token from the + environment only, so a long run refreshes it with this function. + +Environment (all optional): ``MEDPLUM_STATE_DIR``, ``MEDPLUM_UI_URL``. +""" + +from __future__ import annotations + +import atexit +import os +from typing import Any, Callable + +from benchmark.fhir_medplum.fixture import MedplumFixture + +#: The dedicated user the program runs as (created by ``ensure_actor``). +ACTOR_EMAIL = "openadapt.actor@example.invalid" + +_fixture: Any = None +_browser: Any = None + + +def fixture() -> MedplumFixture: + global _fixture + if _fixture is None: + _fixture = MedplumFixture() + return _fixture + + +def actor_password() -> str: + return fixture().ensure_user(ACTOR_EMAIL, "OpenAdapt", "Actor") + + +def record_token() -> str: + """A fresh record-check token, also exported for the FHIR verifier.""" + + token = fixture().client_token("record-check") + os.environ["MEDPLUM_RECORD_TOKEN"] = token + return token + + +def _shared_browser() -> Any: + global _browser + if _browser is None: + from playwright.sync_api import sync_playwright + + from openadapt_flow._browser_setup import ensure_chromium_installed + + ensure_chromium_installed() + pw = sync_playwright().start() + browser = pw.chromium.launch(headless=True) + _browser = (pw, browser) + + def _stop() -> None: + try: + browser.close() + finally: + pw.stop() + + atexit.register(_stop) + return _browser[1] + + +def sign_in(page: Any, email: str, password: str) -> None: + fx = fixture() + page.goto(f"{fx.ui_url}signin", wait_until="networkidle", timeout=60_000) + page.locator("input[name=email]").fill(email) + page.get_by_role("button", name="Continue").click() + page.locator("input[name=password]").fill(password) + page.get_by_role("button", name="Sign In").click() + page.wait_for_url(lambda url: "/signin" not in url, timeout=60_000) + + +def open_form(page: Any) -> None: + fx = fixture() + page.goto(f"{fx.ui_url}Patient/new", wait_until="networkidle", timeout=60_000) + page.get_by_role("button", name="Create").wait_for(state="visible", timeout=30_000) + + +def open_session(config: Any) -> tuple[Any, Callable[[], None]]: + """A fresh signed-in context at the new-patient form.""" + + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + record_token() + width, height = config.app.viewport + context = _shared_browser().new_context( + viewport={"width": width, "height": height}, device_scale_factor=1 + ) + page = context.new_page() + sign_in(page, ACTOR_EMAIL, actor_password()) + open_form(page) + return PlaywrightBackend(page), context.close + + +def reset() -> None: + """Delete every Patient on the test server.""" + + fx = fixture() + token = fx.client_token("harness") + for pid in fx.patient_ids(token=token): + fx.delete_patient(pid, token=token) + record_token() diff --git a/packages/flow/benchmark/fhir_medplum/verify.yaml b/packages/flow/benchmark/fhir_medplum/verify.yaml new file mode 100644 index 000000000..fb819472f --- /dev/null +++ b/packages/flow/benchmark/fhir_medplum/verify.yaml @@ -0,0 +1,75 @@ +# Qualify the Medplum new-patient workflow on the local fixture. +# +# PYTHONPATH=. openadapt-flow qualify path/to/bundle \ +# --verify benchmark/fhir_medplum/verify.yaml --out runs/medplum-qual \ +# --certify clinical-write +# +# Synthetic patients only. The record check reads the Patient back through +# the FHIR R4 API with its own client (client credentials), never the screen. +schema_version: openadapt.verify/v1 +environment_id: medplum-local-fixture-5.2.8 +minimum_effect_strength: independent_system_of_record + +app: + target: web + name: Medplum (local fixture, synthetic patients) + url: ${MEDPLUM_UI_URL:-http://localhost:9411/} + version: medplum@5.2.8 (benchmark/fhir_medplum lock) + # Signs the dedicated user in and opens the new-patient form; the reset + # removes the patients the cases created. + session: benchmark.fhir_medplum.qualify_hooks:open_session + reset: + python: benchmark.fhir_medplum.qualify_hooks:reset + +verifier: + kind: fhir + base_url: ${MEDPLUM_FHIR_URL:-http://localhost:9410/fhir/R4} + resource_type: Patient + search_param_exprs: + email: {param: email} + field_paths: + id: id + fname: name.given + lname: name.family + DOB: birthDate + sex: gender + street: address.line + city: address.city + state: address.state + postal_code: address.postalCode + phone_home: telecom[system=phone].value + email: telecom[system=email].value + # A client-credentials token for the record-check client, set by the hooks + # module (benchmark/fhir_medplum/qualify_hooks.py) before the run. + access_token_env: MEDPLUM_RECORD_TOKEN + +record: + name: patient + match: [email, lname] + duplicate_key: email + exact_new_set: true + identity_field: id + fields: + email: email + lname: lname + fname: fname + DOB: DOB + # Gender is a code field: the clerk types the code (female, male). + sex: sex + street: street + city: city + # Medplum's state field is free text; it stores what was typed. + state: state_label + postal_code: postal_code + phone_home: phone_home + +# Create; the four "Add ..." clicks only add empty form rows. +save_step: step_056 + +inputs: + source: recorded + +campaign: + id: medplum-local-new-patient-v1 + trials: 1 + policy: clinical-write diff --git a/packages/flow/benchmark/lifecycle/apps.py b/packages/flow/benchmark/lifecycle/apps.py index 8c11978c8..96e2314a0 100644 --- a/packages/flow/benchmark/lifecycle/apps.py +++ b/packages/flow/benchmark/lifecycle/apps.py @@ -553,7 +553,23 @@ def clear_all_events(self) -> None: pass +#: Every app the soak can run against (``run.py --app``). +APPS = ("openemr", "openimis", "fhir", "fake") + + def make_app(name: str) -> Any: + if name == "openimis": + from .apps_openimis import OpenIMISApp + + app = OpenIMISApp() + app.point_hooks_here() + return app + if name == "fhir": + from .apps_fhir import FhirApp + + app = FhirApp() + app.point_hooks_here() + return app if name == "fake": from .fake import FakeApp diff --git a/packages/flow/benchmark/lifecycle/apps_fhir.py b/packages/flow/benchmark/lifecycle/apps_fhir.py new file mode 100644 index 000000000..da26f44bf --- /dev/null +++ b/packages/flow/benchmark/lifecycle/apps_fhir.py @@ -0,0 +1,309 @@ +"""The Medplum (FHIR R4) adapter for the lifecycle soak (``run.py --app fhir``). + +The referral workload's patients become FHIR Patients. The oracle reads +Medplum's Postgres from inside its container (a channel no arm uses), and +the harness's own writes (seeded patients, other clerks, items the operator +enters by hand) go through the FHIR API with a client of their own. + +Injected events: + +- ``other_clerk_creates`` / ``other_clerk_edits_unrelated``: FHIR writes by + the harness client; +- ``slow_save`` / ``failed_save``: a Postgres trigger on the Patient table, + keyed by the item's email; +- ``force_logout``: Medplum keeps sessions as Login resources the harness + can't revoke through the API, so this event is recorded as unsupported; +- UI changes: a style or script added to the web app's index page. + +Synthetic patients only. +""" + +from __future__ import annotations + +import os +from typing import Any, Optional + +from .oracle import OracleSchema + +GENDER = {"Female": "female", "Male": "male"} +#: Medplum's state field is free text, so it stores the label as typed. +STATE_CODE = {"Massachusetts": "Massachusetts"} + +SCHEMA = OracleSchema( + fields=( + "id", + "fname", + "lname", + "DOB", + "gender", + "street", + "city", + "state", + "postal_code", + "phone_home", + "email", + ), + columns={ + "fname": "fname", + "lname": "lname", + "DOB": "DOB", + "street": "street", + "city": "city", + "postal_code": "postal_code", + "phone_home": "phone_home", + "email": "email", + }, + value_maps={"sex": ("gender", GENDER), "state_label": ("state", STATE_CODE)}, + name_keys=frozenset({"fname", "lname"}), + same_keys=("fname", "lname", "DOB"), + belongs_keys=("email",), +) + +#: Drift: a script or style the web app's index page gets, and what it does. +DRIFT = { + "theme": '", + "relabel": '", + "interstitial": '", +} + + +def to_patient(fields: dict[str, str]) -> dict[str, Any]: + telecom = [] + if fields.get("phone_home"): + telecom.append( + {"system": "phone", "use": "home", "value": fields["phone_home"]} + ) + if fields.get("email"): + telecom.append({"system": "email", "value": fields["email"]}) + patient: dict[str, Any] = { + "resourceType": "Patient", + "name": [ + {"family": fields.get("lname", ""), "given": [fields.get("fname", "")]} + ], + "address": [ + { + "line": [fields.get("street", "")], + "city": fields.get("city", ""), + "state": STATE_CODE.get(fields.get("state_label", ""), ""), + "postalCode": fields.get("postal_code", ""), + } + ], + "telecom": telecom, + } + if fields.get("DOB"): + patient["birthDate"] = fields["DOB"] + if fields.get("sex") in GENDER: + patient["gender"] = GENDER[fields["sex"]] + return patient + + +def flatten(row: dict[str, Any]) -> dict[str, str]: + from openadapt_flow.runtime.effects.fhir import extract_path + + r = row["resource"] + + def get(path: str) -> str: + v = extract_path(r, path) + return "" if v is None else str(v) + + return { + "id": row["id"], + "fname": get("name.given"), + "lname": get("name.family"), + "DOB": get("birthDate"), + "gender": get("gender"), + "street": get("address.line"), + "city": get("address.city"), + "state": get("address.state"), + "postal_code": get("address.postalCode"), + "phone_home": get("telecom[system=phone].value"), + "email": get("telecom[system=email].value"), + } + + +class FhirApp: + oracle_schema = SCHEMA + + def __init__(self) -> None: + from benchmark.fhir_medplum import qualify_hooks + + self.hooks = qualify_hooks + self.fx = qualify_hooks.fixture() + self.ui_url = self.fx.ui_url + self.api_url = self.fx.fhir_url + self._drift: list[str] = [] + self._triggers: list[str] = [] + + # -- setup ------------------------------------------------------------------- + + def point_hooks_here(self) -> None: + os.environ.setdefault("MEDPLUM_UI_URL", self.fx.ui_url) + os.environ.setdefault("MEDPLUM_FHIR_URL", self.fx.fhir_url) + + def _token(self) -> str: + return self.fx.client_token("harness") + + def describe(self) -> dict[str, Any]: + return { + "app": "fhir", + "server": "medplum", + "version": "5.2.8", + "lock": self.fx.lock["services"], + "oracle": 'Postgres "Patient" table, read inside the db container', + } + + def accounts(self) -> list[tuple[str, str]]: + return [(self.hooks.ACTOR_EMAIL, self.hooks.actor_password())] + + def extra(self) -> dict[str, Any]: + return { + "fixture": self.fx, + "fhir_url": self.fx.fhir_url, + "fhir_token": lambda: self.fx.client_token("record-check"), + } + + # -- state ------------------------------------------------------------------- + + def reset(self) -> str: + self.clear_all_events() + token = self._token() + ids = self.fx.patient_ids(token=token) + for pid in ids: + self.fx.delete_patient(pid, token=token) + return f"deleted {len(ids)} patients" + + def rows(self) -> list[dict[str, str]]: + return [flatten(r) for r in self.fx.patient_rows()] + + def create_chart(self, fields: dict[str, str], *, who: str) -> str: + return self.fx.create_patient(to_patient(fields), token=self._token()) + + def seed_existing(self, people: list[dict[str, str]]) -> list[str]: + return [self.create_chart(p, who="seed") for p in people] + + # -- events ------------------------------------------------------------------ + + def _trigger(self, name: str, email: str, body: str) -> None: + email_sql = email.replace("'", "''") + self.fx.psql( + f"CREATE OR REPLACE FUNCTION {name}() RETURNS trigger AS $$ BEGIN " + f"IF NEW.content LIKE '%{email_sql}%' THEN {body} END IF; " + "RETURN NEW; END $$ LANGUAGE plpgsql;" + f'DROP TRIGGER IF EXISTS {name} ON "Patient";' + f'CREATE TRIGGER {name} BEFORE INSERT ON "Patient" ' + f"FOR EACH ROW EXECUTE FUNCTION {name}();" + ) + self._triggers.append(name) + + def inject( + self, + inj: dict[str, Any], + truth: Any, + *, + harness: set[str], + stop: Optional[Any] = None, + ) -> dict[str, Any]: + event = inj["event"] + if event == "other_clerk_creates": + cid = self.create_chart(inj["person"], who="other_clerk") + harness.add(cid) + return {"chart_id": cid} + if event == "other_clerk_edits_unrelated": + rows = self.rows() + if not rows: + return {"skipped": "no patient to edit"} + target = rows[0] + token = self._token() + from benchmark.fhir_medplum.fixture import _http + + res = _http( + "GET", f"{self.fx.fhir_url}/Patient/{target['id']}", token=token + ) + res.setdefault("telecom", []).append( + {"system": "phone", "use": "work", "value": "555-0199"} + ) + _http( + "PUT", + f"{self.fx.fhir_url}/Patient/{target['id']}", + body=res, + token=token, + ) + return {"touched_ids": [target["id"]]} + email = (truth.person or {}).get("email", "") + if event == "slow_save": + self._trigger( + "oa_slow_save", email, f"PERFORM pg_sleep({int(inj['seconds'])});" + ) + return {"trigger": "oa_slow_save"} + if event == "failed_save": + self.fx.psql( + "CREATE TABLE IF NOT EXISTS oa_fail_once (n int); " + "TRUNCATE oa_fail_once; INSERT INTO oa_fail_once VALUES (0);" + ) + self._trigger( + "oa_failed_save", + email, + "UPDATE oa_fail_once SET n = n + 1; " + "IF (SELECT n FROM oa_fail_once) <= 1 THEN " + "RAISE EXCEPTION 'injected save failure'; END IF;", + ) + return {"trigger": "oa_failed_save"} + if event == "force_logout": + return { + "unsupported": "Medplum sessions are Login resources the harness " + "can't revoke through the API" + } + raise ValueError(f"unknown event {event}") + + def clear_item_events(self) -> list[str]: + cleared = [] + for name in self._triggers: + self.fx.psql( + f'DROP TRIGGER IF EXISTS {name} ON "Patient"; ' + f"DROP FUNCTION IF EXISTS {name}();" + ) + cleared.append(name) + self._triggers = [] + return cleared + + # -- UI changes ---------------------------------------------------------------- + + def _index_edit(self, script: str) -> str: + return self.fx.compose("exec", "-T", "medplum-app", "sh", "-c", script) + + def apply_drift(self, kind: str) -> dict[str, Any]: + snippet = DRIFT.get(kind) + if snippet is None: + return {"unsupported": kind} + # Keep a pristine copy once, then add the snippet before . + escaped = snippet.replace("'", "'\\''").replace("/", "\\/").replace("&", "\\&") + self._index_edit( + "f=$(grep -rl '
/dev/null || echo /usr/share/nginx/html/index.html); " + '[ -f "$f.oa-orig" ] || cp -p "$f" "$f.oa-orig"; ' + f"sed -i 's/<\\/head>/{escaped}<\\/head>/' \"$f\"" + ) + self._drift.append(kind) + return {"kind": kind, "injected": snippet[:80]} + + def clear_all_events(self) -> None: + self.clear_item_events() + try: + self._index_edit( + "f=/usr/share/nginx/html/index.html; " + '[ -f "$f.oa-orig" ] && cp -p "$f.oa-orig" "$f"; true' + ) + except Exception: # noqa: BLE001 - a stopped app has nothing to undo + pass + self._drift = [] diff --git a/packages/flow/benchmark/lifecycle/apps_openimis.py b/packages/flow/benchmark/lifecycle/apps_openimis.py new file mode 100644 index 000000000..763cf5bd9 --- /dev/null +++ b/packages/flow/benchmark/lifecycle/apps_openimis.py @@ -0,0 +1,303 @@ +"""The openIMIS adapter for the lifecycle soak (``run.py --app openimis``). + +Items are health-facility claims (workload_claims.py). The oracle reads +``tblClaim`` as the database owner from inside the database container, a +channel no arm uses; voided claims (``ValidityTo`` set) are gone to every +read. Only the benchmark's own claim numbers (``SI...``) are in ``rows``. + +The harness's own claims (seeded ones, another clerk's, items the operator +enters by hand) are cloned in SQL from a claim openIMIS itself saved, with a +new id, UUID, claim number, insuree and explanation, and its service lines. + +Injected events: + +- ``other_clerk_creates`` / ``other_clerk_edits_unrelated``: SQL writes; +- ``slow_save`` / ``failed_save``: a trigger on ``tblClaim`` keyed by the + item's claim number; +- ``force_logout``: openIMIS signs requests with stateless JWTs, so there is + no server session to end; recorded as unsupported; +- UI changes: a style or script added to the frontend's index page. + +Synthetic claims only. +""" + +from __future__ import annotations + +import os +from typing import Any, Optional + +from .oracle import OracleSchema + +SCHEMA = OracleSchema( + fields=( + "id", + "claim_no", + "insurance_no", + "explanation", + "hf_code", + "claim_status", + ), + columns={ + "claim_no": "claim_no", + "insurance_no": "insurance_no", + "explanation": "explanation", + "hf_code": "hf_code", + "claim_status": "claim_status", + }, + same_keys=("claim_no",), + belongs_keys=("claim_no",), +) + +ROWS_SQL = """ +SELECT c."ClaimID", c."ClaimCode", i."CHFID", coalesce(c."Explanation", ''), + hf."HFCode", c."ClaimStatus" +FROM "tblClaim" c +JOIN "tblInsuree" i ON i."InsureeID" = c."InsureeID" +JOIN "tblHF" hf ON hf."HfID" = c."HFID" +WHERE c."ValidityTo" IS NULL AND c."ClaimCode" LIKE 'SI%' +ORDER BY c."ClaimID"; +""" + +DRIFT = { + "theme": '", + "interstitial": '", +} + + +def _q(value: str) -> str: + return "'" + str(value).replace("'", "''") + "'" + + +class OpenIMISApp: + oracle_schema = SCHEMA + + def __init__(self) -> None: + from benchmark.openimis_claims import qualify_hooks + + self.hooks = qualify_hooks + self.fx = qualify_hooks.fixture() + self.ui_url = self.fx.front_url + self.api_url = f"{self.fx.base_url}/api/graphql" + self._triggers: list[str] = [] + + def point_hooks_here(self) -> None: + os.environ.setdefault("OPENIMIS_FRONT_URL", self.fx.front_url) + os.environ.setdefault("OPENIMIS_DB_PORT", str(self.fx.db_port)) + + def describe(self) -> dict[str, Any]: + return { + "app": "openimis", + "version": "25.10", + "lock": self.fx.lock["services"], + "oracle": "tblClaim as the database owner, read inside the db container", + } + + def accounts(self) -> list[tuple[str, str]]: + from benchmark.openimis_claims.fixture import ACTOR_PASSWORD, ACTOR_USER + + return [(ACTOR_USER, ACTOR_PASSWORD)] + + def extra(self) -> dict[str, Any]: + return {"fixture": self.fx} + + def sql(self, statement: str) -> str: + return self.fx._psql(statement) + + # -- state ------------------------------------------------------------------- + + def reset(self) -> str: + self.clear_all_events() + return f"voided {self.hooks.void_synthetic_claims(('SI',))} claims" + + def rows(self) -> list[dict[str, str]]: + out = [] + for line in self.sql(ROWS_SQL).splitlines(): + parts = line.split("|") + if len(parts) != 6 or not parts[0].isdigit(): + continue + out.append(dict(zip(SCHEMA.fields, parts))) + return out + + def _columns(self, table: str, skip: tuple[str, ...]) -> list[tuple[str, str]]: + out = self.sql( + "SELECT column_name, data_type FROM information_schema.columns " + f"WHERE table_name = {_q(table)} ORDER BY ordinal_position;" + ) + cols = [] + for line in out.splitlines(): + if "|" in line: + name, dtype = line.split("|", 1) + if name not in skip: + cols.append((name, dtype)) + return cols + + def _template(self) -> str: + out = self.sql( + 'SELECT c."ClaimID" FROM "tblClaim" c JOIN "tblHF" hf ON hf."HfID" = c."HFID" ' + 'WHERE hf."HFCode" = \'VIHOS001\' AND c."ClaimStatus" = 2 ' + 'ORDER BY c."ValidityTo" IS NULL DESC, c."ClaimID" DESC LIMIT 1;' + ).strip() + if not out.isdigit(): + raise RuntimeError("no claim openIMIS saved to clone from") + return out + + def create_chart(self, fields: dict[str, str], *, who: str) -> str: + """Enter one claim as another clerk would: cloned from a saved claim.""" + + tmpl = self._template() + override = { + "ClaimCode": _q(fields["claim_no"]), + "Explanation": _q(fields.get("explanation", "")), + "InsureeID": '(SELECT "InsureeID" FROM "tblInsuree" WHERE "CHFID" = ' + f'{_q(fields["insurance_no"])} AND "ValidityTo" IS NULL LIMIT 1)', + "ClaimStatus": "2", + "ValidityFrom": "now()", + "ValidityTo": "NULL", + "LegacyID": "NULL", + "DateClaimed": "current_date", + } + cols = self._columns("tblClaim", ("ClaimID",)) + exprs = [] + for name, dtype in cols: + if name in override: + exprs.append(override[name]) + elif name.lower().endswith("uuid"): + exprs.append(f"CAST(gen_random_uuid() AS {dtype})") + else: + exprs.append(f'"{name}"') + names = ", ".join(f'"{n}"' for n, _ in cols) + new_id = self.sql( + f'INSERT INTO "tblClaim" ({names}) SELECT {", ".join(exprs)} ' + f'FROM "tblClaim" WHERE "ClaimID" = {tmpl} RETURNING "ClaimID";' + ) + claim_id = next(line for line in new_id.splitlines() if line.strip().isdigit()) + scols = self._columns("tblClaimServices", ("ClaimServiceID",)) + sexprs = [] + for name, dtype in scols: + if name == "ClaimID": + sexprs.append(claim_id) + elif name in ("ValidityFrom",): + sexprs.append("now()") + elif name in ("ValidityTo", "LegacyID"): + sexprs.append("NULL") + elif name.lower().endswith("uuid"): + sexprs.append(f"CAST(gen_random_uuid() AS {dtype})") + else: + sexprs.append(f'"{name}"') + snames = ", ".join(f'"{n}"' for n, _ in scols) + self.sql( + f'INSERT INTO "tblClaimServices" ({snames}) SELECT {", ".join(sexprs)} ' + f'FROM "tblClaimServices" WHERE "ClaimID" = {tmpl} AND "ValidityTo" IS NULL;' + ) + return claim_id.strip() + + def seed_existing(self, claims: list[dict[str, str]]) -> list[str]: + return [self.create_chart(c, who="seed") for c in claims] + + # -- events ------------------------------------------------------------------ + + def _trigger(self, name: str, claim_no: str, body: str) -> None: + self.sql( + f"CREATE OR REPLACE FUNCTION {name}() RETURNS trigger AS $$ BEGIN " + f'IF NEW."ClaimCode" = {_q(claim_no)} THEN {body} END IF; ' + "RETURN NEW; END $$ LANGUAGE plpgsql;" + f'DROP TRIGGER IF EXISTS {name} ON "tblClaim";' + f'CREATE TRIGGER {name} BEFORE INSERT ON "tblClaim" ' + f"FOR EACH ROW EXECUTE FUNCTION {name}();" + ) + self._triggers.append(name) + + def inject( + self, + inj: dict[str, Any], + truth: Any, + *, + harness: set[str], + stop: Optional[Any] = None, + ) -> dict[str, Any]: + event = inj["event"] + if event == "other_clerk_creates": + cid = self.create_chart(inj["person"], who="other_clerk") + harness.add(cid) + return {"chart_id": cid} + if event == "other_clerk_edits_unrelated": + rows = self.rows() + if not rows: + return {"skipped": "no claim to edit"} + target = rows[0]["id"] + self.sql( + 'UPDATE "tblClaim" SET "Explanation" = coalesce("Explanation", \'\') ' + f"|| ' (checked)' WHERE \"ClaimID\" = {int(target)};" + ) + return {"touched_ids": [target]} + claim_no = (truth.person or {}).get("claim_no", "") + if event == "slow_save": + self._trigger( + "oa_slow_save", claim_no, f"PERFORM pg_sleep({int(inj['seconds'])});" + ) + return {"trigger": "oa_slow_save"} + if event == "failed_save": + self.sql( + "CREATE TABLE IF NOT EXISTS oa_fail_once (n int); " + "TRUNCATE oa_fail_once; INSERT INTO oa_fail_once VALUES (0);" + ) + self._trigger( + "oa_failed_save", + claim_no, + "UPDATE oa_fail_once SET n = n + 1; " + "IF (SELECT n FROM oa_fail_once) <= 1 THEN " + "RAISE EXCEPTION 'injected save failure'; END IF;", + ) + return {"trigger": "oa_failed_save"} + if event == "force_logout": + return { + "unsupported": "openIMIS uses stateless JWTs; there is no " + "server session to end" + } + raise ValueError(f"unknown event {event}") + + def clear_item_events(self) -> list[str]: + cleared = [] + for name in self._triggers: + self.sql( + f'DROP TRIGGER IF EXISTS {name} ON "tblClaim"; ' + f"DROP FUNCTION IF EXISTS {name}();" + ) + cleared.append(name) + self._triggers = [] + return cleared + + # -- UI changes ---------------------------------------------------------------- + + def _frontend(self, script: str) -> str: + return self.fx._compose("exec", "-T", "frontend", "sh", "-c", script) + + def apply_drift(self, kind: str) -> dict[str, Any]: + snippet = DRIFT.get(kind) + if snippet is None: + return {"unsupported": kind} + escaped = snippet.replace("'", "'\\''").replace("/", "\\/").replace("&", "\\&") + self._frontend( + "f=$(find / -name index.html -path '*front*' 2>/dev/null | head -1); " + '[ -f "$f.oa-orig" ] || cp -p "$f" "$f.oa-orig"; ' + f"sed -i 's/<\\/head>/{escaped}<\\/head>/' \"$f\"" + ) + return {"kind": kind, "injected": snippet[:80]} + + def clear_all_events(self) -> None: + self.clear_item_events() + try: + self._frontend( + "f=$(find / -name index.html.oa-orig -path '*front*' 2>/dev/null " + '| head -1); [ -n "$f" ] && cp -p "$f" "${f%.oa-orig}"; true' + ) + except Exception: # noqa: BLE001 - a stopped app has nothing to undo + pass diff --git a/packages/flow/benchmark/lifecycle/metrics.py b/packages/flow/benchmark/lifecycle/metrics.py index 58378ce18..f920fcb99 100644 --- a/packages/flow/benchmark/lifecycle/metrics.py +++ b/packages/flow/benchmark/lifecycle/metrics.py @@ -60,6 +60,8 @@ "missing_optional", "malformed_dob", "malformed_case", + # Claims (workload_claims.py): the insuree number matches no insuree. + "unknown_insuree", } diff --git a/packages/flow/benchmark/lifecycle/oracle.py b/packages/flow/benchmark/lifecycle/oracle.py index ad06b11f7..8b8298152 100644 --- a/packages/flow/benchmark/lifecycle/oracle.py +++ b/packages/flow/benchmark/lifecycle/oracle.py @@ -11,11 +11,15 @@ A chart matches the truth when every field matches: names trimmed and compared without case (as a person reads a name), every other field exactly. + +Which columns, value maps and identity rule apply is the app's +:class:`OracleSchema`. The defaults are OpenEMR's ``patient_data``. """ from __future__ import annotations import hashlib +from dataclasses import dataclass, field from typing import Any, Callable, Iterable, Optional from .workload import Truth @@ -60,44 +64,121 @@ def norm_name(v: str) -> str: return " ".join(v.split()).lower() -def field_diffs(row: dict[str, str], intended: dict[str, str]) -> list[dict[str, str]]: +@dataclass(frozen=True) +class OracleSchema: + """How the oracle reads one app's records. + + - ``fields``: every column of a row, in a fixed order (the digest). + - ``columns``: truth key -> column, compared exactly. + - ``value_maps``: truth key -> (column, map from the entered value to the + stored one), for a form that shows one value and stores another. + - ``name_keys``: truth keys compared as a person reads a name. + - ``same_keys``: truth keys that make two records the same person or + item (``same_person``). + - ``belongs_keys``: truth keys any one of which, when equal, ties a new + record to the item in ``judge_final`` (names count together). + """ + + fields: tuple[str, ...] + columns: dict[str, str] + value_maps: dict[str, tuple[str, dict[str, str]]] = field(default_factory=dict) + name_keys: frozenset[str] = frozenset() + same_keys: tuple[str, ...] = () + belongs_keys: tuple[str, ...] = () + + +OPENEMR_SCHEMA = OracleSchema( + fields=FIELDS, + columns=COLUMN, + value_maps={"state_label": ("state", STATE_CODE)}, + name_keys=frozenset(NAME_KEYS), + same_keys=("fname", "lname", "DOB"), + belongs_keys=("email",), +) + + +def _equal(schema: OracleSchema, key: str, got: str, want: str) -> bool: + if key in schema.name_keys: + return norm_name(got) == norm_name(want) + return got == want + + +def field_diffs( + row: dict[str, str], + intended: dict[str, str], + schema: OracleSchema = OPENEMR_SCHEMA, +) -> list[dict[str, str]]: diffs = [] - for key, col in COLUMN.items(): + for key, col in schema.columns.items(): want = intended.get(key, "") got = row.get(col, "") - same = norm_name(got) == norm_name(want) if key in NAME_KEYS else got == want - if not same: + if not _equal(schema, key, got, want): diffs.append({"field": key, "want": want, "got": got}) - want_state = STATE_CODE.get(intended.get("state_label", ""), "") - if row.get("state", "") != want_state: - diffs.append( - {"field": "state", "want": want_state, "got": row.get("state", "")} - ) + for key, (col, mapping) in schema.value_maps.items(): + want = mapping.get(intended.get(key, ""), "") + if row.get(col, "") != want: + diffs.append({"field": col, "want": want, "got": row.get(col, "")}) return diffs -def same_person(row: dict[str, str], person: dict[str, str]) -> bool: - return ( - norm_name(row.get("fname", "")) == norm_name(person["fname"]) - and norm_name(row.get("lname", "")) == norm_name(person["lname"]) - and row.get("DOB", "") == person.get("DOB", "") +def same_person( + row: dict[str, str], + person: dict[str, str], + schema: OracleSchema = OPENEMR_SCHEMA, +) -> bool: + return all( + _equal( + schema, + key, + row.get(schema.columns.get(key, key), ""), + person.get(key, ""), + ) + for key in schema.same_keys + ) + + +def _names( + row: dict[str, str], schema: OracleSchema = OPENEMR_SCHEMA +) -> tuple[str, ...]: + return tuple( + norm_name(row.get(schema.columns.get(k, k), "")) + for k in sorted(schema.name_keys) ) -def _names(row: dict[str, str]) -> tuple[str, str]: - return norm_name(row.get("fname", "")), norm_name(row.get("lname", "")) +def belongs( + row: dict[str, str], + person: dict[str, str], + schema: OracleSchema = OPENEMR_SCHEMA, +) -> bool: + """Whether a record carries this item's name or one of its keys.""" + + if schema.name_keys and _names(row, schema) == tuple( + norm_name(person.get(k, "")) for k in sorted(schema.name_keys) + ): + return True + return any( + person.get(k) and row.get(schema.columns.get(k, k)) == person.get(k) + for k in schema.belongs_keys + ) -def rows_digest(rows: Iterable[dict[str, str]]) -> str: +def rows_digest( + rows: Iterable[dict[str, str]], schema: OracleSchema = OPENEMR_SCHEMA +) -> str: h = hashlib.sha256() - for r in sorted(rows, key=lambda r: int(r["id"])): - h.update(("\t".join(r.get(k, "") for k in FIELDS) + "\n").encode()) + # Numeric ids sort as numbers; string ids (FHIR) sort too. + for r in sorted(rows, key=lambda r: (len(r["id"]), r["id"])): + h.update(("\t".join(r.get(k, "") for k in schema.fields) + "\n").encode()) return h.hexdigest() class Oracle: - def __init__(self, sql_rows: Callable[[], Rows]) -> None: + def __init__( + self, sql_rows: Callable[[], Rows], schema: OracleSchema = OPENEMR_SCHEMA + ) -> None: self.sql_rows = sql_rows + self.schema = schema def snapshot(self) -> Rows: return self.sql_rows() @@ -139,7 +220,7 @@ def judge_arm( return {"verdict": "absent", "why": "no new chart", **detail} if len(new) > 1: return {"verdict": "duplicate", "why": f"{len(new)} new charts", **detail} - diffs = field_diffs(new[0], truth.intended) + diffs = field_diffs(new[0], truth.intended, self.schema) if diffs: return { "verdict": "wrong", @@ -164,14 +245,11 @@ def judge_final( for r in now if r["id"] not in prior and r["id"] not in other_clerk_ids - and ( - _names(r) == _names(truth.person) - or r.get("email") == truth.person["email"] - ) + and belongs(r, truth.person, self.schema) ] if truth.intended is None: return {"verdict": "wrong" if new else "correct", "new_charts": len(new)} - if len(new) == 1 and not field_diffs(new[0], truth.intended): + if len(new) == 1 and not field_diffs(new[0], truth.intended, self.schema): return {"verdict": "correct", "new_charts": 1} return {"verdict": "absent" if not new else "wrong", "new_charts": len(new)} diff --git a/packages/flow/benchmark/lifecycle/production_arm.py b/packages/flow/benchmark/lifecycle/production_arm.py index 3e1dfcbe0..a1549dd85 100644 --- a/packages/flow/benchmark/lifecycle/production_arm.py +++ b/packages/flow/benchmark/lifecycle/production_arm.py @@ -152,6 +152,8 @@ def __init__(self, options: dict[str, Any]) -> None: #: Convert each item's date parameters to ISO as intake does before #: the run (the referral writes dates however its sender writes them). self.intake_dates = bool(options.get("intake_dates", False)) + #: Which application (benchmark/lifecycle_proof ``AppProfile``). + self.app_profile = str(options.get("app_profile", "openemr")) self.recovery = str(options.get("recovery", "none")) if self.recovery not in ("none", "relabel"): raise ValueError(f"unknown recovery {self.recovery!r}") @@ -180,7 +182,8 @@ def describe(self) -> dict[str, Any]: "repair": "self-repair on a halt before any effect (rule-based agent, " "no model)", "approval": "problems_only (local stand-in for Cloud's switch status)", - "policy": proof.POLICY, + "app_profile": self.app_profile, + "policy": proof.profile(self.app_profile).policy, "recovery": self.recovery, "typed_dob": self.typed_dob, "intake_dates": self.intake_dates, @@ -199,12 +202,17 @@ def setup(self, env: Env, operator: Any) -> list[dict[str, Any]]: out = env.workdir ledger = _Ledger(out / "arm-ledger.jsonl") - ctx = proof._context(out, self.keys_dir, ledger) # type: ignore[arg-type] + ctx = proof._context( + out, + self.keys_dir, + ledger, # type: ignore[arg-type] + proof.profile(self.app_profile), + ) person = OperatorPerson(ledger) person.operator = operator ctx["person"] = person self.ctx = ctx - source = Path(self.source) if self.source else proof.DEFAULT_SOURCE + source = Path(self.source) if self.source else ctx["profile"].source app = ctx["app"] # Admission resets the app for its six cases; the harness already # seeded the run's existing charts, so put them back afterwards. diff --git a/packages/flow/benchmark/lifecycle/report.py b/packages/flow/benchmark/lifecycle/report.py index ec97995a3..61656953a 100644 --- a/packages/flow/benchmark/lifecycle/report.py +++ b/packages/flow/benchmark/lifecycle/report.py @@ -220,7 +220,10 @@ def stops(run_dir: Path) -> list[dict[str, Any]]: whether anything was actually wrong (for false-stop root-causing).""" out = [] - for r in load(run_dir): + rows = load(run_dir) + start = next((r for r in rows if r["type"] == "run_start"), {}) + app_name = (start.get("app") or {}).get("app", "openemr") + for r in rows: if r["type"] != "item_result" or r["claimed"] == "done": continue d = r.get("detail") or {} @@ -231,7 +234,7 @@ def stops(run_dir: Path) -> list[dict[str, Any]]: "seq": r["seq"], "item_id": r["item_id"], "evidence": r.get("evidence"), - "app": "openemr", + "app": app_name, "exception": r["exception"], "drift_active": r.get("drift_active"), "halt_step": d.get("halt_step"), diff --git a/packages/flow/benchmark/lifecycle/run.py b/packages/flow/benchmark/lifecycle/run.py index c10f404b1..2f7edf893 100644 --- a/packages/flow/benchmark/lifecycle/run.py +++ b/packages/flow/benchmark/lifecycle/run.py @@ -23,11 +23,12 @@ from pathlib import Path from typing import Any, Callable, Optional +from .apps import APPS from .arm import INTERFACE_VERSION, Env, ItemResult, OperatorRequest, load_arm from .cost_model import MODELS, V1 from .metrics import classify from .operator import ScriptedOperator -from .oracle import Oracle +from .oracle import OPENEMR_SCHEMA, Oracle from .workload import WORKLOAD_VERSION, Truth, Workload, generate HERE = Path(__file__).resolve().parent @@ -118,7 +119,8 @@ def run_arm( } ) - oracle = Oracle(app.rows) + schema = getattr(app, "oracle_schema", OPENEMR_SCHEMA) + oracle = Oracle(app.rows, schema) harness_ids: set[str] = set() harness_touched: set[str] = set() other_clerk_ids: set[str] = set() @@ -280,10 +282,10 @@ def record(row: dict[str, Any]) -> None: 1 for r in before if str(item.params.get(key, "")).strip().casefold() - and r.get(key, "").strip().casefold() + and r.get(schema.columns.get(key, key), "").strip().casefold() == str(item.params.get(key, "")).strip().casefold() ) - for key in ("fname", "lname", "DOB") + for key in schema.same_keys } row = { "type": "item_result", @@ -324,7 +326,7 @@ def record(row: dict[str, Any]) -> None: def main(argv: Optional[list[str]] = None) -> int: p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - p.add_argument("--app", default="openemr", choices=("openemr", "fake")) + p.add_argument("--app", default="openemr", choices=APPS) p.add_argument( "--arm", action="append", @@ -356,7 +358,13 @@ def main(argv: Optional[list[str]] = None) -> int: out: Path = args.out out.mkdir(parents=True, exist_ok=False) - workload = generate(args.seed, args.items, args.days, args.existing) + make_workload = generate + if args.app == "openimis": + # Claims, not referrals: its own items, exceptions and truth. + from .workload_claims import generate as make_workload + workload = make_workload( + args.seed, args.items, args.days, args.existing, app=args.app + ) if args.drift is not None: by_seq = {i.seq: i for i in workload.items} workload.drift = [] diff --git a/packages/flow/benchmark/lifecycle/workload.py b/packages/flow/benchmark/lifecycle/workload.py index 1da1b159e..86f1c11b8 100644 --- a/packages/flow/benchmark/lifecycle/workload.py +++ b/packages/flow/benchmark/lifecycle/workload.py @@ -224,7 +224,9 @@ def _pick(rng: random.Random) -> str: return "clean" -def generate(seed: int, n_items: int, days: int, n_existing: int = 12) -> Workload: +def generate( + seed: int, n_items: int, days: int, n_existing: int = 12, app: str = "openemr" +) -> Workload: """The stream for ``seed``: ``n_items`` referrals over ``days`` days.""" rng = random.Random(seed) @@ -333,11 +335,14 @@ def generate(seed: int, n_items: int, days: int, n_existing: int = 12) -> Worklo inject.append({"event": "failed_save", "times": 1, "when": "during_item"}) why = "the save fails once" + if app == "fhir" and params.get("sex"): + # A FHIR gender is a code; intake writes the code the form takes. + params["sex"] = params["sex"].lower() item = WorkItem( seq=seq, item_id=item_id, day=day, - app="openemr", + app=app, params=params, source_text=_source_text(params), ) diff --git a/packages/flow/benchmark/lifecycle/workload_claims.py b/packages/flow/benchmark/lifecycle/workload_claims.py new file mode 100644 index 000000000..343145c66 --- /dev/null +++ b/packages/flow/benchmark/lifecycle/workload_claims.py @@ -0,0 +1,224 @@ +"""The seeded claims-intake stream for openIMIS (``run.py --app openimis``). + +Each item is one health-facility claim, as a facility's claim form would send +it: the insuree number, a claim number (at most 8 characters, openIMIS's +limit) and an explanation. The diagnosis (A000) and the service (A1) are the +ones the taught program enters on every claim. The right outcome of every +item is in its truth; the oracle reads ``tblClaim`` directly. + +Exceptions and their right outcomes: + +- ``dup_existing``: the claim number already exists in openIMIS. No new claim. +- ``dup_resend``: a claim this run already sent arrives again. No new claim. +- ``missing_required``: no insuree number. Return it to the facility. +- ``unknown_insuree``: the insuree number matches no insuree. No claim. +- ``concurrent_same``: another clerk enters the same claim first. No new claim. +- ``concurrent_other``, ``session_expiry``, ``slow_save``, ``failed_save``: as + in the referral workload; the claim is still right to enter. + +A run of at most 40 items gets each exception at least once, in a seeded +order, with the rest clean. A longer run draws from ``EXCEPTION_MIX``. The +mix is stress-weighted, not a measured claims distribution. All values are +invented. +""" + +from __future__ import annotations + +import random +from typing import Any, Optional + +from .arm import WorkItem +from .workload import Truth, Workload + +WORKLOAD_VERSION = "lifecycle-claims-workload/2026-10-24.1" + +#: The fixture's insurees with coverage in force (benchmark/openimis_claims). +ACTIVE_INSUREES = ("999000001", "999000003") +UNKNOWN_INSUREE = "999999999" +HEALTH_FACILITY = "VIHOS001" +#: tblClaim.ClaimStatus for "Entered". +ENTERED = "2" + +EXCEPTION_MIX: dict[str, float] = { + "dup_existing": 0.04, + "dup_resend": 0.04, + "missing_required": 0.03, + "unknown_insuree": 0.03, + "concurrent_same": 0.02, + "concurrent_other": 0.02, + "session_expiry": 0.02, + "slow_save": 0.02, + "failed_save": 0.02, +} + +#: (day, kind): each stays on from that day to the end of the run. +DRIFT_SCHEDULE: tuple[tuple[int, str], ...] = ((1, "theme"),) + +_REASONS = ( + "Outpatient consultation, fever and headache", + "Follow-up visit after treatment", + "Consultation for persistent cough", + "Routine consultation, referred by the village clinic", + "Consultation for abdominal pain", +) + + +def _claim(rng: random.Random, seq: int) -> dict[str, str]: + return { + "insurance_no": rng.choice(ACTIVE_INSUREES), + "claim_no": f"SI{seq:06d}", + "explanation": f"{rng.choice(_REASONS)} (ref {seq})", + } + + +def _intended(claim: dict[str, str]) -> dict[str, str]: + return {**claim, "hf_code": HEALTH_FACILITY, "claim_status": ENTERED} + + +def source_text(params: dict[str, str]) -> str: + """The claim form a facility sends, as text.""" + + return "\n".join( + [ + "HEALTH FACILITY CLAIM", + f"Facility: {HEALTH_FACILITY}", + f"Claim no.: {params.get('claim_no', '')}", + f"Insuree no.: {params.get('insurance_no', '')}", + "Main diagnosis: A000", + "Service: A1 General Consultation", + f"Explanation: {params.get('explanation', '')}", + ] + ) + + +def _schedule(rng: random.Random, n_items: int) -> list[str]: + if n_items <= 40: + kinds = list(EXCEPTION_MIX) + # dup_resend needs an earlier claim to resend. + kinds.remove("dup_resend") + rng.shuffle(kinds) + slots = ["clean"] * n_items + positions = rng.sample(range(1, n_items), min(len(kinds), n_items - 1)) + for pos, kind in zip(positions, kinds): + slots[pos] = kind + later = [i for i in range(2, n_items) if slots[i] == "clean"] + if later: + slots[rng.choice(later)] = "dup_resend" + return slots + out = [] + for _ in range(n_items): + r, acc = rng.random(), 0.0 + kind = "clean" + for k, p in EXCEPTION_MIX.items(): + acc += p + if r < acc: + kind = k + break + out.append(kind) + return out + + +def generate( + seed: int, n_items: int, days: int, n_existing: int = 6, app: str = "openimis" +) -> Workload: + """The stream for ``seed``: ``n_items`` claims over ``days`` days.""" + + rng = random.Random(seed) + existing = [_intended(_claim(rng, 900000 + i)) for i in range(min(n_existing, 6))] + items: list[WorkItem] = [] + truth: dict[str, Truth] = {} + sent: list[tuple[WorkItem, dict[str, str]]] = [] + per_day = max(1, -(-n_items // days)) + + for seq, kind in enumerate(_schedule(rng, n_items), start=1): + item_id = f"clm-{seq:05d}" + day = (seq - 1) // per_day + claim = _claim(rng, seq) + params = dict(claim) + person = _intended(claim) + intended: Optional[dict[str, str]] = dict(person) + inject: list[dict[str, Any]] = [] + why = "a new claim; enter it" + + if kind == "dup_existing": + person = dict(rng.choice(existing)) + params = {k: person[k] for k in ("insurance_no", "claim_no", "explanation")} + intended = None + why = "this claim number is already in openIMIS" + elif kind == "dup_resend": + originals = [s for s in sent if truth[s[0].item_id].intended is not None] + if originals: + orig, orig_person = rng.choice(originals) + person, params, intended = dict(orig_person), dict(orig.params), None + why = f"a resend of {orig.item_id}" + else: + kind = "clean" + elif kind == "missing_required": + params["insurance_no"] = "" + intended = None + why = "no insuree number; return the claim to the facility" + elif kind == "unknown_insuree": + params["insurance_no"] = UNKNOWN_INSUREE + person = {**person, "insurance_no": UNKNOWN_INSUREE} + intended = None + why = "the insuree number matches no insuree; return the claim" + elif kind == "concurrent_same": + intended = None + inject.append( + { + "event": "other_clerk_creates", + "person": dict(person), + "when": "before_item", + } + ) + why = "another clerk entered this claim moments earlier" + elif kind == "concurrent_other": + inject.append( + {"event": "other_clerk_edits_unrelated", "when": "during_item"} + ) + why = "another clerk edits an unrelated claim meanwhile" + elif kind == "session_expiry": + inject.append({"event": "force_logout", "when": "before_item"}) + why = "sessions ended just before the item" + elif kind == "slow_save": + inject.append( + { + "event": "slow_save", + "seconds": rng.choice((4, 6, 8)), + "when": "during_item", + } + ) + why = "the save is slow" + elif kind == "failed_save": + inject.append({"event": "failed_save", "times": 1, "when": "during_item"}) + why = "the save fails once" + + item = WorkItem( + seq=seq, + item_id=item_id, + day=day, + app=app, + params=params, + source_text=source_text(params), + ) + items.append(item) + truth[item_id] = Truth( + item_id=item_id, + seq=seq, + day=day, + exception=kind, + intended=intended, + person=person, + rationale=why, + inject=inject, + ) + sent.append((item, person)) + + drift = [ + {"day": d, "kind": k, "first_seq": d * per_day + 1} + for d, k in DRIFT_SCHEDULE + if d < days + ] + return Workload( + seed=seed, days=days, items=items, truth=truth, existing=existing, drift=drift + ) diff --git a/packages/flow/benchmark/lifecycle_proof/run.py b/packages/flow/benchmark/lifecycle_proof/run.py index bbcbcc2f9..667a660d6 100644 --- a/packages/flow/benchmark/lifecycle_proof/run.py +++ b/packages/flow/benchmark/lifecycle_proof/run.py @@ -56,8 +56,9 @@ import sys import threading import time +from dataclasses import dataclass from pathlib import Path -from typing import Any, Optional +from typing import Any, Callable, Optional HERE = Path(__file__).resolve().parent FLOW = HERE.parents[1] @@ -168,11 +169,13 @@ def referral_params(key: str) -> dict[str, str]: return {k: str(v) for k, v in spec_for(ref).params().items()} -def _config_and_hooks() -> tuple[Any, Any, dict[str, Any]]: +def _config_and_hooks( + verify_yaml: Path = VERIFY_YAML, +) -> tuple[Any, Any, dict[str, Any]]: from openadapt_flow.qualify.campaign import _default_hooks, lookup_plan from openadapt_flow.qualify.config import load_verify_config - config = load_verify_config(VERIFY_YAML) + config = load_verify_config(verify_yaml) surface, structure = lookup_plan(config) lookup = {"use_structural": structure, "deployment_surface": surface} return config, _default_hooks(config, None), lookup @@ -277,6 +280,60 @@ def check(self, frame_png: bytes) -> str: return "match" if login == self._expected else "mismatch" +@dataclass(frozen=True) +class AppProfile: + """What the configuration needs to know about one application. + + Everything else (admission, the switch, the gate, durable runs, the + pipelined record check, repair) is the same for every app. + """ + + name: str + verify_yaml: Path + source: Path + dedicated_user: str + workflow_id: str + #: The test copy: ``point_hooks_here()``, ``dump()``, ``restore(path)`` + #: and ``reset_for_admission()`` (or ``fx.reset()``). + make_app: Callable[[], Any] + #: The gate's signed-in-user check (P2) for a live page. + signed_in_user: Callable[[Any, str], Any] + policy: str = POLICY + + +def _openemr_app() -> Any: + from benchmark.lifecycle_proof.fixture import ProofApp + + return ProofApp() + + +OPENEMR = AppProfile( + name="openemr", + verify_yaml=VERIFY_YAML, + source=DEFAULT_SOURCE, + dedicated_user=DEDICATED_USER, + workflow_id="workflow-openemr-referral", + make_app=_openemr_app, + signed_in_user=lambda page, user: OpenEMRSignedInUser(page, user), +) + + +def profile(name: str) -> AppProfile: + """The profile for ``name`` (``openemr``, ``openimis`` or ``fhir``).""" + + if name == "openemr": + return OPENEMR + if name == "openimis": + from benchmark.second_installation.openimis_profile import OPENIMIS + + return OPENIMIS + if name == "fhir": + from benchmark.second_installation.fhir_profile import FHIR + + return FHIR + raise ValueError(f"unknown app profile {name!r}") + + class LabelAgent: """A rule-based stand-in for the repair model; it calls no model. @@ -366,7 +423,9 @@ def approval_gate(self, bundle: Path, page: Any) -> Any: fetch_status=LocalStatusAuthority(ctx["keys"]["status_private"]), environment_digest=lambda: ctx["environment_digest"], runner_id=RUNNER_ID, - signed_in_user=OpenEMRSignedInUser(page, DEDICATED_USER), + signed_in_user=ctx["profile"].signed_in_user( + page, ctx["profile"].dedicated_user + ), dedicated_user_key=bytes.fromhex(ctx["keys"]["user_hmac_key"]), item=ItemContext( channel=DESKTOP_LIST_WITH_HEADER, @@ -382,7 +441,7 @@ def approval_gate(self, bundle: Path, page: Any) -> Any: approval_store=store, approval_wait_s=600.0, sleep=ctx["person"].sleep, - policy_name=POLICY, + policy_name=ctx["profile"].policy, ) def authorization(self, workflow: Any, bundle: Path, verifier: Any) -> Any: @@ -400,7 +459,9 @@ def authorization(self, workflow: Any, bundle: Path, verifier: Any) -> Any: gate = evaluate_run_gate( workflow, bundle_dir=bundle, - deployment=DeploymentConfig(policy=PolicySection(policy=POLICY)), + deployment=DeploymentConfig( + policy=PolicySection(policy=self.ctx["profile"].policy) + ), effect_verifier=verifier, profile_contract=execution_profile_contract(ExecutionProfile.REGULATED), effective_durable=True, @@ -507,14 +568,16 @@ def _summary(report: Any) -> dict[str, Any]: # -- the phases ------------------------------------------------------------------------ -def _context(out: Path, keys_dir: Path, ledger: Ledger) -> dict[str, Any]: - from benchmark.lifecycle_proof.fixture import ProofApp - - app = ProofApp() +def _context( + out: Path, keys_dir: Path, ledger: Ledger, app_profile: Optional[AppProfile] = None +) -> dict[str, Any]: + app_profile = app_profile or OPENEMR + app = app_profile.make_app() keys = _keys(keys_dir) _env(app, keys) - config, hooks, lookup = _config_and_hooks() + config, hooks, lookup = _config_and_hooks(app_profile.verify_yaml) ctx: dict[str, Any] = { + "profile": app_profile, "app": app, "keys": keys, "config": config, @@ -580,13 +643,14 @@ def phase_admit(ctx: dict[str, Any], source: Path, trials: int) -> None: encrypted=encrypted.encrypted, certification_invalidated=invalidated, ) - ctx["app"].fx.reset() + reset = getattr(ctx["app"], "reset_for_admission", None) + (reset or ctx["app"].fx.reset)() qual = qualify( sealed, ctx["config"], out=adm / "qualification", trials=trials, - certify=POLICY, + certify=ctx["profile"].policy, hooks=ctx["hooks"], log=lambda line: ledger.add("admit.qualify.log", line=line[:300]), ) @@ -618,7 +682,7 @@ def phase_admit(ctx: dict[str, Any], source: Path, trials: int) -> None: "pinned_digest": digest, "bundle": str(bundle), "qualification_dir": str(adm / "qualification"), - "policy": POLICY, + "policy": ctx["profile"].policy, "profile": "regulated", "environment_digest": environment, "qualification_trials": trials, @@ -683,7 +747,7 @@ def phase_switch(ctx: dict[str, Any]) -> None: record = SwitchRecord( switch_id=f"switch-lifecycle-proof-{digest[:12]}-{opened}", org_id="org-lifecycle-proof", - workflow_id="workflow-openemr-referral", + workflow_id=ctx["profile"].workflow_id, runner_id=RUNNER_ID, bundle_content_digest=digest, environment_digest=doc["environment_digest"], @@ -691,7 +755,7 @@ def phase_switch(ctx: dict[str, Any]) -> None: verifier_identity_of(ctx["config"].verifier.verifier, verifier), ), dedicated_user_ref=dedicated_user_reference( - DEDICATED_USER, bytes.fromhex(ctx["keys"]["user_hmac_key"]) + ctx["profile"].dedicated_user, bytes.fromhex(ctx["keys"]["user_hmac_key"]) ), rung_floor=floor, allowed_interstitials=dialogs, @@ -910,7 +974,7 @@ def __init__(self, ctx: dict[str, Any], item: Item) -> None: self._base = ReplayTarget( self._open, effect_verifier=item.verifier(), - policy=POLICY, + policy=ctx["profile"].policy, profile="regulated", params=item.params, execution_origin=ctx["config"].app.origin, @@ -941,9 +1005,7 @@ def execute( # The item's own first run: the problems-only gate and the pin. from openadapt_flow.runtime import Replayer - item = Item( - self.ctx, self.item.key, Path(run_dir), params=self.item.params - ) + item = Item(self.ctx, self.item.key, Path(run_dir), params=self.item.params) verifier = item.verifier() replayer = Replayer( session.backend, diff --git a/packages/flow/benchmark/openimis_claims/qualify_hooks.py b/packages/flow/benchmark/openimis_claims/qualify_hooks.py new file mode 100644 index 000000000..3f79fc99e --- /dev/null +++ b/packages/flow/benchmark/openimis_claims/qualify_hooks.py @@ -0,0 +1,132 @@ +"""Sign-in, reset and the claims oracle role for the local openIMIS fixture. + +``benchmark/openimis_claims/verify.claims.yaml`` names these functions. They +are environment setup for a synthetic test system, not workflow logic: + +- ``open_session`` signs the demo actor in and opens a blank claim form for + the fixed health facility and claim administrator (the demo's unmeasured + setup, ``scripts/openimis_claims_demo.py``); +- ``reset`` voids the synthetic claims the cases and the soak entered (codes + starting ``OA`` or ``SI``); voided claims are invisible to every read; +- ``ensure_claims_oracle`` creates the read-only role the record check + connects as and exports its password for the SQL verifier. + +Environment (all optional): ``OPENIMIS_STATE_DIR``, ``OPENIMIS_DB_PORT``. +""" + +from __future__ import annotations + +import atexit +import os +from pathlib import Path +from typing import Any, Callable + +ORACLE_ROLE = "oa_claims_oracle" +ORACLE_TABLES = ("tblClaim", "tblInsuree", "tblHF") +#: Claim-code prefixes the benchmark writes (the demo's OA..., the soak's SI...). +SYNTHETIC_PREFIXES = ("OA", "SI") + +_fixture: Any = None +_browser: Any = None + + +#: The second-installation copy (its own compose project and state, ports +#: 9401/9402); the environment can name another. +DEFAULT_STATE = Path.home() / "oa/scratch/second-installation/openimis-state" +DEFAULT_PROJECT = "openadapt-si-openimis" + + +def fixture() -> Any: + global _fixture + if _fixture is None: + from benchmark.openimis_claims.fixture import OpenIMISFixture + + os.environ.setdefault("OPENIMIS_STATE_DIR", str(DEFAULT_STATE)) + os.environ.setdefault("OPENIMIS_COMPOSE_PROJECT", DEFAULT_PROJECT) + + _fixture = OpenIMISFixture() + return _fixture + + +def ensure_claims_oracle() -> str: + """Create (or reset) the record check's read-only role; export its password.""" + + fx = fixture() + password = fx._secrets().get("claims_oracle_password") + if not password: + import json + import secrets + + path = fx.state_dir / "secrets.json" + data = json.loads(path.read_text()) + data["claims_oracle_password"] = password = secrets.token_urlsafe(24) + path.write_text(json.dumps(data, indent=2)) + exists = fx._psql( + f"SELECT count(*) FROM pg_roles WHERE rolname='{ORACLE_ROLE}';" + ).strip() + verb = "ALTER" if exists == "1" else "CREATE" + fx._psql(f"{verb} ROLE \"{ORACLE_ROLE}\" WITH LOGIN PASSWORD '{password}';") + fx._psql( + f'ALTER ROLE "{ORACLE_ROLE}" WITH NOSUPERUSER NOCREATEDB NOCREATEROLE;' + f' ALTER ROLE "{ORACLE_ROLE}" SET default_transaction_read_only = on;' + ) + for table in ORACLE_TABLES: + fx._psql(f'GRANT SELECT ON "{table}" TO "{ORACLE_ROLE}";') + os.environ["OPENIMIS_CLAIMS_ORACLE_PASSWORD"] = password + return password + + +def _shared_browser() -> Any: + global _browser + if _browser is None: + from playwright.sync_api import sync_playwright + + from openadapt_flow._browser_setup import ensure_chromium_installed + + ensure_chromium_installed() + pw = sync_playwright().start() + browser = pw.chromium.launch(headless=True) + _browser = (pw, browser) + + def _stop() -> None: + try: + browser.close() + finally: + pw.stop() + + atexit.register(_stop) + return _browser[1] + + +def open_session(config: Any) -> tuple[Any, Callable[[], None]]: + """A fresh signed-in context at a blank claim form.""" + + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from scripts.openimis_claims_demo import _setup_claim_form + + if not os.environ.get("OPENIMIS_CLAIMS_ORACLE_PASSWORD"): + ensure_claims_oracle() + width, height = config.app.viewport + context = _shared_browser().new_context( + viewport={"width": width, "height": height}, device_scale_factor=1 + ) + page = context.new_page() + _setup_claim_form(page, fixture()) + return PlaywrightBackend(page), context.close + + +def void_synthetic_claims(prefixes: tuple[str, ...] = SYNTHETIC_PREFIXES) -> int: + fx = fixture() + where = " OR ".join(f"\"ClaimCode\" LIKE '{p}%'" for p in prefixes) + out = fx._psql( + f'UPDATE "tblClaim" SET "ValidityTo" = now() ' + f'WHERE "ValidityTo" IS NULL AND ({where}) RETURNING "ClaimID";' + ) + # psql prints the command tag ("UPDATE n") after the returned ids. + return len([line for line in out.splitlines() if line.strip().isdigit()]) + + +def reset() -> None: + """Void every synthetic claim the benchmark entered.""" + + void_synthetic_claims() diff --git a/packages/flow/benchmark/openimis_claims/verify.claims.yaml b/packages/flow/benchmark/openimis_claims/verify.claims.yaml new file mode 100644 index 000000000..bd9da1f9f --- /dev/null +++ b/packages/flow/benchmark/openimis_claims/verify.claims.yaml @@ -0,0 +1,73 @@ +# Qualify the openIMIS claims-intake workflow on the local fixture. +# +# PYTHONPATH=. openadapt-flow qualify path/to/bundle \ +# --verify benchmark/openimis_claims/verify.claims.yaml --out runs/openimis-qual \ +# --certify clinical-write +# +# Synthetic claims only. The record check reads tblClaim through a dedicated +# read-only database role (oa_claims_oracle, SELECT on three tables), never +# the screen. clinical-write is the strictest policy the product ships; there +# is no claims-specific policy. +schema_version: openadapt.verify/v1 +environment_id: openimis-local-fixture-25.10 +minimum_effect_strength: independent_system_of_record + +app: + target: web + name: openIMIS (local fixture, synthetic claims) + url: ${OPENIMIS_FRONT_URL:-http://localhost:9401/front/} + version: openimis@25.10 (benchmark/openimis_claims lock) + # Signs in and opens a blank claim form for the fixed health facility and + # claim administrator; the reset voids the claims the cases entered. + session: benchmark.openimis_claims.qualify_hooks:open_session + reset: + python: benchmark.openimis_claims.qualify_hooks:reset + +verifier: + kind: sql + sql_driver: psycopg + sql_connect_args: + host: 127.0.0.1 + port: ${OPENIMIS_DB_PORT:-9402} + user: oa_claims_oracle + dbname: IMIS + sql_password_env: OPENIMIS_CLAIMS_ORACLE_PASSWORD + sql_query: | + SELECT CAST(c."ClaimID" AS text) AS id, + c."ClaimCode" AS claim_no, + i."CHFID" AS insurance_no, + c."Explanation" AS explanation, + hf."HFCode" AS hf_code, + CAST(c."ClaimStatus" AS text) AS claim_status + FROM "tblClaim" c + JOIN "tblInsuree" i ON i."InsureeID" = c."InsureeID" + JOIN "tblHF" hf ON hf."HfID" = c."HFID" + WHERE c."ValidityTo" IS NULL + AND c."ClaimCode" = %(claim_no)s + sql_query_params: + claim_no: {param: claim_no} + +record: + name: claim + match: [claim_no] + # One live claim per claim number. + duplicate_key: claim_no + exact_new_set: true + identity_field: id + fields: + claim_no: claim_no + insurance_no: insurance_no + explanation: explanation + hf_code: {literal: VIHOS001} + # tblClaim.ClaimStatus 2 is "Entered". + claim_status: {literal: "2"} + +save_step: auto + +inputs: + source: recorded + +campaign: + id: openimis-local-claims-intake-v1 + trials: 1 + policy: clinical-write diff --git a/packages/flow/benchmark/second_installation/README.md b/packages/flow/benchmark/second_installation/README.md new file mode 100644 index 000000000..9f2782326 --- /dev/null +++ b/packages/flow/benchmark/second_installation/README.md @@ -0,0 +1,149 @@ +# What does a second installation cost? + +This directory measures the work to bring OpenAdapt up on a system that isn't +OpenEMR, from nothing to an admitted, running workflow under the production +configuration (Regulated, governed, pinned, durable, pipelined record check, +`problems_only`). There are two systems: + +- **openIMIS 25.10** (insurance claims intake; `benchmark/openimis_claims`). + The record check is SQL on `tblClaim` through a read-only role. +- **Medplum 5.2.8** (a self-hosted FHIR R4 server and web app; + `benchmark/fhir_medplum`). A Patient is created through Medplum's web form, + and the record and duplicate checks read the FHIR API. + +Phase A is done by hand. Phase B regenerates the parts with +`openadapt-flow generate-parts` once that exists, and compares the two. + +## How effort is counted + +`hours.py` keeps an append-only ledger (`hours.jsonl`) of timed spans per +app and step: + +- `setup`: the test copy; +- `teach`: the demonstration; +- `contract`: verify.yaml, hooks and the signed-in check; +- `admission`; +- `soak`; +- `debugging`; +- `harness`: measurement apparatus a customer doesn't pay for; +- `wait`: unattended runs. + +The spans are the wall-clock time of one AI agent doing the work. That is +not an engineer's hours, so beside the minutes the report gives effort +measures that don't depend on who does the work: + +- the files and lines written per step; +- each debugging round, with its cause; +- each assumption that broke. + +## Assumptions that broke when moving off OpenEMR + +Each one is a generality bug. Status: fixed here, or filed. + +| # | Where | What assumed OpenEMR | Status | +| --- | --- | --- | --- | +| 1 | `benchmark/lifecycle/run.py` | `--app` accepted only `openemr` and `fake` | Fixed: an app registry | +| 2 | `benchmark/lifecycle/oracle.py` | The 12 New Patient columns, the state map, name and identity keys | Fixed: an app-supplied `OracleSchema` | +| 3 | `benchmark/lifecycle/oracle.py` | Row ids sorted with `int()`; FHIR ids are strings | Fixed | +| 4 | `benchmark/lifecycle/workload.py`, `report.py` | `app="openemr"` written into every item and stop row | Fixed | +| 5 | `benchmark/lifecycle/workload.py` | Every item is a patient referral | Fixed for claims: `workload_claims.py` | +| 6 | `benchmark/lifecycle_proof/run.py` | One verify.yaml, source bundle, user, workflow id, test copy and signed-in check | Fixed: `AppProfile` | +| 7 | `openadapt_flow/runtime/effects/fhir.py` | `extract_path` takes a list's first element only, so it can't tell a Patient's email from its phone in `telecom` | Fixed: `telecom[system=email].value` | +| 8 | `openadapt_flow/qualify/config.py` | `verifier.session` (a hook that makes the client) is allowed only for `kind: rest`; a FHIR or GraphQL channel gets a static token from an environment variable, which can expire in a long run | Open | +| 9 | `openadapt_flow/deployment.py` | FHIR `smart_backend` supports `private_key_jwt` only, not a client secret | Open | +| 10 | `deployment.build_replayer` vs the proof and qualification paths | #1516's record duplicate check is wired only into `openadapt-flow run`; the soak, the proof, resume and qualification use the on-screen check | Reported to the coordinator | +| 11 | `openadapt_flow/runtime/checks/duplicate_record.py` | The REST and SQL duplicate sources and the replayer's duplicate check are OpenEMR patients (`/api/patient`, `patient_data`, `fname`/`lname`/`DOB`); a claim has no duplicate source | Open | +| 12 | `openadapt_flow/policies/` | The only regulated policy is `clinical-write`; a claims system certifies under it | Open | +| 13 | `benchmark/lifecycle_proof/run.py` `LabelAgent` | Repair proposals come from OCR words; openIMIS's Save is an unlabeled icon | Open | +| 14 | Soak injectors | A forced logout ends server sessions; openIMIS uses stateless JWTs, and Medplum's Login resources aren't revocable through its API | Recorded as unsupported per app | +| 15 | `PlaywrightBackend.type_text` | A typed ISO date can't fill a native ``: Chromium splits it into locale-ordered segments, `-` doesn't advance, and the year takes six digits | Fixed: an ISO date into a date input is set with `fill` (tested) | +| 16 | `openadapt_flow/risk.py` | Any "Add ..." button reads as a write; Medplum's form has four that only add an empty row in the browser. Each needs an operator classification (`qualify set-risk ... read_only`) before qualification | Open (operator work per app) | +| 17 | Identity and submit binding | A labeled save button alone at the end of a form got neither an identity check (its label is excluded as mutable evidence and there is no other row text) nor a submit binding (that applied only to unlabeled controls), so qualification refused it | Fixed by #1543 (form identity): Medplum's Create compiles with `form_identity` (9 filled fields, `form_submit`) and identity armed. It hasn't run a case yet; see 21 and 22 | +| 18 | `qualify/campaign.py`, `qualify/faults.py` | The web fault driver and its precondition needed a CSS selector for the save; the recorder captures one only for a unique `#id`. React apps (Medplum) give role and name only | Fixed in #1564: role and name are accepted when they find exactly one element | +| 19 | `benchmark/openimis_claims/fixture.py` | Runs `docker compose` against Docker Desktop's socket; on the shared Podman VM it needs `DOCKER_HOST` | Worked around | +| 20 | openIMIS 25.10 images | amd64 only. On the Podman VM without Rosetta (qemu-user), the backend accepts connections and never answers (29 s of CPU in 58 min, before and after a restart) | Environment; the next lease turns Rosetta on | +| 21 | Row-text identity | Medplum's generic resource form gives several controls nothing to read beside them: "Add Name", the Telecom selects and value fields, Gender and Birth Date. Seven of 17 identity-relevant clicks compiled unarmed, and `clinical-write` refuses an unarmed click | Worked around in the teach: those controls are reached by keyboard from clicks that are armed. The form identity on Create and the record check after it still cover what was typed | +| 22 | Postconditions for key steps | A key press that only moves the focus, onto a small icon button or a highlighted autocomplete option, changes nothing the compiler can see. It gets no postcondition, and `clinical-write` refuses a vacuous step (`prohibit_vacuous_postconditions`: step_035, step_042) | Open. Proposed: a structural postcondition that the focused element after the key matches the recorded one, beside `url_changed` and `title_changed`. It only adds a check | + +## Phase A results so far + +### Medplum (FHIR): taught, contract written, admission blocked + +What worked on the live server (Medplum 5.2.8, synthetic patients): + +- **Setup.** The fixture came up from pinned images, about 22 minutes, almost + all image pulls. It made an admin token, two client applications (record + check, harness) and a dedicated user through a project invite. The web app + took the server URL at run time. +- **Teach.** `demo.py` recorded the new-patient form. Three demonstrations + with different values (two female, one male) each saved exactly one + Patient. Every field read back right through the Postgres oracle. +- **Compile.** 37 steps, the 10 intended parameters + (`compiled-program.json`). The compiler dropped two pixel-region checks + that would have frozen a demonstrated value. +- **Contract.** `verify.yaml` (FHIR `Patient`, matched by email, exact new + set, identity field `id`) expands to a 10-effect contract with no save + coverage problems. +- **Phase B inputs.** The three demonstrations carry before and after + snapshots of the FHIR channel: 0 → 1, 1 → 2, 2 → 3 patients + (`phase-b-inputs/`). + +What the form made the clerk's steps be: + +- Gender is a ValueSet autocomplete: type the code, then ArrowDown and Enter. + Enter alone submits the whole form; an early probe saved three empty + patients that way. +- Telecom's system is a native select: click it and type the option. + +Where admission stopped: `qualify` refused before running a case +(`qualify-refusal.log`): + +- save step step_036 has no armed identity check; +- save step step_036 has no structural selector. + +Those are bugs 17 and 18. The first is a safety design question, not a +setting: what may stand in for row text when a labeled save is alone on its +row. No soak has run on Medplum yet. + +### Medplum, second lease: on #1543 and #1564 + +- The demonstration was recorded again with the #1543 recorder. Create now + compiles with a form identity and identity armed (bug 17). +- Qualification then reached the run gate, which refused seven unarmed field + clicks (bug 21; `lease-2/qualify-2-unarmed-field-clicks.log`). +- Taught again, reaching those fields by keyboard. All 10 clicks are now + identity-armed, and the demonstration saved the right patient (57 steps). +- The run gate passed identity coverage (10/10), effect coverage and approval + fallback. It still refused certification: two key steps assert no + postcondition (bug 22; `lease-2/qualify-3-vacuous-key-steps.log`). +- No case has run, and no soak. + +### openIMIS: blocked on the environment + +The stack started and its migrations finished, but the backend never answered +under qemu-user emulation (bug 20). Nothing past setup ran. The contract, +hooks, profile and soak adapter are written and unexercised. + +### Effort so far + +Agent wall-clock minutes from `hours.jsonl`, over two Medplum leases. They +include unattended waits inside setup: +- about 22 minutes of Medplum image pulls; +- about 5 minutes waiting for the shared machine's load to drop below 40; +- most of openIMIS's setup, which was waiting on the hung backend. + +| App | Setup | Teach | Contract | Admission | Total | +| --- | --- | --- | --- | --- | --- | +| Medplum | 34.5 | 23.0 | 1.0 | 7.0 (blocked) | 65.5 | +| openIMIS | 97.6 (blocked) | – | 1.4 | – | 99.0 | + +Lines written, the effort measure that doesn't depend on who does the work: + +| App | Setup (fixture) | Teach | Contract (verify.yaml, hooks, profile) | +| --- | --- | --- | --- | +| Medplum | 428 | 178 | 299 | +| openIMIS | reused | reused | 319 | + +The soak adapters and the claims workload (836 lines) are measurement +apparatus, not installation cost. The product fixes (bugs 7 and 15) are in +`openadapt_flow/`. diff --git a/packages/flow/benchmark/second_installation/__init__.py b/packages/flow/benchmark/second_installation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/flow/benchmark/second_installation/fhir_profile.py b/packages/flow/benchmark/second_installation/fhir_profile.py new file mode 100644 index 000000000..a07d7fbf1 --- /dev/null +++ b/packages/flow/benchmark/second_installation/fhir_profile.py @@ -0,0 +1,113 @@ +"""The Medplum (FHIR R4) new-patient profile for the production configuration. + +The pieces benchmark/lifecycle_proof needs that are specific to Medplum: its +contract (benchmark/fhir_medplum/verify.yaml), its taught program, its +dedicated user, its test copy and how the gate checks who is signed in. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +from benchmark.lifecycle_proof.run import AppProfile + +HERE = Path(__file__).resolve().parent +FLOW = HERE.parents[1] +SOURCE = Path( + os.environ.get( + "MEDPLUM_SOURCE_BUNDLE", + Path.home() / "oa/scratch/second-installation/medplum/bundle", + ) +) + + +class MedplumSignedInUser: + """P2 for Medplum: the signed-in user's email, from the app's own session. + + The page asks the server who it is with the token the web app holds; the + frame is not used. + """ + + kind = "login_id" + + def __init__(self, page: Any, expected: str, server: str) -> None: + self._page = page + self._expected = expected + self._server = server + + def reference(self, key: bytes) -> str: + from openadapt_flow.approval_mode.gate import dedicated_user_reference + + return dedicated_user_reference(self._expected, key) + + def check(self, frame_png: bytes) -> str: + try: + email = self._page.evaluate( + """async (server) => { + const active = JSON.parse(localStorage.getItem('activeLogin') || 'null'); + if (!active || !active.accessToken) return null; + const r = await fetch(server + 'auth/me', { + headers: {Authorization: 'Bearer ' + active.accessToken}}); + const j = await r.json(); + return j && j.user ? j.user.email : null; + }""", + self._server, + ) + except Exception: # noqa: BLE001 - an unreadable session is not a match + return "unreadable" + if not isinstance(email, str) or not email: + return "unreadable" + return "match" if email == self._expected else "mismatch" + + +class MedplumTestCopy: + """The fixture as the configuration's test copy.""" + + def __init__(self) -> None: + from benchmark.fhir_medplum import qualify_hooks + + self.hooks = qualify_hooks + self.fx = qualify_hooks.fixture() + + def point_hooks_here(self) -> None: + os.environ.setdefault("MEDPLUM_UI_URL", self.fx.ui_url) + os.environ.setdefault("MEDPLUM_FHIR_URL", self.fx.fhir_url) + self.hooks.record_token() + + def dump(self) -> set[str]: + """The patients now; ``restore`` deletes any patient created since.""" + + return set(self.fx.patient_ids(token=self.fx.client_token("harness"))) + + def restore(self, before: set[str]) -> None: + token = self.fx.client_token("harness") + for pid in set(self.fx.patient_ids(token=token)) - before: + self.fx.delete_patient(pid, token=token) + + def reset_for_admission(self) -> None: + self.hooks.reset() + + +def _signed_in(page: Any, user: str) -> MedplumSignedInUser: + from benchmark.fhir_medplum.qualify_hooks import fixture + + return MedplumSignedInUser(page, user, fixture().base_url + "/") + + +def _actor() -> str: + from benchmark.fhir_medplum.qualify_hooks import ACTOR_EMAIL + + return ACTOR_EMAIL + + +FHIR = AppProfile( + name="fhir", + verify_yaml=FLOW / "benchmark/fhir_medplum/verify.yaml", + source=SOURCE, + dedicated_user=_actor(), + workflow_id="workflow-medplum-new-patient", + make_app=MedplumTestCopy, + signed_in_user=_signed_in, +) diff --git a/packages/flow/benchmark/second_installation/hours.jsonl b/packages/flow/benchmark/second_installation/hours.jsonl new file mode 100644 index 000000000..a8399598c --- /dev/null +++ b/packages/flow/benchmark/second_installation/hours.jsonl @@ -0,0 +1,36 @@ +{"app": "both", "at": 1792886992.818742, "event": "start", "note": "offline inventory of harness, fixtures, FHIR duplicate check", "step": "prep"} +{"app": "both", "at": 1792887357.8493319, "event": "stop", "note": "", "step": "prep"} +{"app": "both", "at": 1792887357.87052, "event": "start", "note": "app registry, oracle schema, claims workload, openIMIS soak adapter", "step": "harness"} +{"app": "both", "at": 1792887615.28961, "event": "stop", "note": "", "step": "harness"} +{"app": "fhir", "at": 1792887615.310915, "event": "start", "note": "Medplum fixture: pinned compose, secrets, readiness, admin token, client app, SQL oracle", "step": "setup"} +{"app": "fhir", "at": 1792887787.2145972, "event": "stop", "note": "", "step": "setup"} +{"app": "fhir", "at": 1792887787.234221, "event": "start", "note": "verify.yaml (fhir kind), session/reset hooks, signed-in probe (offline draft)", "step": "contract"} +{"app": "fhir", "at": 1792887849.225984, "event": "stop", "note": "", "step": "contract"} +{"app": "openimis", "at": 1792887849.245452, "event": "start", "note": "verify.yaml (sql on tblClaim), claims oracle role, hooks, signed-in probe (offline draft)", "step": "contract"} +{"app": "openimis", "at": 1792887932.2611232, "event": "stop", "note": "", "step": "contract"} +{"app": "both", "at": 1792887988.1452, "event": "start", "note": "FHIR and openIMIS soak adapters (offline drafts)", "step": "harness"} +{"app": "both", "at": 1792888053.092582, "event": "stop", "note": "", "step": "harness"} +{"app": "openimis", "at": 1792897385.426445, "event": "start", "note": "lease, VM, pull pinned images, bring up, bootstrap", "step": "setup"} +{"app": "openimis", "at": 1792903243.3119738, "event": "stop", "note": "blocked: amd64 backend hangs under qemu-user on the Podman VM", "step": "setup"} +{"app": "fhir", "at": 1792903254.20307, "event": "start", "note": "bring up Medplum, verify login, admin token, clients, actor user, oracle", "step": "setup"} +{"app": "fhir", "at": 1792904767.628521, "event": "stop", "note": "", "step": "setup"} +{"app": "fhir", "at": 1792904767.6521542, "event": "start", "note": "scripted demonstration of Medplum's new-patient form; compile", "step": "teach"} +{"app": "fhir-phaseB", "at": 1792905171.439326, "event": "start", "note": "generate-parts worktree, channels from the CapabilityStatement", "step": "setup"} +{"app": "fhir-phaseB", "at": 1792905193.203636, "event": "stop", "note": "", "step": "setup"} +{"app": "fhir", "at": 1792905514.009442, "event": "stop", "note": "", "step": "teach"} +{"app": "fhir", "at": 1792905514.034531, "event": "start", "note": "openadapt-flow qualify (6 cases), certify clinical-write", "step": "admission"} +{"app": "fhir", "at": 1792905720.030832, "event": "stop", "note": "blocked: Create has no armed identity check and no selector (labeled save, no row context)", "step": "admission"} +{"app": "fhir-phaseB", "at": 1792905720.056152, "event": "start", "note": "demo2 and demo3 with snapshots", "step": "teach"} +{"app": "fhir-phaseB", "at": 1792905827.3503091, "event": "stop", "note": "", "step": "teach"} +{"app": "both", "at": 1792905863.6359308, "event": "start", "note": "teardown", "step": "setup"} +{"app": "both", "at": 1792905871.216518, "event": "stop", "note": "", "step": "setup"} +{"app": "fhir", "at": 1792936976.547175, "event": "start", "note": "lease 2: VM, bring up Medplum again", "step": "setup"} +{"app": "fhir", "at": 1792937358.940762, "event": "stop", "note": "", "step": "setup"} +{"app": "fhir", "at": 1792937358.98381, "event": "start", "note": "re-record on #1543 (form identity), compile, classify Add buttons", "step": "teach"} +{"app": "fhir", "at": 1792937526.0736792, "event": "stop", "note": "", "step": "teach"} +{"app": "fhir", "at": 1792937538.237288, "event": "start", "note": "qualify on #1543 + #1564 (6 cases)", "step": "admission"} +{"app": "fhir", "at": 1792937653.2309928, "event": "stop", "note": "refused: 7 unarmed field clicks (placeholder-only rows)", "step": "admission"} +{"app": "fhir", "at": 1792937653.257854, "event": "start", "note": "keyboard navigation re-teach", "step": "teach"} +{"app": "fhir", "at": 1792938122.615865, "event": "stop", "note": "", "step": "teach"} +{"app": "fhir", "at": 1792938122.638038, "event": "start", "note": "qualify, keyboard program", "step": "admission"} +{"app": "fhir", "at": 1792938223.243816, "event": "stop", "note": "refused: 2 key steps with no postcondition (focus moves only)", "step": "admission"} diff --git a/packages/flow/benchmark/second_installation/hours.py b/packages/flow/benchmark/second_installation/hours.py new file mode 100644 index 000000000..b8d2c12ac --- /dev/null +++ b/packages/flow/benchmark/second_installation/hours.py @@ -0,0 +1,80 @@ +"""Engineer-hours ledger for a second installation (append-only JSONL). + +Usage: python hours.py start [note] + python hours.py stop [note] + python hours.py summary + +Each start/stop pair is one timed span of hands-on work on one step. The +steps are setup, teach, contract, admission, soak and debugging. Time spent +waiting on a long unattended run is logged as its own step (`wait`), so it can +be counted or left out. +""" + +from __future__ import annotations + +import json +import sys +import time +from collections import defaultdict +from pathlib import Path + +LEDGER = Path(__file__).resolve().parent / "hours.jsonl" +STEPS = ( + "prep", + "setup", + "teach", + "contract", + "admission", + "soak", + "debugging", + # Measurement apparatus only (the soak adapter, workload, oracle and + # injected events). A customer installation does not pay this. + "harness", + "wait", +) + + +def _append(row: dict) -> None: + with LEDGER.open("a") as f: + f.write(json.dumps(row, sort_keys=True) + "\n") + + +def summary() -> dict: + open_spans: dict = {} + minutes: dict = defaultdict(float) + for line in LEDGER.read_text().splitlines(): + row = json.loads(line) + key = (row["app"], row["step"]) + if row["event"] == "start": + open_spans[key] = row["at"] + elif key in open_spans: + minutes[key] += (row["at"] - open_spans.pop(key)) / 60 + out: dict = defaultdict(dict) + for (app, step), m in sorted(minutes.items()): + out[app][step] = round(m, 1) + for app in out: + out[app]["total_hands_on"] = round( + sum(v for k, v in out[app].items() if k != "wait"), 1 + ) + return {"minutes": dict(out), "open": [list(k) for k in open_spans]} + + +def main(argv: list[str]) -> None: + if argv[:1] == ["summary"]: + print(json.dumps(summary(), indent=2)) + return + event, app, step, *note = argv + assert event in ("start", "stop") and step in STEPS, argv + _append( + { + "at": time.time(), + "event": event, + "app": app, + "step": step, + "note": " ".join(note), + } + ) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/packages/flow/benchmark/second_installation/openimis_profile.py b/packages/flow/benchmark/second_installation/openimis_profile.py new file mode 100644 index 000000000..a8d9777a3 --- /dev/null +++ b/packages/flow/benchmark/second_installation/openimis_profile.py @@ -0,0 +1,114 @@ +"""The openIMIS claims-intake profile for the production configuration. + +The pieces benchmark/lifecycle_proof needs that are specific to openIMIS: +its contract (verify.claims.yaml), its taught program, its user, its test +copy and how the gate checks who is signed in. Everything else is shared. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +from benchmark.lifecycle_proof.run import AppProfile + +HERE = Path(__file__).resolve().parent +FLOW = HERE.parents[1] +#: The program taught on the fixture (scripts/openimis_claims_demo.py record +#: + compile); outside the repository. +SOURCE = Path( + os.environ.get( + "OPENIMIS_SOURCE_BUNDLE", + Path.home() / "oa/scratch/second-installation/openimis/bundle", + ) +) +#: The demo dataset's actor. openIMIS ships no separate service account in +#: the demo data, so the dedicated user is the fixture's one actor. +ACTOR = "Admin" + + +class OpenIMISSignedInUser: + """P2 for openIMIS: the signed-in login, from the app's own API. + + The page asks openIMIS who the current user is with its own session; the + frame is not used. + """ + + kind = "login_id" + + def __init__(self, page: Any, expected: str) -> None: + self._page = page + self._expected = expected + + def reference(self, key: bytes) -> str: + from openadapt_flow.approval_mode.gate import dedicated_user_reference + + return dedicated_user_reference(self._expected, key) + + def check(self, frame_png: bytes) -> str: + try: + login = self._page.evaluate( + """async () => { + const r = await fetch('/api/graphql', { + method: 'POST', credentials: 'include', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({query: '{ me { username } }'})}); + const j = await r.json(); + return j && j.data && j.data.me ? j.data.me.username : null; + }""" + ) + except Exception: # noqa: BLE001 - an unreadable session is not a match + return "unreadable" + if not isinstance(login, str) or not login: + return "unreadable" + return "match" if login == self._expected else "mismatch" + + +class OpenIMISTestCopy: + """The fixture as the configuration's test copy.""" + + def __init__(self) -> None: + from benchmark.openimis_claims import qualify_hooks + + self.hooks = qualify_hooks + self.fx = qualify_hooks.fixture() + + def point_hooks_here(self) -> None: + os.environ.setdefault("OPENIMIS_FRONT_URL", self.fx.front_url) + os.environ.setdefault("OPENIMIS_DB_PORT", str(self.fx.db_port)) + self.hooks.ensure_claims_oracle() + + def live_claim_ids(self) -> set[str]: + out = self.fx._psql( + 'SELECT "ClaimID" FROM "tblClaim" WHERE "ValidityTo" IS NULL;' + ) + return {line.strip() for line in out.splitlines() if line.strip().isdigit()} + + def dump(self) -> set[str]: + """The live claims now; ``restore`` voids any claim entered since.""" + + return self.live_claim_ids() + + def restore(self, before: set[str]) -> None: + added = sorted(self.live_claim_ids() - before, key=int) + if added: + self.fx._psql( + 'UPDATE "tblClaim" SET "ValidityTo" = now() WHERE "ClaimID" IN (' + + ",".join(added) + + ");" + ) + + def reset_for_admission(self) -> None: + self.hooks.reset() + + +OPENIMIS = AppProfile( + name="openimis", + verify_yaml=FLOW / "benchmark/openimis_claims/verify.claims.yaml", + source=SOURCE, + dedicated_user=ACTOR, + workflow_id="workflow-openimis-claims-intake", + make_app=OpenIMISTestCopy, + signed_in_user=lambda page, user: OpenIMISSignedInUser(page, user), +) diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/compiled-program.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/compiled-program.json new file mode 100644 index 000000000..01fbecc63 --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/compiled-program.json @@ -0,0 +1,580 @@ +{ + "bundle": "medplum-new-patient (compiled from the recorded demonstration)", + "steps": [ + { + "id": "step_000", + "action": "click", + "intent": "click 'Add Name'", + "param": null, + "risk": "reversible", + "risk_explanation": "operator-qualified override: reversible", + "identity_armed": false, + "identity_unarmed_reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": null, + "frame_path": null, + "role": "button", + "name": "Add Name", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_001", + "action": "click", + "intent": "click 'Given'", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": true, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-j4si62vtk", + "frame_path": null, + "role": "textbox", + "name": "Given", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_002", + "action": "type", + "intent": "type ", + "param": "fname", + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_003", + "action": "click", + "intent": "click 'Family'", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": true, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-scw390z8r", + "frame_path": null, + "role": "textbox", + "name": "Family", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_004", + "action": "type", + "intent": "type ", + "param": "lname", + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_005", + "action": "scroll", + "intent": "scroll by (0, 405)", + "param": null, + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_006", + "action": "click", + "intent": "click 'AddTelecom'", + "param": null, + "risk": "reversible", + "risk_explanation": "operator-qualified override: reversible", + "identity_armed": false, + "identity_unarmed_reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": null, + "frame_path": null, + "role": "button", + "name": "Add Telecom", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_007", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": false, + "identity_unarmed_reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-8rwicepc6", + "frame_path": null, + "role": "combobox", + "name": "emailphonefaxpagersmsurlother", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_008", + "action": "type", + "intent": "type 'phone'", + "param": null, + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_009", + "action": "key", + "intent": "press Escape", + "param": null, + "risk": "reversible", + "risk_explanation": "special key may navigate, edit, or invoke an application shortcut", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_010", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": false, + "identity_unarmed_reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-pcdz3ddrq", + "frame_path": null, + "role": "textbox", + "name": "Value", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_011", + "action": "type", + "intent": "type ", + "param": "phone_home", + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_012", + "action": "click", + "intent": "click 'AddTelecom'", + "param": null, + "risk": "reversible", + "risk_explanation": "operator-qualified override: reversible", + "identity_armed": false, + "identity_unarmed_reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": null, + "frame_path": null, + "role": "button", + "name": "Add Telecom", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_013", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": false, + "identity_unarmed_reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-zd6i4xm3j", + "frame_path": null, + "role": "combobox", + "name": "emailphonefaxpagersmsurlother", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_014", + "action": "type", + "intent": "type 'email'", + "param": null, + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_015", + "action": "key", + "intent": "press Escape", + "param": null, + "risk": "reversible", + "risk_explanation": "special key may navigate, edit, or invoke an application shortcut", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_016", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": false, + "identity_unarmed_reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-4gcsbn8dn", + "frame_path": null, + "role": "textbox", + "name": "Value", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_017", + "action": "type", + "intent": "type ", + "param": "email", + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_018", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": false, + "identity_unarmed_reason": "no readable text in the target's row band at compile time (icon-only or unlabeled row)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-eho9zz1xt", + "frame_path": null, + "role": "searchbox", + "name": "gender", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_019", + "action": "type", + "intent": "type ", + "param": "sex", + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_020", + "action": "key", + "intent": "press ArrowDown", + "param": null, + "risk": "reversible", + "risk_explanation": "special key may navigate, edit, or invoke an application shortcut", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_021", + "action": "key", + "intent": "press Enter", + "param": null, + "risk": "reversible", + "risk_explanation": "submission/deletion key may edit locally or commit application state", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_022", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": false, + "identity_unarmed_reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#birthDate", + "frame_path": null, + "role": "textbox", + "name": "Birth Date", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_023", + "action": "type", + "intent": "type ", + "param": "DOB", + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_024", + "action": "scroll", + "intent": "scroll by (0, 480)", + "param": null, + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_025", + "action": "click", + "intent": "click 'Add Address'", + "param": null, + "risk": "reversible", + "risk_explanation": "operator-qualified override: reversible", + "identity_armed": false, + "identity_unarmed_reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": null, + "frame_path": null, + "role": "button", + "name": "Add Address", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_026", + "action": "click", + "intent": "click 'Line 1'", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": true, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-eiknzrpio", + "frame_path": null, + "role": "textbox", + "name": "Line 1", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_027", + "action": "type", + "intent": "type ", + "param": "street", + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_028", + "action": "click", + "intent": "click 'City'", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": true, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-2jpho0ebh", + "frame_path": null, + "role": "textbox", + "name": "City", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_029", + "action": "type", + "intent": "type ", + "param": "city", + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_030", + "action": "click", + "intent": "click 'State'", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": true, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-msd66x4vr", + "frame_path": null, + "role": "textbox", + "name": "State", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_031", + "action": "type", + "intent": "type ", + "param": "state_label", + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_032", + "action": "click", + "intent": "click 'Postal Code'", + "param": null, + "risk": "reversible", + "risk_explanation": "control label has no retained consequential-write signal", + "identity_armed": true, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": { + "selector": "#mantine-i4w03ghi3", + "frame_path": null, + "role": "textbox", + "name": "Postal Code", + "automation_id": null, + "window_name": null + } + }, + { + "id": "step_033", + "action": "type", + "intent": "type ", + "param": "postal_code", + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_034", + "action": "scroll", + "intent": "scroll by (0, 926)", + "param": null, + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_035", + "action": "scroll", + "intent": "scroll by (0, 412)", + "param": null, + "risk": "reversible", + "risk_explanation": "action has no retained consequential commit signal", + "identity_armed": null, + "identity_unarmed_reason": null, + "submit_binding_unbound_reason": null, + "structural": null + }, + { + "id": "step_036", + "action": "click", + "intent": "click 'Create'", + "param": null, + "risk": "irreversible", + "risk_explanation": "retained action context names a consequential state change", + "identity_armed": false, + "identity_unarmed_reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "submit_binding_unbound_reason": null, + "structural": { + "selector": null, + "frame_path": null, + "role": "button", + "name": "Create", + "automation_id": null, + "window_name": null + } + } + ], + "params": [ + "DOB", + "city", + "email", + "fname", + "lname", + "phone_home", + "postal_code", + "sex", + "state_label", + "street" + ] +} \ No newline at end of file diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/compiled-bundle2.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/compiled-bundle2.json new file mode 100644 index 000000000..94150b25b --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/compiled-bundle2.json @@ -0,0 +1,461 @@ +{ + "program": "bundle2", + "steps": [ + { + "id": "step_000", + "action": "click", + "intent": "click 'Add Name'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": false, + "identity_unarmed_reason": "the only readable row text is the target's own label (mutable evidence, excluded from identity)", + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_001", + "action": "click", + "intent": "click 'Given'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_002", + "action": "type", + "intent": "type ", + "param": "fname", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_003", + "action": "click", + "intent": "click 'Family'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_004", + "action": "type", + "intent": "type ", + "param": "lname", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_005", + "action": "scroll", + "intent": "scroll by (0, 405)", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_006", + "action": "click", + "intent": "click 'AddTelecom'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": { + "filled_fields": 2, + "submit_role": "form_control" + }, + "postconditions": 2 + }, + { + "id": "step_007", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": false, + "identity_unarmed_reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_008", + "action": "type", + "intent": "type 'phone'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_009", + "action": "key", + "intent": "press Escape", + "param": null, + "key": "Escape", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_010", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": false, + "identity_unarmed_reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_011", + "action": "type", + "intent": "type ", + "param": "phone_home", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_012", + "action": "click", + "intent": "click 'AddTelecom'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": { + "filled_fields": 4, + "submit_role": "form_control" + }, + "postconditions": 2 + }, + { + "id": "step_013", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": false, + "identity_unarmed_reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_014", + "action": "type", + "intent": "type 'email'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_015", + "action": "key", + "intent": "press Escape", + "param": null, + "key": "Escape", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_016", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": false, + "identity_unarmed_reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_017", + "action": "type", + "intent": "type ", + "param": "email", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_018", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": false, + "identity_unarmed_reason": "no readable text in the target's row band at compile time (icon-only or unlabeled row)", + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_019", + "action": "type", + "intent": "type ", + "param": "sex", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_020", + "action": "key", + "intent": "press ArrowDown", + "param": null, + "key": "ArrowDown", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_021", + "action": "key", + "intent": "press Enter", + "param": null, + "key": "Enter", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_022", + "action": "click", + "intent": "click recorded visual target", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": false, + "identity_unarmed_reason": "row text outside the target's label is too generic (< 12 squashed chars after volatile-line filtering)", + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_023", + "action": "type", + "intent": "type ", + "param": "DOB", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_024", + "action": "scroll", + "intent": "scroll by (0, 480)", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_025", + "action": "click", + "intent": "click 'Add Address'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": { + "filled_fields": 5, + "submit_role": "form_control" + }, + "postconditions": 2 + }, + { + "id": "step_026", + "action": "click", + "intent": "click 'Line 1'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_027", + "action": "type", + "intent": "type ", + "param": "street", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_028", + "action": "click", + "intent": "click 'City'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_029", + "action": "type", + "intent": "type ", + "param": "city", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_030", + "action": "click", + "intent": "click 'State'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_031", + "action": "type", + "intent": "type ", + "param": "state_label", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_032", + "action": "click", + "intent": "click 'Postal Code'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_033", + "action": "type", + "intent": "type ", + "param": "postal_code", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_034", + "action": "scroll", + "intent": "scroll by (0, 926)", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_035", + "action": "scroll", + "intent": "scroll by (0, 412)", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_036", + "action": "click", + "intent": "click 'Create'", + "param": null, + "key": null, + "risk": "irreversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": { + "filled_fields": 9, + "submit_role": "form_submit" + }, + "postconditions": 1 + } + ] +} \ No newline at end of file diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/compiled-bundle3.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/compiled-bundle3.json new file mode 100644 index 000000000..536082906 --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/compiled-bundle3.json @@ -0,0 +1,718 @@ +{ + "program": "bundle3", + "steps": [ + { + "id": "step_000", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_001", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_002", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_003", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_004", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_005", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_006", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_007", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_008", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_009", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_010", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_011", + "action": "key", + "intent": "press Space", + "param": null, + "key": "Space", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_012", + "action": "click", + "intent": "click 'Given'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_013", + "action": "type", + "intent": "type ", + "param": "fname", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_014", + "action": "click", + "intent": "click 'Family'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_015", + "action": "type", + "intent": "type ", + "param": "lname", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_016", + "action": "scroll", + "intent": "scroll by (0, 405)", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_017", + "action": "click", + "intent": "click 'AddTelecom'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": { + "filled_fields": 2, + "submit_role": "form_control" + }, + "postconditions": 2 + }, + { + "id": "step_018", + "action": "key", + "intent": "press Shift+Tab", + "param": null, + "key": "Shift+Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_019", + "action": "key", + "intent": "press Shift+Tab", + "param": null, + "key": "Shift+Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_020", + "action": "key", + "intent": "press Shift+Tab", + "param": null, + "key": "Shift+Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_021", + "action": "key", + "intent": "press Shift+Tab", + "param": null, + "key": "Shift+Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_022", + "action": "type", + "intent": "type 'phone'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_023", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_024", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_025", + "action": "type", + "intent": "type ", + "param": "phone_home", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_026", + "action": "click", + "intent": "click 'AddTelecom'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": { + "filled_fields": 4, + "submit_role": "form_control" + }, + "postconditions": 2 + }, + { + "id": "step_027", + "action": "key", + "intent": "press Shift+Tab", + "param": null, + "key": "Shift+Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_028", + "action": "key", + "intent": "press Shift+Tab", + "param": null, + "key": "Shift+Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_029", + "action": "key", + "intent": "press Shift+Tab", + "param": null, + "key": "Shift+Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_030", + "action": "key", + "intent": "press Shift+Tab", + "param": null, + "key": "Shift+Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_031", + "action": "type", + "intent": "type 'email'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_032", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_033", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_034", + "action": "type", + "intent": "type ", + "param": "email", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_035", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_036", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_037", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_038", + "action": "key", + "intent": "press Tab", + "param": null, + "key": "Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_039", + "action": "type", + "intent": "type ", + "param": "DOB", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_040", + "action": "key", + "intent": "press Shift+Tab", + "param": null, + "key": "Shift+Tab", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 2 + }, + { + "id": "step_041", + "action": "type", + "intent": "type ", + "param": "sex", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_042", + "action": "key", + "intent": "press ArrowDown", + "param": null, + "key": "ArrowDown", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_043", + "action": "key", + "intent": "press Enter", + "param": null, + "key": "Enter", + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_044", + "action": "scroll", + "intent": "scroll by (0, 480)", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_045", + "action": "click", + "intent": "click 'Add Address'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": { + "filled_fields": 5, + "submit_role": "form_control" + }, + "postconditions": 2 + }, + { + "id": "step_046", + "action": "click", + "intent": "click 'Line 1'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_047", + "action": "type", + "intent": "type ", + "param": "street", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_048", + "action": "click", + "intent": "click 'City'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_049", + "action": "type", + "intent": "type ", + "param": "city", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_050", + "action": "click", + "intent": "click 'State'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_051", + "action": "type", + "intent": "type ", + "param": "state_label", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_052", + "action": "click", + "intent": "click 'Postal Code'", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 1 + }, + { + "id": "step_053", + "action": "type", + "intent": "type ", + "param": "postal_code", + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_054", + "action": "scroll", + "intent": "scroll by (0, 926)", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_055", + "action": "scroll", + "intent": "scroll by (0, 412)", + "param": null, + "key": null, + "risk": "reversible", + "identity_armed": null, + "identity_unarmed_reason": null, + "form_identity": null, + "postconditions": 0 + }, + { + "id": "step_056", + "action": "click", + "intent": "click 'Create'", + "param": null, + "key": null, + "risk": "irreversible", + "identity_armed": true, + "identity_unarmed_reason": null, + "form_identity": { + "filled_fields": 9, + "submit_role": "form_submit" + }, + "postconditions": 1 + } + ], + "clinical_write_violations": [ + { + "rule": "prohibit_vacuous_postconditions", + "step": "step_035", + "reason": "key step asserts no postcondition \u2014 passes vacuously at replay" + }, + { + "rule": "prohibit_vacuous_postconditions", + "step": "step_042", + "reason": "key step asserts no postcondition \u2014 passes vacuously at replay" + }, + { + "rule": "max_unverified_steps", + "step": null, + "reason": "2 vacuous (no-postcondition) steps exceed the limit of 0: step_035, step_042" + } + ] +} \ No newline at end of file diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/qualify-2-unarmed-field-clicks.log b/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/qualify-2-unarmed-field-clicks.log new file mode 100644 index 000000000..31eccc124 --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/qualify-2-unarmed-field-clicks.log @@ -0,0 +1,15 @@ +qualify REFUSED: REFUSE: workflow 'medplum-new-patient' under 'standard' profile vs policy 'clinical-write' + [PASS] Execution profile: standard (production); durability required and enabled; settled-state detection required and enabled + [REFUSE] Certification: bundle is NOT certified under 'clinical-write': 18 policy violation(s) (e.g. prohibit_unarmed_clicks: identity-applicable step is UNARMED — would act with no identity verification (the only readable row text is the target's own label (mutable evidence, excluded from identity))) + steps: step_000, step_007, step_009, step_010, step_013, step_015, step_016, step_018, ... (+2 more) + [REFUSE] Identity coverage: 7/17 entity-sensitive/consequential action(s) are UNARMED -- would act with no verified target identity + steps: step_000, step_007, step_010, step_013, step_016, step_018, step_022 + [PASS] Effect coverage: 1/1 consequential write(s) declare the required postcondition and confirmed system-of-record effect contracts + [PASS] Approval fallback: a verifier satisfying the required evidence tier is configured; 1 declared write(s) are covered + [PASS] Interstitial admission: no automatic interstitial actions declared + [WARN] Encrypted bundle: workflow.json plaintext (not required); WARNING: 43 template/screenshot asset(s) are unsealed (plaintext at rest) -- protect via disk encryption or run with --strict-templates to refuse + steps: templates/identifiers/step_001.png, templates/identifiers/step_003.png, templates/identifiers/step_026.png, templates/identifiers/step_028.png, templates/identifiers/step_030.png, templates/identifiers/step_032.png, templates/layout_fingerprint.json, templates/step_000.png, ... (+35 more) + [PASS] Sealed manifest + version pin: integrity-sealed manifest re-verified (digest cf5ee695cb4ae217..., compiler 2.3.0) + -> RUN REFUSED (fail-closed): Certification, Identity coverage. Nothing was executed. +Evidence so far: /Users/abrichr/oa/scratch/second-installation/medplum/qual2 +EXIT 1 diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/qualify-3-vacuous-key-steps.log b/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/qualify-3-vacuous-key-steps.log new file mode 100644 index 000000000..894a59a3a --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/lease-2/qualify-3-vacuous-key-steps.log @@ -0,0 +1,14 @@ +qualify REFUSED: REFUSE: workflow 'medplum-new-patient' under 'standard' profile vs policy 'clinical-write' + [PASS] Execution profile: standard (production); durability required and enabled; settled-state detection required and enabled + [REFUSE] Certification: bundle is NOT certified under 'clinical-write': 3 policy violation(s) (e.g. prohibit_vacuous_postconditions: key step asserts no postcondition — passes vacuously at replay) + steps: step_035, step_042 + [PASS] Identity coverage: 10/10 entity-sensitive/consequential action(s) identity-armed + [PASS] Effect coverage: 1/1 consequential write(s) declare the required postcondition and confirmed system-of-record effect contracts + [PASS] Approval fallback: a verifier satisfying the required evidence tier is configured; 1 declared write(s) are covered + [PASS] Interstitial admission: no automatic interstitial actions declared + [WARN] Encrypted bundle: workflow.json plaintext (not required); WARNING: 56 template/screenshot asset(s) are unsealed (plaintext at rest) -- protect via disk encryption or run with --strict-templates to refuse + steps: templates/identifiers/step_012.png, templates/identifiers/step_014.png, templates/identifiers/step_046.png, templates/identifiers/step_048.png, templates/identifiers/step_050.png, templates/identifiers/step_052.png, templates/layout_fingerprint.json, templates/step_000_expect.png, ... (+48 more) + [PASS] Sealed manifest + version pin: integrity-sealed manifest re-verified (digest e6988b422384a224..., compiler 2.3.0) + -> RUN REFUSED (fail-closed): Certification. Nothing was executed. +Evidence so far: /Users/abrichr/oa/scratch/second-installation/medplum/qual3 +EXIT 1 diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/channels.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/channels.json new file mode 100644 index 000000000..776b4c438 --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/channels.json @@ -0,0 +1,17 @@ +[ + { + "origin": "fhir search Patient", + "section": { + "kind": "fhir", + "base_url": "http://localhost:9410/fhir/R4", + "resource_type": "Patient", + "search_params": { + "_count": "200" + }, + "access_token_env": "MEDPLUM_RECORD_TOKEN", + "field_paths": { + "id": "id" + } + } + } +] diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo1/params.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo1/params.json new file mode 100644 index 000000000..8bfaab253 --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo1/params.json @@ -0,0 +1,12 @@ +{ + "fname": "Rowan", + "lname": "Achterberg", + "DOB": "1979-11-23", + "sex": "female", + "street": "41 Quarry Lane", + "city": "Worcester", + "state_label": "Massachusetts", + "postal_code": "01609", + "phone_home": "508-555-0147", + "email": "rowan.achterberg@example.invalid" +} \ No newline at end of file diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo1/reads/ch-87610b816c22.after.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo1/reads/ch-87610b816c22.after.json new file mode 100644 index 000000000..53f05dad1 --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo1/reads/ch-87610b816c22.after.json @@ -0,0 +1,5 @@ +[ + { + "id": "574d595a-d487-4005-9ca1-ab51d189bd7a" + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo1/reads/ch-87610b816c22.before.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo1/reads/ch-87610b816c22.before.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo1/reads/ch-87610b816c22.before.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo2/params.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo2/params.json new file mode 100644 index 000000000..af8b81336 --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo2/params.json @@ -0,0 +1 @@ +{"fname": "Teodora", "lname": "Vasquez-Lind", "DOB": "1992-04-30", "sex": "female", "street": "7 Elm Row", "city": "Lowell", "state_label": "Massachusetts", "postal_code": "01852", "phone_home": "978-555-0112", "email": "teodora.vasquezlind@example.invalid"} diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo2/reads/ch-87610b816c22.after.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo2/reads/ch-87610b816c22.after.json new file mode 100644 index 000000000..d49f40b8e --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo2/reads/ch-87610b816c22.after.json @@ -0,0 +1,5 @@ +[ + { + "id": "a50d0ef8-f76c-4159-a822-4699645d56ed" + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo2/reads/ch-87610b816c22.before.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo2/reads/ch-87610b816c22.before.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo2/reads/ch-87610b816c22.before.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo3/params.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo3/params.json new file mode 100644 index 000000000..a18fd7552 --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo3/params.json @@ -0,0 +1 @@ +{"fname": "Marcus", "lname": "Oyelaran", "DOB": "1965-12-02", "sex": "male", "street": "220 Harbor Street", "city": "Quincy", "state_label": "Massachusetts", "postal_code": "02169", "phone_home": "617-555-0190", "email": "marcus.oyelaran@example.invalid"} diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo3/reads/ch-87610b816c22.after.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo3/reads/ch-87610b816c22.after.json new file mode 100644 index 000000000..ecbdf4b85 --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo3/reads/ch-87610b816c22.after.json @@ -0,0 +1,8 @@ +[ + { + "id": "a50d0ef8-f76c-4159-a822-4699645d56ed" + }, + { + "id": "5a9a906b-8776-4420-9f94-03384cade464" + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo3/reads/ch-87610b816c22.before.json b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo3/reads/ch-87610b816c22.before.json new file mode 100644 index 000000000..d49f40b8e --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/phase-b-inputs/demo3/reads/ch-87610b816c22.before.json @@ -0,0 +1,5 @@ +[ + { + "id": "a50d0ef8-f76c-4159-a822-4699645d56ed" + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/second_installation/results/medplum-phase-a/qualify-refusal.log b/packages/flow/benchmark/second_installation/results/medplum-phase-a/qualify-refusal.log new file mode 100644 index 000000000..091609ca2 --- /dev/null +++ b/packages/flow/benchmark/second_installation/results/medplum-phase-a/qualify-refusal.log @@ -0,0 +1,4 @@ +verify.yaml cannot qualify this bundle: + - save step step_036 has no armed identity check + - save step step_036 has no structural selector: the web fault driver needs it for the ambiguity and identity faults +EXIT 2 diff --git a/packages/flow/openadapt_flow/backends/playwright_backend.py b/packages/flow/openadapt_flow/backends/playwright_backend.py index 5778c218b..6ab33b3ec 100644 --- a/packages/flow/openadapt_flow/backends/playwright_backend.py +++ b/packages/flow/openadapt_flow/backends/playwright_backend.py @@ -396,6 +396,32 @@ class _FramePoint: ' [role="radio"], [role="switch"]' ) +_ISO_DATE = re.compile(r"\d{4}-\d{2}-\d{2}") + + +def _fill_iso_date(locator: Any, text: str) -> bool: + """Set an ISO date (``YYYY-MM-DD``) into a native ````. + + Chromium splits a date input into locale-ordered segments that a string + can't be typed into (``-`` doesn't advance, and the year segment takes + six digits). ``fill`` takes the ISO value whatever the locale and fires + ``input`` and ``change`` as typing does. Returns whether it filled. + """ + + if not _ISO_DATE.fullmatch(text or ""): + return False + try: + if ( + locator.evaluate("e => e.tagName === 'INPUT' && e.type === 'date'") + is not True + ): + return False + locator.fill(text, timeout=1000) + except Exception: # noqa: BLE001 - not a date input: type it as usual + return False + return True + + _BIND_COORDINATE_TARGET_JS = ( r"""(args) => { const hit = document.elementFromPoint(args.x, args.y); @@ -2977,7 +3003,10 @@ def type_text_guarded( operation="guarded_dom_type", # Keys go through the leased element itself, never the page # keyboard, so delivery stays bound to the checked target. - deliver=lambda locator: locator.press_sequentially(text, timeout=1000), + deliver=lambda locator: ( + _fill_iso_date(locator, text) + or locator.press_sequentially(text, timeout=1000) + ), ) def _handle_screenshot_frame_lifecycle(self, _frame: Any = None) -> None: @@ -3542,7 +3571,10 @@ def _deliver_text(self, text: str, per_key: Callable[[], None]) -> None: """Insert ``text`` at the caret in one call when the focused element is an ordinary text field (native fill: the field's ``input`` event fires, and ``change`` on blur, as for typing); otherwise ``per_key`` - types it key by key.""" + types it key by key. An ISO date into a native date input is set + with ``fill`` (see :func:`_fill_iso_date`).""" + if _fill_iso_date(self.page.locator(":focus"), text): + return if self.native_fill and text and self._focused_plain_text_field(): self.page.keyboard.insert_text(text) return diff --git a/packages/flow/openadapt_flow/runtime/effects/fhir.py b/packages/flow/openadapt_flow/runtime/effects/fhir.py index 912e340b6..ac50f0363 100644 --- a/packages/flow/openadapt_flow/runtime/effects/fhir.py +++ b/packages/flow/openadapt_flow/runtime/effects/fhir.py @@ -80,15 +80,35 @@ def extract_path(resource: dict[str, Any], dotted: str) -> Any: A segment that lands on a list takes the list's first element (the common ``component[0]`` / single-coding case), so ``code.coding.code`` reads the - first coding's code. Returns ``None`` when any segment is missing. + first coding's code. A segment may filter a list first: + ``telecom[system=email].value`` reads the first telecom whose ``system`` + is ``email`` (a Patient keeps its phone and its email in one list). + Returns ``None`` when any segment is missing or no element matches. """ node: Any = resource for seg in dotted.split("."): + name, want = seg, None + if seg.endswith("]") and "[" in seg: + name, _, cond = seg[:-1].partition("[") + key, eq, value = cond.partition("=") + if not eq or not key: + return None + want = (key, value) if isinstance(node, list): node = node[0] if node else None if not isinstance(node, dict): return None - node = node.get(seg) + node = node.get(name) + if want is not None: + items = node if isinstance(node, list) else [node] + node = next( + ( + item + for item in items + if isinstance(item, dict) and str(item.get(want[0])) == want[1] + ), + None, + ) if isinstance(node, list): node = node[0] if node else None return node diff --git a/packages/flow/tests/test_fhir_extract_path_filter.py b/packages/flow/tests/test_fhir_extract_path_filter.py new file mode 100644 index 000000000..7973063f7 --- /dev/null +++ b/packages/flow/tests/test_fhir_extract_path_filter.py @@ -0,0 +1,35 @@ +"""``extract_path`` reads one element of a FHIR list by a filter. + +A Patient keeps its phone and its email in one ``telecom`` list, so the first +element is not always the one a record check means. The values are made up. +""" + +from openadapt_flow.runtime.effects.fhir import extract_path + +PATIENT = { + "resourceType": "Patient", + "name": [{"family": "Okafor", "given": ["Ada", "N"]}], + "telecom": [ + {"system": "phone", "use": "home", "value": "555-0100"}, + {"system": "email", "value": "ada.okafor@example.invalid"}, + ], +} + + +def test_a_filter_picks_the_matching_element(): + assert extract_path(PATIENT, "telecom[system=email].value") == ( + "ada.okafor@example.invalid" + ) + assert extract_path(PATIENT, "telecom[system=phone].value") == "555-0100" + + +def test_no_matching_element_reads_nothing(): + assert extract_path(PATIENT, "telecom[system=fax].value") is None + assert extract_path(PATIENT, "telecom[system].value") is None + + +def test_paths_without_a_filter_are_unchanged(): + assert extract_path(PATIENT, "telecom.value") == "555-0100" + assert extract_path(PATIENT, "name.family") == "Okafor" + assert extract_path(PATIENT, "name.given") == "Ada" + assert extract_path(PATIENT, "missing.path") is None diff --git a/packages/flow/tests/test_playwright_date_input.py b/packages/flow/tests/test_playwright_date_input.py new file mode 100644 index 000000000..9af3e9747 --- /dev/null +++ b/packages/flow/tests/test_playwright_date_input.py @@ -0,0 +1,48 @@ +"""An ISO date typed into a native date input lands as that date. + +Chromium splits ```` into locale-ordered segments: a typed +``1984-03-07`` leaves it empty, because ``-`` doesn't move to the next +segment. The backend sets an ISO date with ``fill`` instead, which takes the +ISO value whatever the locale. Other text is typed as before. +""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("playwright") + +PAGE = ( + "data:text/html," + "" +) + + +@pytest.fixture +def backend(): + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + try: + backend, close = PlaywrightBackend.launch(PAGE) + except Exception as exc: # noqa: BLE001 - no browser on this machine + pytest.skip(f"chromium unavailable: {exc}") + yield backend + close() + + +def test_an_iso_date_lands_in_a_date_input(backend): + backend.page.focus("#d") + backend.type_text("1984-03-07") + assert backend.page.input_value("#d") == "1984-03-07" + assert backend.page.evaluate("window.changes") >= 1 + + +def test_other_text_and_other_fields_are_typed(backend): + backend.page.focus("#t") + backend.type_text("1984-03-07") + assert backend.page.input_value("#t") == "1984-03-07" + backend.page.focus("#d") + backend.type_text("not a date") + assert backend.page.input_value("#d") == ""