diff --git a/CAPABILITIES.md b/CAPABILITIES.md index abe725d13..cd646be8e 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -249,7 +249,7 @@ improvising or retrying when something is off. | See a compiled program as a map (every step, exact edge, loop and stop rule, with each step's checks) and play a recorded run over it; one React package that Desktop, app.openadapt.ai and openadapt.ai share | On main ([packages/program-graph](packages/program-graph/README.md)) | Real compiled OpenEMR referral program and a recorded scoreboard run, both on synthetic data | `@openadapt/program-graph`; `openadapt-flow visualize --format json` | | Encrypt PHI at rest: every bundle file that can hold patient data under one sealed index (unknown files refused), web recordings from the first frame, and the run directories of sealed bundles; compile, induce, scaffold-verifier, qualify, admit, run, resume and undo read them in memory and refuse without the key | On main | Synthetic (no-plaintext audit-hook test over record, compile and a VERIFIED run; per-frame cost measured) | `openadapt-flow seal`, `record`; [phi_at_rest.md](packages/flow/docs/phi_at_rest.md) | | Signed receipts (Seals), countersigned and timestamped, offline by default | Released | Unit tests, real TSA tokens | `openadapt-flow seal`, `countersign`, `local-seal` | -| Offline Seals signed with a key the customer holds: a key file, the OS keychain, or a PKCS#11 token or HSM; pinned keys with rotation and revocation; no fallback to a generated key | On main ([#1392](https://github.com/OpenAdaptAI/OpenAdapt/pull/1392)) | Unit tests with synthetic data (keychain and PKCS#11 through stand-ins) | `openadapt-flow local-seal --key`, `local-seal key create`, `pin`, `revoke`; [OFFLINE_SEALS.md](packages/flow/docs/OFFLINE_SEALS.md#customer-held-keys) | +| Offline Seals signed with a key the customer holds: a key file, the OS keychain, or a PKCS#11 token or HSM; pinned keys with rotation and revocation; no fallback to a generated key | On main ([#1392](https://github.com/OpenAdaptAI/OpenAdapt/pull/1392)) | Unit tests with synthetic data (keychain and PKCS#11 through stand-ins) | `openadapt-flow local-seal --key`, `local-seal key create`, `pin`, `revoke`; `openadapt seal verify FOLDER` checks every sealed run in a folder; [OFFLINE_SEALS.md](packages/flow/docs/OFFLINE_SEALS.md#customer-held-keys) | | Anchor an offline Seal in the public Sigstore Rekor transparency log (opt-in; sends one digest and the public key), and verify the anchor offline from the stored inclusion proof | On main ([#1394](https://github.com/OpenAdaptAI/OpenAdapt/pull/1394)) | Unit tests against a simulated Rekor log; Ed25519ph matches the RFC 8032 test vector | `openadapt-flow local-seal anchor`, `local-seal verify --require-anchor`; [OFFLINE_SEALS.md](packages/flow/docs/OFFLINE_SEALS.md#optional-anchor-a-seal-in-a-public-transparency-log) | | Strict no-outbound mode and an unattended on-prem service | Released | Synthetic | `openadapt-flow no-outbound`, `service`; [ON_PREM.md](packages/flow/docs/ON_PREM.md) | diff --git a/docs/quickstart/README.md b/docs/quickstart/README.md new file mode 100644 index 000000000..e7ef7bd20 --- /dev/null +++ b/docs/quickstart/README.md @@ -0,0 +1,12 @@ +# Quickstart capture and timing + +- `quickstart.gif`: `openadapt quickstart` start to finish, recorded with + [VHS](https://github.com/charmbracelet/vhs) from `quickstart.tape`. The + browser is hidden in the recording; the output is otherwise the real command + at real speed. Re-record with + `PATH="$PWD/.venv/bin:$PATH" vhs docs/quickstart/quickstart.tape`. +- `inline-screenshot.png`: the third act on a terminal that draws images + (WezTerm), with the halted run's screenshot beside the database's answer. +- `timing-2026-10-20.json`: the engine tutorial with `--break-it` (what + `openadapt quickstart --break-it` ran before) against the new quickstart, + three rounds each, interleaved, from `scripts/quickstart_timing.py`. diff --git a/docs/quickstart/inline-screenshot.png b/docs/quickstart/inline-screenshot.png new file mode 100644 index 000000000..d3cc19916 Binary files /dev/null and b/docs/quickstart/inline-screenshot.png differ diff --git a/docs/quickstart/quickstart.gif b/docs/quickstart/quickstart.gif new file mode 100644 index 000000000..2eecca333 Binary files /dev/null and b/docs/quickstart/quickstart.gif differ diff --git a/docs/quickstart/quickstart.tape b/docs/quickstart/quickstart.tape new file mode 100644 index 000000000..d8033ce25 --- /dev/null +++ b/docs/quickstart/quickstart.tape @@ -0,0 +1,33 @@ +# Records docs/quickstart/quickstart.gif with VHS (https://github.com/charmbracelet/vhs, MIT). +# +# PATH="$PWD/.venv/bin:$PATH" vhs docs/quickstart/quickstart.tape +# +# The browser stays hidden (no display in the recording) and the side-by-side +# page is not opened, so nothing appears outside the terminal. Everything else +# is the real command, start to finish, at real speed. + +Output docs/quickstart/quickstart.gif + +Set Shell bash +Set FontSize 15 +Set Width 1240 +Set Height 1060 +Set Padding 24 +Set Framerate 24 +Set TypingSpeed 45ms +Set Theme "Builtin Dark" + +Hide +Type "unset DISPLAY WAYLAND_DISPLAY; export SSH_CONNECTION=recording BROWSER=true OPENADAPT_SEAL_HOME=$(mktemp -d) PS1='$ '; cd $(mktemp -d) && clear" +Enter +Show + +Sleep 500ms +Type "openadapt quickstart" +Sleep 400ms +Enter +Wait+Screen@180s /Press Enter/ +Sleep 2s +Enter +Wait+Screen@180s /Give your agent/ +Sleep 6s diff --git a/docs/quickstart/timing-2026-10-20-parallel.json b/docs/quickstart/timing-2026-10-20-parallel.json new file mode 100644 index 000000000..f04cda84e --- /dev/null +++ b/docs/quickstart/timing-2026-10-20-parallel.json @@ -0,0 +1,94 @@ +{ + "machine": "Darwin arm64", + "python": "3.12.13", + "commands": { + "before": "flow tutorial --break-it --out", + "after": "quickstart --headless --out" + }, + "runs": [ + { + "command": "before", + "round": 0, + "exit_code": 0, + "wall_s": 21.6, + "load_1m": 9.3, + "lines": 54 + }, + { + "command": "after", + "round": 0, + "exit_code": 0, + "wall_s": 22.58, + "load_1m": 10.0, + "lines": 39 + }, + { + "command": "before", + "round": 1, + "exit_code": 0, + "wall_s": 21.02, + "load_1m": 9.8, + "lines": 54 + }, + { + "command": "after", + "round": 1, + "exit_code": 0, + "wall_s": 22.09, + "load_1m": 9.7, + "lines": 39 + }, + { + "command": "before", + "round": 2, + "exit_code": 0, + "wall_s": 22.26, + "load_1m": 10.3, + "lines": 54 + }, + { + "command": "after", + "round": 2, + "exit_code": 0, + "wall_s": 22.31, + "load_1m": 30.5, + "lines": 39 + }, + { + "command": "before", + "round": 3, + "exit_code": 0, + "wall_s": 21.45, + "load_1m": 25.8, + "lines": 54 + }, + { + "command": "after", + "round": 3, + "exit_code": 0, + "wall_s": 21.56, + "load_1m": 28.4, + "lines": 39 + }, + { + "command": "before", + "round": 4, + "exit_code": 0, + "wall_s": 22.95, + "load_1m": 33.9, + "lines": 54 + }, + { + "command": "after", + "round": 4, + "exit_code": 0, + "wall_s": 24.19, + "load_1m": 35.9, + "lines": 39 + } + ], + "median_wall_s": { + "before": 21.6, + "after": 22.31 + } +} diff --git a/docs/quickstart/timing-2026-10-20.json b/docs/quickstart/timing-2026-10-20.json new file mode 100644 index 000000000..fef056da1 --- /dev/null +++ b/docs/quickstart/timing-2026-10-20.json @@ -0,0 +1,62 @@ +{ + "machine": "Darwin arm64", + "python": "3.12.13", + "commands": { + "before": "flow tutorial --break-it --out", + "after": "quickstart --headless --out" + }, + "runs": [ + { + "command": "before", + "round": 0, + "exit_code": 0, + "wall_s": 21.11, + "load_1m": 19.1, + "lines": 54 + }, + { + "command": "after", + "round": 0, + "exit_code": 0, + "wall_s": 31.44, + "load_1m": 22.9, + "lines": 40 + }, + { + "command": "before", + "round": 1, + "exit_code": 0, + "wall_s": 23.01, + "load_1m": 37.6, + "lines": 54 + }, + { + "command": "after", + "round": 1, + "exit_code": 0, + "wall_s": 40.22, + "load_1m": 46.8, + "lines": 39 + }, + { + "command": "before", + "round": 2, + "exit_code": 0, + "wall_s": 22.12, + "load_1m": 36.7, + "lines": 54 + }, + { + "command": "after", + "round": 2, + "exit_code": 0, + "wall_s": 32.87, + "load_1m": 27.4, + "lines": 39 + } + ], + "median_wall_s": { + "before": 22.12, + "after": 32.87 + } +} diff --git a/llms-full.txt b/llms-full.txt index 9aa1b5778..7b204a0c5 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -645,7 +645,7 @@ improvising or retrying when something is off. | See a compiled program as a map (every step, exact edge, loop and stop rule, with each step's checks) and play a recorded run over it; one React package that Desktop, app.openadapt.ai and openadapt.ai share | On main ([packages/program-graph](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/program-graph/README.md)) | Real compiled OpenEMR referral program and a recorded scoreboard run, both on synthetic data | `@openadapt/program-graph`; `openadapt-flow visualize --format json` | | Encrypt PHI at rest: every bundle file that can hold patient data under one sealed index (unknown files refused), web recordings from the first frame, and the run directories of sealed bundles; compile, induce, scaffold-verifier, qualify, admit, run, resume and undo read them in memory and refuse without the key | On main | Synthetic (no-plaintext audit-hook test over record, compile and a VERIFIED run; per-frame cost measured) | `openadapt-flow seal`, `record`; [phi_at_rest.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/phi_at_rest.md) | | Signed receipts (Seals), countersigned and timestamped, offline by default | Released | Unit tests, real TSA tokens | `openadapt-flow seal`, `countersign`, `local-seal` | -| Offline Seals signed with a key the customer holds: a key file, the OS keychain, or a PKCS#11 token or HSM; pinned keys with rotation and revocation; no fallback to a generated key | On main ([#1392](https://github.com/OpenAdaptAI/OpenAdapt/pull/1392)) | Unit tests with synthetic data (keychain and PKCS#11 through stand-ins) | `openadapt-flow local-seal --key`, `local-seal key create`, `pin`, `revoke`; [OFFLINE_SEALS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/OFFLINE_SEALS.md#customer-held-keys) | +| Offline Seals signed with a key the customer holds: a key file, the OS keychain, or a PKCS#11 token or HSM; pinned keys with rotation and revocation; no fallback to a generated key | On main ([#1392](https://github.com/OpenAdaptAI/OpenAdapt/pull/1392)) | Unit tests with synthetic data (keychain and PKCS#11 through stand-ins) | `openadapt-flow local-seal --key`, `local-seal key create`, `pin`, `revoke`; `openadapt seal verify FOLDER` checks every sealed run in a folder; [OFFLINE_SEALS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/OFFLINE_SEALS.md#customer-held-keys) | | Anchor an offline Seal in the public Sigstore Rekor transparency log (opt-in; sends one digest and the public key), and verify the anchor offline from the stored inclusion proof | On main ([#1394](https://github.com/OpenAdaptAI/OpenAdapt/pull/1394)) | Unit tests against a simulated Rekor log; Ed25519ph matches the RFC 8032 test vector | `openadapt-flow local-seal anchor`, `local-seal verify --require-anchor`; [OFFLINE_SEALS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/OFFLINE_SEALS.md#optional-anchor-a-seal-in-a-public-transparency-log) | | Strict no-outbound mode and an unattended on-prem service | Released | Synthetic | `openadapt-flow no-outbound`, `service`; [ON_PREM.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/ON_PREM.md) | diff --git a/openadapt/cli.py b/openadapt/cli.py index 5e5dcaaf8..9f00b71f5 100644 --- a/openadapt/cli.py +++ b/openadapt/cli.py @@ -29,6 +29,7 @@ import re import shlex import sys +from contextlib import contextmanager from pathlib import Path from typing import Optional @@ -71,11 +72,31 @@ def resolve_command(self, ctx, args): f"No such command '{name}'. Did you mean 'openadapt flow {name}'?", ctx, ) + if not ctx.resilient_parsing and self.get_command(ctx, name) is None: + # A command this launcher doesn't know may be one a newer + # openadapt elsewhere on PATH has. + _warn_about_launcher_conflicts() return super().resolve_command(ctx, args) +def _print_version(ctx: click.Context, _param, value: bool) -> None: + """``--version``, then any other ``openadapt`` on PATH that would shadow it.""" + if not value or ctx.resilient_parsing: + return + click.echo(f"openadapt, version {__version__}") + _warn_about_launcher_conflicts() + ctx.exit() + + @click.group(cls=_FlowFirstGroup) -@click.version_option(version=__version__, prog_name="openadapt") +@click.option( + "--version", + is_flag=True, + expose_value=False, + is_eager=True, + callback=_print_version, + help="Show the version and exit.", +) def main(): """OpenAdapt launcher for the openadapt-flow compiler. @@ -87,11 +108,7 @@ def main(): \b Quick Start (Python 3.10-3.12): python -m pip install --upgrade openadapt - openadapt flow tutorial - - \b - Halt demo: - openadapt quickstart --break-it + openadapt quickstart \b Manual Demo Lifecycle: @@ -302,7 +319,7 @@ def _python_fix() -> str: "have it:\n" f"{installer}" " uv tool install --python 3.12 openadapt\n" - f"Then run: openadapt quickstart --break-it{script}" + f"Then run: openadapt quickstart{script}" ) @@ -440,7 +457,17 @@ def _is_externally_managed_error(error: BaseException) -> bool: # Options quickstart parses itself. Click consumes them before passthrough, # so they only matter here as suggestions for a mistyped flag. -_QUICKSTART_OPTIONS = frozenset({"--out", "--headed", "--break-it", "--help"}) +_QUICKSTART_OPTIONS = frozenset( + { + "--out", + "--headless", + "--no-break", + "--verbose", + "--headed", + "--break-it", + "--help", + } +) def _refuse_unknown_tutorial_options(args: list[str]) -> None: @@ -470,6 +497,41 @@ def _refuse_unknown_tutorial_options(args: list[str]) -> None: raise click.UsageError(message) +@contextmanager +def _synthetic_scrub_off(): + """Keep an unconfigured privacy provider from blocking the synthetic app. + + The bundled tutorial contains only fixed synthetic data. An operator's + explicit setting is left alone; the default is restored afterwards. + """ + scrub = os.environ.get("OPENADAPT_FLOW_SCRUB") + if scrub in (None, "auto"): + os.environ["OPENADAPT_FLOW_SCRUB"] = "off" + try: + yield + finally: + if scrub is None: + os.environ.pop("OPENADAPT_FLOW_SCRUB", None) + elif scrub == "auto": + os.environ["OPENADAPT_FLOW_SCRUB"] = scrub + + +def _quickstart_root(out: Optional[Path]) -> Path: + if out is None: + root = Path(_DEFAULT_QUICKSTART_DIR).resolve() + suffix = 2 + while root.exists(): + root = Path(f"{_DEFAULT_QUICKSTART_DIR}-{suffix}").resolve() + suffix += 1 + return root + root = out.expanduser().resolve() + if root.exists(): + raise click.UsageError( + f"Output already exists: {root}. Pass --out with a new directory." + ) + return root + + @main.command( "quickstart", context_settings={ @@ -482,143 +544,145 @@ def _refuse_unknown_tutorial_options(args: list[str]) -> None: type=click.Path(path_type=Path), default=None, help=( - f"Directory for the recording, bundle, and run report. Default: " + f"Directory for the recording, program, and run reports. Default: " f"{_DEFAULT_QUICKSTART_DIR}, suffixed -2, -3, ... when taken." ), ) @click.option( - "--headed", is_flag=True, help="Show the browser while the tutorial runs." + "--headless", + is_flag=True, + help="Hide the browser. It is shown by default when a display is available.", ) @click.option( - "--break-it", - "break_it", + "--no-break", + "no_break", is_flag=True, - help=( - "After the verified run, rerun the same certified bundle against a " - "fault-injecting backend and watch the engine halt when the save " - "doesn't reach the record." - ), + help="Skip the last part, where one save fails silently.", ) +@click.option( + "--verbose", + is_flag=True, + help="Also print each run's engine detail and report path.", +) +@click.option("--headed", is_flag=True, hidden=True) +@click.option("--break-it", "break_it", is_flag=True, hidden=True) @click.pass_context def quickstart( - command_ctx: click.Context, out: Optional[Path], headed: bool, break_it: bool + command_ctx: click.Context, + out: Optional[Path], + headless: bool, + no_break: bool, + verbose: bool, + headed: bool, + break_it: bool, ) -> None: - """Run a verified local tutorial against the bundled synthetic app. - - This is the shortest path to a real OpenAdapt run. It uses the bundled - synthetic tutorial, verifies the write through an independent read-only - system-of-record interface, keeps every artifact on this computer, and - enables no model or Cloud call. The output directory is never overwritten. - Other flags that the engine tutorial accepts (for example --guided or - --interactive-record) pass through to it unchanged. - """ - import os + """Show a task once, run it, then watch it catch a save that didn't happen. + + About 30 seconds, in a synthetic clinic app on this computer. OpenAdapt + turns one demonstration into a program, runs it five times and checks + every save in the app's database, then makes one save fail silently: the + screen says saved, the database has nothing, and OpenAdapt stops. The + program makes no AI calls. Nothing in the output directory is overwritten. + Other flags of the engine tutorial (for example --guided or + --interactive-record) run that tutorial instead, unchanged. + """ _require_supported_python() _refuse_unknown_tutorial_options(command_ctx.args) + root = _quickstart_root(out) - if out is None: - root = Path(_DEFAULT_QUICKSTART_DIR).resolve() - suffix = 2 - while root.exists(): - root = Path(f"{_DEFAULT_QUICKSTART_DIR}-{suffix}").resolve() - suffix += 1 - click.echo(f"Using output directory: {root}") - else: - root = out.expanduser().resolve() - if root.exists(): - raise click.UsageError( - f"Output already exists: {root}. Pass --out with a new directory." + if command_ctx.args or not _flow_runs_quickstart(): + # Engine-owned flags (--guided, --interactive-record, and future + # engine additions) run the engine's own tutorial, as before. So does + # a Flow older than the pieces the three-act quickstart uses. + _engine_tutorial(root, out is None, headed, break_it, command_ctx.args) + return + + from openadapt.quickstart import next_steps, run_quickstart + from openadapt.quickstart.engine import display_available + + visible = not headless and (headed or display_available()) + steps = next_steps( + _hint, + _shell_quote, + _agent_serves_tutorial(), + keep=_keep_command(), + shadow=_shadowing_openadapt if _under_uvx() else None, + ) + try: + with _synthetic_scrub_off(): + outcome = run_quickstart( + root, + visible=visible, + break_it=not no_break, + verbose=verbose, + steps=steps, ) + except KeyboardInterrupt: + click.echo(f"\nStopped. Anything written so far is in {root}", err=True) + raise click.exceptions.Exit(130) + except Exception as error: + if _is_externally_managed_error(error): + _echo_python_remedy() + raise click.exceptions.Exit(1) from error + if type(error).__name__ in ("TutorialError", "DurableAuthorityBusy"): + raise click.ClickException( + f"The quickstart stopped: {error}\nAnything it wrote is in {root}" + ) from error + raise + if outcome.checked != len(outcome.runs) or ( + outcome.broken is not None and not outcome.broken.stopped + ): + raise click.exceptions.Exit(1) - argv = [ - "tutorial", - "--out", - str(root), - "--name", - "local-quickstart", - ] + +# The oldest Flow with every piece the three-act quickstart uses (shared +# browser, record checks bound to run inputs, the side-by-side page). +_QUICKSTART_FLOW_MINIMUM = (2, 2, 0) + + +def _flow_runs_quickstart() -> bool: + """Return whether the installed Flow can run the three-act quickstart.""" + from importlib.metadata import PackageNotFoundError + from importlib.metadata import version as dist_version + + try: + installed = _release_tuple(dist_version("openadapt-flow")) + except PackageNotFoundError: + return False + return installed is not None and installed >= _QUICKSTART_FLOW_MINIMUM + + +def _engine_tutorial( + root: Path, announce: bool, headed: bool, break_it: bool, extra: list[str] +) -> None: + """Run the engine's own tutorial with engine-only flags, as before.""" + if announce: + click.echo(f"Using output directory: {root}") + argv = ["tutorial", "--out", str(root), "--name", "local-quickstart"] if headed: argv.append("--headed") if break_it: argv.append("--break-it") - # Engine-owned flags (--guided, --interactive-record, and future engine - # additions) forward verbatim instead of being whitelisted here. - argv.extend(command_ctx.args) - - # The bundled tutorial contains only fixed synthetic data. Keep an - # installed-but-unconfigured privacy provider from blocking this known-safe - # fixture, then restore the operator's setting immediately. - scrub = os.environ.get("OPENADAPT_FLOW_SCRUB") - if scrub in (None, "auto"): - os.environ["OPENADAPT_FLOW_SCRUB"] = "off" + argv.extend(extra) try: - code = _invoke_flow(argv) + with _synthetic_scrub_off(): + code = _invoke_flow(argv) except Exception as error: if _is_externally_managed_error(error): _echo_python_remedy() raise click.exceptions.Exit(1) from error raise - finally: - if scrub in (None, "auto"): - if scrub is None: - os.environ.pop("OPENADAPT_FLOW_SCRUB", None) - else: - os.environ["OPENADAPT_FLOW_SCRUB"] = scrub if code: raise click.ClickException( f"The verified tutorial stopped with exit code {code}. " f"Any completed artifacts remain in {root}." ) - - click.echo("\nLocal quickstart complete.") - click.echo(f"Bundle: {root / 'bundle'}") + click.echo(f"\nBundle: {root / 'bundle'}") click.echo(f"Run evidence: {root / 'run'}") - click.echo("Outcome: VERIFIED under the Standard profile.") - click.echo( - "The synthetic write was confirmed through a read-only system-of-record API." - ) - click.echo("No model or Cloud call was enabled.") if break_it: - click.echo( - "The rerun against the fault-injecting backend HALTED as designed: " - "the form was submitted, but the system of record had no new note." - ) click.echo(f"Caught-fault evidence: {root / 'run-broken' / 'REPORT.md'}") click.echo(_halted_rerun_result(root / "run-broken")) - click.echo( - f"Inspect qualification gaps: {_hint('flow lint ' + _shell_quote(root / 'bundle'))}" - ) - if not break_it: - click.echo( - "Watch it catch a save that didn't land: " - + _hint(f"quickstart --break-it --out {_shell_quote(f'{root}-break-it')}") - ) - # The tutorial bundle is recorded against MockMed's fault server, and - # `replay --drift` serves plain MockMed, so replaying that bundle halts at - # the login screen with or without drift. Record a demo on plain MockMed. - drift = Path(f"{root}-drift") - rec, bundle = (_shell_quote(drift / name) for name in ("rec", "bundle")) - click.echo("See it stop when the screen changes (a separate MockMed demo):") - click.echo(f" {_hint(f'flow demo-record --out {rec}')}") - click.echo(f" {_hint(f'flow compile {rec} --out {bundle} --name drift-demo')}") - click.echo( - " " - + _hint( - f"flow replay {bundle} --drift modal --run-dir {_shell_quote(drift / 'run')}" - ) - ) - if _agent_serves_tutorial(): - click.echo("Give a local agent the run tool (synthetic MockMed, no frames):") - click.echo(f" {_hint('agent serve --allow-run')}") - click.echo( - "A client sends authorized work and gets a receipt. Frames stay here." - ) - click.echo( - "Connect this computer when you want Cloud history and collaboration: " - "https://app.openadapt.ai/dashboard/settings/ingest" - ) - click.echo("Qualify a consequential workflow: https://openadapt.ai/qualify") def _hint(rest: str) -> str: @@ -629,13 +693,146 @@ def _hint(rest: str) -> str: Repeat the ``uvx`` form there; otherwise keep the plain command. """ - prefix = sys.prefix.replace("\\", "/") - if "/uv/archive-v" in prefix: + if _under_uvx(): major, minor = sys.version_info[:2] return f"uvx --python {major}.{minor} openadapt {rest}" return f"openadapt {rest}" +def _under_uvx() -> bool: + """True when this process runs from a ``uvx`` throwaway environment.""" + return "/uv/archive-v" in sys.prefix.replace("\\", "/") + + +def _keep_command() -> Optional[str]: + """Under ``uvx``: the command that installs this launcher for good.""" + if not _under_uvx(): + return None + major, minor = sys.version_info[:2] + return f"uv tool install --python {major}.{minor} openadapt" + + +def _launcher_version(path: str, run=None) -> Optional[str]: + """Ask the ``openadapt`` at ``path`` for its version; None when it can't say.""" + import subprocess + + try: + result = (run or subprocess.run)( + [path, "--version"], capture_output=True, text=True, timeout=15 + ) + text = f"{result.stdout} {result.stderr}" + except Exception: # noqa: BLE001 - an unrunnable openadapt has no version + return None + match = re.search(r"\b(\d+\.\d+\.\d+\S*)", text) + return match.group(1) if match else None + + +def _shadowing_openadapt(which=None, run=None) -> Optional[str]: + """Warn when the shell's ``openadapt`` is another version than this one. + + Under ``uvx`` the printed commands repeat ``uvx`` and reach this version, + but a person who types plain ``openadapt`` gets whatever is first on PATH, + for example an older install in a Conda ``bin`` ahead of ``~/.local/bin``. + A copy of this same version is fine and gets no warning. + """ + import shutil + + found = (which or shutil.which)("openadapt") + if not found: + return None + version = _launcher_version(found, run) + if version == __version__: + return None + named = f"is {version}" if version else "is an unknown version" + return ( + f"Your shell's openadapt {named} at {found}; the commands above use " + "uvx so they run this version." + ) + + +def _path_launchers(environ=None) -> list[Path]: + """Every ``openadapt`` executable on PATH, in PATH order (``which -a``).""" + env = os.environ if environ is None else environ + names = ["openadapt.exe", "openadapt"] if os.name == "nt" else ["openadapt"] + found: list[Path] = [] + seen: set[Path] = set() + for folder in env.get("PATH", "").split(os.pathsep): + for name in names: + candidate = Path(folder or ".") / name + if candidate.is_file() and os.access(candidate, os.X_OK): + real = candidate.resolve() + if real not in seen: + seen.add(real) + found.append(candidate) + return found + + +def _this_launcher_dir() -> Path: + return (Path(sys.prefix) / ("Scripts" if os.name == "nt" else "bin")).resolve() + + +def _uninstall_fix(path: Path) -> str: + """How to remove the ``openadapt`` at ``path`` from its own environment.""" + folder = path.parent + for name in ("python", "python3", "python.exe"): + python = folder / name + if python.is_file(): + return ( + f"{_shell_quote(python)} -m pip uninstall openadapt openadapt-flow, " + "then run: hash -r" + ) + return ( + f"pip uninstall openadapt openadapt-flow from {folder.parent}, " + "then run: hash -r" + ) + + +def _launcher_conflicts(launchers=None, run=None) -> list[str]: + """Plain warnings when PATH holds another ``openadapt`` that matters. + + Two cases: the first ``openadapt`` on PATH is another version than this + one (so typing ``openadapt`` runs that one), or a newer ``openadapt`` is + on PATH than the one running. Each names the paths and versions; the first + also says how to remove the other install. Copies of this same version + are fine. + """ + here = _this_launcher_dir() + others = [ + path + for path in (_path_launchers() if launchers is None else launchers) + if path.parent.resolve() != here + ] + if not others: + return [] + versions = {path: _launcher_version(str(path), run) for path in others} + lines = [] + first = (_path_launchers() if launchers is None else launchers)[0] + if first in versions and versions[first] != __version__: + named = versions[first] or "an unknown version" + lines.append( + f"Your shell runs openadapt {named} at {first} before this one " + f"({__version__}). To remove it: {_uninstall_fix(first)}" + ) + mine = _release_tuple(__version__) + for path, version in versions.items(): + theirs = _release_tuple(version or "") + if mine is not None and theirs is not None and theirs > mine: + lines.append( + f"A newer openadapt, {version}, is at {path}; this one is " + f"{__version__}." + ) + return lines + + +def _warn_about_launcher_conflicts() -> None: + try: + lines = _launcher_conflicts() + except Exception: # noqa: BLE001 - a PATH check must never break a command + return + for line in lines: + click.echo(line, err=True) + + _SECRET_REFERENCE = re.compile(r"^(?:env:[A-Z][A-Z0-9_]*|keychain:[^/\s]+/[^/\s]+)$") # Flow 1.33 and 1.34 probe four X11 libraries in the wrong case, so on Linux # they refuse to download Chromium on hosts that have them. 1.35.0 fixed it. @@ -997,12 +1194,18 @@ def agent(command_ctx: click.Context) -> None: ) @click.pass_context def seal(command_ctx: click.Context) -> None: - """Check an OpenAdapt Seal offline: `openadapt seal verify `. + """Check an OpenAdapt Seal offline: `openadapt seal verify `. - Also `openadapt seal keys pin` to pin Cloud's Seal keys after a person - confirms them. Every argument passes to `openadapt-agent seal` unchanged. + A folder is a run folder (or a folder of them, such as the quickstart's) + signed on this computer; each run's report is checked against its + `seal.json`. Also `openadapt seal keys pin` to pin Cloud's Seal keys after + a person confirms them. Every other argument passes to `openadapt-agent + seal` unchanged. """ _require_supported_python() + args = list(command_ctx.args) + if len(args) >= 2 and args[0] == "verify" and Path(args[1]).is_dir(): + sys.exit(_verify_sealed_runs(Path(args[1]), args[2:])) try: from openadapt_agent import seal_cli # noqa: F401 from openadapt_agent.cli import main as agent_main @@ -1021,6 +1224,72 @@ def seal(command_ctx: click.Context) -> None: sys.exit(int(agent_main(["seal", *command_ctx.args]) or 0)) +def _sealed_run_dirs(folder: Path) -> list[Path]: + """The run folder itself when it is sealed, else its sealed run folders.""" + if (folder / "seal.json").is_file(): + return [folder] + return sorted( + child + for child in folder.iterdir() + if child.is_dir() and (child / "seal.json").is_file() + ) + + +def _verify_sealed_runs(folder: Path, extra: list[str]) -> int: + """Check each sealed run under ``folder`` with Flow's offline Seal check. + + One sealed run folder goes to `openadapt flow local-seal verify` with + every option. A folder of them (the quickstart's) is checked run by run + against this computer's Seal key, or the keys pinned with ``--keys FILE``. + """ + if any(arg in ("--help", "-h") for arg in extra): + return _invoke_flow(["local-seal", "verify", "--help"]) + runs = _sealed_run_dirs(folder) + if not runs: + click.echo(f"not valid: no sealed run in {folder} (no seal.json)", err=True) + return 1 + if runs == [folder]: + return _invoke_flow(["local-seal", "verify", str(folder), *extra]) + try: + from openadapt_flow.offline_seal import ( + load_registry, + parse_key_registry, + public_key_entry, + seal_home, + verify_run, + ) + except ImportError: + click.echo( + "This openadapt-flow cannot check offline Seals. Upgrade with: " + "pip install --upgrade openadapt", + err=True, + ) + return 1 + if extra[:1] == ["--keys"] and len(extra) == 2: + registry = load_registry(Path(extra[1])) + source = f"keys pinned in {extra[1]}" + elif extra: + raise click.UsageError( + "A folder of runs takes only --keys FILE. Check one run folder for " + "the other options." + ) + else: + entry = public_key_entry(seal_home()) + registry = parse_key_registry([entry]) + source = f"this computer's Seal key, {entry['key_id']}" + click.echo(f"Checking {len(runs)} sealed runs in {folder} against {source}") + valid = 0 + for run in runs: + result = verify_run(run, registry) + if result.ok: + valid += 1 + click.echo(f" {run.name} valid") + else: + click.echo(f" {run.name} not valid: {result.reason}") + click.echo(f"{valid} of {len(runs)} valid") + return 0 if valid == len(runs) else 1 + + @main.command("connect") @click.option( "--pairing", default=None, help="Five-minute pairing code from Cloud settings" diff --git a/openadapt/quickstart/__init__.py b/openadapt/quickstart/__init__.py new file mode 100644 index 000000000..712ad4536 --- /dev/null +++ b/openadapt/quickstart/__init__.py @@ -0,0 +1,94 @@ +"""``openadapt quickstart``: show a task once, run it, break it. + +The first command anyone runs. It demonstrates one task in a synthetic clinic +app (MockMed), compiles it into a program, runs the program five times with a +new note each time and checks every save in the app's database, then makes +one save fail silently: the screen says saved, the database has nothing, and +OpenAdapt stops instead of reporting success. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import IO, Callable, Optional + +from openadapt.quickstart.story import ( + YOUR_APP_URL, + NextSteps, + Outcome, + display_path, + run_story, +) + + +def next_steps( + hint: Callable[[str], str], + quote: Callable[[object], str], + agent_serves_tutorial: bool, + *, + keep: Optional[str] = None, + shadow: Optional[Callable[[], Optional[str]]] = None, +) -> Callable[[Outcome], NextSteps]: + """The finale's commands, spelled by the launcher's ``hint`` helper. + + ``keep`` and ``shadow`` apply under ``uvx``: the install command, and a + check (run once, at the finale) for an older ``openadapt`` on PATH. + """ + + def build(outcome: Outcome) -> NextSteps: + verify = None + if outcome.seal is not None: + verify = hint(f"seal verify {quote(display_path(outcome.root))}") + if agent_serves_tutorial: + agent = hint("agent serve --allow-run") + else: + agent = hint( + f"agent serve --bundles {quote(display_path(outcome.root / 'bundle'))}" + ) + return NextSteps( + verify=verify, + record=hint( + f"flow record --backend web --url {YOUR_APP_URL} --out my-task" + ), + agent=agent, + keep=keep, + shadow=shadow() if shadow is not None else None, + ) + + return build + + +def run_quickstart( + root: Path, + *, + visible: bool, + break_it: bool, + verbose: bool, + steps: Callable[[Outcome], NextSteps], + stream: Optional[IO[str]] = None, + interactive: Optional[bool] = None, +) -> Outcome: + """Run the quickstart into ``root`` and print it to ``stream``.""" + + from openadapt.quickstart.engine import FlowEngine, display_available, open_page + from openadapt.quickstart.view import make_view + + stream = stream or sys.stdout + if interactive is None: + interactive = bool(sys.stdin.isatty() and getattr(stream, "isatty", bool)()) + # A person watching gets a short pause between demonstrated actions; a + # hidden browser runs at full speed. + engine = FlowEngine(root, visible=visible, pace_s=0.3 if visible else 0.0) + return run_story( + engine, + make_view(stream), + next_steps=steps, + break_it=break_it, + interactive=interactive, + verbose=verbose, + open_page=open_page if display_available() else None, + ) + + +__all__ = ["next_steps", "run_quickstart"] diff --git a/openadapt/quickstart/engine.py b/openadapt/quickstart/engine.py new file mode 100644 index 000000000..c2d4e8cd2 --- /dev/null +++ b/openadapt/quickstart/engine.py @@ -0,0 +1,717 @@ +"""Drive the engine's bundled tutorial pieces and return what each run showed. + +Every number the quickstart prints comes from here, and everything here comes +from the engine's own artifacts: the recorder's ``events.jsonl``, the compiled +workflow, each run's report, the demo app's database (read out of band, at a +path the app never calls), and the halted run's screenshot. Nothing is +scripted output. + +The stages are the tutorial's own (:mod:`openadapt_flow.tutorial`): record +against MockMed's database-backed server, compile with mined record checks, +certify under ``clinical-write``, and run under the Standard profile through +the unchanged fail-closed run gate. The quickstart adds two things: the +program runs five times with a new note each time, and the record checks read +that run's note (``bind_effect_params``) so a new note can be checked. +""" + +from __future__ import annotations + +import contextlib +import inspect +import json +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional, Sequence + +#: Synthetic notes for the five runs. None of them is clinical advice. +RUN_NOTES = ( + "Follow-up in two weeks", + "Recheck blood pressure next month", + "Refer to physical therapy", + "Repeat labs before next visit", + "No change to the care plan", +) + +#: The part of the 1280x800 screenshot that holds the patient banner and the +#: saved message (the same crop the side-by-side page shows). +SCREEN_CROP = (0, 0, 640, 320) + +#: How often the step ticker reads the recorder's event log, in seconds. +_TICK_S = 0.05 + + +@dataclass +class DemoStep: + """One action of the demonstration, as the recorder logged it.""" + + label: str + seconds: float + + +@dataclass +class RunFacts: + """One run of the program, read from its own report and the database.""" + + index: int + note: str + run_dir: Path + execution_outcome: str + transaction_outcome: Optional[str] + checked: bool + records: int + model_calls: int + cost_usd: float + effects_required: int + effects_confirmed: int + seconds: float + #: The engine's plain-language result for this run ("Done and checked", + #: "Stopped before saving", ...), from its transaction outcome. + plain_label: str = "" + #: True when this run is a repeat: its first try, in a worker, stalled + #: and was stopped. + repeated: bool = False + + @property + def report_md(self) -> Path: + return self.run_dir / "REPORT.md" + + +@dataclass +class BreakFacts: + """The rerun against a database that rejects the write.""" + + run_dir: Path + fault: str + execution_outcome: str + transaction_outcome: Optional[str] + screen_showed_success: bool + screen_text: Optional[str] + screenshot: Optional[Path] + new_records: int + effects_required: int + effects_refuted: int + halt_reason: str + model_calls: int + seconds: float + result: Any = field(default=None, repr=False) + + @property + def stopped(self) -> bool: + return self.execution_outcome == "HALTED" + + @property + def report_md(self) -> Path: + return self.run_dir / "REPORT.md" + + +@dataclass +class SealFacts: + """Offline Seals over the verified runs' reports.""" + + key_id: str + sealed: int + + +def demo_step_label(event: dict[str, Any]) -> Optional[str]: + """A short label for one recorded action, from what the recorder logged.""" + + kind = event.get("kind") + if kind == "type": + text = str(event.get("text") or "") + return f'Type "{text}"' if text else "Type" + if kind in ("click", "double_click"): + structural = event.get("structural") or {} + name = str(structural.get("name") or "").strip() + return f"Click {name}" if name else "Click" + if kind == "key": + return f"Press {event.get('key') or 'a key'}" + return None + + +class StepTicker: + """Read the recorder's ``events.jsonl`` while it is being written. + + The recorder appends one line per committed action, so a second thread + that tails the file reports each step as it happens without any change to + the engine. + """ + + def __init__(self, events: Path, on_step: Callable[[DemoStep], None]) -> None: + self._events = events + self._on_step = on_step + self._offset = 0 + self._stop = threading.Event() + self._thread = threading.Thread(target=self._loop, name="quickstart-ticker") + self._thread.daemon = True + self.steps: list[DemoStep] = [] + + def __enter__(self) -> "StepTicker": + self._thread.start() + return self + + def __exit__(self, *exc: object) -> None: + self._stop.set() + self._thread.join() + self._drain() + + def _loop(self) -> None: + while not self._stop.wait(_TICK_S): + self._drain() + + def _drain(self) -> None: + try: + with self._events.open("r", encoding="utf-8") as handle: + handle.seek(self._offset) + text = handle.read() + except FileNotFoundError: + return + complete, _, _partial = text.rpartition("\n") + if not complete and not text.endswith("\n"): + return + self._offset += len((complete + "\n").encode("utf-8")) + for line in complete.splitlines(): + try: + event = json.loads(line) + except ValueError: + continue + label = demo_step_label(event) + if label is None: + continue + step = DemoStep(label=label, seconds=float(event.get("t") or 0.0)) + self.steps.append(step) + self._on_step(step) + + +def _accepts(function: Callable[..., Any], name: str) -> bool: + try: + return name in inspect.signature(function).parameters + except (TypeError, ValueError): + return False + + +def _effect_counts(report: Any) -> tuple[int, int]: + envelope = getattr(report, "outcome_envelope", None) + if envelope is None: + return 0, 0 + return int(envelope.required_contracts.effect), int( + envelope.passed_contracts.effect + ) + + +def read_screen_text(screenshot: Path) -> Optional[str]: + """Read the saved message off the halted run's screenshot. + + The engine's halt record keeps the on-screen text as its OCR read it at + screen scale, which can run words together. Reading the same pixels at + twice the scale gives the words back. Returns the line that says + "saved", or None when no line does. + """ + + from openadapt_flow.vision.ocr import ocr, upscale_png + + from openadapt.quickstart.images import crop_png + + try: + png = upscale_png(crop_png(screenshot, SCREEN_CROP), 2) + lines = [str(line.text).strip() for line in ocr(png)] + except Exception: # noqa: BLE001 - the screenshot itself still shows it + return None + return next((line for line in lines if "saved" in line.lower()), None) + + +def run_program( + *, + base_url: str, + workflow: Any, + bundle_dir: Path, + run_dir: Path, + index: int, + note: Optional[str], + visible: bool, +) -> RunFacts: + """Run the compiled program once on a fresh copy of the clinic's database. + + ``note`` is this run's input when the engine accepts run inputs; None + replays the demonstrated note. The facts come from the run's own report + and an out-of-band read of the database. + """ + + from openadapt_flow.plain_outcome import plain_result + from openadapt_flow.receipt import _build_tutorial_receipt, write_receipt + from openadapt_flow.report import render_run_report + from openadapt_flow.tutorial import TUTORIAL_NOTE, _records, run_tutorial_workflow + + extra: dict[str, Any] = {} + if note is not None and _accepts(run_tutorial_workflow, "params"): + extra["params"] = {"note": note} + else: + note = str(workflow.params.get("note", TUTORIAL_NOTE)) + started = time.monotonic() + report = run_tutorial_workflow( + base_url=base_url, + workflow=workflow, + bundle_dir=bundle_dir, + run_dir=run_dir, + headed=visible, + **extra, + ) + records = _records(base_url) + seconds = time.monotonic() - started + render_run_report(run_dir) + required, confirmed = _effect_counts(report) + checked = ( + report.execution_outcome == "VERIFIED" + and report.transaction_outcome == "VERIFIED" + and required > 0 + and confirmed == required + and any(record.get("note") == note for record in records) + ) + if report.execution_outcome == "VERIFIED": + write_receipt(_build_tutorial_receipt(report), run_dir) + return RunFacts( + index=index, + note=note, + run_dir=run_dir, + execution_outcome=str(report.execution_outcome), + transaction_outcome=report.transaction_outcome, + checked=checked, + records=len(records), + model_calls=int(report.model_calls), + cost_usd=float(getattr(report, "est_model_cost_usd", 0.0) or 0.0), + effects_required=required, + effects_confirmed=confirmed, + seconds=seconds, + plain_label=plain_result( + report.transaction_outcome, report.execution_outcome + ).label, + ) + + +class EarlyScreenRead: + """Read a screenshot on a thread as soon as it appears on disk. + + :meth:`text_for` returns the text only for the same file with the same + bytes as were read; anything else is read again by the caller. + """ + + def __init__(self, path: Path) -> None: + self._path = path + self._stop = threading.Event() + self._digest: Optional[str] = None + self._text: Optional[str] = None + self._thread = threading.Thread(target=self._loop, name="quickstart-screen") + self._thread.daemon = True + self._thread.start() + + def _loop(self) -> None: + while not self._stop.wait(_TICK_S): + if self._path.is_file(): + time.sleep(_TICK_S) # let the writer finish the file + digest = _sha256(self._path) + self._text = read_screen_text(self._path) + self._digest = digest + return + + def stop(self) -> None: + self._stop.set() + + def text_for(self, path: Path) -> Optional[str]: + self._thread.join(timeout=10) + if self._digest is None or Path(path).resolve() != self._path.resolve(): + return None + return self._text if _sha256(Path(path)) == self._digest else None + + +def _sha256(path: Path) -> Optional[str]: + import hashlib + + try: + return hashlib.sha256(path.read_bytes()).hexdigest() + except OSError: + return None + + +class FlowEngine: + """The bundled tutorial, one stage per call, inside one browser process.""" + + def __init__(self, root: Path, *, visible: bool, pace_s: float = 0.0) -> None: + self.root = root + self.visible = visible + self.pace_s = pace_s + self.recording_dir = root / "recording" + self.bundle_dir = root / "bundle" + self.workflow: Any = None + self._stack = contextlib.ExitStack() + self._base_url: Optional[str] = None + self._pool: Any = None + self._starter: Any = None + + # -- lifetime --------------------------------------------------------- + + def __enter__(self) -> "FlowEngine": + from openadapt_flow import _prewarm + from openadapt_flow.backends._browser_process import shared_browser + from openadapt_flow.mockmed.fault_server import serve + + # Load OCR while the browser starts; compiling and every run use it. + _prewarm.start() + self.root.mkdir(parents=True, exist_ok=True) + try: + self._stack.enter_context(shared_browser(headless=not self.visible)) + base_url, _db, stop = serve() + self._stack.callback(stop) + except BaseException: + self._stack.close() + raise + self._base_url = base_url + return self + + def __exit__(self, *exc: object) -> None: + self._stack.close() + + # -- act 1 ------------------------------------------------------------ + + def record(self, on_step: Callable[[DemoStep], None]) -> list[DemoStep]: + """Demonstrate the task once in the browser, reporting each step.""" + + from openadapt_flow.tutorial import record_tutorial + + assert self._base_url is not None + with StepTicker(self.recording_dir / "events.jsonl", on_step) as ticker: + record_tutorial( + self._base_url, + self.recording_dir, + headed=self.visible, + presentation_delay_s=self.pace_s, + ) + return ticker.steps + + def compile(self) -> int: + """Compile and certify the recording; return the program's step count.""" + + from openadapt_flow.compiler import compile_recording + from openadapt_flow.tutorial import ( + TUTORIAL_WORKFLOW_NAME, + certify_tutorial, + consequential_step, + ) + + workflow = compile_recording( + self.recording_dir, + self.bundle_dir, + name=TUTORIAL_WORKFLOW_NAME, + mine_effects=True, + bind_effect_params=True, + ) + consequential_step(workflow) + certify_tutorial(workflow) + self.workflow = workflow + return len(workflow.steps) + + @property + def program_name(self) -> str: + return str(getattr(self.workflow, "name", "") or "") + + def note_param(self) -> bool: + """True when the engine can give each run its own note.""" + + from openadapt_flow.tutorial import run_tutorial_workflow + + return _accepts(run_tutorial_workflow, "params") + + # -- act 2 ------------------------------------------------------------ + + def run( + self, index: int, note: Optional[str], folder: Optional[str] = None + ) -> RunFacts: + """Run the program once, in this process, on a fresh database.""" + + assert self._base_url is not None and self.workflow is not None + return run_program( + base_url=self._base_url, + workflow=self.workflow, + bundle_dir=self.bundle_dir, + run_dir=self.root / (folder or f"run-{index}"), + index=index, + note=note, + visible=self.visible, + ) + + def start_workers(self) -> None: + """Start the worker processes for Act 2 while Act 1 is still running.""" + + import threading + + from openadapt.quickstart.workers import ( + WorkerPool, + worker_browser, + worker_count, + ) + + count = worker_count() + if count <= 1 or self._starter is not None: + return + + def start() -> None: + # Checking for the bundled browser takes about a second, so it + # happens here, off the main thread, once for every worker. + browser = worker_browser(self.visible) + if browser is not None: + self._pool = WorkerPool(count, visible=self.visible, browser=browser) + + self._starter = threading.Thread(target=start, name="quickstart-workers") + self._starter.start() + self._stack.callback(self._close_pool) + + def _close_pool(self) -> None: + if self._starter is not None: + self._starter.join() + if self._pool is not None: + self._pool.close() + + def _started_pool(self) -> Any: + if self._starter is not None: + self._starter.join() + return self._pool + + @property + def parallel(self) -> int: + """How many runs can go at once: the workers that didn't give up.""" + + pool = self._started_pool() + return max(1, pool.usable()) if pool is not None else 1 + + def run_all( + self, notes: Sequence[str], on_finish: Callable[[RunFacts], None] + ) -> None: + """Run the program once per note, in the worker processes when started. + + Each run is a complete checked run in its own browser, against its own + copy of the clinic's database. A run a worker can't take (a worker + that failed to start, or one that stalled) runs here instead, so every + note still gets its own checked run. + """ + + assert self.workflow is not None + varied = self.note_param() + jobs = [ + (index, note if varied else None, f"run-{index}") + for index, note in enumerate(notes, 1) + ] + leftover = jobs + pool = self._started_pool() + if pool is not None: + leftover = pool.run( + jobs, + bundle_dir=self.bundle_dir, + root=self.root, + on_finish=on_finish, + ) + # Act 2 is their only job: let them shut down during Act 3. + pool.release() + for index, note, folder in leftover: + run = self.run(index, note, folder) + run.repeated = folder != f"run-{index}" + on_finish(run) + + # -- act 3 ------------------------------------------------------------ + + def before_prompt(self) -> None: + """Give the keyboard back to the terminal before asking for Enter. + + On macOS a visible browser takes focus when its window opens and keeps + it after the window closes, so Enter would go to the browser. The + terminal app's bundle ID is in the environment macOS gives programs + it starts; ``open -b`` brings that app forward without a permission + prompt. Elsewhere, or without the ID, nothing happens. + """ + + return_focus_to_terminal(visible=self.visible) + + def break_it(self) -> BreakFacts: + """Rerun the same program while the database rejects the save.""" + + from openadapt_flow.demo_proof import _final_frame, _load_report + from openadapt_flow.tutorial import _run_break_it, consequential_step + + assert self.workflow is not None + run_dir = self.root / "run-broken" + save = consequential_step(self.workflow).id + # The save step's screenshot lands before the engine's wait on the + # database, so it can be read during that wait. + self._early = EarlyScreenRead(run_dir / "steps" / f"{save}_after.png") + started = time.monotonic() + try: + result = _run_break_it( + workflow=self.workflow, + bundle_dir=self.bundle_dir, + run_dir=run_dir, + headed=self.visible, + say=lambda message: None, + ) + finally: + self._early.stop() + seconds = time.monotonic() - started + report = _load_report(Path(result.run_dir)) + screenshot = _final_frame(Path(result.run_dir), report) if report else None + screen_text = None # read by read_screen(), alongside the page + return BreakFacts( + run_dir=Path(result.run_dir), + fault=result.fault, + execution_outcome=str(result.execution_outcome), + transaction_outcome=result.transaction_outcome, + screen_showed_success=bool(result.screen_claimed_success), + screen_text=screen_text, + screenshot=screenshot, + new_records=int(result.system_of_record_records), + effects_required=int(result.effects_required), + effects_refuted=int(result.effects_refuted), + halt_reason=str(result.halt_reason), + model_calls=int(getattr(report, "model_calls", 0) or 0), + seconds=seconds, + result=result, + ) + + def read_screen(self, broken: BreakFacts) -> Optional[str]: + """What the halted run's screenshot says (see :func:`read_screen_text`).""" + + if broken.screenshot is None: + return None + early = getattr(self, "_early", None) + if early is not None: + text = early.text_for(broken.screenshot) + if text is not None: + return text + return read_screen_text(broken.screenshot) + + # -- finale ----------------------------------------------------------- + + def seal(self, runs: list[RunFacts]) -> Optional[SealFacts]: + """Sign each checked run's report with this computer's Seal key. + + Engines without offline Seals return None, and the quickstart prints + no Seal and no command to check one. + """ + + try: + from openadapt_flow.offline_seal import seal_home, seal_run + except ImportError: + return None + checked = [run for run in runs if run.checked] + if not checked: + return None + key_id = "" + for run in checked: + # The configured Seal key, or one generated under the Seal home + # on first use (the offline-Seal default). + seal = seal_run(run.run_dir, home=seal_home(), enqueue=False) + key_id = str(seal.key_id) + return SealFacts(key_id=key_id, sealed=len(checked)) + + def write_page(self, first: RunFacts, broken: BreakFacts) -> Optional[Path]: + """Write the side-by-side page (``openadapt flow demo``'s page).""" + + from openadapt_flow import demo_proof + from openadapt_flow.tutorial import TutorialResult + + report = demo_proof._load_report(first.run_dir) + tier: Optional[int] = None + for step in getattr(report, "results", []) or []: + for evidence in step.effect_evidence: + if ( + evidence.final_verdict == "confirmed" + and evidence.verification_tier is not None + ): + value = int(evidence.verification_tier) + tier = value if tier is None else min(tier, value) + receipts = { + kind: first.run_dir / f"receipt.{kind}" + for kind in ("json", "md", "png") + if (first.run_dir / f"receipt.{kind}").is_file() + } + result = TutorialResult( + recording_dir=self.recording_dir, + bundle_dir=self.bundle_dir, + run_dir=first.run_dir, + execution_outcome=first.execution_outcome, + transaction_outcome=first.transaction_outcome, + execution_profile=getattr(report, "execution_profile", None), + transaction_billable=getattr(report, "transaction_billable", None), + model_calls=first.model_calls, + effects_required=first.effects_required, + effects_confirmed=first.effects_confirmed, + effect_tier=tier, + bundle_digest=getattr(report, "bundle_content_digest", None), + system_of_record_records=first.records, + receipt_paths=receipts, + break_it=broken.result, + ) + evidence = demo_proof.collect_evidence(result, self.root) + return demo_proof.write_demo_page(evidence) + + +def return_focus_to_terminal( + *, visible: bool, platform: Optional[str] = None, environ=None, run=None +) -> bool: + """Bring the terminal app forward on macOS after a visible browser ran.""" + + import os + import subprocess + import sys + + env = os.environ if environ is None else environ + bundle = env.get("__CFBundleIdentifier", "") + if not visible or (platform or sys.platform) != "darwin" or not bundle: + return False + try: + (run or subprocess.run)(["open", "-b", bundle], check=False, timeout=5) + except Exception: # noqa: BLE001 - focus is a convenience only + return False + return True + + +def display_available( + *, platform: Optional[str] = None, environ=None, isatty=None +) -> bool: + """True when a browser window (or tab) is likely to reach a person. + + The same rule as ``openadapt flow demo``: output on a terminal, and a + display (X11 or Wayland, or a local macOS or Windows session, not SSH). + """ + + import os + import sys + + env = os.environ if environ is None else environ + tty = isatty() if isatty is not None else sys.stdout.isatty() + if not tty: + return False + if env.get("DISPLAY") or env.get("WAYLAND_DISPLAY"): + return True + if env.get("SSH_CONNECTION") or env.get("SSH_TTY"): + return False + return (platform or sys.platform) in ("darwin", "win32") + + +def open_page(page: Path) -> bool: + from openadapt_flow.demo_proof import open_page as opener + + return opener(page) + + +__all__ = [ + "RUN_NOTES", + "BreakFacts", + "DemoStep", + "FlowEngine", + "RunFacts", + "SealFacts", + "StepTicker", + "demo_step_label", + "display_available", + "open_page", + "read_screen_text", + "return_focus_to_terminal", +] diff --git a/openadapt/quickstart/images.py b/openadapt/quickstart/images.py new file mode 100644 index 000000000..c45c625f9 --- /dev/null +++ b/openadapt/quickstart/images.py @@ -0,0 +1,126 @@ +"""Show a screenshot inline on terminals that can draw images. + +Two protocols cover the terminals that support it: iTerm2's inline images +(iTerm2 and WezTerm) and Kitty's graphics protocol (Kitty and Ghostty). Each +terminal is recognized from the variables it sets for its own sessions. Nothing +is assumed: an unknown terminal, a terminal multiplexer (tmux and screen don't +pass images through unless configured to), or output that isn't a terminal +gets ``None``, and the caller draws a text box instead. +""" + +from __future__ import annotations + +import base64 +import io +import os +from pathlib import Path +from typing import Mapping, Optional + +ITERM2 = "iterm2" +KITTY = "kitty" + +#: Set to 0 to turn inline images off, or to iterm2/kitty to choose a protocol +#: that detection can't see (for example inside a tmux passthrough setup). +OVERRIDE_ENV = "OPENADAPT_INLINE_IMAGES" + + +def detect_image_protocol( + environ: Optional[Mapping[str, str]] = None, *, isatty: bool +) -> Optional[str]: + """Return the image protocol this terminal speaks, or None when unsure.""" + + env = os.environ if environ is None else environ + override = (env.get(OVERRIDE_ENV) or "").strip().lower() + if override in ("0", "off", "no", "false"): + return None + if override in (ITERM2, KITTY): + return override + if not isatty: + return None + term = env.get("TERM", "") + if env.get("TMUX") or env.get("STY") or term.startswith(("screen", "tmux")): + return None + program = env.get("TERM_PROGRAM", "") + if program == "iTerm.app" or env.get("LC_TERMINAL") == "iTerm2": + return ITERM2 + if program == "WezTerm": + return ITERM2 + if env.get("KITTY_WINDOW_ID") or term == "xterm-kitty": + return KITTY + if program == "ghostty" or term == "xterm-ghostty": + return KITTY + return None + + +def crop_png(path: Path, box: tuple[int, int, int, int]) -> bytes: + """Return the ``(left, top, right, bottom)`` part of a PNG, as PNG bytes.""" + + from PIL import Image + + with Image.open(path) as image: + width, height = image.size + left, top, right, bottom = box + cropped = image.crop((left, top, min(right, width), min(bottom, height))) + out = io.BytesIO() + cropped.save(out, format="PNG") + return out.getvalue() + + +def image_sequence(png: bytes, protocol: str, *, cols: int, rows: int) -> str: + """The escape sequence that draws ``png`` in a ``cols`` x ``rows`` cell box.""" + + data = base64.b64encode(png).decode("ascii") + if protocol == ITERM2: + return ( + f"\x1b]1337;File=inline=1;size={len(png)};width={cols};height={rows};" + f"preserveAspectRatio=1:{data}\x07" + ) + if protocol == KITTY: + # Kitty takes the payload in chunks of at most 4096 bytes. q=2 keeps + # the terminal from writing a reply into the person's input. + chunks = [data[i : i + 4096] for i in range(0, len(data), 4096)] or [""] + parts = [] + for index, chunk in enumerate(chunks): + more = 1 if index < len(chunks) - 1 else 0 + if index == 0: + head = f"a=T,f=100,q=2,c={cols},r={rows},m={more}" + else: + head = f"m={more}" + parts.append(f"\x1b_G{head};{chunk}\x1b\\") + return "".join(parts) + raise ValueError(f"unknown image protocol {protocol!r}") + + +def beside_image( + image: str, right: list[str], *, indent: int, cols: int, rows: int, gap: int = 3 +) -> str: + """Draw ``image`` at the left and the ``right`` lines next to it. + + The block reserves its rows first, so a screen that has to scroll does it + before anything is drawn. The cursor is saved around the image because + the two protocols leave it in different places. + """ + + height = max(rows, len(right)) + out = ["\n" * height, f"\x1b[{height}A\r"] + out.append("\x1b7") + if indent: + out.append(f"\x1b[{indent}C") + out.append(image) + out.append("\x1b8") + column = indent + cols + gap + for index in range(height): + line = right[index] if index < len(right) else "" + out.append(f"\r\x1b[{column}C{line}\n" if line else "\n") + return "".join(out) + + +__all__ = [ + "ITERM2", + "KITTY", + "OVERRIDE_ENV", + "beside_image", + "crop_png", + "detect_image_protocol", + "image_sequence", +] diff --git a/openadapt/quickstart/story.py b/openadapt/quickstart/story.py new file mode 100644 index 000000000..59d2597d3 --- /dev/null +++ b/openadapt/quickstart/story.py @@ -0,0 +1,317 @@ +"""The quickstart in three acts: show it once, run it, break it. + +This module decides what happens and in what order. The engine adapter +(:mod:`openadapt.quickstart.engine`) does the work and measures it; the view +(:mod:`openadapt.quickstart.view`) decides how it looks. Every number a view +prints arrives here from the engine, so a change in what a run did changes the +output. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional, Protocol, Sequence + +from openadapt.quickstart.engine import ( + RUN_NOTES, + BreakFacts, + DemoStep, + RunFacts, + SealFacts, +) + +#: The placeholder app address in the "try it on your own app" command. +YOUR_APP_URL = "https://your-app.example" + + +def display_path(path: Path, cwd: Optional[Path] = None) -> Path: + """``path`` relative to the current folder when it is inside it. + + Commands printed with it work from the folder the quickstart ran in. + """ + + base = (cwd or Path.cwd()).resolve() + try: + return path.resolve().relative_to(base) + except ValueError: + return path + + +class Engine(Protocol): + """What the story needs from the engine (a fake one in tests).""" + + root: Path + visible: bool + + def __enter__(self) -> "Engine": ... + + def __exit__(self, *exc: object) -> None: ... + + def record(self, on_step: Callable[[DemoStep], None]) -> list[DemoStep]: ... + + def compile(self) -> int: ... + + @property + def program_name(self) -> str: ... + + def note_param(self) -> bool: ... + + def start_workers(self) -> None: ... + + @property + def parallel(self) -> int: ... + + def run_all( + self, notes: Sequence[str], on_finish: Callable[[RunFacts], None] + ) -> None: ... + + def before_prompt(self) -> None: ... + + def break_it(self) -> BreakFacts: ... + + def read_screen(self, broken: BreakFacts) -> Optional[str]: ... + + def seal(self, runs: list[RunFacts]) -> Optional[SealFacts]: ... + + def write_page(self, first: RunFacts, broken: BreakFacts) -> Optional[Path]: ... + + +@dataclass +class NextSteps: + """The commands the finale prints, each spelled to work as printed.""" + + verify: Optional[str] + record: str + agent: str + #: Under ``uvx`` only: the command that installs this version for good. + keep: Optional[str] = None + #: Under ``uvx`` only: a warning when the shell's ``openadapt`` is another + #: version, which the plain commands would reach instead. + shadow: Optional[str] = None + + +@dataclass +class Outcome: + """Everything one quickstart showed, for the finale and the tests.""" + + root: Path + demo_steps: list[DemoStep] = field(default_factory=list) + program_steps: int = 0 + program_name: str = "" + act1_seconds: float = 0.0 + runs: list[RunFacts] = field(default_factory=list) + act2_seconds: float = 0.0 + broken: Optional[BreakFacts] = None + seal: Optional[SealFacts] = None + page: Optional[Path] = None + page_opened: bool = False + total_seconds: float = 0.0 + + @property + def checked(self) -> int: + return sum(1 for run in self.runs if run.checked) + + @property + def model_calls(self) -> int: + calls = sum(run.model_calls for run in self.runs) + if self.broken is not None: + calls += self.broken.model_calls + return calls + + @property + def cost_usd(self) -> float: + return sum(run.cost_usd for run in self.runs) + + +class View(Protocol): + def intro(self, root: Path, visible: bool) -> None: ... + + def starting(self) -> Any: ... + + def act(self, number: int, title: str, subtitle: str) -> None: ... + + def demo_step(self, step: DemoStep) -> None: ... + + def compiling(self) -> Any: ... + + def compiled(self, outcome: Outcome) -> None: ... + + def running(self, total: int, parallel: int) -> Any: ... + + def progress(self, finished: int, total: int, checked: int) -> None: ... + + def run_row(self, run: RunFacts, total: int) -> None: ... + + def runs_total(self, outcome: Outcome) -> None: ... + + def break_setup(self) -> None: ... + + def confirm(self, prompt: str) -> None: ... + + def breaking(self) -> Any: ... + + def break_result(self, broken: BreakFacts) -> None: ... + + def finale(self, outcome: Outcome, steps: NextSteps) -> None: ... + + def detail(self, lines: list[str]) -> None: ... + + +def _engine_detail_run(run: RunFacts) -> list[str]: + return [ + f"run {run.index}: execution {run.execution_outcome}, transaction " + f"{run.transaction_outcome}, profile standard, record checks " + f"{run.effects_confirmed}/{run.effects_required} confirmed by an " + f"independent read of the database, {run.records} record(s) after the " + f"run; report {run.report_md}", + ] + + +def _engine_detail_break(broken: BreakFacts) -> list[str]: + return [ + f"fault mode {broken.fault!r}: the server rejects the write after the " + "app shows its saved message", + f"execution {broken.execution_outcome}, transaction " + f"{broken.transaction_outcome}, record checks refuted " + f"{broken.effects_refuted}/{broken.effects_required}", + f"engine: {broken.halt_reason}", + f"report {broken.report_md}", + ] + + +def run_story( + engine: Engine, + view: View, + *, + next_steps: Callable[[Outcome], NextSteps], + break_it: bool = True, + interactive: bool = False, + verbose: bool = False, + open_page: Optional[Callable[[Path], bool]] = None, + clock: Callable[[], float] = time.monotonic, +) -> Outcome: + """Run the three acts against ``engine`` and show them through ``view``.""" + + outcome = Outcome(root=engine.root) + started = clock() + paused = 0.0 + view.intro(engine.root, engine.visible) + with view.starting(): + engine.__enter__() + try: + # Act 1: show it once. Act 2's workers get ready meanwhile. + engine.start_workers() + view.act(1, "Show it once", _act1_subtitle(engine.visible)) + act_start = clock() + outcome.demo_steps = engine.record(view.demo_step) + with view.compiling(): + outcome.program_steps = engine.compile() + outcome.program_name = engine.program_name + outcome.act1_seconds = clock() - act_start + view.compiled(outcome) + if verbose: + view.detail( + [ + f"recording {engine.root / 'recording'}", + f"program {engine.root / 'bundle'} (certified under " + "clinical-write; record checks mined from the demonstration)", + ] + ) + + # Act 2: run it. + total = len(RUN_NOTES) + view.act( + 2, "Run it", _act2_subtitle(total, engine.note_param(), engine.parallel) + ) + act_start = clock() + + def finished(run: RunFacts) -> None: + outcome.runs.append(run) + view.run_row(run, total) + if verbose: + view.detail(_engine_detail_run(run)) + view.progress(len(outcome.runs), total, outcome.checked) + + with view.running(total, engine.parallel): + engine.run_all(RUN_NOTES, finished) + outcome.act2_seconds = clock() - act_start + view.runs_total(outcome) + # Sign the checked runs' reports while Act 3 runs; the finale waits. + sealing = _in_background(lambda: engine.seal(list(outcome.runs))) + + # Act 3: break it. + paging = None + if break_it: + view.act(3, "Break it", "the next save fails silently") + view.break_setup() + if interactive: + waited = clock() + engine.before_prompt() + view.confirm("Press Enter to make the next save fail silently") + paused += clock() - waited + with view.breaking(): + outcome.broken = engine.break_it() + broken = outcome.broken + if outcome.runs: + # The page needs only the runs; build it while the + # screenshot is read. + paging = _in_background( + lambda: engine.write_page(outcome.runs[0], broken) + ) + broken.screen_text = engine.read_screen(broken) + view.break_result(outcome.broken) + if verbose: + view.detail(_engine_detail_break(outcome.broken)) + + outcome.seal = sealing.result() + if paging is not None: + outcome.page = paging.result() + if outcome.page is not None and open_page is not None: + outcome.page_opened = bool(open_page(outcome.page)) + finally: + engine.__exit__(None, None, None) + outcome.total_seconds = clock() - started - paused + view.finale(outcome, next_steps(outcome)) + return outcome + + +def _in_background(work: Callable[[], Any]) -> Any: + """Start ``work`` on a thread; ``.result()`` waits and re-raises.""" + + from concurrent.futures import ThreadPoolExecutor + + executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="quickstart") + future = executor.submit(work) + executor.shutdown(wait=False) + return future + + +def _act2_subtitle(total: int, varied: bool, parallel: int) -> str: + # An engine that can't give a run its own note replays the demonstrated + # one, and the header says so. + what = ( + f"the program on {total} new notes" if varied else f"the program {total} times" + ) + if parallel >= total: + return f"{what}, all at once" + if parallel > 1: + return f"{what}, {parallel} at a time" + return what + + +def _act1_subtitle(visible: bool) -> str: + where = "watch the browser window" if visible else "in a hidden browser" + return f"a demonstration in a synthetic clinic app, {where}" + + +__all__ = [ + "YOUR_APP_URL", + "display_path", + "Engine", + "NextSteps", + "Outcome", + "View", + "run_story", +] diff --git a/openadapt/quickstart/view.py b/openadapt/quickstart/view.py new file mode 100644 index 000000000..4a6e399da --- /dev/null +++ b/openadapt/quickstart/view.py @@ -0,0 +1,539 @@ +"""How the quickstart looks: a live terminal view, or plain lines. + +Both views say the same things in the same words (the helpers below); the +rich view adds color, spinners, a box around the receipt, and the halted +run's screenshot on terminals that can draw images. The plain view prints one +line per fact, for logs, pipes, CI, ``NO_COLOR`` and ``TERM=dumb``. +""" + +from __future__ import annotations + +import contextlib +import io +import os +import sys +from pathlib import Path +from typing import IO, Any, Iterator, Mapping, Optional + +from openadapt.quickstart.engine import BreakFacts, DemoStep, RunFacts +from openadapt.quickstart.story import NextSteps, Outcome, display_path + +#: Width of the step and note column, so times line up. +_LABEL_WIDTH = 40 + + +# --------------------------------------------------------------------------- +# Words shared by both views +# --------------------------------------------------------------------------- + + +class Marks: + """The two status marks, with ASCII stand-ins for terminals without them.""" + + def __init__(self, encoding: Optional[str]) -> None: + try: + "✓■·—".encode(encoding or "utf-8") + unicode = True + except (LookupError, UnicodeEncodeError): + unicode = False + self.ok = "✓" if unicode else "OK" + self.stop = "■" if unicode else "STOP" + self.dot = "·" if unicode else "-" + + +def seconds(value: float) -> str: + return f"{value:.1f}s" + + +def compiled_line(outcome: Outcome) -> str: + return f"Compiled into a program: {outcome.program_steps} steps." + + +def run_status(run: RunFacts, marks: Marks) -> str: + if run.checked: + again = " (on a second try)" if run.repeated else "" + return f"{marks.ok} checked in the database{again}" + return f"{marks.stop} {run.plain_label or run.execution_outcome}" + + +def runs_total_line(outcome: Outcome, marks: Marks) -> str: + dot = f" {marks.dot} " + return dot.join( + [ + f"{outcome.checked} of {len(outcome.runs)} saved and checked", + f"{outcome.model_calls} AI calls", + f"${outcome.cost_usd:.2f}", + seconds(outcome.act2_seconds), + ] + ) + + +def database_line(broken: BreakFacts) -> str: + count = broken.new_records + return f"{count} new record" + ("" if count == 1 else "s") + + +def bot_verdict(broken: BreakFacts, marks: Marks) -> Optional[str]: + """What a tool that trusts the screen would report, when the screen said so.""" + + return f"{marks.ok} Done" if broken.screen_showed_success else None + + +def openadapt_verdict(broken: BreakFacts, marks: Marks) -> str: + if broken.stopped: + return f"{marks.stop} Stopped before reporting success" + return f"{broken.execution_outcome.title()} (expected a stop)" + + +def why_line(broken: BreakFacts) -> str: + if broken.stopped and broken.new_records == 0: + return ( + "OpenAdapt reads the record back from the database after each save. " + "It found nothing, so it stopped." + ) + if broken.stopped: + return "OpenAdapt read the record back, and it didn't match, so it stopped." + return "OpenAdapt did not stop. Treat this build as broken." + + +def aha_line(broken: BreakFacts) -> Optional[str]: + """The one-line summary, only when the run showed all of it.""" + + if not ( + broken.stopped and broken.screen_showed_success and broken.new_records == 0 + ): + return None + text = broken.screen_text + end = text.lower().find("saved") if text else -1 + if text and end >= 0: + screen = f'The screen said "{text[: end + len("saved")]}".' + else: + screen = "The screen showed its saved message." + return ( + f"{screen} The database had nothing. A bot that trusts the screen " + "reports Done. OpenAdapt stopped." + ) + + +def receipt_rows(outcome: Outcome, marks: Marks) -> list[tuple[str, str]]: + dot = f" {marks.dot} " + rows = [ + ("Program", f"{outcome.program_name}{dot}{outcome.program_steps} steps"), + ( + "Runs", + f"{outcome.checked} of {len(outcome.runs)} saved and checked in the " + "database", + ), + ( + "Cost", + f"{outcome.model_calls} AI calls{dot}${outcome.cost_usd:.2f}{dot}" + f"{seconds(outcome.total_seconds)} in all", + ), + ] + if outcome.seal is not None: + reports = "report" if outcome.seal.sealed == 1 else "reports" + rows.append( + ( + "Seal", + f"{outcome.seal.key_id}{dot}{outcome.seal.sealed} {reports} signed", + ) + ) + return rows + + +def details_line(outcome: Outcome) -> str: + """One line: where every run's full report is, and the side-by-side page.""" + + root = display_path(outcome.root) + line = f"Details: {root}, one REPORT.md per run" + if outcome.page is not None: + page = display_path(outcome.page) + line += f"; side-by-side page {page}" + if outcome.page_opened: + line += ", opened in your browser" + return line + + +def next_step_lines(steps: NextSteps) -> list[str]: + lines = [ + f"Try it on your own app: {steps.record}", + f"Give your agent this as a tool: {steps.agent}", + ] + if steps.keep: + lines.append(f"Keep it: {steps.keep}") + return lines + + +# --------------------------------------------------------------------------- +# Choosing a view +# --------------------------------------------------------------------------- + + +def wants_plain(stream: IO[str], environ: Optional[Mapping[str, str]] = None) -> bool: + """Plain lines when the output isn't a terminal or color is turned off.""" + + env = os.environ if environ is None else environ + if "NO_COLOR" in env and env.get("NO_COLOR", "") != "": + return True + if env.get("TERM") == "dumb": + return True + isatty = getattr(stream, "isatty", None) + return not (callable(isatty) and isatty()) + + +def make_view(stream: Optional[IO[str]] = None, *, environ=None) -> Any: + stream = stream or sys.stdout + if wants_plain(stream, environ): + return PlainView(stream) + try: + return RichView(stream, environ=environ) + except ImportError: # rich is a launcher dependency; plain lines still work + return PlainView(stream) + + +# --------------------------------------------------------------------------- +# Plain lines +# --------------------------------------------------------------------------- + + +class PlainView: + """One line per fact, printed as it happens. No color, no cursor moves.""" + + def __init__(self, stream: IO[str]) -> None: + self.stream = stream + self.marks = Marks(getattr(stream, "encoding", None)) + + def _say(self, text: str = "") -> None: + self.stream.write(text + "\n") + self.stream.flush() + + @contextlib.contextmanager + def _quiet(self) -> Iterator[None]: + yield + + def intro(self, root: Path, visible: bool) -> None: + self._say( + "OpenAdapt quickstart: show a task once, run it, then watch it " + "catch a save that didn't happen." + ) + self._say(f"Synthetic data only. Files go to {display_path(root)}") + + def starting(self): + return self._quiet() + + def act(self, number: int, title: str, subtitle: str) -> None: + self._say() + self._say(f"{number} {title} {self.marks.dot} {subtitle}") + + def demo_step(self, step: DemoStep) -> None: + self._say( + f" {self.marks.ok} {step.label:<{_LABEL_WIDTH}} {seconds(step.seconds)}" + ) + + def compiling(self): + return self._quiet() + + def compiled(self, outcome: Outcome) -> None: + self._say(f" {compiled_line(outcome)} {seconds(outcome.act1_seconds)}") + + def running(self, total: int, parallel: int): + return self._quiet() + + def progress(self, finished: int, total: int, checked: int) -> None: + pass + + def run_row(self, run: RunFacts, total: int) -> None: + self._say( + f" {run.index}/{total} {run.note:<{_LABEL_WIDTH - 6}} " + f"{run_status(run, self.marks)} {seconds(run.seconds)}" + ) + + def runs_total(self, outcome: Outcome) -> None: + self._say(f" {runs_total_line(outcome, self.marks)}") + + def break_setup(self) -> None: + self._say( + " The app will show its saved message. The database will reject the save." + ) + + def confirm(self, prompt: str) -> None: + self.stream.write(f" {prompt} ") + self.stream.flush() + try: + input() + except EOFError: + self._say() + + def breaking(self): + return self._quiet() + + def break_result(self, broken: BreakFacts) -> None: + quote = broken.screen_text + shown = f'"{quote}"' if quote else "a saved message" + self._say(f" What the screen showed: {shown}") + self._say(f" What the database showed: {database_line(broken)}") + bot = bot_verdict(broken, self.marks) + if bot: + self._say(f" A screen-trusting bot: {bot}") + self._say( + f" OpenAdapt: {openadapt_verdict(broken, self.marks)}" + f" {seconds(broken.seconds)}" + ) + self._say(f" Why: {why_line(broken)}") + aha = aha_line(broken) + if aha: + self._say() + self._say(aha) + + def finale(self, outcome: Outcome, steps: NextSteps) -> None: + self._say() + self._say("Receipt") + for name, value in receipt_rows(outcome, self.marks): + self._say(f" {name:<9} {value}") + if steps.verify: + self._say(f"Check it: {steps.verify}") + self._say(details_line(outcome)) + for line in next_step_lines(steps): + self._say(line) + if steps.shadow: + self._say(steps.shadow) + + def detail(self, lines: list[str]) -> None: + for line in lines: + self._say(f" {line}") + + +# --------------------------------------------------------------------------- +# Live terminal +# --------------------------------------------------------------------------- + + +class RichView: + """Color, spinners, a boxed receipt, and the screenshot where possible.""" + + def __init__( + self, stream: IO[str], *, environ=None, width: Optional[int] = None + ) -> None: + from rich.console import Console + + from openadapt.quickstart.images import detect_image_protocol + + self.console = Console( + file=stream, highlight=False, soft_wrap=False, width=width + ) + self.marks = Marks(getattr(stream, "encoding", None) or "utf-8") + self.image_protocol = detect_image_protocol(environ, isatty=True) + + def _status(self, text: str): + return self.console.status(f"[dim]{text}[/]", spinner="dots") + + def intro(self, root: Path, visible: bool) -> None: + self.console.print( + "[bold]OpenAdapt quickstart[/]: show a task once, run it, then watch " + "it catch a save that didn't happen." + ) + self.console.print( + f"[dim]Synthetic data only. Files go to {display_path(root)}[/]" + ) + + def starting(self): + return self._status("Starting the synthetic clinic app") + + def act(self, number: int, title: str, subtitle: str) -> None: + self.console.print() + self.console.print( + f"[bold reverse] {number} [/] [bold]{title}[/] [dim]{subtitle}[/]" + ) + + def demo_step(self, step: DemoStep) -> None: + self.console.print( + f" [green]{self.marks.ok}[/] {_escape(step.label):<{_LABEL_WIDTH}} " + f"[dim]{seconds(step.seconds)}[/]" + ) + + def compiling(self): + return self._status("Compiling the demonstration into a program") + + def compiled(self, outcome: Outcome) -> None: + self.console.print( + f" [bold]{compiled_line(outcome)}[/] [dim]{seconds(outcome.act1_seconds)}[/]" + ) + + def running(self, total: int, parallel: int): + self._live = self._status(self._running_text(0, total, 0, parallel)) + self._parallel = parallel + return self._live + + def _running_text( + self, finished: int, total: int, checked: int, parallel: int + ) -> str: + how = "at once" if parallel >= total else f"{parallel} at a time" + if parallel <= 1: + how = "one after another" + return ( + f"Running {total} {how} {self.marks.dot} {finished} done " + f"{self.marks.dot} {checked} checked in the database" + ) + + def progress(self, finished: int, total: int, checked: int) -> None: + live = getattr(self, "_live", None) + if live is not None: + live.update( + f"[dim]{self._running_text(finished, total, checked, self._parallel)}[/]" + ) + + def run_row(self, run: RunFacts, total: int) -> None: + color = "green" if run.checked else "red" + self.console.print( + f" [dim]{run.index}/{total}[/] {_escape(run.note):<{_LABEL_WIDTH - 6}} " + f"[{color}]{run_status(run, self.marks)}[/] [dim]{seconds(run.seconds)}[/]" + ) + + def runs_total(self, outcome: Outcome) -> None: + self.console.print(f" [bold]{runs_total_line(outcome, self.marks)}[/]") + + def break_setup(self) -> None: + self.console.print( + " The app will show its saved message. The database will reject the save." + ) + + def confirm(self, prompt: str) -> None: + try: + self.console.input(f" [bold]{prompt}[/] [dim](Enter)[/] ") + except EOFError: + self.console.print() + + def breaking(self): + return self._status("Running the program once more, checking the database") + + def _right_column(self, broken: BreakFacts) -> list[str]: + width = 52 + lines = [ + "[bold]What the database showed[/]", + f"[bold red]{database_line(broken)}[/]", + "", + ] + bot = bot_verdict(broken, self.marks) + if bot: + lines.append(f"A screen-trusting bot [red]{bot}[/]") + lines.append( + f"OpenAdapt [bold green]{openadapt_verdict(broken, self.marks)}[/]" + f" [dim]{seconds(broken.seconds)}[/]" + ) + lines.append("") + lines.extend(f"[dim]{part}[/]" for part in _wrap(why_line(broken), width)) + return lines + + def break_result(self, broken: BreakFacts) -> None: + from rich.panel import Panel + from rich.table import Table + + right = self._right_column(broken) + drawn = False + if self.image_protocol and broken.screenshot is not None: + drawn = self._draw_beside_screenshot(broken, right) + if not drawn: + quote = broken.screen_text or "a saved message" + box = Panel( + f"[green]{_escape(quote)}[/]", + title="What the screen showed", + title_align="left", + width=36, + padding=(1, 2), + ) + grid = Table.grid(padding=(0, 3)) + grid.add_column() + grid.add_column() + grid.add_row(box, "\n".join(right)) + self.console.print() + self.console.print(grid, crop=False) + aha = aha_line(broken) + if aha: + self.console.print() + self.console.print(f"[bold]{_escape(aha)}[/]") + + def _draw_beside_screenshot(self, broken: BreakFacts, right: list[str]) -> bool: + from openadapt.quickstart.engine import SCREEN_CROP + from openadapt.quickstart.images import beside_image, crop_png, image_sequence + + try: + png = crop_png(broken.screenshot, SCREEN_CROP) + except Exception: # noqa: BLE001 - fall back to the text box + return False + cols, rows = 40, 10 + rendered = [self._ansi(line) for line in right] + self.console.print() + self.console.print( + " [bold]What the screen showed[/] [dim](its screenshot)[/]" + ) + block = beside_image( + image_sequence(png, self.image_protocol, cols=cols, rows=rows), + rendered, + indent=3, + cols=cols, + rows=rows, + ) + self.console.file.write(block) + self.console.file.flush() + return True + + def _ansi(self, markup: str) -> str: + from rich.console import Console + + buffer = io.StringIO() + Console( + file=buffer, + force_terminal=True, + color_system=self.console.color_system, + width=200, + highlight=False, + ).print(markup, end="") + return buffer.getvalue() + + def finale(self, outcome: Outcome, steps: NextSteps) -> None: + from rich.panel import Panel + from rich.table import Table + + table = Table.grid(padding=(0, 2)) + table.add_column(style="dim") + table.add_column() + for name, value in receipt_rows(outcome, self.marks): + table.add_row(name, _escape(value)) + self.console.print() + self.console.print( + Panel(table, title="Receipt", title_align="left", expand=False) + ) + if steps.verify: + self.console.print(f"[dim]Check it:[/] [bold]{_escape(steps.verify)}[/]") + self.console.print(f"[dim]{_escape(details_line(outcome))}[/]") + for line in next_step_lines(steps): + label, _, command = line.partition(": ") + self.console.print(f"{label}: [bold cyan]{_escape(command)}[/]") + if steps.shadow: + self.console.print(f"[yellow]{_escape(steps.shadow)}[/]") + + def detail(self, lines: list[str]) -> None: + for line in lines: + self.console.print(f" [dim]{_escape(line)}[/]") + + +def _escape(text: str) -> str: + from rich.markup import escape + + return escape(text) + + +def _wrap(text: str, width: int) -> list[str]: + import textwrap + + return textwrap.wrap(text, width) or [""] + + +__all__ = [ + "Marks", + "PlainView", + "RichView", + "aha_line", + "make_view", + "wants_plain", +] diff --git a/openadapt/quickstart/workers.py b/openadapt/quickstart/workers.py new file mode 100644 index 000000000..53d2388aa --- /dev/null +++ b/openadapt/quickstart/workers.py @@ -0,0 +1,340 @@ +"""Act 2's runs, in parallel: one worker process per run. + +Each worker starts while Act 1 is still demonstrating and compiling: it loads +the engine, starts its own browser, and serves its own copy of the clinic app +with its own database. When the program is ready, each worker runs it once +for one note, through the same governed path as any run (the Standard run +gate, identity checks, postconditions, the database read-back), and sends +back the facts. Nothing is shared between runs but the program on disk, which +every run checks against its sealed manifest before it starts. + +Processes, not threads: the browser driver's objects belong to the thread that +made them, and each run needs its own database because every run of the +tutorial program writes the same logical record. + +Workers drive Playwright's own Chromium. Several installed-Chrome instances +started at once stalled about one run in ten on a loaded Mac (2 of 20 runs), +while the bundled build ran 30 of 30. So a worker starts only when the bundled +build is already on disk (or the person chose a browser with +``OPENADAPT_FLOW_BROWSER``); otherwise the quickstart downloads nothing extra +and runs the five runs one after another in its own process. +""" + +from __future__ import annotations + +import multiprocessing +import os +import time +from dataclasses import asdict +from multiprocessing.connection import Connection, wait +from pathlib import Path +from typing import Any, Callable, Optional, Sequence + +#: Act 2 runs five notes; more workers than runs would only cost memory. +MAX_WORKERS = 5 + +#: How long workers may take to get ready before their runs are done in the +#: main process instead. +READY_TIMEOUT_S = 120.0 + +#: A run takes about 3 to 6 seconds. One that hasn't reported after this long +#: has stalled: its worker is stopped (its browser and its database go with +#: it) and the run is repeated in the main process, in a new run folder. +STALL_S = 45.0 + +#: How much lower than the main process a worker's CPU priority is. +WORKER_NICENESS = 10 + +#: The OCR engine's thread count (read once, when the engine loads). +OCR_THREADS_ENV = "OPENADAPT_FLOW_OCR_THREADS" + +#: One job: run number, note (None replays the demonstrated note), and the +#: name of its run folder under the quickstart's output directory. +Job = tuple[int, Optional[str], str] + + +def worker_count(cpus: Optional[int] = None) -> int: + """Workers for this computer: one per two CPUs, at most one per run.""" + + cpus = cpus if cpus is not None else (os.cpu_count() or 2) + return max(1, min(MAX_WORKERS, cpus // 2)) + + +def worker_browser(visible: bool) -> Optional[str]: + """The browser setting workers should use, or None when they shouldn't run. + + Checked once, in the main process: ``""`` keeps the person's own + ``OPENADAPT_FLOW_BROWSER`` (or ``auto``, which means the bundled build on + a computer without Chrome or Edge); ``"bundled"`` points workers at + Playwright's own Chromium, already on disk. None means only an installed + Chrome or Edge is available, so the runs go one after another. + """ + + from openadapt_flow import _browser_setup as setup + + if os.environ.get(setup.BROWSER_ENV): + return "" + if setup.installed_browser_channel() is None: + return "" + try: + present = setup._bundled_present(only_shell=not visible) + except Exception: # noqa: BLE001 - unknown means run one after another + present = False + return "bundled" if present else None + + +def _worker_main(conn: Connection, visible: bool, browser: str = "") -> None: + """One worker: get ready, then run the program for each job it is sent.""" + + # Five engines at once: two OCR threads each, as Flow's parallel workers + # use, so they don't crowd out the main process's compile. + os.environ.setdefault(OCR_THREADS_ENV, "2") + if browser: + os.environ["OPENADAPT_FLOW_BROWSER"] = browser + # Workers get ready while the main process demonstrates and compiles. + # Lower priority keeps them on spare cores then; in Act 2 the main + # process only waits, so the workers have the computer to themselves. + if hasattr(os, "nice"): + try: + os.nice(WORKER_NICENESS) + except OSError: + pass + try: + from openadapt_flow import _prewarm + from openadapt_flow.backends._browser_process import shared_browser + from openadapt_flow.ir import Workflow + from openadapt_flow.mockmed.fault_server import serve + + from openadapt.quickstart.engine import run_program + + _prewarm.start() + with shared_browser(headless=not visible): + base_url, _db, stop = serve() + try: + conn.send(("ready",)) + while True: + message = conn.recv() + if message[0] != "run": + return + _, index, note, bundle_dir, run_dir = message + try: + facts = run_program( + base_url=base_url, + workflow=Workflow.load(Path(bundle_dir)), + bundle_dir=Path(bundle_dir), + run_dir=Path(run_dir), + index=index, + note=note, + visible=visible, + ) + except Exception as exc: # noqa: BLE001 - reported to the parent + conn.send(("failed", f"{type(exc).__name__}: {exc}")) + continue + payload = asdict(facts) + payload["run_dir"] = str(facts.run_dir) + conn.send(("done", payload)) + finally: + stop() + except (EOFError, KeyboardInterrupt, BrokenPipeError): + return + except Exception as exc: # noqa: BLE001 - the parent runs the job itself + try: + conn.send(("broken", f"{type(exc).__name__}: {exc}")) + except (BrokenPipeError, OSError): + pass + finally: + conn.close() + + +def _facts(fields: dict[str, Any]) -> Any: + from openadapt.quickstart.engine import RunFacts + + fields = dict(fields) + fields["run_dir"] = Path(fields["run_dir"]) + return RunFacts(**fields) + + +def _unfinished(root: Path, job: Job, reason: str) -> Any: + """A run that raised in its worker: not checked, with the reason.""" + + from openadapt.quickstart.engine import RunFacts + + index, note, folder = job + return RunFacts( + index=index, + note=note or "", + run_dir=root / folder, + execution_outcome="ERROR", + transaction_outcome=None, + checked=False, + records=0, + model_calls=0, + cost_usd=0.0, + effects_required=0, + effects_confirmed=0, + seconds=0.0, + plain_label=f"Didn't finish: {reason}"[:80], + ) + + +class WorkerPool: + """Worker processes, started early, each taking one run at a time.""" + + def __init__( + self, + size: int, + *, + visible: bool, + browser: str = "", + target: Callable[..., None] = _worker_main, + ) -> None: + context = multiprocessing.get_context("spawn") + self._workers: dict[Connection, Any] = {} + for _ in range(size): + parent, child = context.Pipe() + process = context.Process( + target=target, args=(child, visible, browser), daemon=True + ) + process.start() + child.close() + self._workers[parent] = process + self._ready: list[Connection] = [] + self._gone: set[Connection] = set() + + @property + def size(self) -> int: + return len(self._workers) + + def usable(self) -> int: + """Workers that are ready or still starting (not ones that gave up).""" + + for conn in wait(list(self._starting()), timeout=0): + self._note_start(conn) + return self.size - len(self._gone) + + def _starting(self) -> set[Connection]: + return set(self._workers) - set(self._ready) - self._gone + + def _note_start(self, conn: Connection) -> None: + """Read a starting worker's first message: ready, or gave up.""" + + try: + message = conn.recv() + except (EOFError, OSError): + message = ("broken", "the worker process exited") + if message[0] == "ready": + self._ready.append(conn) + else: + self._drop(conn) + + def _drop(self, conn: Connection) -> None: + self._gone.add(conn) + process = self._workers.get(conn) + if process is not None and process.is_alive(): + process.terminate() + + def run( + self, + jobs: Sequence[Job], + *, + bundle_dir: Path, + root: Path, + on_finish: Callable[[Any], None], + clock: Callable[[], float] = time.monotonic, + ) -> list[Job]: + """Run ``jobs`` across the workers; return the jobs to run here instead. + + Returned jobs are those no worker started, and those whose worker + stalled. A stalled job comes back with a new run folder, since the + engine never reuses one; its worker, with its browser and its own + database, is stopped first. A run that raised in its worker is + reported as not finished and not repeated. + """ + + pending = list(jobs) + given: dict[Connection, tuple[Job, float]] = {} + back: list[Job] = [] + started = clock() + + def give(conn: Connection) -> None: + if pending: + job = pending.pop(0) + index, note, folder = job + conn.send(("run", index, note, str(bundle_dir), str(root / folder))) + given[conn] = (job, clock()) + + for conn in list(self._ready): + give(conn) + while (self._starting() and pending) or given: + now = clock() + deadlines = [since + STALL_S for _job, since in given.values()] + starting = self._starting() if pending else set() + if starting: + deadlines.append(started + READY_TIMEOUT_S) + timeout = max(0.0, min(deadlines) - now) if deadlines else None + for conn in wait(list(starting | set(given)), timeout=timeout): + if conn not in given: + self._note_start(conn) + if conn in self._ready: + give(conn) + continue + try: + message = conn.recv() + except (EOFError, OSError): + message = ("broken", "the worker process exited") + kind = message[0] + if kind == "done": + given.pop(conn, None) + on_finish(_facts(message[1])) + give(conn) + elif kind == "failed": + job, _since = given.pop(conn) + on_finish(_unfinished(root, job, message[1])) + give(conn) + else: # broken: this worker exited mid-run + job, _since = given.pop(conn) + self._drop(conn) + back.append(_again(job)) + now = clock() + for conn, (job, since) in list(given.items()): + if now - since >= STALL_S: + given.pop(conn) + self._drop(conn) + back.append(_again(job)) + if pending and now - started >= READY_TIMEOUT_S: + for conn in self._starting(): + self._drop(conn) + return sorted(back + pending) + + def release(self) -> None: + """Tell every worker to shut down, without waiting for it.""" + + for conn in self._workers: + try: + conn.send(("close",)) + except (BrokenPipeError, OSError): + pass + + def close(self) -> None: + self.release() + for conn, process in self._workers.items(): + process.join(timeout=10) + if process.is_alive(): + process.terminate() + process.join(timeout=5) + conn.close() + + +def _again(job: Job) -> Job: + index, note, folder = job + return (index, note, f"{folder}-again") + + +__all__ = [ + "MAX_WORKERS", + "STALL_S", + "Job", + "WorkerPool", + "worker_browser", + "worker_count", +] diff --git a/packages/flow/openadapt_flow/tutorial.py b/packages/flow/openadapt_flow/tutorial.py index 395ef784b..ac5e476b9 100644 --- a/packages/flow/openadapt_flow/tutorial.py +++ b/packages/flow/openadapt_flow/tutorial.py @@ -472,6 +472,7 @@ def run_tutorial_workflow( headed: bool = False, entry_query: Optional[str] = None, presentation_delay_s: float = 0.0, + params: Optional[dict[str, str]] = None, ) -> Any: """Admit and execute the tutorial under the ``standard`` profile. @@ -479,6 +480,10 @@ def run_tutorial_workflow( ``--break-it`` rerun passes :data:`TUTORIAL_BREAK_ENTRY_QUERY` instead. Nothing else differs between the two runs: same bundle, same policy, same gate, same verifier. + + ``params`` supplies this run's inputs (the note); by default the run + replays the demonstrated values. The run authorization binds them, and + the bundle on disk is unchanged. """ from openadapt_flow.backends.playwright_backend import PlaywrightBackend @@ -521,7 +526,8 @@ def run_tutorial_workflow( f"executed:\n{gate.render()}" ) replay_params = { - "note": str(getattr(workflow, "params", {}).get("note", TUTORIAL_NOTE)) + "note": str(getattr(workflow, "params", {}).get("note", TUTORIAL_NOTE)), + **(params or {}), } authorization = build_runtime_authorization( workflow, diff --git a/packages/flow/tests/test_tutorial_run_params.py b/packages/flow/tests/test_tutorial_run_params.py new file mode 100644 index 000000000..b827fc3ac --- /dev/null +++ b/packages/flow/tests/test_tutorial_run_params.py @@ -0,0 +1,57 @@ +"""A tutorial run can take its own note, and its record check reads that note. + +The launcher's quickstart runs the compiled tutorial five times with a new +note each time. The bundle on disk never changes: the note reaches the run +through ``run_tutorial_workflow(params=...)``, the run authorization binds it, +and a record check compiled with ``bind_effect_params`` compares the saved +record with this run's note instead of the demonstrated one. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from openadapt_flow import tutorial +from openadapt_flow.backends._browser_process import shared_browser +from openadapt_flow.compiler import compile_recording +from openadapt_flow.mockmed.fault_server import serve + + +@pytest.mark.timeout(600) +def test_a_run_with_a_new_note_is_checked_against_that_note(tmp_path: Path) -> None: + base_url, _db, stop = serve() + try: + with shared_browser(headless=True): + tutorial.record_tutorial(base_url, tmp_path / "recording") + bundle_dir = tmp_path / "bundle" + workflow = compile_recording( + tmp_path / "recording", + bundle_dir, + name=tutorial.TUTORIAL_WORKFLOW_NAME, + mine_effects=True, + bind_effect_params=True, + ) + tutorial.certify_tutorial(workflow) + digest = (bundle_dir / "workflow.json").read_bytes() + + note = "Recheck blood pressure next month" + report = tutorial.run_tutorial_workflow( + base_url=base_url, + workflow=workflow, + bundle_dir=bundle_dir, + run_dir=tmp_path / "run", + params={"note": note}, + ) + records = tutorial._records(base_url) + finally: + stop() + + assert report.execution_outcome == "VERIFIED" + assert report.params["note"] == note + assert report.outcome_envelope.passed_contracts.effect == 2 + assert [record["note"] for record in records] == [note] + # The program itself is unchanged; only this run's input differs. + assert (bundle_dir / "workflow.json").read_bytes() == digest + assert workflow.params["note"] == tutorial.TUTORIAL_NOTE diff --git a/pyproject.toml b/pyproject.toml index 8ac243b7c..26912824d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,9 @@ classifiers = [ # tests/test_python_support.py keeps these in step. dependencies = [ "click>=8.0.0", + # The quickstart's terminal view (MIT). Pure Python; the engine doesn't + # depend on it. Gated like the engine: without it the quickstart can't run. + "rich>=13.0.0; python_version < '3.13'", # Every Flow floor is 1.35.0: Flow 1.33 and 1.34 probe four X11 libraries in # the wrong case and refuse Chromium on Linux hosts that have them # (tests/test_flow_floor.py). diff --git a/scripts/install_weight/budgets.json b/scripts/install_weight/budgets.json index ab8714df9..55c62b0a9 100644 --- a/scripts/install_weight/budgets.json +++ b/scripts/install_weight/budgets.json @@ -323,8 +323,10 @@ "jsonschema", "jsonschema-specifications", "keyring", + "markdown-it-py", "mcp", "mcp-types", + "mdurl", "more-itertools", "numpy", "onnxruntime", @@ -343,11 +345,13 @@ "pydantic", "pydantic-core", "pyee", + "pygments", "pyjwt", "python-multipart", "pyyaml", "rapidocr-onnxruntime", "referencing", + "rich", "rpds-py", "secretstorage", "shapely", diff --git a/scripts/quickstart_lifecycle.py b/scripts/quickstart_lifecycle.py index 2300109c8..8b53a06bc 100644 --- a/scripts/quickstart_lifecycle.py +++ b/scripts/quickstart_lifecycle.py @@ -144,12 +144,22 @@ def _install_browser_system_dependencies( ) -def _inspect_quickstart(root: Path) -> dict[str, object]: - run = root / "run" +def _quickstart_runs(root: Path) -> list[Path]: + """The quickstart's checked runs: run-1, run-2, ... (or the older run/).""" + numbered = sorted( + (path for path in root.glob("run-*") if path.name[4:].isdigit()), + key=lambda path: int(path.name[4:]), + ) + return numbered or [root / "run"] + + +def _inspect_run(run: Path) -> int: + """Check one run is VERIFIED under Standard with its receipt; return effects.""" report = _load_object(run / "report.json") if report.get("execution_outcome") != "VERIFIED": raise AssertionError( - f"quickstart outcome is {report.get('execution_outcome')!r}, not VERIFIED" + f"quickstart {run.name} outcome is " + f"{report.get('execution_outcome')!r}, not VERIFIED" ) if report.get("execution_profile") != "standard": raise AssertionError( @@ -174,12 +184,30 @@ def _inspect_quickstart(root: Path) -> dict[str, object]: for path in (run / "REPORT.md", run / "receipt.png", run / "receipt.md"): if not path.is_file(): raise AssertionError(f"quickstart is missing {path.name}") + return passed + + +def _inspect_quickstart(root: Path) -> dict[str, object]: + runs = _quickstart_runs(root) + effects = [_inspect_run(run) for run in runs] + broken = root / "run-broken" / "report.json" + stopped = None + if broken.is_file(): + report = _load_object(broken) + if report.get("execution_outcome") != "HALTED": + raise AssertionError( + "the save that failed silently was not caught: the rerun ended " + f"{report.get('execution_outcome')!r}, not HALTED" + ) + stopped = True return { "outcome": "VERIFIED", "profile": "standard", "transaction_outcome": "VERIFIED", - "effect_contracts": passed, + "effect_contracts": min(effects), + "verified_runs": len(runs), + "silent_failure_stopped": stopped, "model_calls": 0, "receipt_emitted": True, } diff --git a/scripts/quickstart_timing.py b/scripts/quickstart_timing.py new file mode 100644 index 000000000..dfcbdfc86 --- /dev/null +++ b/scripts/quickstart_timing.py @@ -0,0 +1,100 @@ +"""Time the quickstart against the tutorial it replaced, interleaved. + +The quickstart used to run the engine's tutorial with ``--break-it``: one +checked run and one run with the save set to fail. It now runs the program +five times and then the failing save. This script runs the two commands in +turn, ``--rounds`` times each, headless and not on a terminal, and writes +each wall time with the machine's load average, so a busy machine slows both +sides alike. + + python scripts/quickstart_timing.py --rounds 3 --out timing.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +#: A run that takes longer than this is recorded as a timeout, not waited on. +RUN_TIMEOUT_S = 300 + +COMMANDS = { + "before": ["flow", "tutorial", "--break-it", "--out"], + "after": ["quickstart", "--headless", "--out"], +} + + +def _launcher() -> str: + found = Path(sys.executable).parent / "openadapt" + return str(found) if found.exists() else (shutil.which("openadapt") or "openadapt") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + + launcher = _launcher() + rows = [] + with tempfile.TemporaryDirectory() as work: + env = { + **os.environ, + "OPENADAPT_SEAL_HOME": str(Path(work) / "seal"), + "BROWSER": "true", + } + for round_index in range(args.rounds): + for name, argv in COMMANDS.items(): + out = Path(work) / f"{name}-{round_index}" + started = time.monotonic() + try: + result = subprocess.run( + [launcher, *argv, str(out)], + env=env, + capture_output=True, + text=True, + timeout=RUN_TIMEOUT_S, + ) + code, lines = result.returncode, len(result.stdout.splitlines()) + except subprocess.TimeoutExpired: + code, lines = "timeout", 0 + seconds = time.monotonic() - started + rows.append( + { + "command": name, + "round": round_index, + "exit_code": code, + "wall_s": round(seconds, 2), + "load_1m": round(os.getloadavg()[0], 1), + "lines": lines, + } + ) + print(json.dumps(rows[-1]), flush=True) + summary = { + name: sorted(row["wall_s"] for row in rows if row["command"] == name) + for name in COMMANDS + } + document = { + "machine": f"{platform.system()} {platform.machine()}", + "python": platform.python_version(), + "commands": {name: " ".join(argv) for name, argv in COMMANDS.items()}, + "runs": rows, + "median_wall_s": { + name: values[len(values) // 2] for name, values in summary.items() + }, + } + args.out.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8") + print(json.dumps(document["median_wall_s"])) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index 3c0db32f4..9187e88f3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,3 +23,36 @@ def _import_torch_before_find_spec_is_stubbed(): if importlib.util.find_spec("torch") is not None: import torch # noqa: F401 yield + + +@pytest.fixture(autouse=True) +def _no_real_quickstart_runs(request, monkeypatch): + """Keep unit tests from starting a real browser run by accident. + + The three-act quickstart drives a browser for about 30 seconds. Tests that + mean to run it are marked ``real_quickstart``; everything else gets a + clear failure instead of a silent half-minute run. + """ + if request.node.get_closest_marker("real_quickstart"): + yield + return + try: + from openadapt.quickstart import engine + except ImportError: + yield + return + + def refuse(self): + raise AssertionError( + "this test started a real quickstart run; stub " + "openadapt.quickstart.run_quickstart or mark it real_quickstart" + ) + + monkeypatch.setattr(engine.FlowEngine, "__enter__", refuse) + yield + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "real_quickstart: runs the real quickstart in a browser" + ) diff --git a/tests/quickstart_fake_workers.py b/tests/quickstart_fake_workers.py new file mode 100644 index 000000000..ee358b1d1 --- /dev/null +++ b/tests/quickstart_fake_workers.py @@ -0,0 +1,66 @@ +"""Stand-in worker processes for tests/test_quickstart_workers.py. + +Spawned children import this module by name, so it lives at the top level of +tests/ (on sys.path under pytest) and imports nothing heavy. +""" + +from __future__ import annotations + +import time +from pathlib import Path + + +def _facts(index: int, note, run_dir: str) -> dict: + return { + "index": index, + "note": note or "", + "run_dir": run_dir, + "execution_outcome": "VERIFIED", + "transaction_outcome": "VERIFIED", + "checked": True, + "records": 1, + "model_calls": 0, + "cost_usd": 0.0, + "effects_required": 2, + "effects_confirmed": 2, + "seconds": 0.1, + "plain_label": "Done and checked", + } + + +def _serve(conn, behave) -> None: + conn.send(("ready",)) + while True: + message = conn.recv() + if message[0] != "run": + return + _, index, note, _bundle, run_dir = message + Path(run_dir).mkdir(parents=True, exist_ok=True) + behave(conn, index, note, run_dir) + + +def good(conn, visible, browser="") -> None: + _serve(conn, lambda c, i, n, d: c.send(("done", _facts(i, n, d)))) + + +def stalls_on_run_2(conn, visible, browser="") -> None: + def behave(c, i, n, d): + if i == 2: + time.sleep(600) + c.send(("done", _facts(i, n, d))) + + _serve(conn, behave) + + +def fails_run_3(conn, visible, browser="") -> None: + def behave(c, i, n, d): + if i == 3: + c.send(("failed", "TutorialError: the gate refused")) + else: + c.send(("done", _facts(i, n, d))) + + _serve(conn, behave) + + +def never_ready(conn, visible, browser="") -> None: + conn.send(("broken", "Playwright's own Chromium isn't installed")) diff --git a/tests/snapshots/quickstart_plain.snapshot b/tests/snapshots/quickstart_plain.snapshot new file mode 100644 index 000000000..f3a2c1633 --- /dev/null +++ b/tests/snapshots/quickstart_plain.snapshot @@ -0,0 +1,39 @@ +OpenAdapt quickstart: show a task once, run it, then watch it catch a save that didn't happen. +Synthetic data only. Files go to openadapt-quickstart + +1 Show it once · a demonstration in a synthetic clinic app, in a hidden browser + ✓ Click Open 0.4s + ✓ Click New Encounter 0.7s + ✓ Click Triage 0.9s + ✓ Click Note 1.1s + ✓ Type "Synthetic follow-up in two weeks" 1.3s + ✓ Click Save Encounter 1.6s + Compiled into a program: 6 steps. 0.5s + +2 Run it · the program on 5 new notes, all at once + 1/5 Follow-up in two weeks ✓ checked in the database 2.6s + 2/5 Recheck blood pressure next month ✓ checked in the database 2.7s + 3/5 Refer to physical therapy ✓ checked in the database 2.8s + 4/5 Repeat labs before next visit ✓ checked in the database 2.9s + 5/5 No change to the care plan ✓ checked in the database 3.0s + 5 of 5 saved and checked · 0 AI calls · $0.00 · 0.5s + +3 Break it · the next save fails silently + The app will show its saved message. The database will reject the save. + What the screen showed: "Encounter saved — Synthetic follow-up in two weeks" + What the database showed: 0 new records + A screen-trusting bot: ✓ Done + OpenAdapt: ■ Stopped before reporting success 7.7s + Why: OpenAdapt reads the record back from the database after each save. It found nothing, so it stopped. + +The screen said "Encounter saved". The database had nothing. A bot that trusts the screen reports Done. OpenAdapt stopped. + +Receipt + Program local-quickstart · 6 steps + Runs 5 of 5 saved and checked in the database + Cost 0 AI calls · $0.00 · 2.5s in all + Seal seal-ed25519-0123456789abcdef · 5 reports signed +Check it: openadapt seal verify openadapt-quickstart +Details: openadapt-quickstart, one REPORT.md per run; side-by-side page openadapt-quickstart/index.html +Try it on your own app: openadapt flow record --backend web --url https://your-app.example --out my-task +Give your agent this as a tool: openadapt agent serve --allow-run diff --git a/tests/test_agent_command.py b/tests/test_agent_command.py index 4193a37c0..a071f4537 100644 --- a/tests/test_agent_command.py +++ b/tests/test_agent_command.py @@ -56,7 +56,16 @@ def test_top_level_help_prints_the_launcher_agent_command(): def test_quickstart_prints_the_launcher_agent_command(monkeypatch, supported_python): - monkeypatch.setattr("openadapt.cli._invoke_flow", lambda _argv: 0) + from openadapt.quickstart.story import Outcome + + printed = [] + + def fake(root, *, steps, **kwargs): + printed.append(steps(Outcome(root=root)).agent) + return Outcome(root=root) + + monkeypatch.setattr("openadapt.cli._flow_runs_quickstart", lambda: True) + monkeypatch.setattr("openadapt.quickstart.run_quickstart", fake) monkeypatch.setattr("openadapt.cli._agent_serves_tutorial", lambda: True) runner = CliRunner() @@ -64,8 +73,8 @@ def test_quickstart_prints_the_launcher_agent_command(monkeypatch, supported_pyt result = runner.invoke(cli_main, ["quickstart", "--out", "qs"]) assert result.exit_code == 0, result.output - assert "openadapt agent serve --allow-run" in result.output - assert not _ENGINE_AGENT_COMMAND.search(result.output), result.output + assert printed == ["openadapt agent serve --allow-run"] + assert not _ENGINE_AGENT_COMMAND.search(printed[0]) def test_agent_forwards_every_argument_and_the_exit_code(fake_agent, supported_python): diff --git a/tests/test_cli_quickstart_polish.py b/tests/test_cli_quickstart_polish.py index 57242d702..aaaaa73dc 100644 --- a/tests/test_cli_quickstart_polish.py +++ b/tests/test_cli_quickstart_polish.py @@ -7,13 +7,14 @@ from __future__ import annotations -import shlex +import os from pathlib import Path import pytest from click.testing import CliRunner from openadapt.cli import main as cli_main +from openadapt.quickstart.story import Outcome @pytest.fixture() @@ -26,7 +27,21 @@ def engine_calls(monkeypatch): return calls -def test_default_output_auto_suffixes_the_first_taken_name(engine_calls): +@pytest.fixture() +def story_calls(monkeypatch): + """Stand in for the three-act quickstart; record what it was asked to do.""" + calls = [] + + def fake(root, **kwargs): + calls.append({"root": root, **kwargs}) + return Outcome(root=root) + + monkeypatch.setattr("openadapt.cli._flow_runs_quickstart", lambda: True) + monkeypatch.setattr("openadapt.quickstart.run_quickstart", fake) + return calls + + +def test_default_output_auto_suffixes_the_first_taken_name(story_calls): runner = CliRunner() with runner.isolated_filesystem(): Path("openadapt-quickstart").mkdir() @@ -34,12 +49,11 @@ def test_default_output_auto_suffixes_the_first_taken_name(engine_calls): expected = Path("openadapt-quickstart-2").resolve() assert result.exit_code == 0, result.output - assert len(engine_calls) == 1 - assert engine_calls[0][2] == str(expected) - assert "openadapt-quickstart-2" in result.output + assert len(story_calls) == 1 + assert story_calls[0]["root"] == expected -def test_default_output_keeps_counting_past_repeated_runs(engine_calls): +def test_default_output_keeps_counting_past_repeated_runs(story_calls): runner = CliRunner() with runner.isolated_filesystem(): Path("openadapt-quickstart").mkdir() @@ -49,10 +63,10 @@ def test_default_output_keeps_counting_past_repeated_runs(engine_calls): expected = Path("openadapt-quickstart-4").resolve() assert result.exit_code == 0, result.output - assert engine_calls[0][2] == str(expected) + assert story_calls[0]["root"] == expected -def test_explicit_out_is_honored_verbatim(engine_calls): +def test_explicit_out_is_honored_verbatim(story_calls): runner = CliRunner() with runner.isolated_filesystem(): Path("openadapt-quickstart").mkdir() @@ -60,7 +74,99 @@ def test_explicit_out_is_honored_verbatim(engine_calls): expected = Path("my-dir").resolve() assert result.exit_code == 0, result.output - assert engine_calls[0][2] == str(expected) + assert story_calls[0]["root"] == expected + + +@pytest.mark.parametrize( + ("argv", "expected"), + [ + ([], {"break_it": True, "verbose": False}), + (["--no-break"], {"break_it": False}), + (["--verbose"], {"verbose": True}), + (["--break-it"], {"break_it": True}), + ], +) +def test_quickstart_options_reach_the_story(story_calls, monkeypatch, argv, expected): + monkeypatch.setattr("openadapt.quickstart.engine.display_available", lambda: False) + with CliRunner().isolated_filesystem(): + result = CliRunner().invoke(cli_main, ["quickstart", *argv, "--out", "qs"]) + assert result.exit_code == 0, result.output + for key, value in expected.items(): + assert story_calls[0][key] == value + + +@pytest.mark.parametrize( + ("argv", "display", "visible"), + [ + ([], True, True), + ([], False, False), + (["--headless"], True, False), + (["--headed"], False, True), + ], +) +def test_the_browser_shows_when_a_display_is_available( + story_calls, monkeypatch, argv, display, visible +): + monkeypatch.setattr( + "openadapt.quickstart.engine.display_available", lambda: display + ) + with CliRunner().isolated_filesystem(): + result = CliRunner().invoke(cli_main, ["quickstart", *argv, "--out", "qs"]) + assert result.exit_code == 0, result.output + assert story_calls[0]["visible"] is visible + + +def test_an_older_flow_runs_its_own_tutorial(monkeypatch, engine_calls): + monkeypatch.setattr("openadapt.cli._flow_runs_quickstart", lambda: False) + with CliRunner().isolated_filesystem(): + result = CliRunner().invoke(cli_main, ["quickstart", "--out", "qs"]) + assert result.exit_code == 0, result.output + assert engine_calls[0][:1] == ["tutorial"] + + +@pytest.mark.parametrize( + ("installed", "runs"), + [("2.1.0", False), ("2.2.0", True), ("2.10.1", True)], +) +def test_the_three_act_quickstart_needs_flow_2_2(monkeypatch, installed, runs): + from openadapt import cli + + monkeypatch.setattr("importlib.metadata.version", lambda _name: installed) + assert cli._flow_runs_quickstart() is runs + + +def test_quickstart_restores_the_operator_scrub_setting(monkeypatch, story_calls): + seen = [] + monkeypatch.setenv("OPENADAPT_FLOW_SCRUB", "auto") + monkeypatch.setattr( + "openadapt.quickstart.run_quickstart", + lambda root, **kw: ( + seen.append(os.environ["OPENADAPT_FLOW_SCRUB"]) or Outcome(root=root) + ), + ) + with CliRunner().isolated_filesystem(): + result = CliRunner().invoke(cli_main, ["quickstart", "--out", "qs"]) + assert result.exit_code == 0, result.output + assert seen == ["off"] + assert os.environ["OPENADAPT_FLOW_SCRUB"] == "auto" + + +def test_a_stopped_stage_names_where_its_files_are(monkeypatch, story_calls): + class TutorialError(RuntimeError): + pass + + def stop(root, **kwargs): + root.mkdir(parents=True) + (root / "partial.json").write_text("{}") + raise TutorialError("the compiler emitted a placeholder") + + monkeypatch.setattr("openadapt.quickstart.run_quickstart", stop) + with CliRunner().isolated_filesystem(): + result = CliRunner().invoke(cli_main, ["quickstart", "--out", "qs"]) + assert Path("qs/partial.json").is_file() + assert result.exit_code == 1 + assert "The quickstart stopped: the compiler emitted a placeholder" in result.output + assert "Anything it wrote is in" in result.output @pytest.mark.parametrize("version_info", [(3, 13), (3, 14)]) @@ -84,14 +190,14 @@ def test_unsupported_python_preflight_prints_remedy_before_delegation( assert engine_calls == [] -def test_pep668_error_prints_the_same_remedy(monkeypatch): - def raise_externally_managed(_argv): +def test_pep668_error_prints_the_same_remedy(monkeypatch, story_calls): + def raise_externally_managed(*_args, **_kwargs): raise RuntimeError( "error: externally-managed-environment\n\n" "This environment is externally managed" ) - monkeypatch.setattr("openadapt.cli._invoke_flow", raise_externally_managed) + monkeypatch.setattr("openadapt.quickstart.run_quickstart", raise_externally_managed) runner = CliRunner() with runner.isolated_filesystem(): @@ -129,50 +235,3 @@ def test_unknown_flags_pass_through_to_engine_verbatim( argv = engine_calls[0] assert argv[0] == "tutorial" assert argv[-len(extra) :] == extra - - -def _printed_commands(output: str) -> list[list[str]]: - commands = [] - for line in output.splitlines(): - _, found, rest = line.partition("openadapt flow ") - if found: - commands.append(shlex.split(rest)) - return commands - - -def _option(argv: list[str], name: str) -> str: - return argv[argv.index(name) + 1] - - -def test_drift_hint_replays_a_bundle_recorded_on_plain_mockmed(engine_calls): - """The tutorial bundle is recorded against MockMed's fault server. - - `flow replay --drift` serves plain MockMed, which opens at the login - screen, so replaying the tutorial bundle halts at step_000 with or - without drift. The drift hint must record and compile its own demo on - plain MockMed and replay that bundle. - """ - runner = CliRunner() - with runner.isolated_filesystem(): - result = runner.invoke(cli_main, ["quickstart", "--out", "qs"]) - root = Path("qs").resolve() - - assert result.exit_code == 0, result.output - assert "fail-safe" not in result.output - commands = _printed_commands(result.output) - drift_replays = [ - argv for argv in commands if argv[0] == "replay" and "--drift" in argv - ] - assert len(drift_replays) == 1, result.output - replay = drift_replays[0] - assert Path(replay[1]) != root / "bundle" - - (record,) = [argv for argv in commands if argv[0] == "demo-record"] - (compile_,) = [argv for argv in commands if argv[0] == "compile"] - assert compile_[1] == _option(record, "--out") - assert replay[1] == _option(compile_, "--out") - assert ( - result.output.index("flow demo-record") - < result.output.index("flow compile") - < result.output.index("--drift") - ) diff --git a/tests/test_cli_shell_hints.py b/tests/test_cli_shell_hints.py index 86b3f50ad..443f32329 100644 --- a/tests/test_cli_shell_hints.py +++ b/tests/test_cli_shell_hints.py @@ -33,26 +33,15 @@ def supported_python(monkeypatch): monkeypatch.setattr("openadapt.cli._python_minor", lambda: cli._SUPPORTED_PYTHON[1]) -def _quickstart_output(monkeypatch, out: Path) -> str: - monkeypatch.setattr("openadapt.cli._invoke_flow", lambda _argv: 0) - monkeypatch.setattr( - "openadapt.cli._flow_tutorial_options", lambda: frozenset(), raising=False - ) - result = CliRunner().invoke(cli_main, ["quickstart", "--out", str(out)]) - assert result.exit_code == 0, result.output - return result.output +def _commands(root: Path) -> list[str]: + """The quickstart's printed commands that carry its output path.""" + from openadapt.quickstart import next_steps + from openadapt.quickstart.engine import SealFacts + from openadapt.quickstart.story import Outcome - -def _commands(output: str, tmp_path: Path) -> list[str]: - """Return each printed `openadapt ...` command that carries a path.""" - commands = [] - for line in output.splitlines(): - start = line.find("openadapt ") - if start >= 0 and str(tmp_path) in line: - commands.append(line[start:]) - # lint, quickstart --break-it, demo-record, compile, replay. - assert len(commands) == 5, output - return commands + outcome = Outcome(root=root, seal=SealFacts("seal-ed25519-0", 5)) + steps = next_steps(cli._hint, cli._shell_quote, False)(outcome) + return [steps.verify, steps.agent] def test_posix_quickstart_hints_keep_a_spaced_path_in_one_argument( @@ -61,18 +50,10 @@ def test_posix_quickstart_hints_keep_a_spaced_path_in_one_argument( monkeypatch.setattr("sys.platform", "darwin") root = tmp_path / "dir with space" / "qs" - output = _quickstart_output(monkeypatch, root) + verify, agent = _commands(root) - commands = _commands(output, tmp_path) - lint = next(c for c in commands if " lint " in c) - assert shlex.split(lint)[-1] == str(root / "bundle") - break_it = next(c for c in commands if "--break-it" in c) - assert shlex.split(break_it)[-1] == f"{root}-break-it" - for command in commands: - argv = shlex.split(command) - paths = [arg for arg in argv if arg.startswith(str(tmp_path))] - assert paths, command - assert all("dir with space" in arg for arg in paths), command + assert shlex.split(verify)[-1] == str(root) + assert shlex.split(agent)[-1] == str(root / "bundle") def test_windows_quickstart_hints_double_quote_a_spaced_path( @@ -81,9 +62,7 @@ def test_windows_quickstart_hints_double_quote_a_spaced_path( monkeypatch.setattr("sys.platform", "win32") root = tmp_path / "dir with space" / "qs" - output = _quickstart_output(monkeypatch, root) - - for command in _commands(output, tmp_path): + for command in _commands(root): assert "'" not in command, command tokens = shlex.split(command, posix=False) paths = [t for t in tokens if str(tmp_path) in t] diff --git a/tests/test_cli_smoke.py b/tests/test_cli_smoke.py index ea0aae7ce..4aaafe8fe 100644 --- a/tests/test_cli_smoke.py +++ b/tests/test_cli_smoke.py @@ -77,7 +77,15 @@ def test_version_command(): assert result.exit_code == 0 -def test_quickstart_runs_one_local_lifecycle_without_overwriting(monkeypatch): +@pytest.fixture() +def engine_tutorial(monkeypatch): + """Select the engine's own tutorial (the path for older Flow releases).""" + monkeypatch.setattr("openadapt.cli._flow_runs_quickstart", lambda: False) + + +def test_quickstart_runs_one_local_lifecycle_without_overwriting( + engine_tutorial, monkeypatch +): calls = [] monkeypatch.setattr( "openadapt.cli._invoke_flow", @@ -93,10 +101,10 @@ def test_quickstart_runs_one_local_lifecycle_without_overwriting(monkeypatch): assert calls[0][0] == "tutorial" assert calls[0][1] == "--out" assert calls[0][3:] == ["--name", "local-quickstart"] - assert "openadapt quickstart --break-it" in result.output + assert "Run evidence:" in result.output -def test_quickstart_forwards_the_headed_tutorial_option(monkeypatch): +def test_quickstart_forwards_the_headed_tutorial_option(engine_tutorial, monkeypatch): calls = [] monkeypatch.setattr( "openadapt.cli._invoke_flow", @@ -131,6 +139,7 @@ def invoke(argv): def test_quickstart_forwards_the_break_it_option_and_names_the_evidence( + engine_tutorial, monkeypatch, ): calls = [] @@ -155,7 +164,6 @@ def test_quickstart_forwards_the_break_it_option_and_names_the_evidence( assert calls[0][0] == "tutorial" assert calls[0][-1] == "--break-it" assert "run-broken" in result.output - assert "HALTED" in result.output # The halted rerun ends RECONCILIATION_REQUIRED. Nothing may claim the # store is unchanged or that a retry is safe. assert "unchanged" not in result.output @@ -197,7 +205,7 @@ def test_quickstart_forwards_the_break_it_option_and_names_the_evidence( ], ) def test_quickstart_derives_the_halted_rerun_result_from_its_report( - monkeypatch, report_text, expected + engine_tutorial, monkeypatch, report_text, expected ): calls = [] monkeypatch.setattr( @@ -218,7 +226,7 @@ def test_quickstart_derives_the_halted_rerun_result_from_its_report( assert "A save may have gone through" in result.output -def test_quickstart_restores_the_operator_scrub_setting(monkeypatch): +def test_quickstart_restores_the_operator_scrub_setting(engine_tutorial, monkeypatch): monkeypatch.setenv("OPENADAPT_FLOW_SCRUB", "auto") monkeypatch.setattr("openadapt.cli._invoke_flow", lambda _argv: 0) @@ -230,7 +238,9 @@ def test_quickstart_restores_the_operator_scrub_setting(monkeypatch): assert os.environ["OPENADAPT_FLOW_SCRUB"] == "auto" -def test_quickstart_retains_artifacts_when_the_engine_halts(monkeypatch): +def test_quickstart_retains_artifacts_when_the_engine_halts( + engine_tutorial, monkeypatch +): def halt_after_writing(argv): root = Path(argv[argv.index("--out") + 1]) root.mkdir(parents=True) @@ -806,8 +816,7 @@ def test_top_level_help_leads_with_flow(): assert result.exit_code == 0 # Quick Start headline and Commands listing both lead with flow. assert "openadapt flow demo-record" in result.output - assert "openadapt flow tutorial" in result.output - assert "openadapt quickstart --break-it" in result.output + assert "openadapt quickstart" in result.output # openadapt-agent 2.0.1 requires --bundles; the help names a command that # works with every released agent and every install method. assert "openadapt agent serve --bundles DIR --mode production" in result.output diff --git a/tests/test_cli_uvx_hints.py b/tests/test_cli_uvx_hints.py index d91b5dd1b..9c04ba90d 100644 --- a/tests/test_cli_uvx_hints.py +++ b/tests/test_cli_uvx_hints.py @@ -16,3 +16,99 @@ def test_hint_repeats_uvx_form_in_a_uvx_environment(monkeypatch) -> None: def test_hint_keeps_plain_form_when_installed(monkeypatch) -> None: monkeypatch.setattr(sys, "prefix", "/Users/x/.local/share/uv/tools/openadapt") assert cli._hint("flow lint bundle") == "openadapt flow lint bundle" + + +# --------------------------------------------------------------------------- +# Under uvx the quickstart says how to keep this version, and warns when the +# shell's own `openadapt` is another one. +# --------------------------------------------------------------------------- + +import subprocess # noqa: E402 +from pathlib import Path # noqa: E402 + +import pytest # noqa: E402 + +from openadapt.quickstart import next_steps # noqa: E402 +from openadapt.quickstart.story import Outcome # noqa: E402 +from openadapt.version import __version__ # noqa: E402 + +UVX_PREFIX = "/Users/x/.cache/uv/archive-v0/kj_30_Pw78" + + +def _version_run(output: str): + def run(argv, **kwargs): + return subprocess.CompletedProcess(argv, 0, stdout=output, stderr="") + + return run + + +def test_keep_it_appears_only_under_uvx(monkeypatch) -> None: + py = f"{sys.version_info[0]}.{sys.version_info[1]}" + monkeypatch.setattr(sys, "prefix", UVX_PREFIX) + assert cli._keep_command() == f"uv tool install --python {py} openadapt" + monkeypatch.setattr(sys, "prefix", "/Users/x/.local/share/uv/tools/openadapt") + assert cli._keep_command() is None + + +def test_an_older_openadapt_on_path_gets_one_warning() -> None: + found = "/opt/homebrew/anaconda3/bin/openadapt" + warning = cli._shadowing_openadapt( + which=lambda name: found, run=_version_run("openadapt, version 1.5.1\n") + ) + assert warning == ( + f"Your shell's openadapt is 1.5.1 at {found}; the commands above use uvx " + "so they run this version." + ) + + +@pytest.mark.parametrize( + ("which", "output"), + [ + (lambda name: None, ""), + ( + lambda name: "/Users/x/.local/bin/openadapt", + f"openadapt, version {__version__}", + ), + ], +) +def test_no_warning_without_a_shadow(which, output) -> None: + assert cli._shadowing_openadapt(which=which, run=_version_run(output)) is None + + +def test_an_openadapt_that_does_not_answer_is_still_named() -> None: + def broken(argv, **kwargs): + raise OSError("exec format error") + + warning = cli._shadowing_openadapt( + which=lambda name: "/usr/bin/openadapt", run=broken + ) + assert warning is not None and "unknown version at /usr/bin/openadapt" in warning + + +def test_the_finale_prints_keep_it_and_the_shadow_warning( + tmp_path, monkeypatch +) -> None: + import io + + from openadapt.quickstart.view import PlainView + + monkeypatch.setattr(sys, "prefix", UVX_PREFIX) + steps = next_steps( + cli._hint, + cli._shell_quote, + True, + keep=cli._keep_command(), + shadow=lambda: cli._shadowing_openadapt( + which=lambda name: "/opt/conda/bin/openadapt", + run=_version_run("openadapt, version 1.5.1"), + ), + )(Outcome(root=tmp_path / "qs")) + stream = io.StringIO() + PlainView(stream).finale(Outcome(root=Path(tmp_path / "qs")), steps) + lines = stream.getvalue().splitlines() + assert lines[-2].startswith("Keep it: uv tool install --python ") + assert lines[-1].startswith( + "Your shell's openadapt is 1.5.1 at /opt/conda/bin/openadapt" + ) + finale = lines[lines.index("Receipt") :] + assert len(finale) <= 12 diff --git a/tests/test_launcher_conflicts.py b/tests/test_launcher_conflicts.py new file mode 100644 index 000000000..035423207 --- /dev/null +++ b/tests/test_launcher_conflicts.py @@ -0,0 +1,108 @@ +"""Another `openadapt` on PATH that shadows this one, or is newer, is named. + +`uv tool install openadapt` puts the launcher in ~/.local/bin, but a Conda +`bin` earlier on PATH can keep running an old 1.x launcher, which fails with +"invalid choice" on every new command. When this launcher runs (`--version`, +an unknown command, or under uvx), it names the other install and the fix. +""" + +from __future__ import annotations + +import os +import stat +import subprocess +import sys +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from openadapt import cli +from openadapt.version import __version__ + + +def _launcher(folder: Path, version: str, *, python: bool = True) -> Path: + folder.mkdir(parents=True, exist_ok=True) + path = folder / "openadapt" + path.write_text(f"#!/bin/sh\necho 'openadapt, version {version}'\n") + path.chmod(path.stat().st_mode | stat.S_IEXEC) + if python: + (folder / "python").write_text("") + return path + + +def _run(argv, **kwargs): + return subprocess.run(argv, capture_output=True, text=True, timeout=15) + + +pytestmark = pytest.mark.skipif(os.name == "nt", reason="POSIX shell stand-ins") + + +def test_path_launchers_follow_path_order_without_duplicates(tmp_path) -> None: + first = _launcher(tmp_path / "conda" / "bin", "1.5.1") + second = _launcher(tmp_path / "local" / "bin", __version__) + path = os.pathsep.join([str(first.parent), str(second.parent), str(first.parent)]) + assert cli._path_launchers({"PATH": path}) == [first, second] + + +def test_an_older_launcher_first_on_path_is_named_with_the_fix(tmp_path) -> None: + old = _launcher(tmp_path / "conda" / "bin", "1.5.1") + lines = cli._launcher_conflicts([old], run=_run) + assert lines == [ + f"Your shell runs openadapt 1.5.1 at {old} before this one ({__version__}). " + f"To remove it: {old.parent / 'python'} -m pip uninstall openadapt " + "openadapt-flow, then run: hash -r" + ] + + +def test_without_a_python_beside_it_the_fix_names_the_environment(tmp_path) -> None: + old = _launcher(tmp_path / "env" / "bin", "1.5.1", python=False) + (line,) = cli._launcher_conflicts([old], run=_run) + assert line.endswith( + f"pip uninstall openadapt openadapt-flow from {old.parent.parent}, " + "then run: hash -r" + ) + + +def test_a_newer_launcher_later_on_path_is_named(tmp_path, monkeypatch) -> None: + here = Path(sys.prefix) / "bin" + newer = _launcher(tmp_path / "new" / "bin", "99.0.0") + lines = cli._launcher_conflicts([here / "openadapt", newer], run=_run) + assert lines == [ + f"A newer openadapt, 99.0.0, is at {newer}; this one is {__version__}." + ] + + +def test_copies_of_this_version_are_fine(tmp_path) -> None: + same = _launcher(tmp_path / "local" / "bin", __version__) + assert cli._launcher_conflicts([same], run=_run) == [] + assert cli._launcher_conflicts([], run=_run) == [] + + +def test_version_prints_the_version_then_any_conflict(monkeypatch) -> None: + monkeypatch.setattr( + cli, "_launcher_conflicts", lambda: ["Your shell runs openadapt 1.5.1"] + ) + result = CliRunner().invoke(cli.main, ["--version"]) + assert result.exit_code == 0 + assert result.output.splitlines()[0] == f"openadapt, version {__version__}" + assert "Your shell runs openadapt 1.5.1" in result.output + + +def test_an_unknown_command_names_a_conflict(monkeypatch) -> None: + monkeypatch.setattr( + cli, "_launcher_conflicts", lambda: ["A newer openadapt, 99.0.0"] + ) + result = CliRunner().invoke(cli.main, ["no-such-verb"]) + assert result.exit_code == 2 + assert "A newer openadapt, 99.0.0" in result.output + assert "No such command 'no-such-verb'" in result.output + + +def test_a_failing_path_check_never_breaks_version(monkeypatch) -> None: + def boom(): + raise OSError("unreadable PATH entry") + + monkeypatch.setattr(cli, "_launcher_conflicts", boom) + result = CliRunner().invoke(cli.main, ["--version"]) + assert result.exit_code == 0 diff --git a/tests/test_python_support.py b/tests/test_python_support.py index 156633452..a51b48e05 100644 --- a/tests/test_python_support.py +++ b/tests/test_python_support.py @@ -353,6 +353,9 @@ def test_tutorial_next_step_prints_the_launcher_form(monkeypatch, supported_pyth def test_quickstart_next_step_prints_the_launcher_form(monkeypatch, supported_python): + # The engine's own tutorial (older Flow, or engine-only flags) prints its + # next steps; the launcher respells them. + monkeypatch.setattr("openadapt.cli._flow_runs_quickstart", lambda: False) _fake_flow(monkeypatch, " openadapt-flow record --backend web --url ") runner = CliRunner() @@ -403,7 +406,16 @@ def missing(_name): def test_quickstart_offers_the_agent_command_only_when_it_works( monkeypatch, supported_python, serves ): - monkeypatch.setattr("openadapt.cli._invoke_flow", lambda _argv: 0) + from openadapt.quickstart.story import Outcome + + printed = [] + + def fake(root, *, steps, **kwargs): + printed.append(steps(Outcome(root=root)).agent) + return Outcome(root=root) + + monkeypatch.setattr("openadapt.cli._flow_runs_quickstart", lambda: True) + monkeypatch.setattr("openadapt.quickstart.run_quickstart", fake) monkeypatch.setattr("openadapt.cli._agent_serves_tutorial", lambda: serves) runner = CliRunner() @@ -411,4 +423,6 @@ def test_quickstart_offers_the_agent_command_only_when_it_works( result = runner.invoke(cli_main, ["quickstart", "--out", "qs"]) assert result.exit_code == 0, result.output - assert ("openadapt agent serve --allow-run" in result.output) is serves + assert (printed == ["openadapt agent serve --allow-run"]) is serves + if not serves: + assert printed == ["openadapt agent serve --bundles qs/bundle"] diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py new file mode 100644 index 000000000..7e91560fb --- /dev/null +++ b/tests/test_quickstart.py @@ -0,0 +1,852 @@ +"""`openadapt quickstart`: three acts, plain and live output, honest numbers. + +The story runs against a fake engine here, so these tests take milliseconds. +The fake returns whatever facts a test gives it, and the output must follow: +nothing the quickstart prints may be a constant standing in for a run. The +real run is covered by ``test_real_quickstart`` (opt in with +``OPENADAPT_REAL_QUICKSTART=1``) and the weekly quickstart-lifecycle job. +""" + +from __future__ import annotations + +import io +import json +import os +import shlex +import struct +import subprocess +import sys +import threading +import time +import zlib +from dataclasses import replace +from pathlib import Path +from typing import Optional + +import pytest +from click.testing import CliRunner + +from openadapt import cli +from openadapt.quickstart import next_steps, run_quickstart +from openadapt.quickstart.engine import ( + RUN_NOTES, + BreakFacts, + DemoStep, + RunFacts, + SealFacts, + StepTicker, + demo_step_label, +) +from openadapt.quickstart.images import ( + ITERM2, + KITTY, + beside_image, + detect_image_protocol, + image_sequence, +) +from openadapt.quickstart.story import NextSteps, Outcome, run_story +from openadapt.quickstart.view import Marks, PlainView, RichView, make_view + +SNAPSHOT = Path(__file__).parent / "snapshots" / "quickstart_plain.snapshot" + +SCREEN_TEXT = "Encounter saved — Synthetic follow-up in two weeks" + +DEMO = [ + ("Click Open", 0.4), + ("Click New Encounter", 0.7), + ("Click Triage", 0.9), + ("Click Note", 1.1), + ('Type "Synthetic follow-up in two weeks"', 1.3), + ("Click Save Encounter", 1.6), +] + + +def _png(path: Path, width: int = 64, height: int = 32) -> Path: + def chunk(tag: bytes, data: bytes) -> bytes: + crc = zlib.crc32(tag + data) & 0xFFFFFFFF + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc) + + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + rows = b"".join(b"\x00" + b"\x80" * (width * 3) for _ in range(height)) + path.write_bytes( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(rows)) + + chunk(b"IEND", b"") + ) + return path + + +def _run(root: Path, index: int, **overrides) -> RunFacts: + facts = RunFacts( + index=index, + note=RUN_NOTES[index - 1], + run_dir=root / f"run-{index}", + execution_outcome="VERIFIED", + transaction_outcome="VERIFIED", + checked=True, + records=1, + model_calls=0, + cost_usd=0.0, + effects_required=2, + effects_confirmed=2, + seconds=2.5 + index / 10, + plain_label="Done and checked", + ) + return replace(facts, **overrides) + + +def _broken(root: Path, **overrides) -> BreakFacts: + facts = BreakFacts( + run_dir=root / "run-broken", + fault="optimistic", + execution_outcome="HALTED", + transaction_outcome="RECONCILIATION_REQUIRED", + screen_showed_success=True, + screen_text=SCREEN_TEXT, + screenshot=None, + new_records=0, + effects_required=2, + effects_refuted=1, + halt_reason="record_written refuted", + model_calls=0, + seconds=7.7, + ) + return replace(facts, **overrides) + + +class FakeEngine: + """Stands in for the browser and the engine; returns the facts it's given.""" + + def __init__( + self, + root: Path, + *, + visible: bool = False, + runs: Optional[dict[int, dict]] = None, + broken: Optional[dict] = None, + program_steps: int = 6, + seal: Optional[SealFacts] = SealFacts("seal-ed25519-0123456789abcdef", 5), + page: bool = True, + demo=DEMO, + note_param: bool = True, + parallel: int = 5, + **_ignored, + ) -> None: + self._note_param = note_param + self._parallel = parallel + self.root = root + self.visible = visible + self._runs = runs or {} + self._broken = broken or {} + self._program_steps = program_steps + self._seal = seal + self._page = page + self._demo = demo + self.calls: list[str] = [] + + def __enter__(self): + self.calls.append("enter") + return self + + def __exit__(self, *exc): + self.calls.append("exit") + + def record(self, on_step): + self.calls.append("record") + steps = [DemoStep(label, at) for label, at in self._demo] + for step in steps: + on_step(step) + return steps + + def compile(self): + self.calls.append("compile") + return self._program_steps + + @property + def program_name(self): + return "local-quickstart" + + def note_param(self): + return self._note_param + + def start_workers(self): + self.calls.append("start workers") + + @property + def parallel(self): + return self._parallel + + def run_all(self, notes, on_finish): + for index, _note in enumerate(notes, start=1): + self.calls.append(f"run {index}") + on_finish(_run(self.root, index, **self._runs.get(index, {}))) + + def before_prompt(self): + self.calls.append("before prompt") + + def break_it(self): + self.calls.append("break") + broken = dict(self._broken) + self._screen = broken.pop("screen_text", SCREEN_TEXT) + return _broken(self.root, **broken) + + def read_screen(self, broken): + return self._screen + + def seal(self, runs): + self.calls.append("seal") + if self._seal is None: + return None + return SealFacts(self._seal.key_id, sum(1 for run in runs if run.checked)) + + def write_page(self, first, broken): + self.calls.append("page") + return self.root / "index.html" if self._page else None + + +def _clock(step: float = 0.5): + now = [0.0] + + def tick() -> float: + now[0] += step + return now[0] + + return tick + + +def _steps(outcome: Outcome) -> NextSteps: + return next_steps(cli._hint, cli._shell_quote, True)(outcome) + + +def _plain(engine, **kwargs) -> tuple[str, Outcome]: + stream = io.StringIO() + outcome = run_story( + engine, + PlainView(stream), + next_steps=_steps, + clock=_clock(), + **kwargs, + ) + return stream.getvalue(), outcome + + +@pytest.fixture() +def root(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "prefix", "/opt/openadapt-env") + return tmp_path / "openadapt-quickstart" + + +# --------------------------------------------------------------------------- +# Plain output (not a terminal) +# --------------------------------------------------------------------------- + + +def test_plain_output_matches_the_snapshot(root): + output, _ = _plain(FakeEngine(root)) + if os.environ.get("UPDATE_SNAPSHOTS") == "1": + SNAPSHOT.parent.mkdir(exist_ok=True) + SNAPSHOT.write_text(output, encoding="utf-8") + assert output == SNAPSHOT.read_text(encoding="utf-8") + + +def test_plain_output_runs_the_three_acts_in_order(root): + engine = FakeEngine(root) + output, _ = _plain(engine) + # Signing runs on a thread during Act 3, after the five runs. + assert [call for call in engine.calls if call != "seal"] == [ + "enter", + "start workers", + "record", + "compile", + *(f"run {index}" for index in range(1, 6)), + "break", + "page", + "exit", + ] + assert engine.calls.index("seal") > engine.calls.index("run 5") + acts = [output.index(f"\n{n} ") for n in (1, 2, 3)] + assert acts == sorted(acts) + assert "\x1b[" not in output + + +def test_plain_output_has_no_internal_vocabulary(root): + output, _ = _plain(FakeEngine(root)) + for term in ( + "persistence boundary", + "effect contract", + "Seal Oracle", + "metering", + "billable", + "success rail", + "RECONCILIATION_REQUIRED", + "HALTED", + "VERIFIED", + "system of record", + "tier", + "fixture", + ): + assert term not in output, term + + +def test_verbose_adds_the_engine_detail(root): + output, _ = _plain(FakeEngine(root), verbose=True) + assert "transaction RECONCILIATION_REQUIRED" in output + assert "engine: record_written refuted" in output + assert str(root / "run-1" / "REPORT.md") in output + + +def test_finale_is_at_most_twelve_lines_with_exactly_two_next_steps(root): + output, _ = _plain(FakeEngine(root)) + finale = output[output.index("\nReceipt\n") + 1 :].rstrip("\n").splitlines() + assert len(finale) <= 12, finale + assert [line for line in finale if line.startswith(("Try it", "Give your"))] == [ + "Try it on your own app: openadapt flow record --backend web --url " + "https://your-app.example --out my-task", + "Give your agent this as a tool: openadapt agent serve --allow-run", + ] + assert sum(1 for line in finale if line.startswith("Details: ")) == 1 + + +def test_no_break_skips_act_three(root): + engine = FakeEngine(root) + output, outcome = _plain(engine, break_it=False) + assert "Break it" not in output + assert "break" not in engine.calls + assert "page" not in engine.calls + assert outcome.broken is None + assert "Receipt" in output + + +# --------------------------------------------------------------------------- +# Every number comes from the runs +# --------------------------------------------------------------------------- + + +def test_numbers_follow_the_runs(root): + engine = FakeEngine( + root, + program_steps=9, + runs={ + 2: {"model_calls": 3, "cost_usd": 0.04, "seconds": 11.25}, + 4: { + "checked": False, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "plain_label": "Stopped before saving", + }, + }, + demo=[("Click Go", 0.25), ('Type "x"', 3.75)], + ) + output, outcome = _plain(engine) + + assert "Compiled into a program: 9 steps." in output + assert "Click Go" in output and "3.8s" in output + assert "11.2s" in output # run 2's own time + assert "■ Stopped before saving" in output + assert "4 of 5 saved and checked · 3 AI calls · $0.04" in output + assert "5 of 5" not in output + assert "0 AI calls" not in output + assert "Seal seal-ed25519-0123456789abcdef · 4 reports signed" in output + assert "Cost 3 AI calls · $0.04 · " in output + assert outcome.checked == 4 + + +def test_a_repeated_run_says_so(root): + output, _ = _plain(FakeEngine(root, runs={3: {"repeated": True}})) + assert "✓ checked in the database (on a second try)" in output + assert output.count("second try") == 1 + + +def test_the_aha_line_appears_only_when_the_run_showed_it(root): + shown, _ = _plain(FakeEngine(root)) + assert ( + 'The screen said "Encounter saved". The database had nothing. A bot that ' + "trusts the screen reports Done. OpenAdapt stopped." in shown + ) + + landed, _ = _plain(FakeEngine(root, broken={"new_records": 1})) + assert "The database had nothing" not in landed + assert "1 new record" in landed + + quiet, _ = _plain(FakeEngine(root, broken={"screen_showed_success": False})) + assert "A screen-trusting bot" not in quiet + assert "reports Done" not in quiet + + missed, _ = _plain(FakeEngine(root, broken={"execution_outcome": "VERIFIED"})) + assert "Stopped before reporting success" not in missed + assert "did not stop" in missed + + +def test_an_engine_without_run_inputs_does_not_claim_new_notes(root): + output, _ = _plain(FakeEngine(root, note_param=False)) + assert "new notes" not in output + assert "2 Run it · the program 5 times, all at once" in output + + +@pytest.mark.parametrize( + ("parallel", "subtitle"), + [(5, "all at once"), (3, "3 at a time"), (1, "new notes\n")], +) +def test_act_two_says_how_many_run_at_once(root, parallel, subtitle): + output, _ = _plain(FakeEngine(root, parallel=parallel)) + assert subtitle in output + + +def test_no_seal_means_no_seal_line_and_no_verify_command(root): + output, _ = _plain(FakeEngine(root, seal=None)) + assert "Seal" not in output + assert "seal verify" not in output + + +def test_quickstart_exit_code_follows_the_runs(monkeypatch, tmp_path): + def fake(root, **kwargs): + outcome = Outcome(root=root) + outcome.runs = [_run(root, 1), _run(root, 2, checked=False)] + return outcome + + monkeypatch.setattr("openadapt.quickstart.run_quickstart", fake) + monkeypatch.setattr("openadapt.cli._flow_runs_quickstart", lambda: True) + with CliRunner().isolated_filesystem(temp_dir=tmp_path): + result = CliRunner().invoke(cli.main, ["quickstart", "--out", "qs"]) + assert result.exit_code == 1 + + +# --------------------------------------------------------------------------- +# Interactive prompt +# --------------------------------------------------------------------------- + + +def test_no_prompt_without_a_terminal(root, monkeypatch): + monkeypatch.setattr( + "builtins.input", lambda *a: pytest.fail("prompted without a terminal") + ) + monkeypatch.setattr("openadapt.quickstart.engine.FlowEngine", FakeEngine) + monkeypatch.setattr("openadapt.quickstart.engine.display_available", lambda: False) + stream = io.StringIO() + run_quickstart( + root, visible=False, break_it=True, verbose=False, steps=_steps, stream=stream + ) + assert "Press Enter" not in stream.getvalue() + assert "Break it" in stream.getvalue() + + +def test_a_terminal_gets_the_prompt_and_waiting_is_not_counted(root, monkeypatch): + asked = [] + + def enter(*args): + asked.append(True) + time.sleep(0.01) + return "" + + monkeypatch.setattr("builtins.input", enter) + clock_values = iter([0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 105.0, 106.0]) + stream = io.StringIO() + outcome = run_story( + FakeEngine(root), + PlainView(stream), + next_steps=_steps, + interactive=True, + clock=lambda: next(clock_values), + ) + assert asked == [True] + assert "Press Enter to make the next save fail silently" in stream.getvalue() + # 106 s elapsed, 100 s of it waiting at the prompt. + assert outcome.total_seconds == pytest.approx(6.0) + + +@pytest.mark.parametrize( + ("environ", "platform", "tty", "shown"), + [ + ({}, "darwin", True, True), + ({}, "linux", True, False), + ({"DISPLAY": ":0"}, "linux", True, True), + ({"SSH_CONNECTION": "x"}, "darwin", True, False), + ({}, "darwin", False, False), + ], +) +def test_display_available_needs_no_engine(environ, platform, tty, shown): + from openadapt.quickstart.engine import display_available + + assert ( + display_available(platform=platform, environ=environ, isatty=lambda: tty) + is shown + ) + + +@pytest.mark.parametrize( + ("visible", "platform", "environ", "opened"), + [ + (True, "darwin", {"__CFBundleIdentifier": "com.apple.Terminal"}, True), + (False, "darwin", {"__CFBundleIdentifier": "com.apple.Terminal"}, False), + (True, "linux", {"__CFBundleIdentifier": "com.apple.Terminal"}, False), + (True, "darwin", {}, False), + ], +) +def test_focus_returns_to_the_terminal_before_the_prompt( + visible, platform, environ, opened +): + from openadapt.quickstart.engine import return_focus_to_terminal + + calls = [] + assert ( + return_focus_to_terminal( + visible=visible, + platform=platform, + environ=environ, + run=lambda argv, **kw: calls.append(argv), + ) + is opened + ) + assert calls == ([["open", "-b", "com.apple.Terminal"]] if opened else []) + + +def test_end_of_input_at_the_prompt_continues(root, monkeypatch): + def eof(*args): + raise EOFError + + monkeypatch.setattr("builtins.input", eof) + output = io.StringIO() + outcome = run_story( + FakeEngine(root), PlainView(output), next_steps=_steps, interactive=True + ) + assert outcome.broken is not None + + +# --------------------------------------------------------------------------- +# Choosing a view +# --------------------------------------------------------------------------- + + +class _Terminal(io.StringIO): + encoding = "utf-8" + + def isatty(self) -> bool: + return True + + +@pytest.mark.parametrize( + ("stream", "environ", "plain"), + [ + (io.StringIO(), {}, True), + (_Terminal(), {}, False), + (_Terminal(), {"NO_COLOR": "1"}, True), + (_Terminal(), {"NO_COLOR": ""}, False), + (_Terminal(), {"TERM": "dumb"}, True), + ], +) +def test_plain_lines_off_a_terminal_or_without_color(stream, environ, plain): + view = make_view(stream, environ=environ) + assert isinstance(view, PlainView) is plain + + +def test_terminals_without_unicode_get_ascii_marks(root): + stream = io.TextIOWrapper(io.BytesIO(), encoding="cp1252") + view = PlainView(stream) + assert (view.marks.ok, view.marks.stop) == ("OK", "STOP") + assert Marks("utf-8").ok == "✓" + + +# --------------------------------------------------------------------------- +# Live terminal view +# --------------------------------------------------------------------------- + + +def _rich(root, environ, **engine_kwargs) -> str: + stream = _Terminal() + view = RichView(stream, environ=environ, width=110) + run_story( + FakeEngine(root, **engine_kwargs), + view, + next_steps=_steps, + clock=_clock(), + ) + return stream.getvalue() + + +def test_terminal_finale_fits_twelve_lines_with_every_optional_line(root, monkeypatch): + stream = _Terminal() + outcome = Outcome(root=root, seal=SealFacts("seal-ed25519-0", 5)) + outcome.runs = [_run(root, index) for index in range(1, 6)] + steps = NextSteps( + verify="openadapt seal verify qs", + record="openadapt flow record", + agent="openadapt agent serve --allow-run", + keep="uv tool install --python 3.12 openadapt", + shadow="Your shell's openadapt is 1.5.1 at /opt/x; the commands above use uvx.", + ) + RichView(stream, environ={}, width=110).finale(outcome, steps) + lines = [line for line in stream.getvalue().splitlines() if line.strip()] + assert len(lines) <= 12, lines + + +def test_terminal_view_renders_color_boxes_and_the_comparison(root): + output = _rich(root, {"TERM_PROGRAM": "Apple_Terminal"}) + assert "\x1b[" in output + assert "╭─ Receipt" in output + assert "What the screen showed" in output + assert "Encounter saved — Synthetic" in output + assert "What the database showed" in output + assert "0 new records" in output + assert "A screen-trusting bot" in output and "✓ Done" in output + assert "■ Stopped before reporting success" in output + assert "\x1b]1337;" not in output and "\x1b_G" not in output + + +@pytest.mark.parametrize( + ("environ", "marker"), + [ + ({"TERM_PROGRAM": "iTerm.app"}, "\x1b]1337;File=inline=1;"), + ({"TERM_PROGRAM": "WezTerm"}, "\x1b]1337;File=inline=1;"), + ({"KITTY_WINDOW_ID": "1", "TERM": "xterm-kitty"}, "\x1b_Ga=T,f=100"), + ({"TERM_PROGRAM": "ghostty"}, "\x1b_Ga=T,f=100"), + ], +) +def test_terminal_view_draws_the_screenshot_where_supported(root, environ, marker): + root.mkdir() + shot = _png(root / "shot.png", 1280, 800) + output = _rich(root, environ, broken={"screenshot": shot}) + assert marker in output + assert "What the database showed" in output + # The text box is the fallback, not drawn as well. + assert "╭─ What the screen showed" not in output + + +def test_terminal_view_falls_back_when_the_screenshot_is_missing(root): + output = _rich(root, {"TERM_PROGRAM": "iTerm.app"}, broken={"screenshot": None}) + assert "\x1b]1337;" not in output + assert "╭─ What the screen showed" in output + + +# --------------------------------------------------------------------------- +# Image protocol detection +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("environ", "isatty", "expected"), + [ + ({"TERM_PROGRAM": "iTerm.app"}, True, ITERM2), + ({"LC_TERMINAL": "iTerm2"}, True, ITERM2), + ({"TERM_PROGRAM": "WezTerm"}, True, ITERM2), + ({"KITTY_WINDOW_ID": "3"}, True, KITTY), + ({"TERM": "xterm-kitty"}, True, KITTY), + ({"TERM_PROGRAM": "ghostty"}, True, KITTY), + ({"TERM": "xterm-ghostty"}, True, KITTY), + ({"TERM_PROGRAM": "Apple_Terminal"}, True, None), + ({"TERM_PROGRAM": "vscode"}, True, None), + ({}, True, None), + ({"TERM_PROGRAM": "iTerm.app"}, False, None), + ({"TERM_PROGRAM": "iTerm.app", "TMUX": "/tmp/tmux"}, True, None), + ({"TERM_PROGRAM": "iTerm.app", "TERM": "screen-256color"}, True, None), + ({"TERM_PROGRAM": "iTerm.app", "OPENADAPT_INLINE_IMAGES": "0"}, True, None), + ({"TMUX": "x", "OPENADAPT_INLINE_IMAGES": "kitty"}, True, KITTY), + ], +) +def test_image_protocol_detection(environ, isatty, expected): + assert detect_image_protocol(environ, isatty=isatty) == expected + + +def test_kitty_payload_is_chunked(): + sequence = image_sequence(b"x" * 10_000, KITTY, cols=40, rows=10) + chunks = sequence.split("\x1b\\")[:-1] + assert len(chunks) > 1 + assert chunks[0].startswith("\x1b_Ga=T,f=100,q=2,c=40,r=10,m=1;") + assert chunks[-1].startswith("\x1b_Gm=0;") + assert all(len(chunk.split(";", 1)[1]) <= 4096 for chunk in chunks) + + +def test_beside_image_reserves_rows_and_offsets_the_text(): + block = beside_image("IMG", ["one", "two"], indent=3, cols=40, rows=4) + assert block.startswith("\n" * 4 + "\x1b[4A") + assert "\x1b7\x1b[3CIMG\x1b8" in block + assert "\r\x1b[46Cone\n" in block and "\r\x1b[46Ctwo\n" in block + assert block.count("\n") == 4 + 4 + + +# --------------------------------------------------------------------------- +# The demonstration ticker reads the recorder's own log +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("event", "label"), + [ + ( + {"kind": "click", "structural": {"name": "Save Encounter"}}, + "Click Save Encounter", + ), + ({"kind": "click"}, "Click"), + ({"kind": "type", "text": "hello"}, 'Type "hello"'), + ({"kind": "key", "key": "Enter"}, "Press Enter"), + ({"kind": "scroll"}, None), + ], +) +def test_demo_step_labels(event, label): + assert demo_step_label(event) == label + + +def test_step_ticker_reports_steps_as_the_recorder_writes_them(tmp_path): + events = tmp_path / "events.jsonl" + seen: list[DemoStep] = [] + lines = [ + {"kind": "click", "structural": {"name": "Open"}, "t": 0.31}, + {"kind": "type", "text": "note", "t": 1.02}, + ] + + def write(): + for line in lines: + with events.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(line)) + time.sleep(0.02) # a half-written line must wait + handle.write("\n") + time.sleep(0.08) + + with StepTicker(events, seen.append) as ticker: + writer = threading.Thread(target=write) + writer.start() + time.sleep(0.05) + writer.join() + assert [(s.label, s.seconds) for s in seen] == [ + ("Click Open", 0.31), + ('Type "note"', 1.02), + ] + assert ticker.steps == seen + + +# --------------------------------------------------------------------------- +# Printed commands work as printed +# --------------------------------------------------------------------------- + + +def _seal_available() -> bool: + try: + from openadapt_flow.offline_seal import seal_run # noqa: F401 + except ImportError: + return False + return True + + +def _console_script() -> Optional[str]: + candidate = Path(sys.executable).parent / ( + "openadapt.exe" if os.name == "nt" else "openadapt" + ) + return str(candidate) if candidate.exists() else None + + +def test_uvx_spelling_reaches_every_printed_command(root, monkeypatch): + monkeypatch.setattr(sys, "prefix", "/Users/x/.cache/uv/archive-v0/abc") + outcome = Outcome(root=root, seal=SealFacts("k", 5)) + steps = next_steps(cli._hint, cli._shell_quote, True)(outcome) + py = f"{sys.version_info[0]}.{sys.version_info[1]}" + for command in (steps.verify, steps.record, steps.agent): + assert command.startswith(f"uvx --python {py} openadapt "), command + + +def test_every_printed_command_answers_help_in_a_fresh_environment(tmp_path): + """Run each next-step command as printed, plus --help, in a new process. + + The agent command is the one the launcher would print with the installed + agent. The Seal check is printed only when the engine can sign Seals, so + it is tested only then. + """ + pytest.importorskip("openadapt_flow", reason="openadapt-flow not installed") + executable = _console_script() + if executable is None: + pytest.skip("the openadapt console script is not installed") + root = tmp_path / "openadapt-quickstart" + root.mkdir() + outcome = Outcome(root=root, seal=SealFacts("k", 1) if _seal_available() else None) + steps = next_steps(cli._hint, cli._shell_quote, cli._agent_serves_tutorial())( + outcome + ) + commands = [steps.record, steps.agent] + ([steps.verify] if steps.verify else []) + env = { + "PATH": os.environ.get("PATH", ""), + "HOME": str(tmp_path), + "OPENADAPT_SEAL_HOME": str(tmp_path / "seal"), + "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""), + } + for command in commands: + argv = shlex.split(command) + assert argv[0] == "openadapt", command + result = subprocess.run( + [executable, *argv[1:], "--help"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, (command, result.stdout, result.stderr) + assert "usage" in (result.stdout + result.stderr).lower(), command + + +def test_seal_verify_checks_each_sealed_run_in_a_folder(tmp_path, monkeypatch): + if not _seal_available(): + pytest.skip("this openadapt-flow cannot sign offline Seals") + from openadapt_flow.offline_seal import seal_run + + monkeypatch.setenv("OPENADAPT_SEAL_HOME", str(tmp_path / "seal")) + root = tmp_path / "openadapt-quickstart" + for index in (1, 2): + run = root / f"run-{index}" + run.mkdir(parents=True) + (run / "report.json").write_text( + json.dumps({"execution_outcome": "VERIFIED", "n": index}), + encoding="utf-8", + ) + seal_run(run, home=tmp_path / "seal", enqueue=False) + (root / "bundle").mkdir() + + result = CliRunner().invoke(cli.main, ["seal", "verify", str(root)]) + assert result.exit_code == 0, result.output + assert "run-1 valid" in result.output and "2 of 2 valid" in result.output + + (root / "run-2" / "report.json").write_text("{}", encoding="utf-8") + result = CliRunner().invoke(cli.main, ["seal", "verify", str(root)]) + assert result.exit_code == 1 + assert "run-2 not valid" in result.output and "1 of 2 valid" in result.output + + +def test_seal_verify_names_a_folder_with_no_sealed_run(tmp_path): + result = CliRunner().invoke(cli.main, ["seal", "verify", str(tmp_path)]) + assert result.exit_code == 1 + assert "no sealed run" in result.output + + +# --------------------------------------------------------------------------- +# The real thing +# --------------------------------------------------------------------------- + + +@pytest.mark.real_quickstart +@pytest.mark.skipif( + os.environ.get("OPENADAPT_REAL_QUICKSTART") != "1", + reason="set OPENADAPT_REAL_QUICKSTART=1 to drive a real browser (~30 s)", +) +def test_real_quickstart(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("OPENADAPT_SEAL_HOME", str(tmp_path / "seal")) + root = tmp_path / "openadapt-quickstart" + stream = io.StringIO() + outcome = run_quickstart( + root, + visible=False, + break_it=True, + verbose=False, + steps=_steps, + stream=stream, + interactive=False, + ) + output = stream.getvalue() + assert outcome.checked == len(RUN_NOTES), output + assert outcome.model_calls == 0 + assert outcome.broken is not None and outcome.broken.stopped + assert outcome.broken.new_records == 0 + assert f"{len(RUN_NOTES)} of {len(RUN_NOTES)} saved and checked" in output + for index in range(1, len(RUN_NOTES) + 1): + report = json.loads((root / f"run-{index}" / "report.json").read_text()) + assert report["execution_outcome"] == "VERIFIED" + assert report["params"]["note"] in output diff --git a/tests/test_quickstart_lifecycle.py b/tests/test_quickstart_lifecycle.py index 64ebef1be..2c2d127e0 100644 --- a/tests/test_quickstart_lifecycle.py +++ b/tests/test_quickstart_lifecycle.py @@ -22,8 +22,8 @@ def _module(): return module -def _write_verified_quickstart(root: Path) -> None: - run = root / "run" +def _write_verified_quickstart(root: Path, name: str = "run-1") -> None: + run = root / name run.mkdir(parents=True) (run / "report.json").write_text( json.dumps( @@ -53,12 +53,15 @@ def test_inspector_requires_a_verified_standard_effect(tmp_path): lifecycle = _module() _write_verified_quickstart(tmp_path) + _write_verified_quickstart(tmp_path, "run-2") + summary = lifecycle._inspect_quickstart(tmp_path) assert summary["outcome"] == "VERIFIED" assert summary["effect_contracts"] == 2 + assert summary["verified_runs"] == 2 - report = tmp_path / "run" / "report.json" + report = tmp_path / "run-2" / "report.json" payload = json.loads(report.read_text(encoding="utf-8")) payload["transaction_outcome"] = "COMPLETED_UNVERIFIED" report.write_text(json.dumps(payload), encoding="utf-8") @@ -66,6 +69,28 @@ def test_inspector_requires_a_verified_standard_effect(tmp_path): lifecycle._inspect_quickstart(tmp_path) +def test_inspector_reads_the_engine_tutorial_layout(tmp_path): + lifecycle = _module() + _write_verified_quickstart(tmp_path, "run") + assert lifecycle._inspect_quickstart(tmp_path)["verified_runs"] == 1 + + +def test_inspector_requires_the_silent_failure_to_stop(tmp_path): + lifecycle = _module() + _write_verified_quickstart(tmp_path) + broken = tmp_path / "run-broken" + broken.mkdir() + (broken / "report.json").write_text( + json.dumps({"execution_outcome": "HALTED"}), encoding="utf-8" + ) + assert lifecycle._inspect_quickstart(tmp_path)["silent_failure_stopped"] is True + (broken / "report.json").write_text( + json.dumps({"execution_outcome": "VERIFIED"}), encoding="utf-8" + ) + with pytest.raises(AssertionError, match="not caught"): + lifecycle._inspect_quickstart(tmp_path) + + def test_zero_exit_with_an_unhandled_async_error_fails(tmp_path, monkeypatch): lifecycle = _module() monkeypatch.setattr( diff --git a/tests/test_quickstart_workers.py b/tests/test_quickstart_workers.py new file mode 100644 index 000000000..3fb569fe9 --- /dev/null +++ b/tests/test_quickstart_workers.py @@ -0,0 +1,84 @@ +"""Act 2's worker pool: parallel runs, and what happens when a worker can't. + +The workers here are stand-ins (tests/quickstart_fake_workers.py) in real +spawned processes, so the pool's scheduling, stall handling and fallbacks run +for real without a browser. +""" + +from __future__ import annotations + +import quickstart_fake_workers as fake + +from openadapt.quickstart import workers +from openadapt.quickstart.workers import WorkerPool, worker_count + +JOBS = [(index, f"note {index}", f"run-{index}") for index in range(1, 6)] + + +def _run(tmp_path, target, size=5): + pool = WorkerPool(size, visible=False, target=target) + finished = [] + try: + back = pool.run( + JOBS, bundle_dir=tmp_path, root=tmp_path, on_finish=finished.append + ) + finally: + pool.close() + return finished, back + + +def test_worker_count_is_one_per_two_cpus_and_at_most_one_per_run(): + assert [worker_count(cpus) for cpus in (1, 2, 4, 8, 12, 64)] == [1, 1, 2, 4, 5, 5] + + +def test_every_run_finishes_in_the_workers(tmp_path): + finished, back = _run(tmp_path, fake.good) + assert back == [] + assert sorted(run.index for run in finished) == [1, 2, 3, 4, 5] + assert all(run.checked for run in finished) + assert finished[0].run_dir.parent == tmp_path + + +def test_fewer_workers_than_runs_take_turns(tmp_path): + finished, back = _run(tmp_path, fake.good, size=2) + assert back == [] + assert sorted(run.index for run in finished) == [1, 2, 3, 4, 5] + + +def test_a_stalled_run_comes_back_in_a_new_folder(tmp_path, monkeypatch): + monkeypatch.setattr(workers, "STALL_S", 3.0) + finished, back = _run(tmp_path, fake.stalls_on_run_2) + assert back == [(2, "note 2", "run-2-again")] + assert 2 not in {run.index for run in finished} + + +def test_a_run_that_raised_is_reported_and_not_repeated(tmp_path): + finished, back = _run(tmp_path, fake.fails_run_3) + assert back == [] + (failed,) = [run for run in finished if run.index == 3] + assert not failed.checked + assert failed.plain_label.startswith("Didn't finish: TutorialError") + + +def test_workers_that_cannot_start_hand_every_run_back(tmp_path): + finished, back = _run(tmp_path, fake.never_ready) + assert finished == [] + assert back == JOBS + + +def test_usable_counts_only_workers_that_did_not_give_up(tmp_path): + import time + + pool = WorkerPool(3, visible=False, target=fake.never_ready) + try: + deadline = time.monotonic() + 30 + while pool.usable() and time.monotonic() < deadline: + time.sleep(0.1) + assert pool.usable() == 0 + finally: + pool.close() + good = WorkerPool(2, visible=False, target=fake.good) + try: + assert good.usable() == 2 + finally: + good.close()