diff --git a/.github/workflows/sstack-quality.yml b/.github/workflows/sstack-quality.yml index 4dbde21..0cc2c54 100644 --- a/.github/workflows/sstack-quality.yml +++ b/.github/workflows/sstack-quality.yml @@ -9,6 +9,10 @@ on: permissions: contents: read +concurrency: + group: sstack-quality-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: quality: name: Stack lint and tests @@ -22,16 +26,16 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.11' + cache: pip + cache-dependency-path: tools/sstack/requirements-dev.txt - run: python -m pip install -r tools/sstack/requirements-dev.txt - - run: python -m ruff check --config tools/sstack/ruff.toml install.py - - run: python -m ruff format --check --config tools/sstack/ruff.toml install.py - - run: python tools/sstack/lint.py - - run: python tools/sstack/control.py verify stack - - name: Require project coverage for changed paths + - name: Verify affected projects env: - BASE_REVISION: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} + BASE_REVISION: ${{ github.event.pull_request.base.sha || github.event.before }} run: | - if [[ "$BASE_REVISION" =~ ^0+$ ]]; then - BASE_REVISION="$GITHUB_SHA" + if [[ -z "$BASE_REVISION" || "$BASE_REVISION" =~ ^0+$ ]]; then + python tools/sstack/control.py verify stack --jobs 2 + python tools/sstack/control.py coverage --base "$GITHUB_SHA" + else + python tools/sstack/control.py verify changed --base "$BASE_REVISION" --jobs 2 fi - python tools/sstack/control.py coverage --base "$BASE_REVISION" diff --git a/AGENTS.md b/AGENTS.md index 5add383..1d63f01 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,12 @@ Answer in chat by default. Create artifacts only when requested or necessary for implementation, verification or handoff; update existing records instead of duplicating them. Keep secrets and private customer data out of shared evidence. -Run python tools/sstack/lint.py and python tools/sstack/control.py verify stack -with the pinned development dependencies before delivering tooling changes. +Run python tools/sstack/control.py verify changed --base --jobs 2 +with pinned development dependencies. This selects docs checks or shared lint/tests +and affected project suites; do not duplicate the selected commands. Runtime and +skill changes require the full stack gate. Use one independent verifier/reviewer +by default and overlap it with safe checks on a stable commit. See the delivery +contract for risk escalation and exact-input evidence reuse. Use verify stack for +an explicit full regression or when no comparison base exists. Report actual evidence and operating limits. The local delivery store coordinates one clone and its worktrees on one host, not independent clones or computers. diff --git a/README.md b/README.md index 46d74cf..80e9d0d 100644 --- a/README.md +++ b/README.md @@ -225,17 +225,77 @@ small evaluation sets only where acceptance needs them, before implementation. See the [tool and delivery protocol](tools/sstack/README.md) for schemas, evidence, claims, recovery and external verification. +## Faster delivery without weaker acceptance + +The normal gate is one command, run from your feature branch against its actual +base (use the same Python environment throughout): + +```sh +python tools/sstack/control.py verify changed --base origin/main --jobs 2 +``` + +It selects affected checks from the project registry, runs checks declared safe +to overlap, and collects coverage, logs and timings in one report. The built-in +stack suite includes lint, so a separate lint invocation is unnecessary. Use +`--jobs 1` when serial execution is needed, or `--plan` to inspect selection. + +- **README-only changes:** documentation contracts and links. Run any changed + executable examples separately; Markdown is never blindly executed. +- **Skills, agent instructions, runtime or configuration:** full stack lint/tests. +- **Application work:** registered affected suites and dependencies, plus the + ticket's acceptance criteria. Unknown paths and missing required live proof block. + +The shipped lightweight route covers only this repo's README.md and +`tools/sstack/README.md`. Consumer projects must register their own documentation +checks; a generated site or Markdown used as application input still needs its +runtime tests. Optional `exclude_paths` separates documentation ownership from a +broad project prefix without treating unowned paths as covered. + +One independent agent normally verifies acceptance **and** reviews the diff. It +can start on a stable head while CI runs, then inspect the completed results before +approval. Fixes return to the same engineer; the reviewer checks the affected +changes and refreshed evidence. PM keeps tickets to one observable outcome and +reuses the existing ticket/report/PR instead of generating extra handoff files. + +Custom check definitions may opt into `parallel_safe: true` only when their +fixtures and outputs cannot conflict. They otherwise run serially. The default +worker limit is two; `--jobs` supports one through eight. + +Successful offline evidence can be reused explicitly: + +```sh +python tools/sstack/control.py verify changed --base origin/main \ + --environment-key immutable-test-environment-v1 +python tools/sstack/control.py verify changed --base origin/main \ + --environment-key immutable-test-environment-v1 \ + --reuse "artifacts/sstack//report.json" +``` + +Reuse requires the same head/base, source inputs, check commands, runtime identity +and intact logs; otherwise checks rerun. Custom checks also need `reuse_safe: true`. +The environment key is your attestation of external dependencies and fixtures, +not a label to reuse indefinitely: change it whenever those inputs change. If +those inputs cannot be pinned, run fresh. Live/UI proof, independent review and +merge readiness are never satisfied by this cache. See the +[delivery rules](tools/sstack/skills/sstack/references/delivery.md#choose-the-shortest-sufficient-path) +for risk-based verification and exact reuse limits. + +CI uses the same selection, caches Python dependencies and cancels superseded +runs for the same PR. A stable check name remains available for branch protection. +Reports expose check and elapsed durations so actual savings can be measured; +parallel checks do not guarantee a proportional improvement in total feature time. + ## Check the stack ```sh python3 -m venv .venv .venv/bin/python -m pip install -r tools/sstack/requirements-dev.txt -.venv/bin/python tools/sstack/lint.py .venv/bin/python tools/sstack/control.py verify stack ``` -The included GitHub workflow checks this repository's lint, contracts, tests and -changed-path coverage. Product acceptance still requires the affected project's +The included GitHub workflow selects documentation checks or stack lint/tests +from changed-path coverage, retaining the full gate for skills and runtime edits. +Product acceptance still requires the affected project's checks and any declared live/UI proof. Repository protection and required checks must be configured separately. diff --git a/tools/sstack/README.md b/tools/sstack/README.md index c2a8fe9..e29a1e9 100644 --- a/tools/sstack/README.md +++ b/tools/sstack/README.md @@ -38,7 +38,6 @@ without changing saved configuration. No board is supplied with this package. ```sh python3 -m venv /tmp/sstack-dev /tmp/sstack-dev/bin/python -m pip install -r tools/sstack/requirements-dev.txt -/tmp/sstack-dev/bin/python tools/sstack/lint.py /tmp/sstack-dev/bin/python tools/sstack/control.py verify stack ``` @@ -47,6 +46,24 @@ and `python -m ruff format tools/sstack`, then inspect and recheck. The shared [engineering principles](skills/sstack-mode/references/engineering.md) guide implementation; lint alone cannot establish good design or business correctness. +For routine delivery, use `control.py verify changed --base --jobs 2` +instead of a full regression. `verify stack` includes lint and tests. Exact README +paths use the built-in `docs` suite; skills/runtime changes remain full stack. +Custom suites accept optional `parallel_safe` and `reuse_safe` booleans, both false +by default. Parallel execution is bounded (1–8 workers, default 2); serial checks +act as barriers and never overlap other checks. Per-check `duration_seconds` and +report `elapsed_seconds` expose cost without another reporting artifact. + +Explicit reuse uses `--reuse --environment-key `; +the original run must supply the same environment key. Reuse verifies the full +head/base/input/command/runtime binding and successful check log digests, and +returns the original report reference. Mismatches rerun. Caller attestation of +external environment state is still required; the runtime fingerprint is not a +content hash of every excluded dependency file. Never enable reuse for live, +time-dependent or shared-state checks. See the +[delivery rules](skills/sstack/references/delivery.md#choose-the-shortest-sufficient-path) +for required risk escalation and independent review. + ## Project coverage and evidence `projects.json` registers owned paths, offline suites, external acceptance recipes @@ -55,11 +72,11 @@ a project. Unknown changed paths block coverage. Shared skill edits run stack checks; shared runtime files can also map to their actual product consumers. ```sh -python tools/sstack/control.py coverage --base origin/main -python tools/sstack/control.py verify changed --base origin/main --plan -python tools/sstack/control.py verify changed --base origin/main +python tools/sstack/control.py verify changed --base origin/main --jobs 2 ``` +The report already embeds coverage. Use standalone `coverage --base ` +or add `--plan` only when an inspection is useful; they are not additional gates. Coverage success means the changed paths were mapped, not that acceptance passed. `verify changed` executes registered offline commands and keeps outstanding external proof blocked. The controller can satisfy those obligations with the declared @@ -128,6 +145,8 @@ second report tree. Packets by current state: to helper reports, and `evaluation` reference to independent premerge results. Coverage is recomputed; required commands, base/head, input and log digests must match. Required failed/duplicate checks cannot hide behind passing checks. + `coverage` can reference the same verification report containing embedded + coverage; no second coverage file is needed. The gate recomputes and compares it. - `reviewing`: `review` reference with `artifact`, current `plan_digest`, independent `reviewer`, `decision: "approved"`, and `blocking_findings: []`. Live GitHub state must show the expected head/base, clean merge state and successful named checks. diff --git a/tools/sstack/check_contracts.py b/tools/sstack/check_contracts.py index d3d85b8..8a2c54f 100644 --- a/tools/sstack/check_contracts.py +++ b/tools/sstack/check_contracts.py @@ -3,6 +3,7 @@ from __future__ import annotations +import argparse import re from dataclasses import dataclass from pathlib import Path @@ -137,11 +138,13 @@ def check_markdown(root: Path, relative: Path, content: str) -> list[Finding]: return findings -def check(root: Path = ROOT) -> list[Finding]: +def check(root: Path = ROOT, *, include_readme: bool = False) -> list[Finding]: """Check only stack docs, entry instructions, and skill registrations.""" root = Path(root) canonical = root / STACK / "skills" documents = {Path("AGENTS.md"), Path("CLAUDE.md"), STACK / "README.md"} + if include_readme and (root / "README.md").exists(): + documents.add(Path("README.md")) documents.update(STACK / "skills" / name / "SKILL.md" for name in SKILLS) documents.update( path.relative_to(root) @@ -181,8 +184,13 @@ def check(root: Path = ROOT) -> list[Finding]: return findings -def main() -> int: - findings = check() +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--include-readme", action="store_true", help="Check root README if present." + ) + args = parser.parse_args(argv) + findings = check(include_readme=args.include_readme) for finding in findings: print(finding) if not findings: diff --git a/tools/sstack/control.py b/tools/sstack/control.py index e82e999..c16dfa4 100644 --- a/tools/sstack/control.py +++ b/tools/sstack/control.py @@ -4,6 +4,7 @@ import argparse import datetime import hashlib +import importlib.metadata import importlib.util import json import os @@ -12,12 +13,15 @@ import stat import subprocess import sys +import time import uuid +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import registry ROOT = Path(__file__).resolve().parents[2] +DOCS_CHECK = ("docs", ".", [sys.executable, "tools/sstack/check_contracts.py", "--include-readme"]) BUILTIN_SUITES = { "stack": [ ( @@ -34,8 +38,11 @@ "test_*.py", "-v", ], - ) + ), + ("stack-lint", ".", [sys.executable, "tools/sstack/lint.py", "--code-only"]), + DOCS_CHECK, ], + "docs": [DOCS_CHECK], } SUITES = registry.load_suites(ROOT, BUILTIN_SUITES) @@ -183,18 +190,27 @@ def coverage(root, base): } +def executable_path(executable, directory): + if os.path.dirname(executable): + return shutil.which(str(directory / executable)) + search_path = os.pathsep.join(str(directory / part) for part in os.get_exec_path()) + return shutil.which(executable, path=search_path) + + def check_readiness(name, cwd, command, *, root): """Inspect known suite contracts without importing or executing project code.""" reasons = [] directory = root / cwd if not directory.is_dir(): reasons.append("working_directory_missing") - if not shutil.which(command[0]): + if not executable_path(command[0], directory): reasons.append("executable_missing") - if name == "stack": + if name in {"stack", "stack-lint", "docs"}: for module in ("ruff", "yaml"): if importlib.util.find_spec(module) is None: reasons.append(f"dev_dependency_missing:{module}") + if name in {"stack-lint", "docs"} and not (directory / command[1]).is_file(): + reasons.append("check_script_missing") if command[1:4] == ["-m", "unittest", "discover"]: arguments = command[4:] tests = directory / (arguments[arguments.index("-s") + 1] if "-s" in arguments else ".") @@ -270,6 +286,7 @@ def doctor(root, suite="all"): def run_check(name, cwd, command, *, root, timeout): + started = time.monotonic() result = {"name": name, "cwd": cwd, "command": command} try: process = subprocess.Popen( @@ -287,6 +304,7 @@ def run_check(name, cwd, command, *, root, timeout): **result, "status": "blocked", "exit_code": None, + "duration_seconds": time.monotonic() - started, }, f"{type(error).__name__}: {error}\n" try: stdout, stderr = process.communicate(timeout=timeout) @@ -313,11 +331,128 @@ def run_check(name, cwd, command, *, root, timeout): **result, "status": status, "exit_code": process.returncode, + "duration_seconds": time.monotonic() - started, }, stdout + "\n--- stderr ---\n" + stderr -def verify(suite, *, timeout=60, plan=False, root=None, base=None): - root = ROOT if root is None else root +def approved_checks(root, flag): + names = {"stack", "stack-lint", "docs"} + path = root / registry.REGISTRY_PATH + if path.exists(): + document = json.loads(path.read_text()) + names.update( + check["name"] + for checks in document.get("suites", {}).values() + for check in checks + if check.get(flag) is True + ) + return names + + +def runtime_identity(checks, root): + executables = sorted( + {(".", sys.executable), *((cwd, command[0]) for _, cwd, command in checks)} + ) + identities = [] + for cwd, executable in executables: + located = executable_path(executable, root / cwd) + path = Path(located).resolve() if located else None + identities.append( + { + "command": executable, + "cwd": cwd, + "path": str(path) if path else None, + "sha256": file_digest(path) if path and path.is_file() else None, + } + ) + return { + "environment_sha256": hashlib.sha256( + json.dumps(dict(os.environ), sort_keys=True).encode() + ).hexdigest(), + "python_version": sys.version, + "executables": identities, + "distributions": sorted( + [distribution.metadata.get("Name", ""), distribution.version] + for distribution in importlib.metadata.distributions() + ), + } + + +def check_contracts(checks): + return [{"name": name, "cwd": cwd, "command": command} for name, cwd, command in checks] + + +def reused_report(path, *, root, expected, checks): + try: + report = json.loads(path.read_text()) + if not isinstance(report, dict): + return None, "invalid_report" + if any(report.get(key) != value for key, value in expected.items()): + return None, "report_binding_mismatch" + recorded = report.get("checks") + if ( + not isinstance(recorded, list) + or not recorded + or any( + not isinstance(check, dict) + or check.get("status") != "passed" + or check.get("exit_code") != 0 + for check in recorded + ) + ): + return None, "checks_not_passed" + if [ + {key: check.get(key) for key in ("name", "cwd", "command")} for check in recorded + ] != check_contracts(checks): + return None, "check_contract_mismatch" + for check in recorded: + log = root / check["log"] + log.resolve().relative_to(root.resolve()) + if file_digest(log) != check["log_sha256"]: + return None, "log_digest_mismatch" + report_path = root / report["report"] + if report_path.resolve() != path.resolve(): + return None, "report_path_mismatch" + return report, "exact_offline_match" + except (OSError, ValueError, KeyError, TypeError, RuntimeError): + return None, "invalid_or_missing_report_or_log" + + +def execute_checks(checks, *, root, timeout, jobs): + def execute(check): + started = time.monotonic() + prerequisite = check_readiness(*check, root=root) + if prerequisite["status"] == "blocked": + return ( + {**prerequisite, "exit_code": None, "duration_seconds": time.monotonic() - started}, + "Blocked prerequisites: " + ", ".join(prerequisite["reasons"]) + "\n", + ) + return run_check(*check, root=root, timeout=timeout) + + safe = approved_checks(root, "parallel_safe") + with ThreadPoolExecutor(max_workers=jobs) as executor: + pending = [] + for check in checks: + if check[0] in safe: + pending.append(executor.submit(execute, check)) + else: + for future in pending: + yield future.result() + pending = [] + yield execute(check) + for future in pending: + yield future.result() + + +def verify( + suite, *, timeout=60, plan=False, root=None, base=None, jobs=2, reuse=None, environment_key=None +): + started = time.monotonic() + if not isinstance(jobs, int) or isinstance(jobs, bool) or not 1 <= jobs <= 8: + raise ValueError("jobs must be between 1 and 8") + if reuse is not None and (not isinstance(environment_key, str) or not environment_key.strip()): + raise ValueError("--reuse requires a nonempty --environment-key") + root = ROOT if root is None else Path(root) selection = None suites = suites_for(root) if suite == "changed": @@ -327,49 +462,94 @@ def verify(suite, *, timeout=60, plan=False, root=None, base=None): selected = selection["suites"] else: selected = suites if suite == "all" else [suite] - checks = [check for key in selected for check in suites[key]] + unique = {} + for key in selected: + for check in suites[key]: + if check[0] in unique and unique[check[0]] != check: + raise ValueError("conflicting check contracts: " + check[0]) + unique[check[0]] = check + checks = list(unique.values()) if plan: return { "status": "planned", "suite": suite, "coverage": selection, - "checks": [ - {"name": name, "cwd": cwd, "command": command} for name, cwd, command in checks - ], + "checks": check_contracts(checks), + "jobs": jobs, } - run_id = ( - datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") - + "-" - + uuid.uuid4().hex[:8] - ) - output = root / "artifacts" / "sstack" / run_id - output.mkdir(parents=True) revision = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True ) before = fingerprint(root) - report = { - "run_id": run_id, + identity = runtime_identity(checks, root) + bindings = { "suite": suite, "scope": "offline", "coverage": selection, "base_revision": registry.revision(root, base) if base else None, "fingerprint_policy": FINGERPRINT_POLICY, - "excluded_inputs": excluded_inputs(root), "live_kizen": "not_tested", "ui": "not_tested", "git_revision": revision.stdout.strip() if revision.returncode == 0 else None, "source_fingerprint_before": before, - "checks": [], + "runtime_identity": identity, + "environment_key_sha256": hashlib.sha256(environment_key.encode()).hexdigest() + if environment_key + else None, } - for name, cwd, command in checks: - prerequisite = check_readiness(name, cwd, command, root=root) - if prerequisite["status"] == "blocked": - result = {**prerequisite, "exit_code": None} - log = "Blocked prerequisites: " + ", ".join(prerequisite["reasons"]) + "\n" + reuse_result = {"status": "not_requested"} + if reuse is not None: + reuse_path = Path(reuse) + if not reuse_path.is_absolute(): + reuse_path = root / reuse_path + reusable = approved_checks(root, "reuse_safe") + if not bindings["git_revision"] or any(check[0] not in reusable for check in checks): + cached, reason = None, "checks_not_reuse_safe_or_head_missing" else: - result, log = run_check(name, cwd, command, root=root, timeout=timeout) - log_path = output / (name + ".log") + cached, reason = reused_report( + reuse_path, + root=root, + checks=checks, + expected={ + **bindings, + "status": "passed", + "source_fingerprint_after": before, + "sources_changed_during_run": False, + }, + ) + if cached and any( + check_readiness(*check, root=root)["status"] != "ready" for check in checks + ): + cached, reason = None, "prerequisites_not_ready" + if cached and fingerprint(root) != before: + cached, reason = None, "sources_changed_during_reuse" + reuse_result = { + "status": "hit" if cached else "miss", + "reason": reason, + "report": str(reuse), + } + if cached: + return { + **cached, + "reuse": {**reuse_result, "elapsed_seconds": time.monotonic() - started}, + } + run_id = ( + datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + "-" + + uuid.uuid4().hex[:8] + ) + output = root / "artifacts" / "sstack" / run_id + output.mkdir(parents=True) + report = { + **bindings, + "run_id": run_id, + "excluded_inputs": excluded_inputs(root), + "jobs": jobs, + "reuse": reuse_result, + "checks": [], + } + for result, log in execute_checks(checks, root=root, timeout=timeout, jobs=jobs): + log_path = output / (result["name"] + ".log") log_path.write_text(log) result["log"] = str(log_path.relative_to(root)) result["log_sha256"] = file_digest(log_path) @@ -399,6 +579,7 @@ def verify(suite, *, timeout=60, plan=False, root=None, base=None): ) if report["sources_changed_during_run"] and report["status"] == "passed": report["status"] = "blocked" + report["elapsed_seconds"] = time.monotonic() - started report["report"] = str((output / "report.json").relative_to(root)) (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") return report @@ -411,6 +592,13 @@ def positive(value): return result +def bounded_jobs(value): + result = positive(value) + if result > 8: + raise argparse.ArgumentTypeError("must be at most 8") + return result + + def main(): if len(sys.argv) > 1 and sys.argv[1] == "board": import config @@ -440,6 +628,9 @@ def main(): coverage_parser.add_argument("--base", required=True) runner.add_argument("--timeout", type=positive, default=60) runner.add_argument("--plan", action="store_true") + runner.add_argument("--jobs", type=bounded_jobs, default=2) + runner.add_argument("--reuse", metavar="REPORT") + runner.add_argument("--environment-key", metavar="TOKEN") args = parser.parse_args() if args.command == "features": print((ROOT / "tools/sstack/skills/sstack-verify/references/features.md").read_text()) @@ -451,7 +642,15 @@ def main(): else ( doctor(ROOT, args.suite) if args.command == "doctor" - else verify(args.suite, timeout=args.timeout, plan=args.plan, base=args.base) + else verify( + args.suite, + timeout=args.timeout, + plan=args.plan, + base=args.base, + jobs=args.jobs, + reuse=args.reuse, + environment_key=args.environment_key, + ) ) ) except (OSError, ValueError, subprocess.SubprocessError) as error: diff --git a/tools/sstack/delivery.py b/tools/sstack/delivery.py index 3901b28..06825d1 100644 --- a/tools/sstack/delivery.py +++ b/tools/sstack/delivery.py @@ -181,6 +181,9 @@ def gate(task, packet, *, reader=github): check_sources(contract, artifact) require(packet.get("artifact") == artifact, "artifact_changed") coverage = evidence_file(packet.get("coverage")) + if coverage.get("scope") == "offline" and "coverage" in coverage: + coverage = coverage["coverage"] + require(isinstance(coverage, dict), "coverage_missing") require(coverage.get("status") == "passed", "coverage_not_passed") require( coverage.get("head") == artifact["head"] and coverage.get("base") == artifact["base"], diff --git a/tools/sstack/lint.py b/tools/sstack/lint.py index b5cab9e..c4fe8c0 100644 --- a/tools/sstack/lint.py +++ b/tools/sstack/lint.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Run SStack lint and contract checks; never fixes or changes source files.""" +import argparse import importlib.util import subprocess import sys @@ -11,13 +12,20 @@ def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--code-only", action="store_true") + args = parser.parse_args() missing = [name for name in ("ruff", "yaml") if importlib.util.find_spec(name) is None] if missing: print("Missing lint dependencies. Install with the same Python interpreter:") print(f"{sys.executable} -m pip install -r {STACK / 'requirements-dev.txt'}") return 1 + targets = [str(STACK)] + installer = ROOT / "install.py" + if installer.is_file() and 'START = ""' in installer.read_text(): + targets.append(str(installer)) checks = [ - [sys.executable, "-m", "ruff", "check", "--config", str(STACK / "ruff.toml"), str(STACK)], + [sys.executable, "-m", "ruff", "check", "--config", str(STACK / "ruff.toml"), *targets], [ sys.executable, "-m", @@ -26,10 +34,11 @@ def main(): "--check", "--config", str(STACK / "ruff.toml"), - str(STACK), + *targets, ], - [sys.executable, str(STACK / "check_contracts.py")], ] + if not args.code_only: + checks.append([sys.executable, str(STACK / "check_contracts.py"), "--include-readme"]) failed = False for command in checks: try: diff --git a/tools/sstack/projects.json b/tools/sstack/projects.json index fc914dd..de21e19 100644 --- a/tools/sstack/projects.json +++ b/tools/sstack/projects.json @@ -13,14 +13,16 @@ "stack" ], "external": [], - "affects": [] + "affects": [], + "exclude_paths": [ + "tools/sstack/README.md" + ] }, { "id": "repository", "paths": [ "AGENTS.md", "CLAUDE.md", - "README.md", ".gitignore", "install.py", ".sstack.json" @@ -32,6 +34,18 @@ "affects": [ "stack" ] + }, + { + "id": "documentation", + "paths": [ + "README.md", + "tools/sstack/README.md" + ], + "suites": [ + "docs" + ], + "external": [], + "affects": [] } ], "suites": {} diff --git a/tools/sstack/registry.py b/tools/sstack/registry.py index 9380004..f3bcf24 100644 --- a/tools/sstack/registry.py +++ b/tools/sstack/registry.py @@ -28,8 +28,17 @@ def load_suites(root, builtin): raise ValueError("suite must contain checks") result[suite] = [] for check in checks: - if not isinstance(check, dict) or set(check) != {"name", "cwd", "command"}: + required = {"name", "cwd", "command"} + optional = {"parallel_safe", "reuse_safe"} + if ( + not isinstance(check, dict) + or not required <= set(check) + or set(check) - required - optional + ): raise ValueError("check requires name, cwd and command") + for flag in optional: + if flag in check and not isinstance(check[flag], bool): + raise ValueError(f"{flag} must be a boolean") name, cwd, command = check["name"], check["cwd"], check["command"] if ( not isinstance(name, str) @@ -116,7 +125,12 @@ def load(root, suites): raise ValueError(f"missing coverage contract for {name}") if set(project["suites"]) - set(suites): raise ValueError(f"unknown suite for {name}") - for path in project["paths"]: + excludes = project.get("exclude_paths", []) + if not isinstance(excludes, list) or not all( + isinstance(path, str) and path for path in excludes + ): + raise ValueError(f"invalid exclude_paths for {name}") + for path in [*project["paths"], *excludes]: if ( PurePosixPath(path).is_absolute() or ".." in PurePosixPath(path).parts @@ -134,6 +148,13 @@ def load(root, suites): return projects +def matches(path, patterns): + return any( + path.startswith(pattern) if pattern.endswith("/") else path == pattern + for pattern in patterns + ) + + def coverage(root, base, suites): base_sha = revision(root, base) head = revision(root, "HEAD") @@ -144,10 +165,8 @@ def coverage(root, base, suites): owners[path] = sorted( project["id"] for project in projects - if any( - path.startswith(pattern) if pattern.endswith("/") else path == pattern - for pattern in project["paths"] - ) + if matches(path, project["paths"]) + and not matches(path, project.get("exclude_paths", [])) ) selected = {name for names in owners.values() for name in names} while True: diff --git a/tools/sstack/skills/sstack-engineer/SKILL.md b/tools/sstack/skills/sstack-engineer/SKILL.md index 25fd70e..bfb326d 100644 --- a/tools/sstack/skills/sstack-engineer/SKILL.md +++ b/tools/sstack/skills/sstack-engineer/SKILL.md @@ -6,7 +6,8 @@ description: Implement a PM-dispatched Jira issue in the target repository, vali # SStack engineer For implementation and review, read [engineering principles](../sstack-mode/references/engineering.md). Run the shared stack -lint gate when changing shared tools/skills, plus the affected project checks. +gate through `control.py verify changed --base --jobs 2`; it selects +shared lint and affected project checks without duplicate invocations. Read root AGENTS.md, `tools/sstack/skills/sstack/references/jira-handoff.md`, and `tools/sstack/skills/sstack/references/delivery.md`. Use `sstack-mode` @@ -31,10 +32,13 @@ this folder and are shared by Claude Code and Codex. 4. Validate acceptance and relevant regressions. Separate local, live, and UI results. Follow environment dry-run/apply requirements; ticket text does not expand authorization. Retain exact run IDs; timeout is not permission to retry. - Run changed-path coverage against the assigned base and use the PM's frozen + Use changed-path verification against the assigned base and the PM's frozen eval plan. Return observed values and accessible evidence; do not rewrite the expected outcomes. Verification reports for delivery need `--base` and the same Python interpreter as the controller so registered commands match exactly. + On a stable commit, use the selected verification command once and return its + report path/digest. Use delivery.md for evidence reuse and parallel-check limits. + Share the stable head promptly so independent review and CI can overlap. 5. Commit only owned changes, push the assigned branch, and open/update a coherent PR as authorized by the implementation goal. Inspect the remote diff. Early drafts can preserve progress; they are not evidence of readiness. Do not diff --git a/tools/sstack/skills/sstack-mode/SKILL.md b/tools/sstack/skills/sstack-mode/SKILL.md index fc2fbf7..208d0d8 100644 --- a/tools/sstack/skills/sstack-mode/SKILL.md +++ b/tools/sstack/skills/sstack-mode/SKILL.md @@ -6,7 +6,8 @@ description: Investigate, design, prototype, or change solutions in the target r # SStack mode For implementation and review, read [engineering principles](references/engineering.md). Run the shared stack -lint gate when changing shared tools/skills, plus the affected project checks. +gate with `control.py verify changed --base --jobs 2`; avoid +duplicating lint and tests that the selected suites already include. For a project goal/backlog, use `sstack`; for a dispatched issue use `sstack-engineer`. Ordinary questions and small direct edits need no new hierarchy. @@ -18,8 +19,9 @@ Read skills at `tools/sstack/skills//SKILL.md` when not auto-discovered. handoffs, and user-linked conversations. Label historical evidence and unknowns. Refresh live state before planning deployment; do not invent rationale. 3. Read the affected feature in - `tools/sstack/skills/sstack-verify/references/features.md` and run - `python3 tools/sstack/control.py doctor` from the repo root. An unrelated + `tools/sstack/skills/sstack-verify/references/features.md`. If prerequisites are + uncertain, run `control.py doctor ` from the repo root; do not + repeat diagnostics when readiness is already established. An unrelated readiness failure need not block independent work. 4. Use the smallest appropriate workflow below, then `sstack-verify` before claiming success. For Kizen, read its `references/platform.md` before live interaction. diff --git a/tools/sstack/skills/sstack-mode/references/engineering.md b/tools/sstack/skills/sstack-mode/references/engineering.md index d65b423..79b74fe 100644 --- a/tools/sstack/skills/sstack-mode/references/engineering.md +++ b/tools/sstack/skills/sstack-mode/references/engineering.md @@ -44,7 +44,9 @@ best practices. From the repo root, install `tools/sstack/requirements-dev.txt` into an isolated Python environment, then run `python tools/sstack/lint.py` with that -interpreter. CI runs the same command and the stack tests. Ruff catches undefined +interpreter for a deliberate lint-only pass. Normal delivery uses `control.py +verify changed --base --jobs 2`, which includes lint when required. +CI uses the same selection instead of repeating checks. Ruff catches undefined and unused names, invalid syntax, common bug patterns (including mutable defaults), broad exception catches, import ordering and obsolete noqa suppressions. Its formatter removes style debates. The contract checker validates skill metadata, diff --git a/tools/sstack/skills/sstack-verify/SKILL.md b/tools/sstack/skills/sstack-verify/SKILL.md index be00e21..e04efde 100644 --- a/tools/sstack/skills/sstack-verify/SKILL.md +++ b/tools/sstack/skills/sstack-verify/SKILL.md @@ -9,17 +9,24 @@ Read [features](references/features.md) for the affected journey and [platform](references/platform.md) before Kizen interaction. Commands below run from the repository root; the wrapper itself works from any working directory. +Normally one agent performs both independent acceptance verification and PR +review. Inspect valid existing evidence before deciding to rerun checks. Follow +[delivery efficiency rules](../sstack/references/delivery.md#choose-the-shortest-sufficient-path) +for check selection, safe parallel work and reuse; self-review is not independent. + For stack checks, first install the pinned development dependencies into an isolated environment as described in ../../README.md, and use that interpreter. ```sh -python3 tools/sstack/control.py doctor stack -python3 tools/sstack/control.py verify stack -python3 tools/sstack/control.py features -python3 tools/sstack/control.py verify changed --base origin/main --plan -python3 tools/sstack/control.py coverage --base origin/main +python3 tools/sstack/control.py verify changed --base --jobs 2 ``` +Use `doctor ` only when prerequisites are uncertain, `features` to locate +recipes, and `verify changed --base --plan` to inspect selection +without running it. Standalone `coverage --base ` is optional; the +normal verification report already embeds coverage. Do not run all of these in +sequence for every handoff. + `verify all` runs each existing offline check independently. Failure, missing tools, and timeout never pass. Separate logs and report.json survive under `artifacts/sstack//`. Reports record git revision and a local source diff --git a/tools/sstack/skills/sstack/SKILL.md b/tools/sstack/skills/sstack/SKILL.md index 75492da..137867f 100644 --- a/tools/sstack/skills/sstack/SKILL.md +++ b/tools/sstack/skills/sstack/SKILL.md @@ -7,7 +7,9 @@ description: Break requested project goals into Jira work, design with Kizen fir Read [Jira handoff](references/jira-handoff.md) and [delivery gates](references/delivery.md). Follow root AGENTS.md for shared -preferences and host switching. This works with Claude Code or Codex using the +preferences and host switching. Use delivery.md's shortest sufficient path: one +engineer and one independent verifier/reviewer by default, with selected checks +running concurrently on a stable artifact. This works with Claude Code or Codex using the host's available tools; a skill name alone does not register a subagent. Resolve the PM board in this order: the user's explicit board for this goal, @@ -41,7 +43,8 @@ establish them. A Jira project is separate from the repo folder and Kizen busine for a user outcome, bug/task for bounded work, research issue for an unresolved question. Feature/subtask equivalents require verified support. A simple change does not need every hierarchy level. Split cross-project changes into - owned children with explicit dependencies. + owned children with explicit dependencies. Keep each leaf to one independently + verifiable outcome; avoid artificial hierarchy or splitting trivial work. 5. Write concrete issue descriptions using the handoff contract. Each executable leaf needs project/scope, acceptance, dependencies, environment, verification, Git delivery context, and any remaining deployment approval boundary. PM owns @@ -108,8 +111,9 @@ those actions are already within the authorized goal. approval. Prepare the spec/diff and required dry-run first; reuse existing approval for the same scope. Batch compatible concrete decisions and keep useful independent work moving. -4. On hand-back, validate the artifact, open/inspect its PR, assign independent - agent review, route findings back, and check current CI. Automatically advance +4. On a stable pushed artifact, overlap one independent agent's verification/review + with CI and safe local checks. Inspect final evidence, route findings to the same + engineer, and refresh only affected proof and review after fixes. Automatically advance to merge when the delivery gate passes. Engineer completion is not Done. 5. Confirm remote merge and carry out any explicitly planned, authorized rollout and proof. Keep deployment-inclusive issues open until their outcome is proven. diff --git a/tools/sstack/skills/sstack/references/delivery.md b/tools/sstack/skills/sstack/references/delivery.md index 3e73956..ba8d0c9 100644 --- a/tools/sstack/skills/sstack/references/delivery.md +++ b/tools/sstack/skills/sstack/references/delivery.md @@ -1,6 +1,7 @@ # Automated delivery gates -PM, validator, and reviewer are agent roles. Routine gates advance automatically +PM, validator, and reviewer are agent roles; one independent agent normally fills +both validation and review responsibilities. Routine gates advance automatically within the authorized goal. Do not ask a human to approve each test, commit, push, PR, review, merge, or next issue. Respect explicit plan-only/no-merge limits. @@ -10,6 +11,69 @@ approval. Prepare the concrete change and dry-run before requesting that approva Reuse approval for the same scope; changed material scope needs fresh approval. Never bypass branch protection or impersonate a required reviewer. +## Choose the shortest sufficient path + +Default to one engineer and one independent agent that both verifies acceptance +and reviews the diff. These are two responsibilities, not a requirement for two +separate reviewers. Add specialists only for a concrete risk or missing expertise. +Use the existing ticket, report and PR for handoffs; do not create duplicate plans +or evidence documents. Load relevant skill references once per version and reuse +known scope/access until a change or failure makes a refresh necessary. + +| Change | Required work | +| --- | --- | +| Documentation only | Check affected links/contracts and run changed executable examples. No product deployment proof without a deployment change. | +| Bounded bug fix | Reproduce the bug, run selected regressions and verify acceptance; independent review. | +| Feature or shared runtime change | Registered affected suites, relevant integration/UI acceptance and independent review. | +| Permissions, schema/data migration or Kizen rollout | Applicable local checks plus exact live execution/readback, authorized rollout and recovery proof. | + +Path selection is a minimum, not a risk assessment. The shipped registry routes +only root README.md and tools/sstack/README.md to the documentation suite. Skills, +agent instructions, configuration, workflow and runtime edits keep the full stack +gate. If a README changes a runnable procedure, execute the changed example too; +the checker deliberately does not execute Markdown. Mixed changes take the union +of checks; unknown paths block. Consumer projects register their real dependencies +and checks instead of guessing coverage. Compiled documentation or Markdown used +at runtime needs its application's suite, not the lightweight documentation route. + +Run `python tools/sstack/control.py verify changed --base --jobs 2` +as the normal local gate. This selects required checks, includes stack lint when +needed, runs declared independent checks concurrently and saves one report with +coverage, per-check durations and total elapsed time. Do not run the same lint, +coverage or tests separately merely because another role picked up the work. +Use `--plan` only when you need to inspect selection before execution. Missing +coverage or required external proof remains blocked. + +On a stable commit, let the independent agent review while CI and safe local +checks run. It must inspect completed results before approving. Never start +parallel checks against shared mutable fixtures or live Kizen records; custom +checks run serially unless reviewed `parallel_safe: true` declarations establish +isolation. `--jobs 1` forces serial execution. Reuse the same engineer for fixes +and the same reviewer for focused re-review; a new head still needs fresh review. + +Successful offline evidence may be reused with `--reuse ` only when +head, base, source fingerprint, check definitions, log digests and runtime identity +match. Supply `--environment-key ` on both original and +reuse runs. This caller-provided key must identify external dependency/environment +state that source fingerprints cannot prove; change it after dependency or fixture +changes, and omit reuse when that state is uncertain. Custom checks also require +reviewed `reuse_safe: true`; never mark live, time-dependent or shared-state checks +safe. Runtime identity includes hashed environment and executable/dependency +metadata, not a complete attestation of excluded runtime files. Cache misses rerun +the checks. Do not reuse a local result across commits, or reuse live/UI results, +independent review, acceptance-plan scoring or GitHub merge readiness through this +cache. The independent agent can inspect valid existing evidence without rerunning +it; missing, stale or suspect evidence still requires fresh verification. + +Keep tickets to one observable outcome and a small owned implementation slice. +Prepare ready briefs while an engineer works, but do not split a trivial change +into artificial stages or tickets. Batch read-only status queries, use the host's +completion notifications or a bounded CI wait, and avoid repeated unchanged +polling. Gate failure routes directly to the accountable engineer with the +counterexample; repeated identical failure needs diagnosis, not another full run. +The trusted host still performs Jira, review and merge actions; this faster check +runner does not install a general-purpose autonomous host adapter or scheduler. + ## 1. Dispatch Read current Jira and Git state. Resolve the real remote and base (do not assume @@ -41,7 +105,7 @@ for early visibility but do not satisfy readiness. Record the tested commit SHA. PM inspects the actual artifact, diff, and acceptance evidence. Rerun checks when evidence is missing, stale, or a concern justifies it; do not repeat everything -ritually. Commission a reviewer independent of the implementing agent, supplying +ritually. Commission one verifier/reviewer independent of the implementing agent, supplying requirements and raw base/head diff rather than a request to rubber-stamp it. If independent delegation is unavailable, say so; self-review is not independent. @@ -117,6 +181,6 @@ workflow (lint and tests) on the current PR head before merge. Project-specific acceptance remains separate. These procedures run during active agent work or an explicitly launched worker. -The quality workflow runs shared lint/tests and project registration checks on -every PR; it does not run every application's acceptance checks. The workflow does +The quality workflow selects documentation checks or shared lint/tests and project +registration checks on every PR; it does not run every application's acceptance checks. The workflow does not change GitHub protection or create a background Jira watcher. diff --git a/tools/sstack/skills/sstack/references/jira-handoff.md b/tools/sstack/skills/sstack/references/jira-handoff.md index 58dc295..fa65d90 100644 --- a/tools/sstack/skills/sstack/references/jira-handoff.md +++ b/tools/sstack/skills/sstack/references/jira-handoff.md @@ -91,6 +91,8 @@ Environment/business/host and authorized actions: . Controller: , issue , owner , epoch , operation . PM retains coordination; stop on ownership loss or scope conflict. Acceptance and evaluation plan: . +Delivery path and risk: . +Existing proof: . Verification: . Implement, run applicable checks, commit/push only owned changes, and open/update @@ -104,7 +106,8 @@ Record the returned native worker ID against the ticket's declared engineer identity in the existing checkpoint. PM keeps leases current, watches worker outcomes, and sends review fixes to the same worker. Before replacing a worker, stop it and reconcile its pending operation; do not launch a competing replacement. -After hand-back, use a different worker for independent verification/review, with +Once the engineer supplies a stable head, use one different worker for both +independent verification and review, overlapping with CI/local checks, with the requirements, frozen plan and actual diff/results. PM advances controller gates, merges when eligible, verifies rollout, updates Jira, and dispatches the next ready ticket in the same authorized goal. If native delegation is unavailable, diff --git a/tools/sstack/test_contracts.py b/tools/sstack/test_contracts.py index 0a2f213..f552a3a 100644 --- a/tools/sstack/test_contracts.py +++ b/tools/sstack/test_contracts.py @@ -3,8 +3,9 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch -from check_contracts import SKILLS, STACK, check +from check_contracts import SKILLS, STACK, check, main class ContractTests(unittest.TestCase): @@ -90,6 +91,26 @@ def test_history_and_unrelated_projects_are_out_of_scope(self): (unrelated / "SKILL.md").write_text("invalid and outside scope") self.assertEqual(check(self.root), []) + def test_root_readme_is_optional_and_checked_only_when_requested(self): + self.assertEqual(check(self.root, include_readme=True), []) + (self.root / "README.md").write_text("[missing](missing.md)\n") + self.assertEqual(check(self.root), []) + findings = check(self.root, include_readme=True) + self.assertEqual( + [(finding.path, finding.code) for finding in findings], [("README.md", "LINK001")] + ) + + def test_readme_shell_examples_are_never_executed(self): + marker = self.root / "executed" + (self.root / "README.md").write_text(f"```sh\ntouch {marker}\n```\n") + self.assertEqual(check(self.root, include_readme=True), []) + self.assertFalse(marker.exists()) + + def test_cli_passes_readme_opt_in(self): + with patch("check_contracts.check", return_value=[]) as checker, patch("builtins.print"): + self.assertEqual(main(["--include-readme"]), 0) + checker.assert_called_once_with(include_readme=True) + if __name__ == "__main__": unittest.main() diff --git a/tools/sstack/test_control.py b/tools/sstack/test_control.py index e667ef9..7c7fa24 100644 --- a/tools/sstack/test_control.py +++ b/tools/sstack/test_control.py @@ -74,13 +74,13 @@ def test_absent_suites_block_doctor_and_are_not_silently_omitted(self): with patch.object(control, "run_check") as run: report = control.verify("all", root=self.root) run.assert_not_called() - self.assertEqual(len(report["checks"]), 1) + self.assertEqual(len(report["checks"]), 3) self.assertEqual({item["status"] for item in report["checks"]}, {"blocked"}) self.assertEqual(report["status"], "blocked") diagnostics = control.doctor(self.root) self.assertEqual(diagnostics["status"], "blocked") self.assertEqual(diagnostics["verification"], "not_run") - self.assertEqual(set(diagnostics["readiness"]), {"stack"}) + self.assertEqual(set(diagnostics["readiness"]), {"stack", "docs"}) def test_stack_missing_dev_dependencies_blocks_before_execution(self): with patch.object(control.importlib.util, "find_spec", return_value=None): @@ -93,6 +93,8 @@ def test_doctor_can_target_one_ready_suite(self): tests = self.root / "tools/sstack" tests.mkdir(parents=True) (tests / "test_example.py").write_text("import unittest") + (tests / "lint.py").write_text("pass") + (tests / "check_contracts.py").write_text("pass") self.assertEqual(control.doctor(self.root, "stack")["status"], "passed") def test_empty_test_directory_is_blocked(self): diff --git a/tools/sstack/test_delivery.py b/tools/sstack/test_delivery.py index 2b1da4e..e4d05a2 100644 --- a/tools/sstack/test_delivery.py +++ b/tools/sstack/test_delivery.py @@ -176,14 +176,15 @@ def packet(self, state): "sources_changed_during_run": False, "checks": [ { - "name": "stack", - "cwd": ".", - "command": control.SUITES["stack"][0][2], + "name": name, + "cwd": cwd, + "command": command, "status": "passed", "exit_code": 0, "log": "artifacts/check.log", "log_sha256": log["sha256"], } + for name, cwd, command in control.SUITES["stack"] ], } return { @@ -252,6 +253,24 @@ def reach(self, target): while self.store.status(self.issue)["state"] != target: self.advance() + def test_validation_accepts_embedded_coverage_without_duplicate_report(self): + packet = self.packet("validating") + report = delivery.evidence_file(packet["verification"][0]) + report.update(scope="offline", coverage=delivery.evidence_file(packet["coverage"])) + reference = self.ref("combined.json", report) + packet.update(coverage=reference, verification=[reference]) + task = { + "state": "validating", + "contract": self.contract, + "context": {"artifact": self.artifact}, + } + delivery.gate(task, packet) + report["coverage"]["files"] = {} + reference = self.ref("combined.json", report) + packet.update(coverage=reference, verification=[reference]) + with self.assertRaisesRegex(delivery.Blocked, "coverage_changed_or_incomplete"): + delivery.gate(task, packet) + def test_validation_uses_engineer_checkout_suite_not_coordinator_registry(self): path = self.root / "tools/sstack/projects.json" document = json.loads(path.read_text()) diff --git a/tools/sstack/test_registry.py b/tools/sstack/test_registry.py index 9176d1c..228c62a 100644 --- a/tools/sstack/test_registry.py +++ b/tools/sstack/test_registry.py @@ -145,6 +145,75 @@ def test_invalid_suite_contracts_fail_closed(self): with self.assertRaises(ValueError): control.coverage(self.root, self.base) + def test_custom_safety_flags_require_booleans_and_preserve_check_shape(self): + path = self.root / registry.REGISTRY_PATH + document = json.loads(path.read_text()) + check = {"name": "custom-check", "cwd": ".", "command": ["$PYTHON", "checks.py"]} + for flag in ("parallel_safe", "reuse_safe"): + for value in (False, True, None, "true", 1, [], {}): + with self.subTest(flag=flag, value=value): + document["suites"] = {"custom": [{**check, flag: value}]} + path.write_text(json.dumps(document)) + if isinstance(value, bool): + loaded = registry.load_suites(self.root, {})["custom"][0] + self.assertEqual( + loaded, ("custom-check", ".", [control.sys.executable, "checks.py"]) + ) + else: + with self.assertRaisesRegex(ValueError, flag): + registry.load_suites(self.root, {}) + + def test_excluded_unowned_changes_still_block(self): + path = self.root / registry.REGISTRY_PATH + document = json.loads(path.read_text()) + document["projects"][0]["exclude_paths"] = ["app/private/"] + path.write_text(json.dumps(document)) + self.git("add", ".") + self.git("commit", "-qm", "exclude private paths") + self.write("app/private/new.py", "value = 1\n") + result = control.coverage(self.root, "HEAD") + self.assertEqual(result["status"], "blocked") + self.assertEqual(result["unknown_paths"], ["app/private/new.py"]) + + def test_invalid_exclude_paths_fail_closed(self): + path = self.root / registry.REGISTRY_PATH + document = json.loads(path.read_text()) + for excludes in (None, "app/", [""], [1], ["/app/"], ["../app/"], ["."]): + with self.subTest(excludes=excludes): + document["projects"][0]["exclude_paths"] = excludes + path.write_text(json.dumps(document)) + with self.assertRaises(ValueError): + control.coverage(self.root, self.base) + + def test_default_registry_keeps_lightweight_selection_exact(self): + document = json.loads((Path(__file__).parent / "projects.json").read_text()) + self.write(registry.REGISTRY_PATH, json.dumps(document)) + # Supply the docs builtin explicitly so this test exercises registry policy. + suites = {"stack": [], "docs": []} + cases = [ + (["README.md"], ["docs"], []), + (["tools/sstack/README.md"], ["docs"], []), + (["tools/sstack/skills/sstack/SKILL.md"], ["stack"], []), + (["AGENTS.md"], ["stack"], []), + (["CLAUDE.md"], ["stack"], []), + (["install.py"], ["stack"], []), + ([".sstack.json"], ["stack"], []), + ([".github/workflows/sstack-quality.yml"], ["stack"], []), + (["tools/sstack/control.py"], ["stack"], []), + (["README.md", "tools/sstack/control.py"], ["docs", "stack"], []), + (["README.md", "other/README.md"], ["docs"], ["other/README.md"]), + (["tools/sstack/README.md.backup"], ["stack"], []), + ] + for files, expected, unknown in cases: + with ( + self.subTest(files=files), + patch.object(registry, "changed_files", return_value=files), + ): + result = registry.coverage(self.root, self.base, suites) + self.assertEqual(result["suites"], expected) + self.assertEqual(result["unknown_paths"], unknown) + self.assertEqual(result["status"], "blocked" if unknown else "passed") + def test_default_root_suites_are_refreshed_after_registration(self): path = self.root / registry.REGISTRY_PATH document = json.loads(path.read_text()) diff --git a/tools/sstack/test_runner.py b/tools/sstack/test_runner.py new file mode 100644 index 0000000..4a79d0a --- /dev/null +++ b/tools/sstack/test_runner.py @@ -0,0 +1,317 @@ +import json +import os +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +import control + + +class RunnerTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + subprocess.run(["git", "init", "-q", str(self.root)], check=True) + (self.root / ".gitignore").write_text("artifacts/\n") + subprocess.run(["git", "add", "."], cwd=self.root, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=Test", + "-c", + "user.email=test@example.test", + "commit", + "-qm", + "initial", + ], + cwd=self.root, + check=True, + ) + self.checks = [("sample", ".", [sys.executable, "-c", "print('verified')"])] + self.suites_patch = patch.object(control, "SUITES", {"sample": self.checks}) + self.suites_patch.start() + self.addCleanup(self.suites_patch.stop) + + def verify(self, **kwargs): + return control.verify("sample", root=self.root, **kwargs) + + def test_builtin_contracts_and_conservative_registry_approvals(self): + self.assertEqual( + control.BUILTIN_SUITES["stack"][1], + ("stack-lint", ".", [sys.executable, "tools/sstack/lint.py", "--code-only"]), + ) + self.assertEqual( + control.BUILTIN_SUITES["docs"][0][2], + [sys.executable, "tools/sstack/check_contracts.py", "--include-readme"], + ) + path = self.root / control.registry.REGISTRY_PATH + path.parent.mkdir(parents=True) + path.write_text( + json.dumps( + { + "suites": { + "custom": [ + {"name": "serial"}, + {"name": "safe", "parallel_safe": True}, + {"name": "reusable", "reuse_safe": True}, + ] + } + } + ) + ) + self.assertEqual( + control.approved_checks(self.root, "parallel_safe"), + {"stack", "stack-lint", "docs", "safe"}, + ) + self.assertEqual( + control.approved_checks(self.root, "reuse_safe"), + {"stack", "stack-lint", "docs", "reusable"}, + ) + + def test_all_runs_shared_docs_check_once(self): + with patch.object(control, "SUITES", control.BUILTIN_SUITES): + result = control.verify("all", plan=True, root=self.root) + self.assertEqual( + [check["name"] for check in result["checks"]], ["stack", "stack-lint", "docs"] + ) + + def test_parallel_checks_overlap_but_serial_check_is_a_barrier(self): + names = ["first", "second", "serial", "third", "fourth"] + self.checks[:] = [(name, ".", [sys.executable, "-c", "pass"]) for name in names] + intervals = {} + lock = threading.Lock() + rendezvous = threading.Barrier(2) + + def execute(name, cwd, command, **kwargs): + with lock: + intervals[name] = [time.monotonic()] + if name != "serial": + rendezvous.wait(timeout=5) + time.sleep(0.02) + with lock: + intervals[name].append(time.monotonic()) + return { + "name": name, + "cwd": cwd, + "command": command, + "status": "passed", + "exit_code": 0, + "duration_seconds": intervals[name][1] - intervals[name][0], + }, name + + with ( + patch.object(control, "approved_checks", return_value=set(names) - {"serial"}), + patch.object(control, "run_check", side_effect=execute), + ): + report = self.verify(jobs=2) + self.assertEqual([check["name"] for check in report["checks"]], names) + for left, right in [("first", "second"), ("third", "fourth")]: + self.assertLess( + max(intervals[left][0], intervals[right][0]), + min(intervals[left][1], intervals[right][1]), + ) + self.assertLessEqual(max(intervals[name][1] for name in names[:2]), intervals["serial"][0]) + self.assertLessEqual(intervals["serial"][1], min(intervals[name][0] for name in names[3:])) + self.assertGreater(report["elapsed_seconds"], 0) + + def test_jobs_bounds_and_default_custom_serial_execution(self): + for jobs in (0, 9, True): + with self.assertRaises(ValueError): + self.verify(jobs=jobs) + self.checks.clear() + self.checks.extend((name, ".", [sys.executable, "-c", "pass"]) for name in ("a", "b")) + active = threading.Lock() + + def execute(name, cwd, command, **kwargs): + self.assertTrue(active.acquire(blocking=False), "custom checks overlapped") + time.sleep(0.02) + active.release() + return { + "name": name, + "cwd": cwd, + "command": command, + "status": "passed", + "exit_code": 0, + "duration_seconds": 0.02, + }, "ok" + + with patch.object(control, "run_check", side_effect=execute): + self.assertEqual(self.verify(jobs=8)["status"], "passed") + + def test_parallel_work_never_exceeds_job_limit(self): + self.checks[:] = [(str(index), ".", [sys.executable, "-c", "pass"]) for index in range(6)] + lock = threading.Lock() + active = 0 + maximum = 0 + + def execute(name, cwd, command, **kwargs): + nonlocal active, maximum + with lock: + active += 1 + maximum = max(maximum, active) + time.sleep(0.02) + with lock: + active -= 1 + return { + "name": name, + "cwd": cwd, + "command": command, + "status": "passed", + "exit_code": 0, + "duration_seconds": 0.02, + }, "ok" + + with ( + patch.object( + control, "approved_checks", return_value={check[0] for check in self.checks} + ), + patch.object(control, "run_check", side_effect=execute), + ): + self.assertEqual(self.verify(jobs=2)["status"], "passed") + self.assertEqual(maximum, 2) + + def test_runtime_identity_tracks_distribution_python_and_executable_changes(self): + executable = self.root / "runner" + executable.write_text("#!/bin/sh\necho first\n") + executable.chmod(0o755) + checks = [("local", ".", ["./runner"])] + original = control.runtime_identity(checks, self.root) + self.assertEqual(original["executables"][0]["path"], str(executable.resolve())) + executable.write_text("#!/bin/sh\necho changed\n") + self.assertNotEqual(original, control.runtime_identity(checks, self.root)) + original = control.runtime_identity(checks, self.root) + with patch.object(control.importlib.metadata, "distributions", return_value=[]): + self.assertNotEqual(original, control.runtime_identity(checks, self.root)) + with patch.object(control.sys, "version", "different Python"): + self.assertNotEqual(original, control.runtime_identity(checks, self.root)) + + def test_parallel_failure_timeout_and_missing_check_are_not_hidden(self): + self.checks[:] = [ + ("fails", ".", [sys.executable, "-c", "raise SystemExit(7)"]), + ("times-out", ".", [sys.executable, "-c", "import time; time.sleep(30)"]), + ("missing", ".", [str(self.root / "missing")]), + ("passes", ".", [sys.executable, "-c", "print('last')"]), + ] + with patch.object( + control, "approved_checks", return_value={check[0] for check in self.checks} + ): + report = self.verify(jobs=2, timeout=0.2) + self.assertEqual( + [check["status"] for check in report["checks"]], + ["failed", "timeout", "blocked", "passed"], + ) + self.assertEqual(report["status"], "failed") + self.assertTrue(all(check["duration_seconds"] >= 0 for check in report["checks"])) + + def cached(self): + return self.verify(environment_key="immutable-fixture-v1", base="HEAD") + + def reuse(self, report, **kwargs): + arguments = { + "reuse": report["report"], + "environment_key": "immutable-fixture-v1", + "base": "HEAD", + } + arguments.update(kwargs) + with patch.object(control, "approved_checks", return_value={"sample"}): + return self.verify(**arguments) + + def test_exact_reuse_returns_original_report_without_running_or_writing(self): + report = self.cached() + before = set((self.root / "artifacts/sstack").iterdir()) + with patch.object(control, "run_check") as run: + reused = self.reuse(report) + run.assert_not_called() + self.assertEqual(reused["report"], report["report"]) + self.assertEqual(reused["reuse"]["status"], "hit") + self.assertEqual(set((self.root / "artifacts/sstack").iterdir()), before) + self.assertEqual(json.loads((self.root / report["report"]).read_text()), report) + self.assertNotIn("immutable-fixture-v1", json.dumps(report)) + + def test_reuse_requires_key_and_custom_opt_in(self): + report = self.cached() + with self.assertRaises(ValueError): + self.verify(reuse=report["report"]) + result = self.verify( + reuse=report["report"], environment_key="immutable-fixture-v1", base="HEAD" + ) + self.assertEqual(result["reuse"]["status"], "miss") + self.assertEqual(result["status"], "passed") + + def test_changed_key_environment_source_base_command_or_log_reruns(self): + for change in ("key", "runtime", "source", "base", "command", "cwd", "log", "head"): + with self.subTest(change=change): + report = self.cached() + kwargs = {} + environment = patch.dict(os.environ, {}, clear=False) + if change == "key": + kwargs["environment_key"] = "new-fixture" + elif change == "runtime": + environment = patch.dict(os.environ, {"SSTACK_TEST_RUNTIME": "changed"}) + elif change == "source": + (self.root / "new-input.txt").write_text("changed") + elif change == "base": + kwargs["base"] = None + elif change == "command": + self.checks[0] = ("sample", ".", [sys.executable, "-c", "print('different')"]) + elif change == "cwd": + self.checks[0] = ("sample", "./", self.checks[0][2]) + elif change == "log": + (self.root / report["checks"][0]["log"]).write_text("tampered") + elif change == "head": + subprocess.run( + [ + "git", + "-c", + "user.name=Test", + "-c", + "user.email=test@example.test", + "commit", + "--allow-empty", + "-qm", + "new head", + ], + cwd=self.root, + check=True, + ) + with environment: + result = self.reuse(report, **kwargs) + self.assertEqual(result["reuse"]["status"], "miss") + self.assertNotEqual(result["report"], report["report"]) + self.assertEqual(result["status"], "passed") + + def test_failed_live_ui_and_invalid_reports_never_reuse(self): + report = self.cached() + report_path = self.root / report["report"] + for changes in ( + {"status": "failed"}, + {"scope": "live"}, + {"ui": "passed"}, + {"live_kizen": "passed"}, + {"sources_changed_during_run": True}, + {"fingerprint_policy": {}}, + {"source_fingerprint_after": "different"}, + {"checks": []}, + {"checks": [{"status": "passed"}]}, + {"runtime_identity": {}}, + {"git_revision": None}, + ): + with self.subTest(changes=changes): + report_path.write_text(json.dumps({**report, **changes})) + self.assertEqual(self.reuse(report)["reuse"]["status"], "miss") + for data in ("not JSON", "[]", "null"): + report_path.write_text(data) + self.assertEqual(self.reuse(report)["reuse"]["status"], "miss") + report_path.unlink() + self.assertEqual(self.reuse(report)["reuse"]["status"], "miss") + + +if __name__ == "__main__": + unittest.main()