From 697edf822d5b587b77a06b8c8bab06cbbc3b9a84 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 22:13:31 -0500 Subject: [PATCH 01/14] Dependabot: restate the pynetdicom, pydicom and webauthn caps as ignore ranges (BACKLOG #2505) Dependabot widens a pyproject cap unless dependabot.yml restates it as an ignore range. Three caps had no entry: pynetdicom<4, pydicom<3.1 and webauthn<4. hvac<3 stays without one, as pyproject.toml says beside it. tests/test_dependabot_cap_ignores.py holds every upper bound in the three dependency tables to a covering ignore range or a stated exemption, and every ignore entry to a cap that still exists. Work in progress: the /simplify pass and code review follow in later commits. (cherry picked from commit b008f221e047224b08eeed906272a664b8b9413c) --- .github/dependabot.yml | 16 ++ tests/test_dependabot_cap_ignores.py | 315 +++++++++++++++++++++++++++ tests/tooling_manifest.txt | 1 + 3 files changed, 332 insertions(+) create mode 100644 tests/test_dependabot_cap_ignores.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 05cd9c446..eebe63530 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -118,6 +118,13 @@ updates: # exits 2. Why the cap exists is beside it in pyproject.toml; read it there. The bare name # matches `uvicorn[standard]` because dependabot-core's Python NameNormaliser strips extras # before it compares names (read 2026-09-30). Lift this entry in the same PR that lifts the cap. + # + # `pynetdicom`, `pydicom` and `webauthn` are the same WIDENED-CAP case (BACKLOG #2505). Each entry + # below carries a one-line pointer to the reason beside its cap in pyproject.toml; read it there. + # pydicom's `<3.1` is a MINOR cap, so without its entry `python-deps` would widen it in the routine + # minor-and-patch batch. `hvac<3` is the one cap with NO entry, on purpose, and pyproject.toml says + # why beside it. tests/test_dependabot_cap_ignores.py holds every upper bound in pyproject.toml to + # an entry here or to a stated exemption, and every entry here to a cap that still exists. ignore: - dependency-name: "ruff" versions: [">=0.16.0"] @@ -127,6 +134,15 @@ updates: versions: [">=7.4.0"] - dependency-name: "uvicorn" versions: [">=0.50.0"] + # pyproject.toml's [dicom] extra comment: pynetdicom 3.x pairs with the pydicom 3.x line. + - dependency-name: "pynetdicom" + versions: [">=4.0.0"] + # pyproject.toml's [dicom] extra comment: the deflate guard replays pydicom internals (BACKLOG #1926). + - dependency-name: "pydicom" + versions: [">=3.1.0"] + # pyproject.toml's [webauthn] extra comment: the cbor2 ranges of webauthn's majors are disjoint. + - dependency-name: "webauthn" + versions: [">=4.0.0"] groups: # Version-update grouping (applies-to defaults to version-updates), SPLIT BY UPDATE TYPE (owner # decision 2026-09-30) so one bad major cannot hold the routine minors and patches hostage. diff --git a/tests/test_dependabot_cap_ignores.py b/tests/test_dependabot_cap_ignores.py new file mode 100644 index 000000000..fbb8eb460 --- /dev/null +++ b/tests/test_dependabot_cap_ignores.py @@ -0,0 +1,315 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""Every upper bound in pyproject.toml has a Dependabot ignore range, or a stated exemption. + +Dependabot WIDENS a declared cap instead of respecting it. ``.github/dependabot.yml`` says so, and +says a load-bearing cap must be restated there as an ``ignore`` range "or the cap is decorative". +PR #66 widened ruff's and annotated-types' caps that way, and Dependabot PR 1773 tried to widen +uvicorn's. Until BACKLOG #2505 nothing held the two files together, so four caps had no entry. + +THE CONTRACT, both directions: + +* Every upper bound in ``[project.dependencies]``, ``[project.optional-dependencies]`` and + ``[dependency-groups]`` (``<``, ``~=``, ``==``, or an ``==X.*`` wildcard) has a uv-ecosystem + ``ignore`` entry whose range covers what the cap excludes and leaves open what it allows. Or the + package is in one of the two exemption tables below, with its reason. +* Every uv ``ignore`` entry names a package that pyproject.toml still caps. dependabot.yml says to + lift a cap and delete its entry in the same PR; this is what makes that rule fail when skipped. + +AN EXACT ``==`` PIN COVERS THE NEXT MINOR, NOT THE NEXT PATCH. That is the ``sigstore`` entry's +documented scope (``>=4.5.0`` for ``==4.4.0``): it blocks the minor the owner declined and leaves +the patch track open. A pin's entry must not cover the pinned version itself. + +``test_the_checker_reds_on_a_mutated_config`` is the mutation arm. It feeds the checker copies of +the real files with one thing broken, and fails unless each copy is reported. +""" + +from __future__ import annotations + +import copy +import tomllib +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Any, NamedTuple + +import pytest +import yaml +from packaging.requirements import Requirement +from packaging.specifiers import Specifier, SpecifierSet +from packaging.utils import canonicalize_name +from packaging.version import Version + +_ROOT = Path(__file__).resolve().parents[1] +_PYPROJECT = _ROOT / "pyproject.toml" +_DEPENDABOT = _ROOT / ".github" / "dependabot.yml" + +#: The exact ``==`` pins in ``[dependency-groups]`` that are deliberately NOT ignored. Each must stay +#: an ``==`` pin in a dependency group; a name here that becomes a cap anywhere else needs its own +#: decision, not this one. +_GROUP_PIN_EXEMPT: frozenset[str] = frozenset( + {"bandit", "pip-audit", "zizmor", "build", "diff-cover", "mutmut"} +) +_GROUP_PIN_REASON = ( + ".github/dependabot.yml, 'NOT IGNORED, deliberately': ignoring the exact pins in " + "[dependency-groups] would freeze the hash-pinned CI toolchain, which ADR 0034 section 3 wants " + "moving through Dependabot" +) + +#: Any other cap with no ignore entry, by package, with the reason. Read the reason where it points. +_CAP_EXEMPT: dict[str, str] = { + "hvac": ( + "pyproject.toml's [vault] extra comment: majors already go to manual review, and an ignore " + "would suppress hvac's security track for no gain" + ), +} + + +class Cap(NamedTuple): + """One upper bound: where it sits, the first version it excludes, and one version it allows.""" + + package: str + where: str + spec: str + exact_pin: bool + first_excluded: Version + allowed: Version + + +def _bump(release: tuple[int, ...]) -> Version: + """Increment the last component of ``release``: (7, 3) -> 7.4, (4,) -> 5.""" + return Version(".".join(str(n) for n in (*release[:-1], release[-1] + 1))) + + +def _just_below(version: Version) -> Version: + """A release just under ``version``: 0.50 -> 0.49.999, 4 -> 3.999, 3.1.0 -> 3.0.999.""" + release = list(version.release) + while release and release[-1] == 0: + release.pop() + if not release: + raise ValueError(f"no release below {version}") + release[-1] -= 1 + return Version(".".join(str(n) for n in (*release, 999))) + + +def _padded(version: Version) -> str: + """The `>=X.Y.Z` spelling the existing ignore entries use: 3.1 -> 3.1.0.""" + return ".".join(str(n) for n in (*version.release, 0, 0)[: max(3, len(version.release))]) + + +def _cap_of(spec: Specifier) -> tuple[Version, Version] | None: + """(first excluded, one allowed) for an upper-bound specifier, or None for a floor or ``!=``.""" + op, raw = spec.operator, spec.version + if op in (">", ">=", "!="): + return None + if op == "<": + first = Version(raw) + return first, _just_below(first) + if op == "~=": + first = _bump(Version(raw).release[:-1]) + return first, _just_below(first) + if op == "==" and raw.endswith(".*"): + first = _bump(Version(raw[:-2]).release) + return first, _just_below(first) + if op == "==": + pinned = Version(raw) + major, minor = (*pinned.release, 0)[:2] + return Version(f"{major}.{minor + 1}.0"), pinned + # `<=` and `===` have no use in pyproject.toml today. Model one deliberately when it arrives. + raise AssertionError(f"unmodelled upper-bound operator {op!r} in {spec}") + + +def _requirements(pyproject: dict[str, Any]) -> Iterator[tuple[str, str]]: + """(where, requirement string) for every requirement in the three tables the uv updater reads.""" + project = pyproject["project"] + for req in project.get("dependencies", []): + yield "[project.dependencies]", req + for extra, reqs in project.get("optional-dependencies", {}).items(): + for req in reqs: + yield f"[project.optional-dependencies].{extra}", req + for group, reqs in pyproject.get("dependency-groups", {}).items(): + for req in reqs: + if isinstance(req, str): # skip `{include-group = ...}` tables + yield f"[dependency-groups].{group}", req + + +def _caps(pyproject: dict[str, Any]) -> list[Cap]: + """The tightest upper bound of every capped requirement. Markers are not specifiers.""" + caps: list[Cap] = [] + for where, raw in _requirements(pyproject): + req = Requirement(raw) + bounds = [b for s in req.specifier if (b := _cap_of(s)) is not None] + if bounds: + first, allowed = min(bounds) + pin = any(s.operator == "==" and not s.version.endswith(".*") for s in req.specifier) + caps.append(Cap(canonicalize_name(req.name), where, str(req), pin, first, allowed)) + return caps + + +def _uv_ignores(dependabot: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + """The root uv entry's ignore list, keyed by canonical package name.""" + uv = [ + u for u in dependabot["updates"] if u["package-ecosystem"] == "uv" and u["directory"] == "/" + ] + assert len(uv) == 1, f"expected one uv update entry for '/', found {len(uv)}" + ignores: dict[str, list[dict[str, Any]]] = {} + for entry in uv[0].get("ignore", []): + ignores.setdefault(canonicalize_name(entry["dependency-name"]), []).append(entry) + return ignores + + +def _covers(entries: list[dict[str, Any]], version: Version) -> bool: + """Whether any entry's `versions` range ignores ``version``. A list of ranges is a union.""" + return any( + SpecifierSet(rng).contains(version, prereleases=True) + for entry in entries + for rng in entry.get("versions", []) + ) + + +def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[str]: + caps = _caps(pyproject) + ignores = _uv_ignores(dependabot) + problems: list[str] = [] + + for cap in caps: + label = f"{cap.where}: {cap.spec}" + if cap.package in _GROUP_PIN_EXEMPT: + if not (cap.exact_pin and cap.where.startswith("[dependency-groups]")): + problems.append(f"{label} is exempt as a group `==` pin, but is not one") + elif cap.package in ignores: + problems.append(f"{label} is exempt ({_GROUP_PIN_REASON}) yet has an ignore entry") + continue + if cap.package in _CAP_EXEMPT: + if cap.package in ignores: + problems.append( + f"{label} is exempt ({_CAP_EXEMPT[cap.package]}) yet has an ignore entry" + ) + continue + entries = ignores.get(cap.package) + if not entries: + problems.append( + f"{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with " + f'versions [">={_padded(cap.first_excluded)}"], or an exemption here with its reason' + ) + continue + if any("update-types" in e for e in entries): + problems.append( + f"{label}: an `update-types` ignore cannot restate a range; use `versions`" + ) + far = Version(f"{cap.first_excluded.major + 1000}") + if not (_covers(entries, cap.first_excluded) and _covers(entries, far)): + problems.append( + f"{label}: its ignore range leaves versions from {cap.first_excluded} up open" + ) + if _covers(entries, cap.allowed): + problems.append( + f"{label}: its ignore range also blocks {cap.allowed}, which the cap allows" + ) + + capped = {cap.package for cap in caps} + for name in sorted(set(ignores) - capped): + problems.append( + f"{_DEPENDABOT.name} ignores {name!r}, which pyproject.toml no longer caps; " + "delete the entry in the PR that lifted the cap" + ) + for name in sorted((_GROUP_PIN_EXEMPT | set(_CAP_EXEMPT)) - capped): + problems.append(f"exemption for {name!r} names no capped requirement in pyproject.toml") + return problems + + +def _load() -> tuple[dict[str, Any], dict[str, Any]]: + pyproject = tomllib.loads(_PYPROJECT.read_text(encoding="utf-8")) + dependabot = yaml.safe_load(_DEPENDABOT.read_text(encoding="utf-8")) + return pyproject, dependabot + + +def test_every_cap_has_an_ignore_entry_or_a_stated_exemption() -> None: + """RED when: a pyproject cap has no matching ignore range, or an ignore entry outlives its cap.""" + problems = _violations(*_load()) + assert not problems, "\n".join(problems) + + +def test_the_census_finds_the_known_caps() -> None: + """Positive control: a parser that found no caps would make the test above pass vacuously.""" + found = {cap.package for cap in _caps(_load()[0])} + known = { + "uvicorn", + "pynetdicom", + "pydicom", + "webauthn", + "hvac", + "ruff", + "sigstore", + "cyclonedx-bom", + } + assert known <= found, f"census missed {sorted(known - found)}" + assert "atheris" not in found, "an environment marker's `==` was read as a version cap" + + +@pytest.mark.parametrize( + ("spec", "first", "allowed"), + [ + ("<0.50", "0.50", "0.49.999"), + ("<4", "4", "3.999"), + ("<3.1", "3.1", "3.0.999"), + ("~=7.3.1", "7.4", "7.3.999"), + ("~=7.3", "8", "7.999"), + ("==4.4.0", "4.5.0", "4.4.0"), + ("==1.2.*", "1.3", "1.2.999"), + ], +) +def test_cap_arithmetic(spec: str, first: str, allowed: str) -> None: + assert _cap_of(Specifier(spec)) == (Version(first), Version(allowed)) + + +_Mutation = Callable[[dict[str, Any]], None] + + +def _drop_ignore(doc: dict[str, Any], name: str) -> None: + uv = next(u for u in doc["updates"] if u["package-ecosystem"] == "uv") + uv["ignore"] = [e for e in uv["ignore"] if e["dependency-name"] != name] + + +def _set_range(doc: dict[str, Any], name: str, rng: str) -> None: + uv = next(u for u in doc["updates"] if u["package-ecosystem"] == "uv") + next(e for e in uv["ignore"] if e["dependency-name"] == name)["versions"] = [rng] + + +def _add_cap(doc: dict[str, Any], req: str) -> None: + doc["project"]["dependencies"].append(req) + + +def _lift_cap(doc: dict[str, Any], old: str, new: str) -> None: + deps = doc["project"]["optional-dependencies"]["dicom"] + deps[deps.index(old)] = new + + +@pytest.mark.parametrize( + ("mutate_pyproject", "mutate_dependabot", "expect"), + [ + pytest.param(None, lambda d: _drop_ignore(d, "pydicom"), "pydicom", id="entry-deleted"), + pytest.param(None, lambda d: _set_range(d, "pydicom", ">=3.2.0"), "open", id="gap-at-cap"), + pytest.param(None, lambda d: _set_range(d, "pydicom", ">=3.0.0"), "blocks", id="freezes"), + pytest.param( + None, lambda d: _set_range(d, "sigstore", ">=4.4.0"), "blocks", id="blocks-pin" + ), + pytest.param(lambda p: _add_cap(p, "newdep>=1,<2"), None, "newdep", id="new-cap"), + pytest.param( + lambda p: _lift_cap(p, "pydicom>=3.0.2,<3.1", "pydicom>=3.0.2"), + None, + "no longer caps", + id="stale-entry", + ), + ], +) +def test_the_checker_reds_on_a_mutated_config( + mutate_pyproject: _Mutation | None, mutate_dependabot: _Mutation | None, expect: str +) -> None: + """Mutation arm: each broken copy of the real files must produce a violation naming it.""" + pyproject, dependabot = (copy.deepcopy(doc) for doc in _load()) + if mutate_pyproject: + mutate_pyproject(pyproject) + if mutate_dependabot: + mutate_dependabot(dependabot) + problems = _violations(pyproject, dependabot) + assert any(expect in p for p in problems), f"no violation mentioning {expect!r}: {problems}" diff --git a/tests/tooling_manifest.txt b/tests/tooling_manifest.txt index df8006797..f35183fbf 100644 --- a/tests/tooling_manifest.txt +++ b/tests/tooling_manifest.txt @@ -114,6 +114,7 @@ tests/test_dast_claims.py tests/test_defaulted_credential_lint.py tests/test_dep1_lock_resync_lockstep.py tests/test_dependabot_automerge_guardrails.py +tests/test_dependabot_cap_ignores.py tests/test_docs_runbooks.py tests/test_failure_signal.py tests/test_feature_map_claims.py From 16306ca48e81ca764bf8944e1952857226b9ec03 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 22:19:41 -0500 Subject: [PATCH 02/14] feat(generators): emit one alternative of a choice group; ORM^O01 gets an order detail (BACKLOG #2497) _emit walked a choice group like a sequence and emitted every required child, so ORM^O01 left ORDER_DETAIL out: its OBR/RQD/RQ1/RXO/ODS/ODT group is a choice. _emit now picks one alternative, through the strict validator's own _is_choice, so the groups hl7apy mislabels as choices still get every part. MessageSpec.preferred_alternative, OBR by default, wins without drawing from the seeded rng; otherwise a seeded pick among buildable alternatives. ORM adds _ORDER_DETAIL to its group suffixes. Tests: tests/test_generators_choice.py. Generated ORM^O01 (v2.5.1, the only version the generator writes) passes strict validation across seeds; five mutation arms each go red. --- changelog.d/2497.changed.md | 6 ++ messagefoundry/generators/_core.py | 34 ++++++++++- messagefoundry/generators/orm.py | 9 ++- tests/test_generators_choice.py | 95 ++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 changelog.d/2497.changed.md create mode 100644 tests/test_generators_choice.py diff --git a/changelog.d/2497.changed.md b/changelog.d/2497.changed.md new file mode 100644 index 000000000..ceb0cb772 --- /dev/null +++ b/changelog.d/2497.changed.md @@ -0,0 +1,6 @@ +- **`messagefoundry generate` now gives every ORM^O01 an order detail.** The generator used to + leave ORDER_DETAIL out, because it emitted every required child of a group, and the order + detail's OBR/RQD/RQ1/RXO/ODS/ODT group is a choice. It now emits one alternative of a choice + group, OBR by default. Each generated ORM^O01 now ends ORC then OBR and passes strict + validation, so a corpus generated before this change differs in its ORM files. + (`BACKLOG #2497`) diff --git a/messagefoundry/generators/_core.py b/messagefoundry/generators/_core.py index 5640b2f76..94e2761a5 100644 --- a/messagefoundry/generators/_core.py +++ b/messagefoundry/generators/_core.py @@ -26,6 +26,9 @@ from messagefoundry.generators import _hl7data as d from messagefoundry.parsing import validate +# The strict validator's own choice test, so generation and validation agree on what a choice is. +from messagefoundry.parsing.validate import _is_choice + _MESSAGES = _ref.MESSAGES _SEGMENTS = _ref.SEGMENTS @@ -398,6 +401,12 @@ class MessageSpec: # Optional groups to recurse into, matched by name *suffix* (e.g. "_PATIENT") so one spec # covers every structure of its type (ORM_O01_PATIENT, SIU_S12_PATIENT, …). group_suffixes: frozenset[str] = frozenset() + # The alternative a choice group emits when it offers this one (see ``_pick_alternative``). + preferred_alternative: str = "OBR" + + +def _builder_for(spec: MessageSpec, name: str) -> SegmentBuilder | None: + return spec.builders.get(name) or SHARED_BUILDERS.get(name) _REGISTRY: dict[str, MessageSpec] = {} @@ -431,6 +440,20 @@ def control_id(code: str, trigger: str, index: int) -> str: # --- reference-driven assembly ---------------------------------------------- +def _pick_alternative(group: str, alternatives: Any, rng: random.Random, spec: MessageSpec) -> Any: + """The one alternative of choice group ``group`` to emit. + + ``spec.preferred_alternative`` (OBR by default) when the group offers it and it can be built, + without drawing from ``rng``. Otherwise a seeded pick among the alternatives that can be + built, so a seed still reproduces its bytes. + """ + usable = [alt for alt in alternatives if alt[3] == "GRP" or _builder_for(spec, alt[0])] + if not usable: + raise RuntimeError(f"no builder for any alternative of choice group {group}") + preferred = next((alt for alt in usable if alt[0] == spec.preferred_alternative), None) + return preferred if preferred is not None else rng.choice(usable) + + def _emit( children: list[Any], rng: random.Random, @@ -443,7 +466,9 @@ def _emit( Required children (min>=1) are always emitted; optional segments are emitted only if allow-listed (a random 0..N for repeating ones), or if named in ``force``. Groups recurse - only when the group itself is required. + only when the group itself is required or matches ``spec.group_suffixes``. A choice group + means "exactly one of", so it emits one alternative rather than every required child. The + choice test is the strict validator's, which keeps the sequences hl7apy mislabels as choices. """ for child in children: name = child[0] @@ -451,7 +476,7 @@ def _emit( min_card, max_card = child[2][0], child[2][1] if name in _SEGMENTS: - builder = spec.builders.get(name) or SHARED_BUILDERS.get(name) + builder = _builder_for(spec, name) if min_card >= 1 or name in force: if builder is None: raise RuntimeError(f"no builder for required/forced segment {name}") @@ -461,7 +486,10 @@ def _emit( for _ in range(rng.randint(0, max_reps)): out.append(builder(rng, ctx)) elif min_card >= 1 or any(name.endswith(s) for s in spec.group_suffixes): - _emit(child_ref[1], rng, ctx, spec, force, out) + group_children = child_ref[1] + if _is_choice(name, child_ref): + group_children = [_pick_alternative(name, group_children, rng, spec)] + _emit(group_children, rng, ctx, spec, force, out) def generate_message(code: str, trigger: str, index: int, *, seed: str = DEFAULT_SEED) -> str: diff --git a/messagefoundry/generators/orm.py b/messagefoundry/generators/orm.py index 17920989f..6fb4246a4 100644 --- a/messagefoundry/generators/orm.py +++ b/messagefoundry/generators/orm.py @@ -2,10 +2,9 @@ # Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors """Generate conformant HL7 v2.5.1 **ORM** (general order) messages. -ORM_O01 only *requires* MSH + ORC; we include the optional PATIENT group (PID/PV1) for realism. -We deliberately omit ORDER_DETAIL: hl7apy models its OBR/RQD/RQ1/RXO/ODS/ODT subgroup as -all-required rather than a choice, so it can't be populated sensibly — OBR-based orders are -better expressed via OML (the modern lab order) instead. +ORM_O01 only *requires* MSH + ORC; we include the optional PATIENT group (PID/PV1) for realism, +and an ORDER_DETAIL after each ORC. Its OBR/RQD/RQ1/RXO/ODS/ODT group is a choice, so the +generator emits exactly one alternative, OBR (see ``_core._pick_alternative``). """ from __future__ import annotations @@ -18,6 +17,6 @@ code="ORM", trigger_to_structure={"O01": "ORM_O01"}, optional_allowlist=frozenset({"PD1", "PV2"}), - group_suffixes=frozenset({"_PATIENT", "_PATIENT_VISIT"}), + group_suffixes=frozenset({"_PATIENT", "_PATIENT_VISIT", "_ORDER_DETAIL"}), ) ) diff --git a/tests/test_generators_choice.py b/tests/test_generators_choice.py new file mode 100644 index 000000000..add1dfb39 --- /dev/null +++ b/tests/test_generators_choice.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""The generator emits exactly one alternative of a choice group (BACKLOG #2497). + +ORM^O01's order detail holds a choice of OBR, RQD, RQ1, RXO, ODS or ODT. Strict validation +needs exactly one, so the generator picks one instead of emitting every required child. All +data is synthetic. +""" + +from __future__ import annotations + +import random +from typing import Any + +import pytest + +from messagefoundry.generators import _core, orm # noqa: F401 (importing orm registers ORM) +from messagefoundry.parsing import Peek +from messagefoundry.parsing.validate import _SEQUENCES_LABELLED_CHOICE + +_ORDER_ALTERNATIVES = ("OBR", "RQD", "RQ1", "RXO", "ODS", "ODT") + + +def _alternatives(*names: str) -> list[list[Any]]: + return [[name, ("sequence", ()), (1, 1), "SEG"] for name in names] + + +def _spec(*buildable: str) -> _core.MessageSpec: + builders = { + name: (lambda rng, ctx, name=name: f"{name}|{rng.randint(1, 9)}") for name in buildable + } + return _core.MessageSpec(code="T", trigger_to_structure={}, builders=builders) + + +# The generator writes v2.5.1 only (MSH-12), so that is the version strict validation checks. +# A few seeds, because the optional PD1/PV2 around the order vary with the seed. +@pytest.mark.parametrize("index", range(1, 6)) +def test_generated_orm_o01_has_one_order_detail_and_is_strictly_valid(index: int) -> None: + msg = _core.generate_message("ORM", "O01", index) + peek = Peek.parse(msg) + assert peek.version == "2.5.1" + names = peek.segments() + assert names[-2:] == ["ORC", "OBR"], names + assert [n for n in names if n in _ORDER_ALTERNATIVES] == ["OBR"], names + ok, errors = _core.gate("ORM", msg, "ORM_O01") + assert ok, errors + + +def test_obr_is_picked_without_drawing_from_the_seed() -> None: + """OBR leaves the random stream alone, so adding a choice moves no other generated value.""" + rng = random.Random("seed") + before = rng.getstate() + alt = _core._pick_alternative("G", _alternatives(*_ORDER_ALTERNATIVES), rng, _spec("RXO")) + assert alt[0] == "OBR" + assert rng.getstate() == before + + +def test_without_obr_the_pick_is_seeded_and_buildable() -> None: + spec = _spec("RXO", "ODS") + alternatives = _alternatives("RQD", "RXO", "ODS") + picks = { + seed: _core._pick_alternative("G", alternatives, random.Random(seed), spec)[0] + for seed in range(40) + } + assert set(picks.values()) == {"RXO", "ODS"} # never RQD, which has no builder + for seed, pick in picks.items(): # the same seed always picks the same alternative + assert _core._pick_alternative("G", alternatives, random.Random(seed), spec)[0] == pick + + +def test_a_choice_with_no_buildable_alternative_names_the_group() -> None: + with pytest.raises(RuntimeError, match="choice group G_SUPPGRP"): + _core._pick_alternative("G_SUPPGRP", _alternatives("RQD", "RQ1"), random.Random(0), _spec()) + + +def _emitted(group: str) -> list[str]: + ctx = _core.Ctx("ORM", "O01", "ORM_O01", "CID", _core.BASE_DT, "A", "F", "B", "G") + group_ref = ("choice", tuple(_alternatives("PID", "PV1"))) + out: list[str] = [] + _core._emit( + [[group, group_ref, (1, 1), "GRP"]], random.Random(0), ctx, _spec(), frozenset(), out + ) + return [segment[:3] for segment in out] + + +def test_a_choice_group_emits_one_alternative() -> None: + assert len(_emitted("ORM_O01_OBRRQDRQ1RXOODSODT_SUPPGRP")) == 1 + + +def test_a_sequence_hl7apy_labels_choice_emits_every_part() -> None: + """The control arm: QBP_E22_QUERY is QPD then RCP in HL7, though hl7apy labels it a choice. + The generator follows the validator's list of such groups, so it still emits both parts. + The generator walks only v2.5.1 today, where no such group occurs; this pins the rule for + the day it walks a later version.""" + assert "QBP_E22_QUERY" in _SEQUENCES_LABELLED_CHOICE + assert _emitted("QBP_E22_QUERY") == ["PID", "PV1"] From e965f7b329b0f4a65c606f189a1b1f75698db910 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 22:21:21 -0500 Subject: [PATCH 03/14] Dependabot cap test: simplify, derive group-pin exemptions, point not restate (BACKLOG #2505) Applies the simplify pass and the Manager's decisions to the cap test: _load is cached, _cap_of returns one Version, the pin's allowed version is _just_below like every other cap, one _EXEMPT dict, one _uv_entry helper, and one two-document mutation callable. The group-pin exemption is now EXACT_GROUP_PINS from test_ci_venv_pinning.py minus sigstore. The mutation arms break a made-up cap and entry pair, never a real one, behind a control that the unbroken fixture passes. The census test is gone: the stale-entry direction already reds on every real entry if the parser finds no caps. dependabot.yml: the three new entries point at the pyproject reason rather than restating it, and the uvicorn note no longer says pyproject declares uvicorn[standard]. Comment changes only. --- .github/dependabot.yml | 18 +- tests/test_dependabot_cap_ignores.py | 300 ++++++++++++++------------- 2 files changed, 167 insertions(+), 151 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index eebe63530..e4e70bdab 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -112,12 +112,12 @@ updates: # security.yml runs pip-audit over ci/locks/release-tools.lock. Lift this entry in the same PR # that moves the pyproject spec to a validated 7.4. # - # `uvicorn` is the same WIDENED-CAP case. Dependabot proposed widening `uvicorn[standard]`'s - # `<0.50` to `<0.54` in PR 1773, which the Lander closed on 2026-09-28: 0.50 and later pick a - # websockets protocol the protocol-header floor refuses (BACKLOG #1120 arm (b)), so serve - # exits 2. Why the cap exists is beside it in pyproject.toml; read it there. The bare name - # matches `uvicorn[standard]` because dependabot-core's Python NameNormaliser strips extras - # before it compares names (read 2026-09-30). Lift this entry in the same PR that lifts the cap. + # `uvicorn` is the same WIDENED-CAP case. Dependabot proposed widening uvicorn's `<0.50` to + # `<0.54` in PR 1773, which the Lander closed on 2026-09-28: 0.50 and later pick a websockets + # protocol the protocol-header floor refuses (BACKLOG #1120 arm (b)), so serve exits 2. Why the + # cap exists is beside it in pyproject.toml; read it there. pyproject.toml declares plain + # `uvicorn`, not `uvicorn[standard]`, so the bare name here matches it as written. Lift this + # entry in the same PR that lifts the cap. # # `pynetdicom`, `pydicom` and `webauthn` are the same WIDENED-CAP case (BACKLOG #2505). Each entry # below carries a one-line pointer to the reason beside its cap in pyproject.toml; read it there. @@ -134,13 +134,13 @@ updates: versions: [">=7.4.0"] - dependency-name: "uvicorn" versions: [">=0.50.0"] - # pyproject.toml's [dicom] extra comment: pynetdicom 3.x pairs with the pydicom 3.x line. + # Why: the comment above the [dicom] extra in pyproject.toml. - dependency-name: "pynetdicom" versions: [">=4.0.0"] - # pyproject.toml's [dicom] extra comment: the deflate guard replays pydicom internals (BACKLOG #1926). + # Why: the comment above the [dicom] extra in pyproject.toml. - dependency-name: "pydicom" versions: [">=3.1.0"] - # pyproject.toml's [webauthn] extra comment: the cbor2 ranges of webauthn's majors are disjoint. + # Why: the comment above the [webauthn] extra in pyproject.toml. - dependency-name: "webauthn" versions: [">=4.0.0"] groups: diff --git a/tests/test_dependabot_cap_ignores.py b/tests/test_dependabot_cap_ignores.py index fbb8eb460..7dc57d099 100644 --- a/tests/test_dependabot_cap_ignores.py +++ b/tests/test_dependabot_cap_ignores.py @@ -5,30 +5,33 @@ Dependabot WIDENS a declared cap instead of respecting it. ``.github/dependabot.yml`` says so, and says a load-bearing cap must be restated there as an ``ignore`` range "or the cap is decorative". PR #66 widened ruff's and annotated-types' caps that way, and Dependabot PR 1773 tried to widen -uvicorn's. Until BACKLOG #2505 nothing held the two files together, so four caps had no entry. +uvicorn's. Until BACKLOG #2505 nothing held the two files together, and three caps had no entry. THE CONTRACT, both directions: * Every upper bound in ``[project.dependencies]``, ``[project.optional-dependencies]`` and ``[dependency-groups]`` (``<``, ``~=``, ``==``, or an ``==X.*`` wildcard) has a uv-ecosystem ``ignore`` entry whose range covers what the cap excludes and leaves open what it allows. Or the - package is in one of the two exemption tables below, with its reason. + package is in ``_EXEMPT`` below, with its reason. * Every uv ``ignore`` entry names a package that pyproject.toml still caps. dependabot.yml says to lift a cap and delete its entry in the same PR; this is what makes that rule fail when skipped. + It is also this test's positive control: a parser that found no caps would red on every real + entry, so the first test cannot pass vacuously. AN EXACT ``==`` PIN COVERS THE NEXT MINOR, NOT THE NEXT PATCH. That is the ``sigstore`` entry's documented scope (``>=4.5.0`` for ``==4.4.0``): it blocks the minor the owner declined and leaves -the patch track open. A pin's entry must not cover the pinned version itself. +the patch track open. A pin's entry must not cover the pinned version's patch track. -``test_the_checker_reds_on_a_mutated_config`` is the mutation arm. It feeds the checker copies of -the real files with one thing broken, and fails unless each copy is reported. +``test_the_checker_reds_on_a_mutated_config`` is the mutation arm. It breaks a made-up cap and entry +pair added to copies of the real files, never a real one, so lifting a real cap cannot break an arm. """ from __future__ import annotations import copy +import functools import tomllib -from collections.abc import Callable, Iterator +from collections.abc import Callable from pathlib import Path from typing import Any, NamedTuple @@ -39,40 +42,36 @@ from packaging.utils import canonicalize_name from packaging.version import Version +from tests.test_ci_venv_pinning import EXACT_GROUP_PINS + _ROOT = Path(__file__).resolve().parents[1] _PYPROJECT = _ROOT / "pyproject.toml" _DEPENDABOT = _ROOT / ".github" / "dependabot.yml" #: The exact ``==`` pins in ``[dependency-groups]`` that are deliberately NOT ignored. Each must stay -#: an ``==`` pin in a dependency group; a name here that becomes a cap anywhere else needs its own -#: decision, not this one. -_GROUP_PIN_EXEMPT: frozenset[str] = frozenset( - {"bandit", "pip-audit", "zizmor", "build", "diff-cover", "mutmut"} -) +#: an ``==`` pin in a dependency group. ``sigstore`` is the one exact group pin that IS ignored, by +#: owner ruling; dependabot.yml's sigstore note says why. +_GROUP_PINS = frozenset(EXACT_GROUP_PINS) - {"sigstore"} _GROUP_PIN_REASON = ( ".github/dependabot.yml, 'NOT IGNORED, deliberately': ignoring the exact pins in " - "[dependency-groups] would freeze the hash-pinned CI toolchain, which ADR 0034 section 3 wants " - "moving through Dependabot" + "[dependency-groups] would freeze the hash-pinned CI toolchain" ) -#: Any other cap with no ignore entry, by package, with the reason. Read the reason where it points. -_CAP_EXEMPT: dict[str, str] = { - "hvac": ( - "pyproject.toml's [vault] extra comment: majors already go to manual review, and an ignore " - "would suppress hvac's security track for no gain" - ), +#: Every cap with no ignore entry, by package, with the reason. Read the reason where it points. +_EXEMPT: dict[str, str] = { + **dict.fromkeys(_GROUP_PINS, _GROUP_PIN_REASON), + "hvac": "pyproject.toml's comment above the [vault] extra says why hvac's cap is not mirrored", } class Cap(NamedTuple): - """One upper bound: where it sits, the first version it excludes, and one version it allows.""" + """One upper bound: where it sits, and the first version it excludes.""" package: str where: str spec: str exact_pin: bool first_excluded: Version - allowed: Version def _bump(release: tuple[int, ...]) -> Version: @@ -91,68 +90,64 @@ def _just_below(version: Version) -> Version: return Version(".".join(str(n) for n in (*release, 999))) -def _padded(version: Version) -> str: - """The `>=X.Y.Z` spelling the existing ignore entries use: 3.1 -> 3.1.0.""" - return ".".join(str(n) for n in (*version.release, 0, 0)[: max(3, len(version.release))]) - - -def _cap_of(spec: Specifier) -> tuple[Version, Version] | None: - """(first excluded, one allowed) for an upper-bound specifier, or None for a floor or ``!=``.""" +def _cap_of(spec: Specifier) -> Version | None: + """The first version an upper-bound specifier excludes, or None for a floor or ``!=``.""" op, raw = spec.operator, spec.version if op in (">", ">=", "!="): return None if op == "<": - first = Version(raw) - return first, _just_below(first) - if op == "~=": - first = _bump(Version(raw).release[:-1]) - return first, _just_below(first) - if op == "==" and raw.endswith(".*"): - first = _bump(Version(raw[:-2]).release) - return first, _just_below(first) + return Version(raw) + wildcard = op == "==" and raw.endswith(".*") + if op == "~=" or wildcard: + release = Version(raw.removesuffix(".*")).release + return _bump(release if wildcard else release[:-1]) if op == "==": pinned = Version(raw) - major, minor = (*pinned.release, 0)[:2] - return Version(f"{major}.{minor + 1}.0"), pinned + return Version(f"{pinned.major}.{pinned.minor + 1}.0") # `<=` and `===` have no use in pyproject.toml today. Model one deliberately when it arrives. raise AssertionError(f"unmodelled upper-bound operator {op!r} in {spec}") -def _requirements(pyproject: dict[str, Any]) -> Iterator[tuple[str, str]]: - """(where, requirement string) for every requirement in the three tables the uv updater reads.""" - project = pyproject["project"] - for req in project.get("dependencies", []): - yield "[project.dependencies]", req - for extra, reqs in project.get("optional-dependencies", {}).items(): - for req in reqs: - yield f"[project.optional-dependencies].{extra}", req - for group, reqs in pyproject.get("dependency-groups", {}).items(): - for req in reqs: - if isinstance(req, str): # skip `{include-group = ...}` tables - yield f"[dependency-groups].{group}", req - - def _caps(pyproject: dict[str, Any]) -> list[Cap]: - """The tightest upper bound of every capped requirement. Markers are not specifiers.""" + """The tightest upper bound of every capped requirement in the three tables uv reads.""" + project = pyproject["project"] + tables: list[tuple[str, list[Any]]] = [("[project.dependencies]", project["dependencies"])] + tables += [ + (f"[project.optional-dependencies].{name}", reqs) + for name, reqs in project.get("optional-dependencies", {}).items() + ] + tables += [ + (f"[dependency-groups].{name}", reqs) + for name, reqs in pyproject.get("dependency-groups", {}).items() + ] caps: list[Cap] = [] - for where, raw in _requirements(pyproject): - req = Requirement(raw) - bounds = [b for s in req.specifier if (b := _cap_of(s)) is not None] - if bounds: - first, allowed = min(bounds) - pin = any(s.operator == "==" and not s.version.endswith(".*") for s in req.specifier) - caps.append(Cap(canonicalize_name(req.name), where, str(req), pin, first, allowed)) + for where, reqs in tables: + for raw in reqs: + if not isinstance(raw, str): # an `{include-group = ...}` table + continue + req = Requirement(raw) # markers parse apart from the specifier, so never read as caps + bounds = [b for s in req.specifier if (b := _cap_of(s)) is not None] + if bounds: + pin = any( + s.operator == "==" and not s.version.endswith(".*") for s in req.specifier + ) + caps.append(Cap(canonicalize_name(req.name), where, str(req), pin, min(bounds))) return caps -def _uv_ignores(dependabot: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: - """The root uv entry's ignore list, keyed by canonical package name.""" +def _uv_entry(dependabot: dict[str, Any]) -> dict[str, Any]: + """The one uv-ecosystem update entry for the repository root.""" uv = [ u for u in dependabot["updates"] if u["package-ecosystem"] == "uv" and u["directory"] == "/" ] assert len(uv) == 1, f"expected one uv update entry for '/', found {len(uv)}" + return uv[0] + + +def _uv_ignores(dependabot: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + """The root uv entry's ignore list, keyed by canonical package name.""" ignores: dict[str, list[dict[str, Any]]] = {} - for entry in uv[0].get("ignore", []): + for entry in _uv_entry(dependabot).get("ignore", []): ignores.setdefault(canonicalize_name(entry["dependency-name"]), []).append(entry) return ignores @@ -173,37 +168,34 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s for cap in caps: label = f"{cap.where}: {cap.spec}" - if cap.package in _GROUP_PIN_EXEMPT: - if not (cap.exact_pin and cap.where.startswith("[dependency-groups]")): + first = cap.first_excluded + if cap.package in _EXEMPT: + if cap.package in _GROUP_PINS and not ( + cap.exact_pin and cap.where.startswith("[dependency-groups]") + ): problems.append(f"{label} is exempt as a group `==` pin, but is not one") elif cap.package in ignores: - problems.append(f"{label} is exempt ({_GROUP_PIN_REASON}) yet has an ignore entry") - continue - if cap.package in _CAP_EXEMPT: - if cap.package in ignores: problems.append( - f"{label} is exempt ({_CAP_EXEMPT[cap.package]}) yet has an ignore entry" + f"{label} is exempt ({_EXEMPT[cap.package]}) yet has an ignore entry" ) continue entries = ignores.get(cap.package) if not entries: problems.append( - f"{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with " - f'versions [">={_padded(cap.first_excluded)}"], or an exemption here with its reason' + f"{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with versions " + f'[">={first.major}.{first.minor}.{first.micro}"], or an exemption with its reason' ) continue if any("update-types" in e for e in entries): problems.append( f"{label}: an `update-types` ignore cannot restate a range; use `versions`" ) - far = Version(f"{cap.first_excluded.major + 1000}") - if not (_covers(entries, cap.first_excluded) and _covers(entries, far)): - problems.append( - f"{label}: its ignore range leaves versions from {cap.first_excluded} up open" - ) - if _covers(entries, cap.allowed): + if not (_covers(entries, first) and _covers(entries, Version(f"{first.major + 1000}"))): + problems.append(f"{label}: its ignore range leaves versions from {first} up open") + allowed = _just_below(first) + if _covers(entries, allowed): problems.append( - f"{label}: its ignore range also blocks {cap.allowed}, which the cap allows" + f"{label}: its ignore range also blocks {allowed}, which the cap allows" ) capped = {cap.package for cap in caps} @@ -212,12 +204,14 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s f"{_DEPENDABOT.name} ignores {name!r}, which pyproject.toml no longer caps; " "delete the entry in the PR that lifted the cap" ) - for name in sorted((_GROUP_PIN_EXEMPT | set(_CAP_EXEMPT)) - capped): + for name in sorted(set(_EXEMPT) - capped): problems.append(f"exemption for {name!r} names no capped requirement in pyproject.toml") return problems +@functools.cache def _load() -> tuple[dict[str, Any], dict[str, Any]]: + """Parsed once per run. Callers that mutate must deepcopy first.""" pyproject = tomllib.loads(_PYPROJECT.read_text(encoding="utf-8")) dependabot = yaml.safe_load(_DEPENDABOT.read_text(encoding="utf-8")) return pyproject, dependabot @@ -229,87 +223,109 @@ def test_every_cap_has_an_ignore_entry_or_a_stated_exemption() -> None: assert not problems, "\n".join(problems) -def test_the_census_finds_the_known_caps() -> None: - """Positive control: a parser that found no caps would make the test above pass vacuously.""" - found = {cap.package for cap in _caps(_load()[0])} - known = { - "uvicorn", - "pynetdicom", - "pydicom", - "webauthn", - "hvac", - "ruff", - "sigstore", - "cyclonedx-bom", - } - assert known <= found, f"census missed {sorted(known - found)}" - assert "atheris" not in found, "an environment marker's `==` was read as a version cap" - - @pytest.mark.parametrize( - ("spec", "first", "allowed"), + ("spec", "first"), [ - ("<0.50", "0.50", "0.49.999"), - ("<4", "4", "3.999"), - ("<3.1", "3.1", "3.0.999"), - ("~=7.3.1", "7.4", "7.3.999"), - ("~=7.3", "8", "7.999"), - ("==4.4.0", "4.5.0", "4.4.0"), - ("==1.2.*", "1.3", "1.2.999"), + ("<0.50", "0.50"), + ("<4", "4"), + ("<3.1", "3.1"), + ("~=7.3.1", "7.4"), + ("~=7.3", "8"), + ("==4.4.0", "4.5.0"), + ("==1.2.*", "1.3"), + (">=1.0", None), + ("!=1.2.*", None), ], ) -def test_cap_arithmetic(spec: str, first: str, allowed: str) -> None: - assert _cap_of(Specifier(spec)) == (Version(first), Version(allowed)) +def test_cap_arithmetic(spec: str, first: str | None) -> None: + assert _cap_of(Specifier(spec)) == (Version(first) if first else None) + + +@pytest.mark.parametrize( + ("version", "below"), [("0.50", "0.49.999"), ("4", "3.999"), ("3.1.0", "3.0.999")] +) +def test_just_below(version: str, below: str) -> None: + assert _just_below(Version(version)) == Version(below) + + +# A made-up cap and entry pair for each shape the mutation arms break. No real package is named, +# so a legitimate cap lift in pyproject.toml cannot crash an arm. +_FAKE_CAP = "mefor-fake-cap>=1.0,<2.5" +_FAKE_PIN = "mefor-fake-pin==3.2.1" + + +def _fake_entry(dependabot: dict[str, Any], name: str) -> dict[str, Any]: + return next(e for e in _uv_entry(dependabot)["ignore"] if e["dependency-name"] == name) + + +def _fixture() -> tuple[dict[str, Any], dict[str, Any]]: + """Copies of the real files, plus the two made-up caps with correct entries.""" + pyproject, dependabot = copy.deepcopy(_load()) + pyproject["project"]["dependencies"].append(_FAKE_CAP) + pyproject.setdefault("dependency-groups", {})["mefor-fake"] = [_FAKE_PIN] + _uv_entry(dependabot).setdefault("ignore", []).extend( + [ + {"dependency-name": "mefor-fake-cap", "versions": [">=2.5.0"]}, + {"dependency-name": "mefor-fake-pin", "versions": [">=3.3.0"]}, + ] + ) + return pyproject, dependabot + + +def test_the_mutation_fixture_is_clean() -> None: + """Control: the arms below are discriminating only if the unbroken fixture passes.""" + problems = _violations(*_fixture()) + assert not problems, "\n".join(problems) + + +_Mutation = Callable[[dict[str, Any], dict[str, Any]], None] + +def _drop_entry(_p: dict[str, Any], d: dict[str, Any]) -> None: + uv = _uv_entry(d) + uv["ignore"] = [e for e in uv["ignore"] if e["dependency-name"] != "mefor-fake-cap"] -_Mutation = Callable[[dict[str, Any]], None] +def _set_range(name: str, rng: str) -> _Mutation: + def mutate(_p: dict[str, Any], d: dict[str, Any]) -> None: + _fake_entry(d, name)["versions"] = [rng] -def _drop_ignore(doc: dict[str, Any], name: str) -> None: - uv = next(u for u in doc["updates"] if u["package-ecosystem"] == "uv") - uv["ignore"] = [e for e in uv["ignore"] if e["dependency-name"] != name] + return mutate -def _set_range(doc: dict[str, Any], name: str, rng: str) -> None: - uv = next(u for u in doc["updates"] if u["package-ecosystem"] == "uv") - next(e for e in uv["ignore"] if e["dependency-name"] == name)["versions"] = [rng] +def _to_update_types(_p: dict[str, Any], d: dict[str, Any]) -> None: + entry = _fake_entry(d, "mefor-fake-cap") + del entry["versions"] + entry["update-types"] = ["version-update:semver-major"] -def _add_cap(doc: dict[str, Any], req: str) -> None: - doc["project"]["dependencies"].append(req) +def _lift_cap(p: dict[str, Any], _d: dict[str, Any]) -> None: + deps = p["project"]["dependencies"] + deps[deps.index(_FAKE_CAP)] = "mefor-fake-cap>=1.0" -def _lift_cap(doc: dict[str, Any], old: str, new: str) -> None: - deps = doc["project"]["optional-dependencies"]["dicom"] - deps[deps.index(old)] = new +def _exempt_unpinned(p: dict[str, Any], _d: dict[str, Any]) -> None: + """A group-pin exemption whose package stops being an `==` pin.""" + name = sorted(_GROUP_PINS)[0] + p["project"]["dependencies"].append(f"{name}>=1,<99") @pytest.mark.parametrize( - ("mutate_pyproject", "mutate_dependabot", "expect"), + ("mutate", "expect"), [ - pytest.param(None, lambda d: _drop_ignore(d, "pydicom"), "pydicom", id="entry-deleted"), - pytest.param(None, lambda d: _set_range(d, "pydicom", ">=3.2.0"), "open", id="gap-at-cap"), - pytest.param(None, lambda d: _set_range(d, "pydicom", ">=3.0.0"), "blocks", id="freezes"), - pytest.param( - None, lambda d: _set_range(d, "sigstore", ">=4.4.0"), "blocks", id="blocks-pin" - ), - pytest.param(lambda p: _add_cap(p, "newdep>=1,<2"), None, "newdep", id="new-cap"), - pytest.param( - lambda p: _lift_cap(p, "pydicom>=3.0.2,<3.1", "pydicom>=3.0.2"), - None, - "no longer caps", - id="stale-entry", - ), + pytest.param(_drop_entry, "no uv ignore entry", id="entry-deleted"), + pytest.param(_set_range("mefor-fake-cap", ">=2.6.0"), "open", id="gap-at-cap"), + pytest.param(_set_range("mefor-fake-cap", ">=2.4.0"), "blocks", id="freezes"), + pytest.param(_set_range("mefor-fake-pin", ">=3.2.0"), "blocks", id="blocks-pin"), + pytest.param(_set_range("mefor-fake-pin", ">=4.0.0"), "open", id="pin-gap"), + pytest.param(_to_update_types, "update-types", id="update-types"), + pytest.param(_lift_cap, "no longer caps", id="stale-entry"), + pytest.param(_exempt_unpinned, "but is not one", id="exempt-not-a-pin"), ], ) -def test_the_checker_reds_on_a_mutated_config( - mutate_pyproject: _Mutation | None, mutate_dependabot: _Mutation | None, expect: str -) -> None: - """Mutation arm: each broken copy of the real files must produce a violation naming it.""" - pyproject, dependabot = (copy.deepcopy(doc) for doc in _load()) - if mutate_pyproject: - mutate_pyproject(pyproject) - if mutate_dependabot: - mutate_dependabot(dependabot) +def test_the_checker_reds_on_a_mutated_config(mutate: _Mutation, expect: str) -> None: + """Mutation arm: each broken copy must produce a violation naming what broke.""" + pyproject, dependabot = _fixture() + mutate(pyproject, dependabot) problems = _violations(pyproject, dependabot) assert any(expect in p for p in problems), f"no violation mentioning {expect!r}: {problems}" From 8e9134ae77153b367f1c6ad16d3d73cbe57e3bd1 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 22:38:59 -0500 Subject: [PATCH 04/14] Dependabot cap test: review round 1, exact ranges and build-system (BACKLOG #2505) Code review round 1 found that two sample versions could not see a hole in an ignore range or a range that blocks an allowed version. An entry for a capped package must now be exactly ">=" the first version the cap excludes, the one range that blocks all it excludes and nothing it allows. _just_below and _covers go with the sampling. The census now reads [build-system].requires too, so "every upper bound" is true; hatchling's pin is exempt with a pointer to the pyproject comment that expects Dependabot to bump it. Entry names drop extras before matching, as dependabot-core does. Added arms for a hole, a blocked allowed range, a non-specifier range, and the three exemption-table violations, each over the made-up pair only. dependabot.yml: the "the one cap with NO entry" claim becomes an "at least" list, the reason map no longer enumerates two of seven, and the pynetdicom pointer says what the [dicom] comment actually holds. --- .github/dependabot.yml | 18 +-- tests/test_dependabot_cap_ignores.py | 180 ++++++++++++++++----------- 2 files changed, 116 insertions(+), 82 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e4e70bdab..c8f044498 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -62,9 +62,8 @@ updates: # resync then propagated 0.8.0/0.16.0 into constraints.lock, so ci.yml's `--constraint` pinned CI # *to* the broken versions. `python-deps` groups every minor and patch on `*`, and both of those # bumps are minors, so they red the WHOLE batch and hold every benign bump in it hostage. WHY - # each is capped lives beside the cap in pyproject.toml (ruff in [dev]; uvicorn in - # [project.dependencies]) — do not - # restate it here; lift a cap and delete its entry in the SAME PR. + # each is capped lives beside the cap in pyproject.toml, at the requirement the entry names. Do + # not restate it here; lift a cap and delete its entry in the SAME PR. # TRADE-OFF, accepted: `ignore` suppresses the SECURITY track for the named RANGE as well as the # routine one (`update-types` is the version-only knob and cannot express a range). A ruff # 0.15.x or uvicorn 0.49.x advisory fix still flows. Detection is untouched — security.yml's @@ -120,11 +119,12 @@ updates: # entry in the same PR that lifts the cap. # # `pynetdicom`, `pydicom` and `webauthn` are the same WIDENED-CAP case (BACKLOG #2505). Each entry - # below carries a one-line pointer to the reason beside its cap in pyproject.toml; read it there. - # pydicom's `<3.1` is a MINOR cap, so without its entry `python-deps` would widen it in the routine - # minor-and-patch batch. `hvac<3` is the one cap with NO entry, on purpose, and pyproject.toml says - # why beside it. tests/test_dependabot_cap_ignores.py holds every upper bound in pyproject.toml to - # an entry here or to a stated exemption, and every entry here to a cap that still exists. + # below carries a one-line pointer to pyproject.toml; read the comment there. pydicom's `<3.1` is + # a MINOR cap, so without its entry `python-deps` would widen it in the routine minor-and-patch + # batch. Some caps have NO entry on purpose: at least `hvac<3`, `hatchling`'s build-system pin and + # the exact group pins under "NOT IGNORED, deliberately" above. tests/test_dependabot_cap_ignores.py + # names every such exemption with its reason, holds every other upper bound in pyproject.toml to + # an entry here, and every entry here to a cap that still exists. ignore: - dependency-name: "ruff" versions: [">=0.16.0"] @@ -134,7 +134,7 @@ updates: versions: [">=7.4.0"] - dependency-name: "uvicorn" versions: [">=0.50.0"] - # Why: the comment above the [dicom] extra in pyproject.toml. + # The comment above the [dicom] extra in pyproject.toml names the 3.x pairing with pydicom. - dependency-name: "pynetdicom" versions: [">=4.0.0"] # Why: the comment above the [dicom] extra in pyproject.toml. diff --git a/tests/test_dependabot_cap_ignores.py b/tests/test_dependabot_cap_ignores.py index 7dc57d099..6562a9159 100644 --- a/tests/test_dependabot_cap_ignores.py +++ b/tests/test_dependabot_cap_ignores.py @@ -9,21 +9,23 @@ THE CONTRACT, both directions: -* Every upper bound in ``[project.dependencies]``, ``[project.optional-dependencies]`` and - ``[dependency-groups]`` (``<``, ``~=``, ``==``, or an ``==X.*`` wildcard) has a uv-ecosystem - ``ignore`` entry whose range covers what the cap excludes and leaves open what it allows. Or the - package is in ``_EXEMPT`` below, with its reason. -* Every uv ``ignore`` entry names a package that pyproject.toml still caps. dependabot.yml says to - lift a cap and delete its entry in the same PR; this is what makes that rule fail when skipped. - It is also this test's positive control: a parser that found no caps would red on every real - entry, so the first test cannot pass vacuously. +* Every upper bound in ``[build-system].requires``, ``[project.dependencies]``, + ``[project.optional-dependencies]`` and ``[dependency-groups]`` (``<``, ``~=``, ``==``, or an + ``==X.*`` wildcard) has a uv-ecosystem ``ignore`` entry whose range is exactly ``>=`` the first + version the cap excludes. That is the one range that blocks everything the cap excludes and + nothing it allows. Or the package is in ``_EXEMPT`` below, with its reason. +* Every uv ``ignore`` entry names a package that pyproject.toml caps. dependabot.yml says to lift a + cap and delete its entry in the same PR; this is what makes that rule fail when skipped. It is + also this test's positive control: a parser that found no caps would red on every real entry, so + the first test cannot pass vacuously. AN EXACT ``==`` PIN COVERS THE NEXT MINOR, NOT THE NEXT PATCH. That is the ``sigstore`` entry's documented scope (``>=4.5.0`` for ``==4.4.0``): it blocks the minor the owner declined and leaves -the patch track open. A pin's entry must not cover the pinned version's patch track. +the patch track open. It is the only ignored pin today; a new one takes the same scope or a +deliberate change here. -``test_the_checker_reds_on_a_mutated_config`` is the mutation arm. It breaks a made-up cap and entry -pair added to copies of the real files, never a real one, so lifting a real cap cannot break an arm. +The mutation arms break a made-up cap and entry pair added to copies of the real files, never a +real one, so lifting a real cap cannot break an arm. """ from __future__ import annotations @@ -38,7 +40,7 @@ import pytest import yaml from packaging.requirements import Requirement -from packaging.specifiers import Specifier, SpecifierSet +from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet from packaging.utils import canonicalize_name from packaging.version import Version @@ -50,7 +52,8 @@ #: The exact ``==`` pins in ``[dependency-groups]`` that are deliberately NOT ignored. Each must stay #: an ``==`` pin in a dependency group. ``sigstore`` is the one exact group pin that IS ignored, by -#: owner ruling; dependabot.yml's sigstore note says why. +#: owner ruling; dependabot.yml's sigstore note says why. If that ruling is lifted and its entry +#: deleted, drop sigstore from this subtraction in the same PR. _GROUP_PINS = frozenset(EXACT_GROUP_PINS) - {"sigstore"} _GROUP_PIN_REASON = ( ".github/dependabot.yml, 'NOT IGNORED, deliberately': ignoring the exact pins in " @@ -61,6 +64,7 @@ _EXEMPT: dict[str, str] = { **dict.fromkeys(_GROUP_PINS, _GROUP_PIN_REASON), "hvac": "pyproject.toml's comment above the [vault] extra says why hvac's cap is not mirrored", + "hatchling": "pyproject.toml's [build-system] comment expects Dependabot to bump this pin", } @@ -79,17 +83,6 @@ def _bump(release: tuple[int, ...]) -> Version: return Version(".".join(str(n) for n in (*release[:-1], release[-1] + 1))) -def _just_below(version: Version) -> Version: - """A release just under ``version``: 0.50 -> 0.49.999, 4 -> 3.999, 3.1.0 -> 3.0.999.""" - release = list(version.release) - while release and release[-1] == 0: - release.pop() - if not release: - raise ValueError(f"no release below {version}") - release[-1] -= 1 - return Version(".".join(str(n) for n in (*release, 999))) - - def _cap_of(spec: Specifier) -> Version | None: """The first version an upper-bound specifier excludes, or None for a floor or ``!=``.""" op, raw = spec.operator, spec.version @@ -109,9 +102,12 @@ def _cap_of(spec: Specifier) -> Version | None: def _caps(pyproject: dict[str, Any]) -> list[Cap]: - """The tightest upper bound of every capped requirement in the three tables uv reads.""" + """The tightest upper bound of every capped requirement in the tables Dependabot's uv reads.""" project = pyproject["project"] - tables: list[tuple[str, list[Any]]] = [("[project.dependencies]", project["dependencies"])] + tables: list[tuple[str, list[Any]]] = [ + ("[build-system].requires", pyproject.get("build-system", {}).get("requires", [])), + ("[project.dependencies]", project.get("dependencies", [])), + ] tables += [ (f"[project.optional-dependencies].{name}", reqs) for name, reqs in project.get("optional-dependencies", {}).items() @@ -138,27 +134,39 @@ def _caps(pyproject: dict[str, Any]) -> list[Cap]: def _uv_entry(dependabot: dict[str, Any]) -> dict[str, Any]: """The one uv-ecosystem update entry for the repository root.""" uv = [ - u for u in dependabot["updates"] if u["package-ecosystem"] == "uv" and u["directory"] == "/" + u + for u in dependabot["updates"] + if u["package-ecosystem"] == "uv" and u.get("directory") == "/" ] - assert len(uv) == 1, f"expected one uv update entry for '/', found {len(uv)}" + assert len(uv) == 1, f"expected one uv update entry with `directory: /`, found {len(uv)}" return uv[0] def _uv_ignores(dependabot: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: - """The root uv entry's ignore list, keyed by canonical package name.""" + """The root uv entry's ignore list, keyed by canonical package name. + + Extras are dropped first, as dependabot-core's Python name normaliser does, so an entry named + ``uvicorn[standard]`` matches a ``uvicorn`` cap. + """ ignores: dict[str, list[dict[str, Any]]] = {} for entry in _uv_entry(dependabot).get("ignore", []): - ignores.setdefault(canonicalize_name(entry["dependency-name"]), []).append(entry) + name = canonicalize_name(entry["dependency-name"].split("[", 1)[0]) + ignores.setdefault(name, []).append(entry) return ignores -def _covers(entries: list[dict[str, Any]], version: Version) -> bool: - """Whether any entry's `versions` range ignores ``version``. A list of ranges is a union.""" - return any( - SpecifierSet(rng).contains(version, prereleases=True) - for entry in entries - for rng in entry.get("versions", []) - ) +def _ranges(entry: dict[str, Any]) -> list[str]: + versions = entry.get("versions", []) + return [versions] if isinstance(versions, str) else list(versions) + + +def _is_exactly_from(rng: str, first: Version) -> bool: + """Whether ``rng`` is the single specifier ``>=first``.""" + try: + specs = list(SpecifierSet(rng)) + except InvalidSpecifier: + return False + return len(specs) == 1 and specs[0].operator == ">=" and Version(specs[0].version) == first def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[str]: @@ -169,6 +177,7 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s for cap in caps: label = f"{cap.where}: {cap.spec}" first = cap.first_excluded + want = f">={first.major}.{first.minor}.{first.micro}" if cap.package in _EXEMPT: if cap.package in _GROUP_PINS and not ( cap.exact_pin and cap.where.startswith("[dependency-groups]") @@ -182,27 +191,27 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s entries = ignores.get(cap.package) if not entries: problems.append( - f"{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with versions " - f'[">={first.major}.{first.minor}.{first.micro}"], or an exemption with its reason' + f'{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with versions ["{want}"], ' + f"or an exemption with its reason in {Path(__file__).name}" ) continue if any("update-types" in e for e in entries): problems.append( - f"{label}: an `update-types` ignore cannot restate a range; use `versions`" + f"{label}: an `update-types` ignore cannot restate a range; use `versions` only" ) - if not (_covers(entries, first) and _covers(entries, Version(f"{first.major + 1000}"))): - problems.append(f"{label}: its ignore range leaves versions from {first} up open") - allowed = _just_below(first) - if _covers(entries, allowed): + ranges = [rng for e in entries for rng in _ranges(e)] + if not ranges or not all(_is_exactly_from(rng, first) for rng in ranges): problems.append( - f"{label}: its ignore range also blocks {allowed}, which the cap allows" + f"{label}: its ignore range must be exactly {want!r}, which blocks what the cap " + f"excludes and nothing it allows; found {ranges}" ) capped = {cap.package for cap in caps} for name in sorted(set(ignores) - capped): problems.append( - f"{_DEPENDABOT.name} ignores {name!r}, which pyproject.toml no longer caps; " - "delete the entry in the PR that lifted the cap" + f"{_DEPENDABOT.name} ignores {name!r}, which pyproject.toml does not cap. An entry " + "here restates a cap, so delete it in the PR that lifts the cap, or model the new " + f"kind of entry in {Path(__file__).name}" ) for name in sorted(set(_EXEMPT) - capped): problems.append(f"exemption for {name!r} names no capped requirement in pyproject.toml") @@ -211,14 +220,14 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s @functools.cache def _load() -> tuple[dict[str, Any], dict[str, Any]]: - """Parsed once per run. Callers that mutate must deepcopy first.""" + """Parsed once per run. Callers that mutate must deepcopy first, as ``_fixture`` does.""" pyproject = tomllib.loads(_PYPROJECT.read_text(encoding="utf-8")) dependabot = yaml.safe_load(_DEPENDABOT.read_text(encoding="utf-8")) return pyproject, dependabot def test_every_cap_has_an_ignore_entry_or_a_stated_exemption() -> None: - """RED when: a pyproject cap has no matching ignore range, or an ignore entry outlives its cap.""" + """RED when: a pyproject cap has no exact ignore range, or an ignore entry outlives its cap.""" problems = _violations(*_load()) assert not problems, "\n".join(problems) @@ -241,13 +250,6 @@ def test_cap_arithmetic(spec: str, first: str | None) -> None: assert _cap_of(Specifier(spec)) == (Version(first) if first else None) -@pytest.mark.parametrize( - ("version", "below"), [("0.50", "0.49.999"), ("4", "3.999"), ("3.1.0", "3.0.999")] -) -def test_just_below(version: str, below: str) -> None: - assert _just_below(Version(version)) == Version(below) - - # A made-up cap and entry pair for each shape the mutation arms break. No real package is named, # so a legitimate cap lift in pyproject.toml cannot crash an arm. _FAKE_CAP = "mefor-fake-cap>=1.0,<2.5" @@ -272,13 +274,21 @@ def _fixture() -> tuple[dict[str, Any], dict[str, Any]]: return pyproject, dependabot -def test_the_mutation_fixture_is_clean() -> None: - """Control: the arms below are discriminating only if the unbroken fixture passes.""" - problems = _violations(*_fixture()) - assert not problems, "\n".join(problems) +_Mutation = Callable[[dict[str, Any], dict[str, Any]], None] -_Mutation = Callable[[dict[str, Any], dict[str, Any]], None] +def _rename_entry(_p: dict[str, Any], d: dict[str, Any]) -> None: + _fake_entry(d, "mefor-fake-cap")["dependency-name"] = "Mefor_Fake_Cap[extra]" + + +@pytest.mark.parametrize("mutate", [None, _rename_entry], ids=["unbroken", "extras-and-case"]) +def test_the_mutation_fixture_is_clean(mutate: _Mutation | None) -> None: + """Control: the arms below are discriminating only if the unbroken fixture passes.""" + pyproject, dependabot = _fixture() + if mutate: + mutate(pyproject, dependabot) + problems = _violations(pyproject, dependabot) + assert not problems, "\n".join(problems) def _drop_entry(_p: dict[str, Any], d: dict[str, Any]) -> None: @@ -286,9 +296,9 @@ def _drop_entry(_p: dict[str, Any], d: dict[str, Any]) -> None: uv["ignore"] = [e for e in uv["ignore"] if e["dependency-name"] != "mefor-fake-cap"] -def _set_range(name: str, rng: str) -> _Mutation: +def _set_versions(name: str, versions: list[str]) -> _Mutation: def mutate(_p: dict[str, Any], d: dict[str, Any]) -> None: - _fake_entry(d, name)["versions"] = [rng] + _fake_entry(d, name)["versions"] = versions return mutate @@ -304,23 +314,28 @@ def _lift_cap(p: dict[str, Any], _d: dict[str, Any]) -> None: deps[deps.index(_FAKE_CAP)] = "mefor-fake-cap>=1.0" -def _exempt_unpinned(p: dict[str, Any], _d: dict[str, Any]) -> None: - """A group-pin exemption whose package stops being an `==` pin.""" - name = sorted(_GROUP_PINS)[0] - p["project"]["dependencies"].append(f"{name}>=1,<99") +_EXACT = "must be exactly" @pytest.mark.parametrize( ("mutate", "expect"), [ pytest.param(_drop_entry, "no uv ignore entry", id="entry-deleted"), - pytest.param(_set_range("mefor-fake-cap", ">=2.6.0"), "open", id="gap-at-cap"), - pytest.param(_set_range("mefor-fake-cap", ">=2.4.0"), "blocks", id="freezes"), - pytest.param(_set_range("mefor-fake-pin", ">=3.2.0"), "blocks", id="blocks-pin"), - pytest.param(_set_range("mefor-fake-pin", ">=4.0.0"), "open", id="pin-gap"), + pytest.param(_set_versions("mefor-fake-cap", [">=2.6.0"]), _EXACT, id="gap-at-cap"), + pytest.param(_set_versions("mefor-fake-cap", [">=2.4.0"]), _EXACT, id="freezes"), + pytest.param( + _set_versions("mefor-fake-cap", ["==2.5.*", ">=3.0.0"]), _EXACT, id="hole-above-cap" + ), + pytest.param( + _set_versions("mefor-fake-cap", [">=1.2.0,<1.4.0", ">=2.5.0"]), + _EXACT, + id="blocks-an-allowed-range", + ), + pytest.param(_set_versions("mefor-fake-cap", ["2.5.0"]), _EXACT, id="not-a-specifier"), + pytest.param(_set_versions("mefor-fake-pin", [">=3.2.0"]), _EXACT, id="blocks-pin"), + pytest.param(_set_versions("mefor-fake-pin", [">=4.0.0"]), _EXACT, id="pin-gap"), pytest.param(_to_update_types, "update-types", id="update-types"), - pytest.param(_lift_cap, "no longer caps", id="stale-entry"), - pytest.param(_exempt_unpinned, "but is not one", id="exempt-not-a-pin"), + pytest.param(_lift_cap, "does not cap", id="stale-entry"), ], ) def test_the_checker_reds_on_a_mutated_config(mutate: _Mutation, expect: str) -> None: @@ -329,3 +344,22 @@ def test_the_checker_reds_on_a_mutated_config(mutate: _Mutation, expect: str) -> mutate(pyproject, dependabot) problems = _violations(pyproject, dependabot) assert any(expect in p for p in problems), f"no violation mentioning {expect!r}: {problems}" + + +@pytest.mark.parametrize( + ("group_pin", "exempt", "expect"), + [ + pytest.param(False, "mefor-fake-cap", "yet has an ignore entry", id="exempt-with-entry"), + pytest.param(False, "mefor-never-capped", "names no capped", id="dead-exemption"), + pytest.param(True, "mefor-fake-cap", "but is not one", id="group-pin-not-a-pin"), + ], +) +def test_the_checker_reds_on_a_broken_exemption( + monkeypatch: pytest.MonkeyPatch, group_pin: bool, exempt: str, expect: str +) -> None: + """Mutation arm for the exemption table, again over the made-up pair only.""" + monkeypatch.setitem(_EXEMPT, exempt, "test reason") + if group_pin: + monkeypatch.setattr(f"{__name__}._GROUP_PINS", _GROUP_PINS | {exempt}) + problems = _violations(*_fixture()) + assert any(expect in p for p in problems), f"no violation mentioning {expect!r}: {problems}" From a593b56caa5feb2c122eb7fff793e8ade6928aa3 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 22:42:06 -0500 Subject: [PATCH 05/14] perf(cli,tray): defer startup imports with __lazy_modules__ (BACKLOG #2514) PEP 810 lists in four modules. Inert on 3.14; on 3.15 the named imports bind lazily. No `lazy` keyword (a SyntaxError on 3.14). - config/__init__.py: config.models. Every CLI command reached pydantic through logging_setup -> config.tls_policy -> this package's re-export. 3.15.0b3: 273 -> 186 modules for --help. - __main__.py: sqlite3, tomllib. - tray/config.py: service_status, tomllib. - tray/__main__.py: tray.instance. Tray first process 204 -> 120. tests/test_startup_import_budget.py: module-set budget, skipped on 3.14, asserted on 3.15, with an eager control arm. --- messagefoundry/__main__.py | 5 + messagefoundry/config/__init__.py | 5 + messagefoundry/tray/__main__.py | 5 + messagefoundry/tray/config.py | 5 + tests/test_startup_import_budget.py | 150 ++++++++++++++++++++++++++++ 5 files changed, 170 insertions(+) create mode 100644 tests/test_startup_import_budget.py diff --git a/messagefoundry/__main__.py b/messagefoundry/__main__.py index ccb8cfdbf..bb431bbfe 100644 --- a/messagefoundry/__main__.py +++ b/messagefoundry/__main__.py @@ -21,6 +21,11 @@ from __future__ import annotations +# PEP 810 (BACKLOG #2514; inert on 3.14, see tests/test_startup_import_budget.py). Only commands that +# open a store or read a service TOML use these. The rest below runs on every command or is the +# logging chain, so it stays eager. The heavy import is deferred in config/__init__.py. +__lazy_modules__ = ["sqlite3", "tomllib"] + import argparse import functools import json diff --git a/messagefoundry/config/__init__.py b/messagefoundry/config/__init__.py index a9f326ec8..965bc5dec 100644 --- a/messagefoundry/config/__init__.py +++ b/messagefoundry/config/__init__.py @@ -11,6 +11,11 @@ from __future__ import annotations +# PEP 810 (BACKLOG #2514; inert on 3.14, see tests/test_startup_import_budget.py). Importing the +# leaf `config.tls_policy`, which every CLI command reaches through `logging_setup`, then no longer +# loads pydantic and the models: about 80 modules. A use of a re-exported name loads them. +__lazy_modules__ = ["messagefoundry.config.models"] + from messagefoundry.config.models import ( AckMode, ConnectorType, diff --git a/messagefoundry/tray/__main__.py b/messagefoundry/tray/__main__.py index ba9235082..b1b99fdf6 100644 --- a/messagefoundry/tray/__main__.py +++ b/messagefoundry/tray/__main__.py @@ -10,6 +10,11 @@ from __future__ import annotations +# PEP 810 (BACKLOG #2514; inert on 3.14, see tests/test_startup_import_budget.py). The first process +# returns after relaunch_branded() and never takes the mutex. logscrub stays eager on purpose: it is +# tray.log's PHI and credential filter, and the log-scrub chain is kept out of every lazy list. +__lazy_modules__ = ["messagefoundry.tray.instance"] + import logging import logging.handlers import sys diff --git a/messagefoundry/tray/config.py b/messagefoundry/tray/config.py index 26daf3936..0469145f0 100644 --- a/messagefoundry/tray/config.py +++ b/messagefoundry/tray/config.py @@ -30,6 +30,11 @@ from __future__ import annotations +# PEP 810 (BACKLOG #2514; inert on 3.14, see tests/test_startup_import_budget.py). The tray's first +# process only calls default_config_dir() before it re-execs as the branded child, so it need not +# load service_status (asyncio, subprocess, ctypes) or tomllib. +__lazy_modules__ = ["messagefoundry.service_status", "tomllib"] + import re import sys import tomllib diff --git a/tests/test_startup_import_budget.py b/tests/test_startup_import_budget.py new file mode 100644 index 000000000..660bffed2 --- /dev/null +++ b/tests/test_startup_import_budget.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""Startup import budget for the CLI and the tray (BACKLOG #2514, PEP 810). + +Four modules carry a `__lazy_modules__` list: `__main__.py`, `config/__init__.py`, `tray/__main__.py` +and `tray/config.py`. On Python 3.15 the imports they name bind lazily and load on first use. On +3.14, the project's floor, the list is an ordinary variable and nothing changes, so the budget is +ADVISORY there: it skips. The `lazy` keyword is not used because it is a SyntaxError on 3.14. + +The budget is a set of modules, never a wall-clock time, so it cannot flake on a loaded runner. +On 3.15.0b3 the lists take the CLI from 273 modules to about 186 and the tray's first process from +204 to 120, counting the interpreter's own modules. + +Each probe runs in a FRESH interpreter, because the pytest process has already imported most of +the engine. The CLI probe runs the package through runpy, as `python -m messagefoundry` does, so +`messagefoundry.__main__` is never in `sys.modules`. The tray probe imports the entry module, which +is what the first process pays before `main()` re-execs as the branded child. + +The control arm runs the same probe with every import forced eager: natively on 3.14, and through +`sys.set_lazy_imports_filter` on 3.15. It asserts the deferred modules DO load there, so a +misspelled name cannot pass the budget by never loading at all. If the control fails because a +module no longer loads even eagerly, its lazy entry is dead: remove it from both places. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import NamedTuple + +import pytest + +_LAZY_ARM_SKIP = "advisory until the floor is 3.15: __lazy_modules__ has no effect on 3.14 (#2478)" + + +class _Probe(NamedTuple): + kind: str + args: tuple[str, ...] + #: Must NOT load on 3.15, and must load when forced eager. + deferred: frozenset[str] + #: The exact engine modules it may load on 3.15, measured on 3.15.0b3. Exact rather than a + #: ceiling, so one new eager engine import on the startup path fails here instead of being + #: absorbed by headroom. If you add one on purpose, add it here and say why in the commit. + engine: frozenset[str] + + +#: The CLI reaches `config.tls_policy` through `logging_setup`. Before #2514, loading that leaf ran +#: `config/__init__.py`, which loaded pydantic and every model. +_CLI_DEFERRED = frozenset({"pydantic", "messagefoundry.config.models", "sqlite3", "tomllib"}) +_CLI_ENGINE = frozenset( + { + "messagefoundry", + "messagefoundry.cli_common", + "messagefoundry.cli_surface", + "messagefoundry.config", + "messagefoundry.config.tls_policy", + "messagefoundry.console_streams", + "messagefoundry.controlchars", + "messagefoundry.keywrap", + "messagefoundry.last_resort", + "messagefoundry.log_spool", + "messagefoundry.logging_guard", + "messagefoundry.logging_setup", + "messagefoundry.odbc_env", + "messagefoundry.redaction", + "messagefoundry.secretscrub", + } +) + +#: `--help` stops once the parser is built. `_build_parser` builds every subparser, so `check`, +#: `verify` and `serve --help` load the same set today; one stands for all three. Add the others +#: back if parser building ever becomes per-subcommand. +_PROBES = { + "cli-version": _Probe("cli", ("--version",), _CLI_DEFERRED, _CLI_ENGINE), + "cli-check-help": _Probe("cli", ("check", "--help"), _CLI_DEFERRED, _CLI_ENGINE), + "tray-first-process": _Probe( + "tray", + (), + frozenset( + {"messagefoundry.service_status", "messagefoundry.tray.instance", "asyncio", "tomllib"} + ), + frozenset( + { + "messagefoundry", + "messagefoundry.api_tls_source", + "messagefoundry.controlchars", + "messagefoundry.redaction", + "messagefoundry.secretscrub", + "messagefoundry.tray", + "messagefoundry.tray.__main__", + "messagefoundry.tray.config", + "messagefoundry.tray.logscrub", + } + ), + ), +} + +_PROBE_CODE = """ +import json, sys +out, mode, kind, args = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4:] +if mode == "eager" and hasattr(sys, "set_lazy_imports_filter"): + sys.set_lazy_imports_filter(lambda *_: False) +try: + if kind == "cli": + import runpy + sys.argv = ["messagefoundry", *args] + runpy.run_module("messagefoundry", run_name="__main__") + else: + import messagefoundry.tray.__main__ +except SystemExit: + pass +finally: + with open(out, "w", encoding="utf-8") as fh: + json.dump(sorted(sys.modules), fh) +""" + + +def _loaded(tmp_path: Path, probe: _Probe, *, mode: str) -> set[str]: + out = tmp_path / "modules.json" + proc = subprocess.run( + [sys.executable, "-c", _PROBE_CODE, str(out), mode, probe.kind, *probe.args], + capture_output=True, + text=True, + # Under the suite's 60 s pytest-timeout, so a hung probe still reports its stderr. + timeout=50, + ) + assert proc.returncode == 0 and out.exists(), proc.stderr[-2000:] + return set(json.loads(out.read_text(encoding="utf-8"))) + + +@pytest.mark.parametrize("name", list(_PROBES)) +def test_control_arm_loads_every_deferred_module_when_eager(tmp_path: Path, name: str) -> None: + probe = _PROBES[name] + missing = probe.deferred - _loaded(tmp_path, probe, mode="eager") + assert not missing, f"never loaded even eagerly, so the lazy entry is dead: {sorted(missing)}" + + +@pytest.mark.skipif(sys.version_info < (3, 15), reason=_LAZY_ARM_SKIP) +@pytest.mark.parametrize("name", list(_PROBES)) +def test_startup_stays_within_budget(tmp_path: Path, name: str) -> None: + probe = _PROBES[name] + loaded = _loaded(tmp_path, probe, mode="lazy") + early = probe.deferred & loaded + assert not early, f"deferred module loaded at startup: {sorted(early)}" + engine = {m for m in loaded if m.partition(".")[0] == "messagefoundry"} + assert engine == probe.engine, ( + f"added: {sorted(engine - probe.engine)}, removed: {sorted(probe.engine - engine)}" + ) From 32b6d53e36b8d59cef3006936e08153c9302888a Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 22:47:24 -0500 Subject: [PATCH 06/14] fix(generators): OBR repeats its ORC's order numbers; choice predicate made public (BACKLOG #2497) Code review round 1 on the previous commit: - OBR-2/OBR-3 drew fresh numbers instead of repeating ORC-2/ORC-3, which the new ORM order detail made visible. The ORC builder hands its numbers to the next OBR through Ctx, consumed once. The OBR's own draws are kept, so only those two fields move: ORM, OML, MDM, ORU, 3720 messages measured, all still strictly valid. - The generator imported the private _is_choice from the hl7apy issue-151 shim, whose notes say to delete it. Renamed is_choice_group, and both deletion notes now say to keep it and _SEQUENCES_LABELLED_CHOICE. - _pick_alternative no longer counts a group alternative as usable; none ships in hl7apy, and it was never checked for buildability. - The ORM test runs under two seeds and checks OBR-2/3 against ORC-2/3. --- changelog.d/2497.changed.md | 7 ++++-- messagefoundry/generators/_core.py | 36 ++++++++++++++++++++-------- messagefoundry/parsing/validate.py | 14 ++++++++--- tests/test_generators_choice.py | 14 +++++++---- tests/test_validate_choice_groups.py | 3 ++- 5 files changed, 54 insertions(+), 20 deletions(-) diff --git a/changelog.d/2497.changed.md b/changelog.d/2497.changed.md index ceb0cb772..0ae209575 100644 --- a/changelog.d/2497.changed.md +++ b/changelog.d/2497.changed.md @@ -2,5 +2,8 @@ leave ORDER_DETAIL out, because it emitted every required child of a group, and the order detail's OBR/RQD/RQ1/RXO/ODS/ODT group is a choice. It now emits one alternative of a choice group, OBR by default. Each generated ORM^O01 now ends ORC then OBR and passes strict - validation, so a corpus generated before this change differs in its ORM files. - (`BACKLOG #2497`) + validation. (`BACKLOG #2497`) +- **A generated OBR now repeats its order's placer and filler numbers.** OBR-2 and OBR-3 drew + fresh numbers instead of copying ORC-2 and ORC-3, as HL7 requires. Every generated message that + carries an ORC then an OBR changes in those two fields and nowhere else: ORM, OML, MDM and ORU. + A corpus generated before this change differs in those files. (`BACKLOG #2497`) diff --git a/messagefoundry/generators/_core.py b/messagefoundry/generators/_core.py index 94e2761a5..a5c13615e 100644 --- a/messagefoundry/generators/_core.py +++ b/messagefoundry/generators/_core.py @@ -27,7 +27,7 @@ from messagefoundry.parsing import validate # The strict validator's own choice test, so generation and validation agree on what a choice is. -from messagefoundry.parsing.validate import _is_choice +from messagefoundry.parsing.validate import is_choice_group _MESSAGES = _ref.MESSAGES _SEGMENTS = _ref.SEGMENTS @@ -77,6 +77,8 @@ class Ctx: receiving_fac: str current: Patient | None = None seq: dict[str, int] = field(default_factory=dict) + # The current ORC's placer and filler order numbers (ORC-2, ORC-3), which its OBR repeats. + order_numbers: tuple[str, str] | None = None def next_seq(ctx: Ctx, name: str) -> int: @@ -258,12 +260,16 @@ def _build_obx(rng: random.Random, ctx: Ctx) -> str: def _build_orc(rng: random.Random, ctx: Ctx) -> str: + control = rng.choice(d.ORDER_CONTROLS) + placer = d.ei(str(rng.randint(100_000, 999_999))) # placer order number + filler = d.ei(str(rng.randint(100_000, 999_999)), "FILLER") + ctx.order_numbers = (placer, filler) return seg( "ORC", { - 1: rng.choice(d.ORDER_CONTROLS), - 2: d.ei(str(rng.randint(100_000, 999_999))), # placer order number - 3: d.ei(str(rng.randint(100_000, 999_999)), "FILLER"), + 1: control, + 2: placer, + 3: filler, 5: rng.choice(d.ORDER_STATUSES), 9: d.ts(ctx.msg_dt), 12: d.xcn(*rng.choice(d.CLINICIANS)), @@ -273,12 +279,20 @@ def _build_orc(rng: random.Random, ctx: Ctx) -> str: def _build_obr(rng: random.Random, ctx: Ctx) -> str: code, text = rng.choice(d.SERVICES) + # Drawn even when an ORC supplies them, so every later value keeps its place in the stream. + placer = d.ei(str(rng.randint(100_000, 999_999))) + filler = d.ei(str(rng.randint(100_000, 999_999)), "FILLER") + if ctx.order_numbers is not None: + # OBR-2/OBR-3 repeat the order's ORC-2/ORC-3. Consumed, so an OBR in a later order + # without its own ORC does not borrow this one's numbers. + placer, filler = ctx.order_numbers + ctx.order_numbers = None return seg( "OBR", { 1: str(next_seq(ctx, "OBR")), - 2: d.ei(str(rng.randint(100_000, 999_999))), - 3: d.ei(str(rng.randint(100_000, 999_999)), "FILLER"), + 2: placer, + 3: filler, 4: d.cwe(code, text, "LN"), # universal service id (required) 7: d.ts(ctx.msg_dt), }, @@ -444,10 +458,12 @@ def _pick_alternative(group: str, alternatives: Any, rng: random.Random, spec: M """The one alternative of choice group ``group`` to emit. ``spec.preferred_alternative`` (OBR by default) when the group offers it and it can be built, - without drawing from ``rng``. Otherwise a seeded pick among the alternatives that can be - built, so a seed still reproduces its bytes. + without drawing from ``rng``. Otherwise a seeded pick among the segment alternatives that + can be built, so a seed reproduces its bytes for a given set of builders; adding a builder + can change that pick. No shipped hl7apy choice group has a group as an alternative, so only + segments are candidates. """ - usable = [alt for alt in alternatives if alt[3] == "GRP" or _builder_for(spec, alt[0])] + usable = [alt for alt in alternatives if alt[3] == "SEG" and _builder_for(spec, alt[0])] if not usable: raise RuntimeError(f"no builder for any alternative of choice group {group}") preferred = next((alt for alt in usable if alt[0] == spec.preferred_alternative), None) @@ -487,7 +503,7 @@ def _emit( out.append(builder(rng, ctx)) elif min_card >= 1 or any(name.endswith(s) for s in spec.group_suffixes): group_children = child_ref[1] - if _is_choice(name, child_ref): + if is_choice_group(name, child_ref): group_children = [_pick_alternative(name, group_children, rng, spec)] _emit(group_children, rng, ctx, spec, force, out) diff --git a/messagefoundry/parsing/validate.py b/messagefoundry/parsing/validate.py index 920c3876a..d405f6e51 100644 --- a/messagefoundry/parsing/validate.py +++ b/messagefoundry/parsing/validate.py @@ -116,6 +116,8 @@ def _choice_fix_needed() -> bool: release that accepts the valid probe only by no longer checking choice groups at all, and for one that merges PR 152 as written and so rejects the labelled-choice probe. ``tests/test_validate_choice_groups.py`` goes red on a fixed release, so the shim gets deleted. + Keep :func:`is_choice_group` and ``_SEQUENCES_LABELLED_CHOICE`` when it is: the generator + uses them. """ from hl7apy.consts import VALIDATION_LEVEL from hl7apy.exceptions import HL7apyException @@ -150,7 +152,13 @@ def accepted(raw: str) -> bool: _rewrite_failures: set[tuple[str, str | None]] = set() -def _is_choice(name: str, ref: _Reference) -> bool: +def is_choice_group(name: str, ref: _Reference) -> bool: + """True if hl7apy table entry ``ref`` for group ``name`` is a real "exactly one of" choice. + + The synthetic-message generator uses this too, so it emits one alternative where strict + validation wants one. This predicate and ``_SEQUENCES_LABELLED_CHOICE`` describe hl7apy's + tables, not issue 151, so they outlive the shim: deleting the shim must keep them. + """ return ref[0] == "choice" and name not in _SEQUENCES_LABELLED_CHOICE @@ -164,7 +172,7 @@ def _as_sequence(ref: _Reference, *, choice: bool = False) -> _Reference: children = [] for name, sub, (low, high), marker in ref[1]: if marker == "GRP": - sub = _as_sequence(sub, choice=_is_choice(name, sub)) + sub = _as_sequence(sub, choice=is_choice_group(name, sub)) children.append([name, sub, (0, high) if choice else (low, high), marker]) return ("sequence", tuple(children), *ref[2:]) @@ -214,7 +222,7 @@ def _first_choice_error(element: Any, ref: _Reference) -> str | None: if marker != "GRP": continue for group in _occurrences(element, name): - error = _choice_error(group, sub) if _is_choice(name, sub) else None + error = _choice_error(group, sub) if is_choice_group(name, sub) else None if error is None: error = _first_choice_error(group, sub) if error is not None: diff --git a/tests/test_generators_choice.py b/tests/test_generators_choice.py index add1dfb39..ec064664c 100644 --- a/tests/test_generators_choice.py +++ b/tests/test_generators_choice.py @@ -33,15 +33,21 @@ def _spec(*buildable: str) -> _core.MessageSpec: # The generator writes v2.5.1 only (MSH-12), so that is the version strict validation checks. -# A few seeds, because the optional PD1/PV2 around the order vary with the seed. -@pytest.mark.parametrize("index", range(1, 6)) -def test_generated_orm_o01_has_one_order_detail_and_is_strictly_valid(index: int) -> None: - msg = _core.generate_message("ORM", "O01", index) +# A few indices under two seeds, because the optional PD1/PV2 around the order vary with both. +@pytest.mark.parametrize("seed", [_core.DEFAULT_SEED, "another-seed"]) +@pytest.mark.parametrize("index", range(1, 4)) +def test_generated_orm_o01_has_one_order_detail_and_is_strictly_valid( + index: int, seed: str +) -> None: + msg = _core.generate_message("ORM", "O01", index, seed=seed) peek = Peek.parse(msg) assert peek.version == "2.5.1" names = peek.segments() assert names[-2:] == ["ORC", "OBR"], names assert [n for n in names if n in _ORDER_ALTERNATIVES] == ["OBR"], names + # HL7 says OBR-2 and OBR-3 are identical to the order's ORC-2 and ORC-3. + assert peek.field("OBR-2") == peek.field("ORC-2") + assert peek.field("OBR-3") == peek.field("ORC-3") ok, errors = _core.gate("ORM", msg, "ORM_O01") assert ok, errors diff --git a/tests/test_validate_choice_groups.py b/tests/test_validate_choice_groups.py index dbf3ee3b0..abe0f1b3c 100644 --- a/tests/test_validate_choice_groups.py +++ b/tests/test_validate_choice_groups.py @@ -155,7 +155,8 @@ def test_an_empty_choice_group_is_reported() -> None: def test_hl7apy_still_has_the_bug_so_the_shim_is_still_on() -> None: - """Goes red on the first hl7apy release that fixes issue 151. Then delete the shim. + """Goes red on the first hl7apy release that fixes issue 151. Then delete the shim, but keep + ``is_choice_group`` and ``_SEQUENCES_LABELLED_CHOICE``: the generator uses them. The shim switches itself off on such a release, since :func:`_choice_fix_needed` asks hl7apy rather than its version number. The code it leaves behind is dead, and this test is From 643c59ce99ea69adbb2ee92707d8db5402b31fbf Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 22:52:04 -0500 Subject: [PATCH 07/14] Dependabot: restate the pynetdicom, pydicom and webauthn caps as ignore ranges, and test every pyproject cap against one (BACKLOG #2505) Ledger banner: BUILT 2026-09-30 on branch b2505-cap-ignores -- three uv ignore entries (pynetdicom >=4.0.0, pydicom >=3.1.0, webauthn >=4.0.0) and tests/test_dependabot_cap_ignores.py, which holds every pyproject upper bound to an exact ignore range or a stated exemption (hvac, hatchling, the exact group pins) and every entry to a live cap. Code review round 2 changes. Each exemption now names the table it holds in, so a new cap on hvac or hatchling elsewhere is not exempt, and both halves of the group-pin rule have their own arm. An entry name with extras is now a violation rather than matched leniently. The suggested range keeps pre-release and epoch parts, a non-string range reds instead of crashing, the pin test lives in one helper, and the docstring points at dependabot.yml rather than restating its history. dependabot.yml says pynetdicom's cap has no reason of its own in pyproject.toml. --- .github/dependabot.yml | 7 +- tests/test_dependabot_cap_ignores.py | 224 +++++++++++++++------------ 2 files changed, 130 insertions(+), 101 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8f044498..23843e27a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -61,9 +61,10 @@ updates: # `annotated-types<0.8` -> `<0.9` and `ruff>=0.4,<0.16` -> `<0.17`; the # resync then propagated 0.8.0/0.16.0 into constraints.lock, so ci.yml's `--constraint` pinned CI # *to* the broken versions. `python-deps` groups every minor and patch on `*`, and both of those - # bumps are minors, so they red the WHOLE batch and hold every benign bump in it hostage. WHY - # each is capped lives beside the cap in pyproject.toml, at the requirement the entry names. Do - # not restate it here; lift a cap and delete its entry in the SAME PR. + # bumps are minors, so they red the WHOLE batch and hold every benign bump in it hostage. Where + # pyproject.toml records WHY a package is capped, it sits beside the cap; do not restate it here. + # pynetdicom's `<4` has no reason of its own there today. Lift a cap and delete its entry in the + # SAME PR. # TRADE-OFF, accepted: `ignore` suppresses the SECURITY track for the named RANGE as well as the # routine one (`update-types` is the version-only knob and cannot express a range). A ruff # 0.15.x or uvicorn 0.49.x advisory fix still flows. Detection is untouched — security.yml's diff --git a/tests/test_dependabot_cap_ignores.py b/tests/test_dependabot_cap_ignores.py index 6562a9159..174e6f179 100644 --- a/tests/test_dependabot_cap_ignores.py +++ b/tests/test_dependabot_cap_ignores.py @@ -2,27 +2,24 @@ # Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors """Every upper bound in pyproject.toml has a Dependabot ignore range, or a stated exemption. -Dependabot WIDENS a declared cap instead of respecting it. ``.github/dependabot.yml`` says so, and -says a load-bearing cap must be restated there as an ``ignore`` range "or the cap is decorative". -PR #66 widened ruff's and annotated-types' caps that way, and Dependabot PR 1773 tried to widen -uvicorn's. Until BACKLOG #2505 nothing held the two files together, and three caps had no entry. +Dependabot WIDENS a declared cap instead of respecting it; ``.github/dependabot.yml`` says so above +its uv ``ignore`` list, with the history. Until BACKLOG #2505 nothing held the two files together, +and three caps had no entry. THE CONTRACT, both directions: * Every upper bound in ``[build-system].requires``, ``[project.dependencies]``, ``[project.optional-dependencies]`` and ``[dependency-groups]`` (``<``, ``~=``, ``==``, or an - ``==X.*`` wildcard) has a uv-ecosystem ``ignore`` entry whose range is exactly ``>=`` the first - version the cap excludes. That is the one range that blocks everything the cap excludes and - nothing it allows. Or the package is in ``_EXEMPT`` below, with its reason. + ``==X.*`` wildcard) has a uv-ecosystem ``ignore`` entry whose range is exactly ``>=`` the + version ``_cap_of`` derives. Or the package is in ``_EXEMPT`` below, in the table it names, with + its reason. * Every uv ``ignore`` entry names a package that pyproject.toml caps. dependabot.yml says to lift a cap and delete its entry in the same PR; this is what makes that rule fail when skipped. It is also this test's positive control: a parser that found no caps would red on every real entry, so the first test cannot pass vacuously. -AN EXACT ``==`` PIN COVERS THE NEXT MINOR, NOT THE NEXT PATCH. That is the ``sigstore`` entry's -documented scope (``>=4.5.0`` for ``==4.4.0``): it blocks the minor the owner declined and leaves -the patch track open. It is the only ignored pin today; a new one takes the same scope or a -deliberate change here. +For a ``<`` cap the derived version is the bound itself. For an exact ``==`` pin it is the next +minor, which is the scope dependabot.yml's sigstore note records: the pin's patch track stays open. The mutation arms break a made-up cap and entry pair added to copies of the real files, never a real one, so lifting a real cap cannot break an arm. @@ -49,33 +46,45 @@ _ROOT = Path(__file__).resolve().parents[1] _PYPROJECT = _ROOT / "pyproject.toml" _DEPENDABOT = _ROOT / ".github" / "dependabot.yml" +_GROUPS = "[dependency-groups]" -#: The exact ``==`` pins in ``[dependency-groups]`` that are deliberately NOT ignored. Each must stay -#: an ``==`` pin in a dependency group. ``sigstore`` is the one exact group pin that IS ignored, by -#: owner ruling; dependabot.yml's sigstore note says why. If that ruling is lifted and its entry -#: deleted, drop sigstore from this subtraction in the same PR. +#: The exact ``==`` pins in ``[dependency-groups]`` that are deliberately NOT ignored. ``sigstore`` +#: is the one exact group pin that IS ignored, by owner ruling; dependabot.yml's sigstore note says +#: why. If that ruling is lifted and its entry deleted, drop sigstore from this subtraction too. _GROUP_PINS = frozenset(EXACT_GROUP_PINS) - {"sigstore"} _GROUP_PIN_REASON = ( ".github/dependabot.yml, 'NOT IGNORED, deliberately': ignoring the exact pins in " "[dependency-groups] would freeze the hash-pinned CI toolchain" ) -#: Every cap with no ignore entry, by package, with the reason. Read the reason where it points. -_EXEMPT: dict[str, str] = { - **dict.fromkeys(_GROUP_PINS, _GROUP_PIN_REASON), - "hvac": "pyproject.toml's comment above the [vault] extra says why hvac's cap is not mirrored", - "hatchling": "pyproject.toml's [build-system] comment expects Dependabot to bump this pin", +#: Every cap with no ignore entry: package -> (the table it must sit in, the reason). The table is a +#: prefix of ``Cap.where``. A cap on the same package anywhere else is not exempt. An exemption in +#: ``[dependency-groups]`` must also be an exact ``==`` pin. Read each reason where it points. +_EXEMPT: dict[str, tuple[str, str]] = { + **dict.fromkeys(_GROUP_PINS, (_GROUPS, _GROUP_PIN_REASON)), + "hvac": ( + "[project.optional-dependencies].vault", + "pyproject.toml's comment above the [vault] extra says why hvac's cap is not mirrored", + ), + "hatchling": ( + "[build-system].requires", + "pyproject.toml's [build-system] comment expects Dependabot to bump this pin", + ), } class Cap(NamedTuple): - """One upper bound: where it sits, and the first version it excludes.""" + """One upper bound: where it sits, and the version its ignore range must start at.""" package: str where: str spec: str exact_pin: bool - first_excluded: Version + ignore_from: Version + + +def _is_pin(spec: Specifier) -> bool: + return spec.operator == "==" and not spec.version.endswith(".*") def _bump(release: tuple[int, ...]) -> Version: @@ -84,19 +93,18 @@ def _bump(release: tuple[int, ...]) -> Version: def _cap_of(spec: Specifier) -> Version | None: - """The first version an upper-bound specifier excludes, or None for a floor or ``!=``.""" + """Where an upper-bound specifier's ignore range starts, or None for a floor or ``!=``.""" op, raw = spec.operator, spec.version if op in (">", ">=", "!="): return None if op == "<": return Version(raw) - wildcard = op == "==" and raw.endswith(".*") - if op == "~=" or wildcard: - release = Version(raw.removesuffix(".*")).release - return _bump(release if wildcard else release[:-1]) - if op == "==": + if _is_pin(spec): pinned = Version(raw) return Version(f"{pinned.major}.{pinned.minor + 1}.0") + if op in ("~=", "=="): # `==` here is the `==X.*` wildcard + release = Version(raw.removesuffix(".*")).release + return _bump(release if op == "==" else release[:-1]) # `<=` and `===` have no use in pyproject.toml today. Model one deliberately when it arrives. raise AssertionError(f"unmodelled upper-bound operator {op!r} in {spec}") @@ -113,8 +121,7 @@ def _caps(pyproject: dict[str, Any]) -> list[Cap]: for name, reqs in project.get("optional-dependencies", {}).items() ] tables += [ - (f"[dependency-groups].{name}", reqs) - for name, reqs in pyproject.get("dependency-groups", {}).items() + (f"{_GROUPS}.{name}", reqs) for name, reqs in pyproject.get("dependency-groups", {}).items() ] caps: list[Cap] = [] for where, reqs in tables: @@ -124,9 +131,7 @@ def _caps(pyproject: dict[str, Any]) -> list[Cap]: req = Requirement(raw) # markers parse apart from the specifier, so never read as caps bounds = [b for s in req.specifier if (b := _cap_of(s)) is not None] if bounds: - pin = any( - s.operator == "==" and not s.version.endswith(".*") for s in req.specifier - ) + pin = any(_is_pin(s) for s in req.specifier) caps.append(Cap(canonicalize_name(req.name), where, str(req), pin, min(bounds))) return caps @@ -143,30 +148,27 @@ def _uv_entry(dependabot: dict[str, Any]) -> dict[str, Any]: def _uv_ignores(dependabot: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: - """The root uv entry's ignore list, keyed by canonical package name. - - Extras are dropped first, as dependabot-core's Python name normaliser does, so an entry named - ``uvicorn[standard]`` matches a ``uvicorn`` cap. - """ + """The root uv entry's ignore list, keyed by canonical package name.""" ignores: dict[str, list[dict[str, Any]]] = {} for entry in _uv_entry(dependabot).get("ignore", []): - name = canonicalize_name(entry["dependency-name"].split("[", 1)[0]) - ignores.setdefault(name, []).append(entry) + ignores.setdefault(canonicalize_name(entry["dependency-name"]), []).append(entry) return ignores -def _ranges(entry: dict[str, Any]) -> list[str]: +def _ranges(entry: dict[str, Any]) -> list[Any]: versions = entry.get("versions", []) - return [versions] if isinstance(versions, str) else list(versions) + return versions if isinstance(versions, list) else [versions] -def _is_exactly_from(rng: str, first: Version) -> bool: - """Whether ``rng`` is the single specifier ``>=first``.""" +def _is_exactly_from(rng: Any, start: Version) -> bool: + """Whether ``rng`` is the single specifier ``>=start``.""" + if not isinstance(rng, str): # an unquoted YAML number, say + return False try: specs = list(SpecifierSet(rng)) except InvalidSpecifier: return False - return len(specs) == 1 and specs[0].operator == ">=" and Version(specs[0].version) == first + return len(specs) == 1 and specs[0].operator == ">=" and Version(specs[0].version) == start def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[str]: @@ -176,23 +178,21 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s for cap in caps: label = f"{cap.where}: {cap.spec}" - first = cap.first_excluded - want = f">={first.major}.{first.minor}.{first.micro}" + want = f">={cap.ignore_from}" if cap.package in _EXEMPT: - if cap.package in _GROUP_PINS and not ( - cap.exact_pin and cap.where.startswith("[dependency-groups]") - ): + table, reason = _EXEMPT[cap.package] + if not cap.where.startswith(table): + problems.append(f"{label} is exempt only in {table}, and this cap is elsewhere") + elif table == _GROUPS and not cap.exact_pin: problems.append(f"{label} is exempt as a group `==` pin, but is not one") elif cap.package in ignores: - problems.append( - f"{label} is exempt ({_EXEMPT[cap.package]}) yet has an ignore entry" - ) + problems.append(f"{label} is exempt ({reason}) yet has an ignore entry") continue entries = ignores.get(cap.package) if not entries: problems.append( - f'{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with versions ["{want}"], ' - f"or an exemption with its reason in {Path(__file__).name}" + f"{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with " + f'versions ["{want}"], or an exemption with its reason in {Path(__file__).name}' ) continue if any("update-types" in e for e in entries): @@ -200,18 +200,15 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s f"{label}: an `update-types` ignore cannot restate a range; use `versions` only" ) ranges = [rng for e in entries for rng in _ranges(e)] - if not ranges or not all(_is_exactly_from(rng, first) for rng in ranges): - problems.append( - f"{label}: its ignore range must be exactly {want!r}, which blocks what the cap " - f"excludes and nothing it allows; found {ranges}" - ) + if not ranges or not all(_is_exactly_from(rng, cap.ignore_from) for rng in ranges): + problems.append(f"{label}: its ignore range must be exactly {want!r}; found {ranges}") capped = {cap.package for cap in caps} for name in sorted(set(ignores) - capped): problems.append( - f"{_DEPENDABOT.name} ignores {name!r}, which pyproject.toml does not cap. An entry " - "here restates a cap, so delete it in the PR that lifts the cap, or model the new " - f"kind of entry in {Path(__file__).name}" + f"{_DEPENDABOT.name} ignores {name!r}, which pyproject.toml does not cap. Name the bare " + "package; an entry here restates a cap, so delete it in the PR that lifts the cap, " + f"or model the new kind of entry in {Path(__file__).name}" ) for name in sorted(set(_EXEMPT) - capped): problems.append(f"exemption for {name!r} names no capped requirement in pyproject.toml") @@ -233,7 +230,7 @@ def test_every_cap_has_an_ignore_entry_or_a_stated_exemption() -> None: @pytest.mark.parametrize( - ("spec", "first"), + ("spec", "start"), [ ("<0.50", "0.50"), ("<4", "4"), @@ -246,8 +243,8 @@ def test_every_cap_has_an_ignore_entry_or_a_stated_exemption() -> None: ("!=1.2.*", None), ], ) -def test_cap_arithmetic(spec: str, first: str | None) -> None: - assert _cap_of(Specifier(spec)) == (Version(first) if first else None) +def test_cap_arithmetic(spec: str, start: str | None) -> None: + assert _cap_of(Specifier(spec)) == (Version(start) if start else None) # A made-up cap and entry pair for each shape the mutation arms break. No real package is named, @@ -274,31 +271,23 @@ def _fixture() -> tuple[dict[str, Any], dict[str, Any]]: return pyproject, dependabot -_Mutation = Callable[[dict[str, Any], dict[str, Any]], None] - - -def _rename_entry(_p: dict[str, Any], d: dict[str, Any]) -> None: - _fake_entry(d, "mefor-fake-cap")["dependency-name"] = "Mefor_Fake_Cap[extra]" - - -@pytest.mark.parametrize("mutate", [None, _rename_entry], ids=["unbroken", "extras-and-case"]) -def test_the_mutation_fixture_is_clean(mutate: _Mutation | None) -> None: +def test_the_mutation_fixture_is_clean() -> None: """Control: the arms below are discriminating only if the unbroken fixture passes.""" - pyproject, dependabot = _fixture() - if mutate: - mutate(pyproject, dependabot) - problems = _violations(pyproject, dependabot) + problems = _violations(*_fixture()) assert not problems, "\n".join(problems) +_Mutation = Callable[[dict[str, Any], dict[str, Any]], None] + + def _drop_entry(_p: dict[str, Any], d: dict[str, Any]) -> None: uv = _uv_entry(d) uv["ignore"] = [e for e in uv["ignore"] if e["dependency-name"] != "mefor-fake-cap"] -def _set_versions(name: str, versions: list[str]) -> _Mutation: +def _set_entry(name: str, key: str, value: Any) -> _Mutation: def mutate(_p: dict[str, Any], d: dict[str, Any]) -> None: - _fake_entry(d, name)["versions"] = versions + _fake_entry(d, name)[key] = value return mutate @@ -315,27 +304,37 @@ def _lift_cap(p: dict[str, Any], _d: dict[str, Any]) -> None: _EXACT = "must be exactly" +_CAP_VERSIONS = "mefor-fake-cap", "versions" @pytest.mark.parametrize( ("mutate", "expect"), [ pytest.param(_drop_entry, "no uv ignore entry", id="entry-deleted"), - pytest.param(_set_versions("mefor-fake-cap", [">=2.6.0"]), _EXACT, id="gap-at-cap"), - pytest.param(_set_versions("mefor-fake-cap", [">=2.4.0"]), _EXACT, id="freezes"), + pytest.param(_set_entry(*_CAP_VERSIONS, [">=2.6.0"]), _EXACT, id="gap-at-cap"), + pytest.param(_set_entry(*_CAP_VERSIONS, [">=2.4.0"]), _EXACT, id="freezes"), pytest.param( - _set_versions("mefor-fake-cap", ["==2.5.*", ">=3.0.0"]), _EXACT, id="hole-above-cap" + _set_entry(*_CAP_VERSIONS, ["==2.5.*", ">=3.0.0"]), _EXACT, id="hole-above-cap" ), pytest.param( - _set_versions("mefor-fake-cap", [">=1.2.0,<1.4.0", ">=2.5.0"]), + _set_entry(*_CAP_VERSIONS, [">=1.2.0,<1.4.0", ">=2.5.0"]), _EXACT, id="blocks-an-allowed-range", ), - pytest.param(_set_versions("mefor-fake-cap", ["2.5.0"]), _EXACT, id="not-a-specifier"), - pytest.param(_set_versions("mefor-fake-pin", [">=3.2.0"]), _EXACT, id="blocks-pin"), - pytest.param(_set_versions("mefor-fake-pin", [">=4.0.0"]), _EXACT, id="pin-gap"), + pytest.param(_set_entry(*_CAP_VERSIONS, ["2.5.0"]), _EXACT, id="not-a-specifier"), + pytest.param(_set_entry(*_CAP_VERSIONS, [2.5]), _EXACT, id="not-a-string"), + pytest.param(_set_entry(*_CAP_VERSIONS, 4), _EXACT, id="not-a-list"), + pytest.param( + _set_entry("mefor-fake-pin", "versions", [">=3.2.0"]), _EXACT, id="blocks-pin" + ), + pytest.param(_set_entry("mefor-fake-pin", "versions", [">=4.0.0"]), _EXACT, id="pin-gap"), pytest.param(_to_update_types, "update-types", id="update-types"), pytest.param(_lift_cap, "does not cap", id="stale-entry"), + pytest.param( + _set_entry("mefor-fake-cap", "dependency-name", "mefor-fake-cap[extra]"), + "does not cap", + id="name-with-extras", + ), ], ) def test_the_checker_reds_on_a_mutated_config(mutate: _Mutation, expect: str) -> None: @@ -347,19 +346,48 @@ def test_the_checker_reds_on_a_mutated_config(mutate: _Mutation, expect: str) -> @pytest.mark.parametrize( - ("group_pin", "exempt", "expect"), + ("name", "table", "added", "expect"), [ - pytest.param(False, "mefor-fake-cap", "yet has an ignore entry", id="exempt-with-entry"), - pytest.param(False, "mefor-never-capped", "names no capped", id="dead-exemption"), - pytest.param(True, "mefor-fake-cap", "but is not one", id="group-pin-not-a-pin"), + pytest.param( + "mefor-fake-cap", + "[project.dependencies]", + None, + "yet has an ignore entry", + id="exempt-with-entry", + ), + pytest.param( + "mefor-never-capped", "[project.dependencies]", None, "names no", id="dead-exemption" + ), + pytest.param( + "mefor-fake-cap", "[build-system]", None, "is exempt only in", id="exempt-elsewhere" + ), + pytest.param( + "mefor-gp", + _GROUPS, + ("dependencies", "mefor-gp==1.0.0"), + "only in", + id="pin-not-in-group", + ), + pytest.param( + "mefor-gp", _GROUPS, ("group", "mefor-gp>=1,<2"), "is not one", id="group-cap-not-a-pin" + ), ], ) def test_the_checker_reds_on_a_broken_exemption( - monkeypatch: pytest.MonkeyPatch, group_pin: bool, exempt: str, expect: str + monkeypatch: pytest.MonkeyPatch, + name: str, + table: str, + added: tuple[str, str] | None, + expect: str, ) -> None: - """Mutation arm for the exemption table, again over the made-up pair only.""" - monkeypatch.setitem(_EXEMPT, exempt, "test reason") - if group_pin: - monkeypatch.setattr(f"{__name__}._GROUP_PINS", _GROUP_PINS | {exempt}) - problems = _violations(*_fixture()) + """Mutation arm for the exemption table, over made-up packages only.""" + monkeypatch.setitem(_EXEMPT, name, (table, "test reason")) + pyproject, dependabot = _fixture() + if added: + where, req = added + if where == "group": + pyproject["dependency-groups"]["mefor-fake"].append(req) + else: + pyproject["project"]["dependencies"].append(req) + problems = _violations(pyproject, dependabot) assert any(expect in p for p in problems), f"no violation mentioning {expect!r}: {problems}" From 8ef470fc578e474a4674dfcf937f68b2166d14ba Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 22:55:56 -0500 Subject: [PATCH 08/14] Dependabot: restate the pydicom cap as an ignore range, and test every pyproject cap against the class rule (BACKLOG #2505) Manager revision: a cap at minor level or lower needs an exact uv ignore entry, because python-deps would widen it in the routine batch. A major-level cap (=3.1.0, and states the class rule once. The test replaces the hvac exemption with the rule and adds arms: a made-up major cap with no entry is green, an opt-in exact entry for it is green, and a non-exact one is red. pyproject.toml is unchanged: pynetdicom's <4 first appears in the history root 546426285b, whose message does not state it. Ledger banner: BUILT 2026-09-30 on branch b2505-cap-ignores -- pydicom >=3.1.0 uv ignore entry, the minor-or-lower class rule in dependabot.yml, and tests/test_dependabot_cap_ignores.py holding it. --- .github/dependabot.yml | 26 +++++------ tests/test_dependabot_cap_ignores.py | 66 ++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 28 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 23843e27a..3e8c08b89 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -63,8 +63,7 @@ updates: # *to* the broken versions. `python-deps` groups every minor and patch on `*`, and both of those # bumps are minors, so they red the WHOLE batch and hold every benign bump in it hostage. Where # pyproject.toml records WHY a package is capped, it sits beside the cap; do not restate it here. - # pynetdicom's `<4` has no reason of its own there today. Lift a cap and delete its entry in the - # SAME PR. + # Lift a cap and delete its entry in the SAME PR. # TRADE-OFF, accepted: `ignore` suppresses the SECURITY track for the named RANGE as well as the # routine one (`update-types` is the version-only knob and cannot express a range). A ruff # 0.15.x or uvicorn 0.49.x advisory fix still flows. Detection is untouched — security.yml's @@ -119,13 +118,16 @@ updates: # `uvicorn`, not `uvicorn[standard]`, so the bare name here matches it as written. Lift this # entry in the same PR that lifts the cap. # - # `pynetdicom`, `pydicom` and `webauthn` are the same WIDENED-CAP case (BACKLOG #2505). Each entry - # below carries a one-line pointer to pyproject.toml; read the comment there. pydicom's `<3.1` is - # a MINOR cap, so without its entry `python-deps` would widen it in the routine minor-and-patch - # batch. Some caps have NO entry on purpose: at least `hvac<3`, `hatchling`'s build-system pin and - # the exact group pins under "NOT IGNORED, deliberately" above. tests/test_dependabot_cap_ignores.py - # names every such exemption with its reason, holds every other upper bound in pyproject.toml to - # an entry here, and every entry here to a cap that still exists. + # `pydicom` is the same WIDENED-CAP case (BACKLOG #2505). Its `<3.1` is a MINOR cap, so without + # its entry `python-deps` would widen it in the routine minor-and-patch batch. + # + # THE CLASS RULE (BACKLOG #2505): a cap at MINOR level or lower needs an entry here. A MAJOR-level + # cap (`=0.16.0"] @@ -135,15 +137,9 @@ updates: versions: [">=7.4.0"] - dependency-name: "uvicorn" versions: [">=0.50.0"] - # The comment above the [dicom] extra in pyproject.toml names the 3.x pairing with pydicom. - - dependency-name: "pynetdicom" - versions: [">=4.0.0"] # Why: the comment above the [dicom] extra in pyproject.toml. - dependency-name: "pydicom" versions: [">=3.1.0"] - # Why: the comment above the [webauthn] extra in pyproject.toml. - - dependency-name: "webauthn" - versions: [">=4.0.0"] groups: # Version-update grouping (applies-to defaults to version-updates), SPLIT BY UPDATE TYPE (owner # decision 2026-09-30) so one bad major cannot hold the routine minors and patches hostage. diff --git a/tests/test_dependabot_cap_ignores.py b/tests/test_dependabot_cap_ignores.py index 174e6f179..f9d66b34f 100644 --- a/tests/test_dependabot_cap_ignores.py +++ b/tests/test_dependabot_cap_ignores.py @@ -10,9 +10,12 @@ * Every upper bound in ``[build-system].requires``, ``[project.dependencies]``, ``[project.optional-dependencies]`` and ``[dependency-groups]`` (``<``, ``~=``, ``==``, or an - ``==X.*`` wildcard) has a uv-ecosystem ``ignore`` entry whose range is exactly ``>=`` the - version ``_cap_of`` derives. Or the package is in ``_EXEMPT`` below, in the table it names, with - its reason. + ``==X.*`` wildcard) at MINOR level or lower has a uv-ecosystem ``ignore`` entry whose range is + exactly ``>=`` the version ``_cap_of`` derives. Or the package is in ``_EXEMPT`` below, in the + table it names, with its reason. +* A MAJOR-level cap (one whose derived version is ``N.0.0`` with ``N`` above zero, such as ``<4``) + is exempt by class, per the class rule in dependabot.yml. An entry for one is a legal opt-in, and + is then held to the same exact range. * Every uv ``ignore`` entry names a package that pyproject.toml caps. dependabot.yml says to lift a cap and delete its entry in the same PR; this is what makes that rule fail when skipped. It is also this test's positive control: a parser that found no caps would red on every real entry, so @@ -62,10 +65,6 @@ #: ``[dependency-groups]`` must also be an exact ``==`` pin. Read each reason where it points. _EXEMPT: dict[str, tuple[str, str]] = { **dict.fromkeys(_GROUP_PINS, (_GROUPS, _GROUP_PIN_REASON)), - "hvac": ( - "[project.optional-dependencies].vault", - "pyproject.toml's comment above the [vault] extra says why hvac's cap is not mirrored", - ), "hatchling": ( "[build-system].requires", "pyproject.toml's [build-system] comment expects Dependabot to bump this pin", @@ -83,6 +82,14 @@ class Cap(NamedTuple): ignore_from: Version +def _is_major_level(start: Version) -> bool: + """Whether an ignore range from ``start`` would hold back only new majors: 4 yes, 0.50 no. + + A ``0.x`` bump counts as a minor, as Dependabot reads it by version position. + """ + return start.major > 0 and not any(start.release[1:]) and not start.is_prerelease + + def _is_pin(spec: Specifier) -> bool: return spec.operator == "==" and not spec.version.endswith(".*") @@ -189,6 +196,8 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s problems.append(f"{label} is exempt ({reason}) yet has an ignore entry") continue entries = ignores.get(cap.package) + if not entries and _is_major_level(cap.ignore_from): + continue # exempt by class; dependabot.yml's class rule says why if not entries: problems.append( f"{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with " @@ -229,6 +238,21 @@ def test_every_cap_has_an_ignore_entry_or_a_stated_exemption() -> None: assert not problems, "\n".join(problems) +@pytest.mark.parametrize( + ("start", "major"), + [ + ("4", True), + ("4.0.0", True), + ("0.50", False), + ("3.1", False), + ("1.0.1", False), + ("4.0.0rc1", False), + ], +) +def test_major_level(start: str, major: bool) -> None: + assert _is_major_level(Version(start)) is major + + @pytest.mark.parametrize( ("spec", "start"), [ @@ -251,6 +275,7 @@ def test_cap_arithmetic(spec: str, start: str | None) -> None: # so a legitimate cap lift in pyproject.toml cannot crash an arm. _FAKE_CAP = "mefor-fake-cap>=1.0,<2.5" _FAKE_PIN = "mefor-fake-pin==3.2.1" +_FAKE_MAJOR = "mefor-fake-major>=1.0,<3" # exempt by class, so the fixture gives it no entry def _fake_entry(dependabot: dict[str, Any], name: str) -> dict[str, Any]: @@ -260,7 +285,7 @@ def _fake_entry(dependabot: dict[str, Any], name: str) -> dict[str, Any]: def _fixture() -> tuple[dict[str, Any], dict[str, Any]]: """Copies of the real files, plus the two made-up caps with correct entries.""" pyproject, dependabot = copy.deepcopy(_load()) - pyproject["project"]["dependencies"].append(_FAKE_CAP) + pyproject["project"]["dependencies"] += [_FAKE_CAP, _FAKE_MAJOR] pyproject.setdefault("dependency-groups", {})["mefor-fake"] = [_FAKE_PIN] _uv_entry(dependabot).setdefault("ignore", []).extend( [ @@ -271,13 +296,27 @@ def _fixture() -> tuple[dict[str, Any], dict[str, Any]]: return pyproject, dependabot -def test_the_mutation_fixture_is_clean() -> None: - """Control: the arms below are discriminating only if the unbroken fixture passes.""" - problems = _violations(*_fixture()) - assert not problems, "\n".join(problems) +_Mutation = Callable[[dict[str, Any], dict[str, Any]], None] -_Mutation = Callable[[dict[str, Any], dict[str, Any]], None] +def _major_entry(rng: str) -> _Mutation: + def mutate(_p: dict[str, Any], d: dict[str, Any]) -> None: + _uv_entry(d)["ignore"].append({"dependency-name": "mefor-fake-major", "versions": [rng]}) + + return mutate + + +@pytest.mark.parametrize( + "mutate", [None, _major_entry(">=3.0.0")], ids=["unbroken", "major-cap-opt-in-entry"] +) +def test_the_mutation_fixture_is_clean(mutate: _Mutation | None) -> None: + """Control: the arms below discriminate only if the unbroken fixture passes. The fixture holds a + major cap with no entry, and an opt-in exact entry for it must stay legal.""" + pyproject, dependabot = _fixture() + if mutate: + mutate(pyproject, dependabot) + problems = _violations(pyproject, dependabot) + assert not problems, "\n".join(problems) def _drop_entry(_p: dict[str, Any], d: dict[str, Any]) -> None: @@ -328,6 +367,7 @@ def _lift_cap(p: dict[str, Any], _d: dict[str, Any]) -> None: _set_entry("mefor-fake-pin", "versions", [">=3.2.0"]), _EXACT, id="blocks-pin" ), pytest.param(_set_entry("mefor-fake-pin", "versions", [">=4.0.0"]), _EXACT, id="pin-gap"), + pytest.param(_major_entry(">=2.0.0"), _EXACT, id="major-opt-in-not-exact"), pytest.param(_to_update_types, "update-types", id="update-types"), pytest.param(_lift_cap, "does not cap", id="stale-entry"), pytest.param( From 8b079cd0b2fc9ee904c9637f538c1215a09c65d8 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 23:06:26 -0500 Subject: [PATCH 09/14] Dependabot: restate the pydicom cap as an ignore range, and test every pyproject cap against the class rule (BACKLOG #2505) Review round on the class-rule delta. One ignore entry serves a package, so only its tightest cap is matched against it; a looser major cap beside a minor one no longer reds. A post-release or epoch bound is not major-level. Comments that still said "every upper bound" now say "below major level", and the dependabot.yml header and class rule say which specs count as major-level. Ledger banner: BUILT 2026-09-30 on branch b2505-cap-ignores -- pydicom >=3.1.0 uv ignore entry, the minor-or-lower class rule in dependabot.yml, and tests/test_dependabot_cap_ignores.py holding it. --- .github/dependabot.yml | 6 ++-- tests/test_dependabot_cap_ignores.py | 41 ++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3e8c08b89..5473bf086 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -57,7 +57,8 @@ updates: default-days: 5 semver-major-days: 7 # Dependabot WIDENS a declared cap instead of respecting it, so a load-bearing upper bound in - # pyproject.toml must be restated here as a version range or the cap is decorative. PR #66 rewrote + # pyproject.toml below major level must be restated here as a version range or the cap is + # decorative; the class rule further down says why a major-level cap needs none. PR #66 rewrote # `annotated-types<0.8` -> `<0.9` and `ruff>=0.4,<0.16` -> `<0.17`; the # resync then propagated 0.8.0/0.16.0 into constraints.lock, so ci.yml's `--constraint` pinned CI # *to* the broken versions. `python-deps` groups every minor and patch on `*`, and both of those @@ -122,7 +123,8 @@ updates: # its entry `python-deps` would widen it in the routine minor-and-patch batch. # # THE CLASS RULE (BACKLOG #2505): a cap at MINOR level or lower needs an entry here. A MAJOR-level - # cap (` (the table it must sit in, the reason). The table is a +#: Every cap below major level with no ignore entry: package -> (its table, the reason). The table is a #: prefix of ``Cap.where``. A cap on the same package anywhere else is not exempt. An exemption in #: ``[dependency-groups]`` must also be an exact ``==`` pin. Read each reason where it points. _EXEMPT: dict[str, tuple[str, str]] = { @@ -87,7 +87,12 @@ def _is_major_level(start: Version) -> bool: A ``0.x`` bump counts as a minor, as Dependabot reads it by version position. """ - return start.major > 0 and not any(start.release[1:]) and not start.is_prerelease + return ( + start.epoch == 0 + and start.major > 0 + and not any(start.release[1:]) + and not (start.is_prerelease or start.is_postrelease) + ) def _is_pin(spec: Specifier) -> bool: @@ -182,6 +187,9 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s caps = _caps(pyproject) ignores = _uv_ignores(dependabot) problems: list[str] = [] + tightest: dict[str, Version] = {} + for cap in caps: + tightest[cap.package] = min(cap.ignore_from, tightest.get(cap.package, cap.ignore_from)) for cap in caps: label = f"{cap.where}: {cap.spec}" @@ -195,14 +203,15 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s elif cap.package in ignores: problems.append(f"{label} is exempt ({reason}) yet has an ignore entry") continue + if cap.ignore_from != tightest[cap.package]: + continue # one ignore entry serves a package; the tightest cap is the one it restates entries = ignores.get(cap.package) - if not entries and _is_major_level(cap.ignore_from): - continue # exempt by class; dependabot.yml's class rule says why if not entries: - problems.append( - f"{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with " - f'versions ["{want}"], or an exemption with its reason in {Path(__file__).name}' - ) + if not _is_major_level(cap.ignore_from): # a major cap is exempt by class + problems.append( + f"{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with " + f'versions ["{want}"], or an exemption with its reason in {Path(__file__).name}' + ) continue if any("update-types" in e for e in entries): problems.append( @@ -233,7 +242,7 @@ def _load() -> tuple[dict[str, Any], dict[str, Any]]: def test_every_cap_has_an_ignore_entry_or_a_stated_exemption() -> None: - """RED when: a pyproject cap has no exact ignore range, or an ignore entry outlives its cap.""" + """RED when: a sub-major cap has no exact ignore range, or an ignore entry outlives its cap.""" problems = _violations(*_load()) assert not problems, "\n".join(problems) @@ -247,6 +256,8 @@ def test_every_cap_has_an_ignore_entry_or_a_stated_exemption() -> None: ("3.1", False), ("1.0.1", False), ("4.0.0rc1", False), + ("4.post1", False), + ("1!4", False), ], ) def test_major_level(start: str, major: bool) -> None: @@ -283,7 +294,7 @@ def _fake_entry(dependabot: dict[str, Any], name: str) -> dict[str, Any]: def _fixture() -> tuple[dict[str, Any], dict[str, Any]]: - """Copies of the real files, plus the two made-up caps with correct entries.""" + """Copies of the real files, plus the made-up caps: two with correct entries, one major.""" pyproject, dependabot = copy.deepcopy(_load()) pyproject["project"]["dependencies"] += [_FAKE_CAP, _FAKE_MAJOR] pyproject.setdefault("dependency-groups", {})["mefor-fake"] = [_FAKE_PIN] @@ -306,8 +317,14 @@ def mutate(_p: dict[str, Any], d: dict[str, Any]) -> None: return mutate +def _looser_cap_elsewhere(p: dict[str, Any], _d: dict[str, Any]) -> None: + p["dependency-groups"]["mefor-fake"].append("mefor-fake-cap<4") + + @pytest.mark.parametrize( - "mutate", [None, _major_entry(">=3.0.0")], ids=["unbroken", "major-cap-opt-in-entry"] + "mutate", + [None, _major_entry(">=3.0.0"), _looser_cap_elsewhere], + ids=["unbroken", "major-cap-opt-in-entry", "looser-cap-on-same-package"], ) def test_the_mutation_fixture_is_clean(mutate: _Mutation | None) -> None: """Control: the arms below discriminate only if the unbroken fixture passes. The fixture holds a From 230cf26852fdea79f7e4396b3f490731849dc18d Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 23:07:40 -0500 Subject: [PATCH 10/14] test(startup): keep the probe's exit code, cover tray.branding, strip PYTHON_LAZY_IMPORTS (BACKLOG #2514) Code-review round 1 findings 1-3: - the CLI probe swallowed SystemExit, so a refused argument passed; - the tray first process also imports tray.branding before re-exec; - an inherited PYTHON_LAZY_IMPORTS would override both arms. Proposed PR title: Defer CLI and tray startup imports with __lazy_modules__ (PEP 810), plus an advisory startup-budget test (BACKLOG #2514) Proposed ledger banner: BUILT -- __lazy_modules__ in config/__init__, __main__, tray/__main__, tray/config; inert on 3.14, on 3.15.0b3 CLI --help 273 -> 186 modules, tray first process 204 -> 120; budget test skips on 3.14 and asserts on 3.15. --- tests/test_startup_import_budget.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_startup_import_budget.py b/tests/test_startup_import_budget.py index 660bffed2..e977857f8 100644 --- a/tests/test_startup_import_budget.py +++ b/tests/test_startup_import_budget.py @@ -13,8 +13,9 @@ Each probe runs in a FRESH interpreter, because the pytest process has already imported most of the engine. The CLI probe runs the package through runpy, as `python -m messagefoundry` does, so -`messagefoundry.__main__` is never in `sys.modules`. The tray probe imports the entry module, which -is what the first process pays before `main()` re-execs as the branded child. +`messagefoundry.__main__` is never in `sys.modules`, and it keeps the CLI's exit code, so a refused +argument fails the probe. The tray probe imports the entry module and `tray.branding`, which is +what the first process loads before `main()` re-execs as the branded child. The control arm runs the same probe with every import forced eager: natively on 3.14, and through `sys.set_lazy_imports_filter` on 3.15. It asserts the deferred modules DO load there, so a @@ -25,6 +26,7 @@ from __future__ import annotations import json +import os import subprocess import sys from pathlib import Path @@ -90,6 +92,7 @@ class _Probe(NamedTuple): "messagefoundry.secretscrub", "messagefoundry.tray", "messagefoundry.tray.__main__", + "messagefoundry.tray.branding", "messagefoundry.tray.config", "messagefoundry.tray.logscrub", } @@ -109,20 +112,26 @@ class _Probe(NamedTuple): runpy.run_module("messagefoundry", run_name="__main__") else: import messagefoundry.tray.__main__ -except SystemExit: - pass + import messagefoundry.tray.branding # main() imports it before the re-exec + code = 0 +except SystemExit as exc: + code = exc.code finally: with open(out, "w", encoding="utf-8") as fh: json.dump(sorted(sys.modules), fh) +sys.exit(code) """ def _loaded(tmp_path: Path, probe: _Probe, *, mode: str) -> set[str]: out = tmp_path / "modules.json" + # PYTHON_LAZY_IMPORTS would override the lists in both arms, so the probe never inherits it. + env = {k: v for k, v in os.environ.items() if k != "PYTHON_LAZY_IMPORTS"} proc = subprocess.run( [sys.executable, "-c", _PROBE_CODE, str(out), mode, probe.kind, *probe.args], capture_output=True, text=True, + env=env, # Under the suite's 60 s pytest-timeout, so a hung probe still reports its stderr. timeout=50, ) From 602673ad714cabd090b072bccc7cca19da15aa7a Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 23:08:35 -0500 Subject: [PATCH 11/14] test(generators): pin the ORC-to-OBR number hand-over; changelog split (BACKLOG #2497) Code review round 2: no correctness defect. Its test gaps are closed here. One test pins that an OBR takes its ORC's numbers once. Another pins that the hand-over moves only OBR-2/3 and no later random value. is_choice_group joins validate.__all__. The OBR numbering fix moves to its own fixed fragment. Proposed PR title: Generator emits one alternative of a choice group, so ORM^O01 carries an order detail; OBR repeats its ORC numbers (BACKLOG #2497; #2499 measured) Proposed ledger banners: #2497 SHIPPED on this branch: _emit picks one choice alternative, OBR by default; ORM^O01 ends ORC then OBR and passes strict validation at v2.5.1, the only version the generator writes. #2499 OPEN, prescription measured as a no-op: hl7apy's parser places a segment only by exact name and drops one it cannot place, and ANYHL7SEGMENT never matches a real segment. A table alternative changes no verdict: 0 of 120 differ, against a control that differs in 4. The real reject, ORC then a non-OBR detail then NTE/VAR/OBX, needs a parser-side decision on what "etc." names. Owner or spec question. --- changelog.d/2497.changed.md | 7 ++----- changelog.d/2497.fixed.md | 4 ++++ messagefoundry/generators/_core.py | 3 ++- messagefoundry/parsing/validate.py | 2 +- tests/test_generators_choice.py | 33 ++++++++++++++++++++++++++++++ 5 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 changelog.d/2497.fixed.md diff --git a/changelog.d/2497.changed.md b/changelog.d/2497.changed.md index 0ae209575..ceb0cb772 100644 --- a/changelog.d/2497.changed.md +++ b/changelog.d/2497.changed.md @@ -2,8 +2,5 @@ leave ORDER_DETAIL out, because it emitted every required child of a group, and the order detail's OBR/RQD/RQ1/RXO/ODS/ODT group is a choice. It now emits one alternative of a choice group, OBR by default. Each generated ORM^O01 now ends ORC then OBR and passes strict - validation. (`BACKLOG #2497`) -- **A generated OBR now repeats its order's placer and filler numbers.** OBR-2 and OBR-3 drew - fresh numbers instead of copying ORC-2 and ORC-3, as HL7 requires. Every generated message that - carries an ORC then an OBR changes in those two fields and nowhere else: ORM, OML, MDM and ORU. - A corpus generated before this change differs in those files. (`BACKLOG #2497`) + validation, so a corpus generated before this change differs in its ORM files. + (`BACKLOG #2497`) diff --git a/changelog.d/2497.fixed.md b/changelog.d/2497.fixed.md new file mode 100644 index 000000000..3e12ae861 --- /dev/null +++ b/changelog.d/2497.fixed.md @@ -0,0 +1,4 @@ +- **A generated OBR now repeats its order's placer and filler numbers.** OBR-2 and OBR-3 drew + fresh numbers instead of copying ORC-2 and ORC-3, as HL7 requires. In generated OML, MDM and + ORU messages that carry an ORC then an OBR, only those two fields change, so a corpus + generated before this change differs there. (`BACKLOG #2497`) diff --git a/messagefoundry/generators/_core.py b/messagefoundry/generators/_core.py index a5c13615e..8d7166c6b 100644 --- a/messagefoundry/generators/_core.py +++ b/messagefoundry/generators/_core.py @@ -6,7 +6,8 @@ trigger→structure map, its segment builders, and which optional segments to sprinkle in. Generation walks hl7apy's own 2.5.1 reference tree (``MESSAGES[structure]``): for each structure we emit required segments in order plus a valid random subset of allow-listed -optionals, then gate every message through the engine's strict validator before it counts. +optionals, and one alternative of each choice group, then gate every message through the +engine's strict validator before it counts. A type module contributes builders only for its *own* segments; the broadly shared ones (MSH/EVN/PID/PV1/…) live here in :data:`SHARED_BUILDERS`. All data is synthetic — no real PHI. diff --git a/messagefoundry/parsing/validate.py b/messagefoundry/parsing/validate.py index d405f6e51..f51a2780b 100644 --- a/messagefoundry/parsing/validate.py +++ b/messagefoundry/parsing/validate.py @@ -48,7 +48,7 @@ normalize, ) -__all__ = ["ValidationResult", "validate"] +__all__ = ["ValidationResult", "is_choice_group", "validate"] logger = logging.getLogger(__name__) diff --git a/tests/test_generators_choice.py b/tests/test_generators_choice.py index ec064664c..9c699cbf3 100644 --- a/tests/test_generators_choice.py +++ b/tests/test_generators_choice.py @@ -52,6 +52,39 @@ def test_generated_orm_o01_has_one_order_detail_and_is_strictly_valid( assert ok, errors +def test_an_obr_takes_its_orc_numbers_once() -> None: + """The ORC hands its numbers to the next OBR only; a second OBR draws its own.""" + rng = random.Random(0) + ctx = _core.Ctx("OML", "O21", "OML_O21", "CID", _core.BASE_DT, "A", "F", "B", "G") + orc = _core._build_orc(rng, ctx).split("|") + first = _core._build_obr(rng, ctx).split("|") + second = _core._build_obr(rng, ctx).split("|") + assert first[2:4] == orc[2:4] + assert second[2:4] != orc[2:4] + + +def test_handing_over_order_numbers_moves_no_other_value(monkeypatch: pytest.MonkeyPatch) -> None: + """The OBR still draws its own numbers when an ORC supplies them, so the random stream, and + every later value of OML, MDM and ORU messages, stays where it was. Only OBR-2/3 move.""" + real_orc = _core.SHARED_BUILDERS["ORC"] + now = _core.generate_message("OML", "O21", 1) + + def orc_handing_nothing(rng: random.Random, ctx: _core.Ctx) -> str: + text = real_orc(rng, ctx) + ctx.order_numbers = None + return text + + monkeypatch.setitem(_core.SHARED_BUILDERS, "ORC", orc_handing_nothing) + before = _core.generate_message("OML", "O21", 1) + changed = [ + (a.split("|"), b.split("|")) + for a, b in zip(now.split("\r"), before.split("\r"), strict=True) + if a != b + ] + assert [a[0] for a, _ in changed] == ["OBR"] # the control: the hand-over did change it + assert [a[:2] + a[4:] for a, _ in changed] == [b[:2] + b[4:] for _, b in changed] + + def test_obr_is_picked_without_drawing_from_the_seed() -> None: """OBR leaves the random stream alone, so adding a choice moves no other generated value.""" rng = random.Random("seed") From b01af4b81357cc095cfb86e0291dbe90a2b9e020 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 23:10:08 -0500 Subject: [PATCH 12/14] Dependabot: restate the pynetdicom, pydicom and webauthn caps as ignore ranges, and test every pyproject cap against one (BACKLOG #2505) Manager revision: the class rule is withdrawn. python-deps-major is one batch on "*", so a widened major cap holds that batch hostage too, and python-security is not split by update type. Every cap now needs an exact entry unless it is a named exemption: hvac (the [vault] comment), hatchling ([build-system]) and the exact group pins. A named exemption is also a must-not: an entry for it reds, with an arm over hvac and hatchling. dependabot.yml restores the webauthn and pynetdicom >=4.0.0 entries beside pydicom >=3.1.0, states the rule and the named exemptions once, and says pyproject records no reason for pynetdicom<4; its history stops at root 546426285b. Kept: tightest cap per package, table-scoped exemptions, the exact range, the extras violation and the [build-system] census. _is_major_level and the class-rule arms are gone. Ledger banner: BUILT 2026-09-30 on branch b2505-cap-ignores -- uv ignore entries for pydicom >=3.1.0, webauthn >=4.0.0 and pynetdicom >=4.0.0, and tests/test_dependabot_cap_ignores.py holding every pyproject cap to an exact entry or a named exemption (hvac, hatchling, the exact group pins), and every entry to a live cap. --- .github/dependabot.yml | 30 +++++---- tests/test_dependabot_cap_ignores.py | 95 ++++++++++------------------ 2 files changed, 52 insertions(+), 73 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5473bf086..9f06087a2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -57,8 +57,7 @@ updates: default-days: 5 semver-major-days: 7 # Dependabot WIDENS a declared cap instead of respecting it, so a load-bearing upper bound in - # pyproject.toml below major level must be restated here as a version range or the cap is - # decorative; the class rule further down says why a major-level cap needs none. PR #66 rewrote + # pyproject.toml must be restated here as a version range or the cap is decorative. PR #66 rewrote # `annotated-types<0.8` -> `<0.9` and `ruff>=0.4,<0.16` -> `<0.17`; the # resync then propagated 0.8.0/0.16.0 into constraints.lock, so ci.yml's `--constraint` pinned CI # *to* the broken versions. `python-deps` groups every minor and patch on `*`, and both of those @@ -119,17 +118,18 @@ updates: # `uvicorn`, not `uvicorn[standard]`, so the bare name here matches it as written. Lift this # entry in the same PR that lifts the cap. # - # `pydicom` is the same WIDENED-CAP case (BACKLOG #2505). Its `<3.1` is a MINOR cap, so without - # its entry `python-deps` would widen it in the routine minor-and-patch batch. + # `pydicom`, `webauthn` and `pynetdicom` are the same WIDENED-CAP case (BACKLOG #2505). A major + # cap needs its entry as much as a minor one: `python-deps-major` is one batch on `*`, so a + # widened major cap holds that batch hostage too. pyproject.toml records no reason for + # `pynetdicom<4`, and its history stops at the root commit 546426285b; the entry mirrors the cap + # as-is. # - # THE CLASS RULE (BACKLOG #2505): a cap at MINOR level or lower needs an entry here. A MAJOR-level - # cap, one whose first excluded version is N.0.0 for N above zero (`<4`, but also `~=3.0` or - # `==3.*`; so `hvac<3`, `webauthn<4` and `pynetdicom<4`), needs none, and normally gets - # none; the reasoning is the one pyproject.toml gives beside `hvac`'s cap, so read it there. An - # entry for a major cap stays legal as an opt-in. Some other caps have NO entry on purpose: at - # least `hatchling`'s build-system pin and the exact group pins under "NOT IGNORED, deliberately" - # above. tests/test_dependabot_cap_ignores.py holds the rule, names each exemption with its - # reason, and holds every entry here to a cap that still exists. + # THE RULE (BACKLOG #2505): every upper bound in pyproject.toml has an entry here, exactly `>=` the + # first version it excludes, unless it is a NAMED exemption. The named exemptions, which must NOT + # have an entry: `hvac` (the comment above the [vault] extra in pyproject.toml), `hatchling` (the + # [build-system] comment there), and the exact group pins under "NOT IGNORED, deliberately" above. + # tests/test_dependabot_cap_ignores.py holds the rule and the list, and holds every entry here to a + # cap that still exists. ignore: - dependency-name: "ruff" versions: [">=0.16.0"] @@ -142,6 +142,12 @@ updates: # Why: the comment above the [dicom] extra in pyproject.toml. - dependency-name: "pydicom" versions: [">=3.1.0"] + # Why: the comment above the [webauthn] extra in pyproject.toml. + - dependency-name: "webauthn" + versions: [">=4.0.0"] + # No reason recorded in pyproject.toml; see the paragraph above. + - dependency-name: "pynetdicom" + versions: [">=4.0.0"] groups: # Version-update grouping (applies-to defaults to version-updates), SPLIT BY UPDATE TYPE (owner # decision 2026-09-30) so one bad major cannot hold the routine minors and patches hostage. diff --git a/tests/test_dependabot_cap_ignores.py b/tests/test_dependabot_cap_ignores.py index ecc6a7638..e111fee34 100644 --- a/tests/test_dependabot_cap_ignores.py +++ b/tests/test_dependabot_cap_ignores.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors -"""Every pyproject.toml cap below major level has a Dependabot ignore range, or a stated exemption. +"""Every upper bound in pyproject.toml has a Dependabot ignore range, or a named exemption. Dependabot WIDENS a declared cap instead of respecting it; ``.github/dependabot.yml`` says so above its uv ``ignore`` list, with the history. Until BACKLOG #2505 nothing held the two files together, @@ -10,12 +10,11 @@ * Every upper bound in ``[build-system].requires``, ``[project.dependencies]``, ``[project.optional-dependencies]`` and ``[dependency-groups]`` (``<``, ``~=``, ``==``, or an - ``==X.*`` wildcard) at MINOR level or lower has a uv-ecosystem ``ignore`` entry whose range is - exactly ``>=`` the version ``_cap_of`` derives. Or the package is in ``_EXEMPT`` below, in the - table it names, with its reason. -* A MAJOR-level cap (one whose derived version is ``N.0.0`` with ``N`` above zero, such as ``<4``) - is exempt by class, per the class rule in dependabot.yml. An entry for one is a legal opt-in, and - is then held to the same exact range. + ``==X.*`` wildcard) has a uv-ecosystem ``ignore`` entry whose range is exactly ``>=`` the version + ``_cap_of`` derives from the package's tightest cap. Or the package is named in ``_EXEMPT`` + below, in the table it names, with its reason. +* A package named in ``_EXEMPT`` must NOT have an entry. That is the must-not list: hvac's cap is + deliberately not mirrored, and pyproject.toml says why. * Every uv ``ignore`` entry names a package that pyproject.toml caps. dependabot.yml says to lift a cap and delete its entry in the same PR; this is what makes that rule fail when skipped. It is also this test's positive control: a parser that found no caps would red on every real entry, so @@ -60,11 +59,16 @@ "[dependency-groups] would freeze the hash-pinned CI toolchain" ) -#: Every cap below major level with no ignore entry: package -> (its table, the reason). The table is a -#: prefix of ``Cap.where``. A cap on the same package anywhere else is not exempt. An exemption in -#: ``[dependency-groups]`` must also be an exact ``==`` pin. Read each reason where it points. +#: Every cap with no ignore entry, by name: package -> (its table, the reason). Each is also a +#: MUST-NOT: an ignore entry for it reds. The table is a prefix of ``Cap.where``, so a cap on the +#: same package anywhere else is not exempt. An exemption in ``[dependency-groups]`` must also be an +#: exact ``==`` pin. Read each reason where it points. _EXEMPT: dict[str, tuple[str, str]] = { **dict.fromkeys(_GROUP_PINS, (_GROUPS, _GROUP_PIN_REASON)), + "hvac": ( + "[project.optional-dependencies].vault", + "pyproject.toml's comment above the [vault] extra says hvac is deliberately not mirrored", + ), "hatchling": ( "[build-system].requires", "pyproject.toml's [build-system] comment expects Dependabot to bump this pin", @@ -82,19 +86,6 @@ class Cap(NamedTuple): ignore_from: Version -def _is_major_level(start: Version) -> bool: - """Whether an ignore range from ``start`` would hold back only new majors: 4 yes, 0.50 no. - - A ``0.x`` bump counts as a minor, as Dependabot reads it by version position. - """ - return ( - start.epoch == 0 - and start.major > 0 - and not any(start.release[1:]) - and not (start.is_prerelease or start.is_postrelease) - ) - - def _is_pin(spec: Specifier) -> bool: return spec.operator == "==" and not spec.version.endswith(".*") @@ -207,11 +198,10 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s continue # one ignore entry serves a package; the tightest cap is the one it restates entries = ignores.get(cap.package) if not entries: - if not _is_major_level(cap.ignore_from): # a major cap is exempt by class - problems.append( - f"{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with " - f'versions ["{want}"], or an exemption with its reason in {Path(__file__).name}' - ) + problems.append( + f"{label} has no uv ignore entry in {_DEPENDABOT.name}; add one with " + f'versions ["{want}"], or a named exemption with its reason in {Path(__file__).name}' + ) continue if any("update-types" in e for e in entries): problems.append( @@ -242,28 +232,11 @@ def _load() -> tuple[dict[str, Any], dict[str, Any]]: def test_every_cap_has_an_ignore_entry_or_a_stated_exemption() -> None: - """RED when: a sub-major cap has no exact ignore range, or an ignore entry outlives its cap.""" + """RED when: a cap has no exact ignore range, an exempt package has one, or one outlives its cap.""" problems = _violations(*_load()) assert not problems, "\n".join(problems) -@pytest.mark.parametrize( - ("start", "major"), - [ - ("4", True), - ("4.0.0", True), - ("0.50", False), - ("3.1", False), - ("1.0.1", False), - ("4.0.0rc1", False), - ("4.post1", False), - ("1!4", False), - ], -) -def test_major_level(start: str, major: bool) -> None: - assert _is_major_level(Version(start)) is major - - @pytest.mark.parametrize( ("spec", "start"), [ @@ -286,7 +259,6 @@ def test_cap_arithmetic(spec: str, start: str | None) -> None: # so a legitimate cap lift in pyproject.toml cannot crash an arm. _FAKE_CAP = "mefor-fake-cap>=1.0,<2.5" _FAKE_PIN = "mefor-fake-pin==3.2.1" -_FAKE_MAJOR = "mefor-fake-major>=1.0,<3" # exempt by class, so the fixture gives it no entry def _fake_entry(dependabot: dict[str, Any], name: str) -> dict[str, Any]: @@ -294,9 +266,9 @@ def _fake_entry(dependabot: dict[str, Any], name: str) -> dict[str, Any]: def _fixture() -> tuple[dict[str, Any], dict[str, Any]]: - """Copies of the real files, plus the made-up caps: two with correct entries, one major.""" + """Copies of the real files, plus the two made-up caps with correct entries.""" pyproject, dependabot = copy.deepcopy(_load()) - pyproject["project"]["dependencies"] += [_FAKE_CAP, _FAKE_MAJOR] + pyproject["project"]["dependencies"].append(_FAKE_CAP) pyproject.setdefault("dependency-groups", {})["mefor-fake"] = [_FAKE_PIN] _uv_entry(dependabot).setdefault("ignore", []).extend( [ @@ -310,25 +282,18 @@ def _fixture() -> tuple[dict[str, Any], dict[str, Any]]: _Mutation = Callable[[dict[str, Any], dict[str, Any]], None] -def _major_entry(rng: str) -> _Mutation: - def mutate(_p: dict[str, Any], d: dict[str, Any]) -> None: - _uv_entry(d)["ignore"].append({"dependency-name": "mefor-fake-major", "versions": [rng]}) - - return mutate - - def _looser_cap_elsewhere(p: dict[str, Any], _d: dict[str, Any]) -> None: p["dependency-groups"]["mefor-fake"].append("mefor-fake-cap<4") @pytest.mark.parametrize( "mutate", - [None, _major_entry(">=3.0.0"), _looser_cap_elsewhere], - ids=["unbroken", "major-cap-opt-in-entry", "looser-cap-on-same-package"], + [None, _looser_cap_elsewhere], + ids=["unbroken", "looser-cap-on-same-package"], ) def test_the_mutation_fixture_is_clean(mutate: _Mutation | None) -> None: - """Control: the arms below discriminate only if the unbroken fixture passes. The fixture holds a - major cap with no entry, and an opt-in exact entry for it must stay legal.""" + """Control: the arms below discriminate only if the unbroken fixture passes. A looser second cap + on a package is matched by nothing; its tightest cap's entry serves it.""" pyproject, dependabot = _fixture() if mutate: mutate(pyproject, dependabot) @@ -384,7 +349,6 @@ def _lift_cap(p: dict[str, Any], _d: dict[str, Any]) -> None: _set_entry("mefor-fake-pin", "versions", [">=3.2.0"]), _EXACT, id="blocks-pin" ), pytest.param(_set_entry("mefor-fake-pin", "versions", [">=4.0.0"]), _EXACT, id="pin-gap"), - pytest.param(_major_entry(">=2.0.0"), _EXACT, id="major-opt-in-not-exact"), pytest.param(_to_update_types, "update-types", id="update-types"), pytest.param(_lift_cap, "does not cap", id="stale-entry"), pytest.param( @@ -448,3 +412,12 @@ def test_the_checker_reds_on_a_broken_exemption( pyproject["project"]["dependencies"].append(req) problems = _violations(pyproject, dependabot) assert any(expect in p for p in problems), f"no violation mentioning {expect!r}: {problems}" + + +@pytest.mark.parametrize("name", sorted(set(_EXEMPT) - _GROUP_PINS)) +def test_a_named_exemption_must_not_have_an_entry(name: str) -> None: + """Mutation arm for the must-not list, over the real named exemptions (hvac, hatchling).""" + pyproject, dependabot = _fixture() + _uv_entry(dependabot)["ignore"].append({"dependency-name": name, "versions": [">=999"]}) + problems = _violations(pyproject, dependabot) + assert any(f"{name}" in p and "yet has an ignore entry" in p for p in problems), problems From ad670911f2a59b0cdb5ab8b6548dc6f40be5afe5 Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Wed, 30 Sep 2026 23:18:10 -0500 Subject: [PATCH 13/14] Dependabot: restate the pynetdicom, pydicom and webauthn caps as ignore ranges, and test every pyproject cap against one (BACKLOG #2505) Review round on the per-package revert. An exemption's table now matches exactly or at a "." boundary, so an extra named vault-legacy is not covered by hvac's [vault] exemption; an arm holds it. A major cap with no entry has its own red arm. The must-not arm names hvac and hatchling explicitly, so dropping either reds instead of skipping. dependabot.yml: the header admits named exemptions; the rule says how the range is derived for a pin, points at _EXEMPT for the list rather than closing it, and excepts sigstore. The pynetdicom note is corrected: the clone is shallow at 546426285b, and the cap is already in 5fa6db9f42, the 2026-07-06 history-reset snapshot. The [dicom] comment gives pynetdicom's pydicom requirement but no reason for its own <4. Ledger banner: BUILT 2026-09-30 on branch b2505-cap-ignores -- uv ignore entries for pydicom >=3.1.0, webauthn >=4.0.0 and pynetdicom >=4.0.0, and tests/test_dependabot_cap_ignores.py holding every pyproject cap to an exact entry or a named exemption (hvac, hatchling, the exact group pins), and every entry to a live cap. --- .github/dependabot.yml | 23 ++++++++++++----------- tests/test_dependabot_cap_ignores.py | 20 +++++++++++++++++--- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9f06087a2..6df5627cc 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -57,7 +57,8 @@ updates: default-days: 5 semver-major-days: 7 # Dependabot WIDENS a declared cap instead of respecting it, so a load-bearing upper bound in - # pyproject.toml must be restated here as a version range or the cap is decorative. PR #66 rewrote + # pyproject.toml must be restated here as a version range or the cap is decorative, unless it is + # a named exemption (THE RULE, below). PR #66 rewrote # `annotated-types<0.8` -> `<0.9` and `ruff>=0.4,<0.16` -> `<0.17`; the # resync then propagated 0.8.0/0.16.0 into constraints.lock, so ci.yml's `--constraint` pinned CI # *to* the broken versions. `python-deps` groups every minor and patch on `*`, and both of those @@ -120,16 +121,16 @@ updates: # # `pydicom`, `webauthn` and `pynetdicom` are the same WIDENED-CAP case (BACKLOG #2505). A major # cap needs its entry as much as a minor one: `python-deps-major` is one batch on `*`, so a - # widened major cap holds that batch hostage too. pyproject.toml records no reason for - # `pynetdicom<4`, and its history stops at the root commit 546426285b; the entry mirrors the cap - # as-is. + # widened major cap holds that batch hostage too. pyproject.toml's [dicom] comment gives + # pynetdicom's own pydicom>=3,<4 requirement but no reason for `pynetdicom<4` itself; the cap is + # already in 5fa6db9f42, the 2026-07-06 history-reset snapshot, so the entry mirrors it as-is. # - # THE RULE (BACKLOG #2505): every upper bound in pyproject.toml has an entry here, exactly `>=` the - # first version it excludes, unless it is a NAMED exemption. The named exemptions, which must NOT - # have an entry: `hvac` (the comment above the [vault] extra in pyproject.toml), `hatchling` (the - # [build-system] comment there), and the exact group pins under "NOT IGNORED, deliberately" above. - # tests/test_dependabot_cap_ignores.py holds the rule and the list, and holds every entry here to a - # cap that still exists. + # THE RULE (BACKLOG #2505): every upper bound in pyproject.toml has an entry here, exactly `>=` + # the version the test derives (a `<` cap's bound; for an exact `==` pin, the next minor), unless + # it is a NAMED exemption. A named exemption must NOT have an entry. The list, with each reason, + # is `_EXEMPT` in tests/test_dependabot_cap_ignores.py; it holds at least `hvac`, `hatchling` and + # the exact group pins under "NOT IGNORED, deliberately" above other than `sigstore`. The test + # also holds every entry here to a cap that still exists. ignore: - dependency-name: "ruff" versions: [">=0.16.0"] @@ -145,7 +146,7 @@ updates: # Why: the comment above the [webauthn] extra in pyproject.toml. - dependency-name: "webauthn" versions: [">=4.0.0"] - # No reason recorded in pyproject.toml; see the paragraph above. + # No reason for this cap is recorded in pyproject.toml; see the paragraph above. - dependency-name: "pynetdicom" versions: [">=4.0.0"] groups: diff --git a/tests/test_dependabot_cap_ignores.py b/tests/test_dependabot_cap_ignores.py index e111fee34..b344b4076 100644 --- a/tests/test_dependabot_cap_ignores.py +++ b/tests/test_dependabot_cap_ignores.py @@ -187,7 +187,7 @@ def _violations(pyproject: dict[str, Any], dependabot: dict[str, Any]) -> list[s want = f">={cap.ignore_from}" if cap.package in _EXEMPT: table, reason = _EXEMPT[cap.package] - if not cap.where.startswith(table): + if cap.where != table and not cap.where.startswith(f"{table}."): problems.append(f"{label} is exempt only in {table}, and this cap is elsewhere") elif table == _GROUPS and not cap.exact_pin: problems.append(f"{label} is exempt as a group `==` pin, but is not one") @@ -319,6 +319,10 @@ def _to_update_types(_p: dict[str, Any], d: dict[str, Any]) -> None: entry["update-types"] = ["version-update:semver-major"] +def _add_major_cap(p: dict[str, Any], _d: dict[str, Any]) -> None: + p["project"]["dependencies"].append("mefor-fake-major>=1.0,<3") + + def _lift_cap(p: dict[str, Any], _d: dict[str, Any]) -> None: deps = p["project"]["dependencies"] deps[deps.index(_FAKE_CAP)] = "mefor-fake-cap>=1.0" @@ -332,6 +336,7 @@ def _lift_cap(p: dict[str, Any], _d: dict[str, Any]) -> None: ("mutate", "expect"), [ pytest.param(_drop_entry, "no uv ignore entry", id="entry-deleted"), + pytest.param(_add_major_cap, "no uv ignore entry", id="major-cap-without-entry"), pytest.param(_set_entry(*_CAP_VERSIONS, [">=2.6.0"]), _EXACT, id="gap-at-cap"), pytest.param(_set_entry(*_CAP_VERSIONS, [">=2.4.0"]), _EXACT, id="freezes"), pytest.param( @@ -392,6 +397,13 @@ def test_the_checker_reds_on_a_mutated_config(mutate: _Mutation, expect: str) -> pytest.param( "mefor-gp", _GROUPS, ("group", "mefor-gp>=1,<2"), "is not one", id="group-cap-not-a-pin" ), + pytest.param( + "mefor-gp", + "[project.optional-dependencies].fake", + ("extra", "mefor-gp>=1,<2"), + "is exempt only in", + id="extra-name-prefix-is-not-the-table", + ), ], ) def test_the_checker_reds_on_a_broken_exemption( @@ -408,16 +420,18 @@ def test_the_checker_reds_on_a_broken_exemption( where, req = added if where == "group": pyproject["dependency-groups"]["mefor-fake"].append(req) + elif where == "extra": + pyproject["project"].setdefault("optional-dependencies", {})["fake-legacy"] = [req] else: pyproject["project"]["dependencies"].append(req) problems = _violations(pyproject, dependabot) assert any(expect in p for p in problems), f"no violation mentioning {expect!r}: {problems}" -@pytest.mark.parametrize("name", sorted(set(_EXEMPT) - _GROUP_PINS)) +@pytest.mark.parametrize("name", ["hvac", "hatchling"]) # explicit, so dropping one reds here def test_a_named_exemption_must_not_have_an_entry(name: str) -> None: """Mutation arm for the must-not list, over the real named exemptions (hvac, hatchling).""" pyproject, dependabot = _fixture() _uv_entry(dependabot)["ignore"].append({"dependency-name": name, "versions": [">=999"]}) problems = _violations(pyproject, dependabot) - assert any(f"{name}" in p and "yet has an ignore entry" in p for p in problems), problems + assert any(name in p and "yet has an ignore entry" in p for p in problems), problems From 80bd731cc8048b69db0404a6d70ff2d53db1f69b Mon Sep 17 00:00:00 2001 From: wshallwshall Date: Thu, 1 Oct 2026 02:53:22 -0500 Subject: [PATCH 14/14] test(generators): import all_types so the order-number test registers OML alone test_handing_over_order_numbers_moves_no_other_value raised KeyError 'OML' when run by itself: the file imported only generators.orm, so OML was registered only when another test module happened to import all_types first. Import generators.all_types instead, as the other generator test files do; it imports orm too, so ORM still registers. Proved by running the one test alone with -p no:randomly. Also merges origin/main in. --- tests/test_generators_choice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_generators_choice.py b/tests/test_generators_choice.py index 9c699cbf3..5c8a082d4 100644 --- a/tests/test_generators_choice.py +++ b/tests/test_generators_choice.py @@ -14,7 +14,7 @@ import pytest -from messagefoundry.generators import _core, orm # noqa: F401 (importing orm registers ORM) +from messagefoundry.generators import _core, all_types # noqa: F401 (registers ORM and OML) from messagefoundry.parsing import Peek from messagefoundry.parsing.validate import _SEQUENCES_LABELLED_CHOICE