From 3f482e73d0861c552b12f2cb9990c390c4df475a Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Sun, 11 Oct 2026 21:28:19 -0400 Subject: [PATCH 1/8] feat(python): eject hands over the emitters a generator writes through; api-docs is ejectable An ejected Python generator used to be one file over packaged emitters, so a defect in the shared-enum, M:N, prompt-tier or api-reference writers still waited for a release. Eject now copies those emitters with the generator into codegen/runtime/, through each registry entry's existing runtime list, and the copy reads them from there. The emitters moved into metaobjects.codegen.runtime; their old import paths stay as deprecated aliases so a copy ejected earlier keeps running. Naming and type-mapping tables, the model walks and the generator base stay in the package as the engine. api-docs is registered for Python and ejectable. The api reference is a generator (docs_surface = "api") that `metaobjects docs` runs; an owned copy is listed under the new docs.generators config block. `gen` and `verify --codegen` skip a docs generator and say so. The page builder reads the router gates from shared_predicates (new emits_writable_router, moved is_read_only_routed), the module an owned routes and filter-allowlist read, so the reference describes an owned copy. test_eject_alone derives its list from the registry: each ejectable generator is ejected alone into an empty project, checked against a written engine surface, run with every packaged emit module unimportable, and compared byte for byte with the packaged generator. Packaged output is unchanged. --- CHANGELOG.md | 59 ++ docs/features/own-your-codegen.md | 115 +++- docs/ports/python.md | 10 + .../registry.json | 4 +- .../src/metaobjects/apidocs/__init__.py | 34 +- .../src/metaobjects/apidocs/api_model.py | 113 +--- .../python/src/metaobjects/apidocs/builder.py | 510 +----------------- .../python/src/metaobjects/apidocs/naming.py | 2 +- .../python/src/metaobjects/apidocs/paths.py | 77 +-- .../src/metaobjects/apidocs/renderer.py | 187 +------ server/python/src/metaobjects/cli.py | 74 +-- .../python/src/metaobjects/codegen/_moved.py | 23 + .../src/metaobjects/codegen/docs_run.py | 91 ++++ .../python/src/metaobjects/codegen/eject.py | 28 +- .../codegen/extract_delegate_emitter.py | 385 +------------ .../codegen/extract_schema_emitter.py | 141 +---- .../codegen/fr010_field_mapping.py | 230 +------- .../src/metaobjects/codegen/generator.py | 22 + .../metaobjects/codegen/generator_registry.py | 51 +- .../codegen/generators/api_docs_generator.py | 68 +++ .../codegen/generators/entity_model.py | 7 +- .../codegen/generators/extractor_generator.py | 4 +- .../generators/filter_allowlist_generator.py | 2 +- .../codegen/generators/fr019_shared_enum.py | 156 +----- .../codegen/generators/m2m_codegen.py | 242 +-------- .../generators/output_parser_generator.py | 3 +- .../generators/output_prompt_generator.py | 3 +- .../codegen/generators/router_generator.py | 29 +- .../generators/trace_helper_generator.py | 3 +- .../codegen/metaobjects-config.schema.json | 12 + .../codegen/output_format_spec_emitter.py | 85 +-- .../src/metaobjects/codegen/project_config.py | 29 +- .../python/src/metaobjects/codegen/runner.py | 11 + .../metaobjects/codegen/runtime/__init__.py | 18 +- .../codegen/runtime/api_docs_builder.py | 488 +++++++++++++++++ .../codegen/runtime/api_docs_model.py | 111 ++++ .../codegen/runtime/api_docs_paths.py | 75 +++ .../codegen/runtime/api_docs_renderer.py | 185 +++++++ .../runtime/extract_delegate_emitter.py | 383 +++++++++++++ .../codegen/runtime/extract_schema_emitter.py | 139 +++++ .../codegen/runtime/fr010_field_mapping.py | 228 ++++++++ .../codegen/runtime/fr019_shared_enum.py | 154 ++++++ .../codegen/runtime/m2m_codegen.py | 240 +++++++++ .../runtime/output_format_spec_emitter.py | 83 +++ .../codegen/runtime/shared_predicates.py | 28 + .../tests/codegen/test_api_docs_paths.py | 2 +- .../python/tests/codegen/test_cli_registry.py | 4 +- .../tests/codegen/test_docs_generators.py | 205 +++++++ .../python/tests/codegen/test_eject_alone.py | 323 +++++++++++ .../tests/codegen/test_eject_independence.py | 7 +- .../tests/codegen/test_eject_runtime.py | 9 +- .../test_fr010_nested_object_prompt.py | 2 +- .../codegen/test_fr010_output_codegen.py | 2 +- .../test_fr019_shared_provided_conformance.py | 2 +- .../python/tests/codegen/test_m2m_codegen.py | 2 +- .../tests/codegen/test_moved_modules.py | 55 ++ .../tests/codegen/test_reverse_finders.py | 2 +- .../tests/codegen/test_router_autoset.py | 2 +- .../tests/codegen/test_router_generator.py | 2 +- .../test_router_generator_m2m_target_tph.py | 2 +- .../codegen/test_router_generator_tph_m2m.py | 2 +- .../test_router_generator_tph_m2m_shadowed.py | 2 +- .../python/tests/codegen/test_tph_codegen.py | 2 +- .../tests/integration/generated_m2m_app.py | 2 +- .../tests/integration/generated_tph_app.py | 2 +- .../test_m2m_target_tph_generated.py | 2 +- .../integration/test_tph_m2m_generated.py | 2 +- .../test_tph_m2m_shadowed_generated.py | 2 +- server/python/tests/test_reporting_inert.py | 6 +- 69 files changed, 3328 insertions(+), 2257 deletions(-) create mode 100644 server/python/src/metaobjects/codegen/_moved.py create mode 100644 server/python/src/metaobjects/codegen/docs_run.py create mode 100644 server/python/src/metaobjects/codegen/generators/api_docs_generator.py create mode 100644 server/python/src/metaobjects/codegen/runtime/api_docs_builder.py create mode 100644 server/python/src/metaobjects/codegen/runtime/api_docs_model.py create mode 100644 server/python/src/metaobjects/codegen/runtime/api_docs_paths.py create mode 100644 server/python/src/metaobjects/codegen/runtime/api_docs_renderer.py create mode 100644 server/python/src/metaobjects/codegen/runtime/extract_delegate_emitter.py create mode 100644 server/python/src/metaobjects/codegen/runtime/extract_schema_emitter.py create mode 100644 server/python/src/metaobjects/codegen/runtime/fr010_field_mapping.py create mode 100644 server/python/src/metaobjects/codegen/runtime/fr019_shared_enum.py create mode 100644 server/python/src/metaobjects/codegen/runtime/m2m_codegen.py create mode 100644 server/python/src/metaobjects/codegen/runtime/output_format_spec_emitter.py create mode 100644 server/python/tests/codegen/test_docs_generators.py create mode 100644 server/python/tests/codegen/test_eject_alone.py create mode 100644 server/python/tests/codegen/test_moved_modules.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fea01d41..c29e7b7ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -194,6 +194,55 @@ generator catalog and the Atlas evaluation are 1.2 work. calls one. The cross-port manifest lists `api-docs` for `csharp`. +- **Python: an ejected generator owns the emitters it writes through, and `api-docs` can be + ejected.** `metaobjects eject` used to copy one file per generator, so an owned `entity` + still called the shared-enum and M:N writers, and an owned `output-parser`, `extractor`, + `output-prompt` or `trace-helper` still called the prompt tier's field mapping, extract + delegate, extract schema and format-spec writers. A defect in any of them waited for a + release. Eject now copies those emitters with the generator, into `codegen/runtime/`, and the + copy reads them from there: `fr019_shared_enum` and `m2m_codegen` with `entity`; + `m2m_codegen` with `filter-allowlist`; `fr010_field_mapping` and `m2m_codegen` with `routes`; + `extract_delegate_emitter` and `fr010_field_mapping` with `output-parser` and `extractor`; + `output_format_spec_emitter` and `fr010_field_mapping` with `output-prompt`; + `extract_schema_emitter` and `fr010_field_mapping` with `trace-helper`. A file two generators + share is copied once and never overwritten, and `metaobjects gen --list` marks each copy + `identical` or `DIFFERS`. What stays in the package is the engine: the generator base and + file writer, `metaobjects.apidocs.naming`, `type_map` and `value_objects` (the naming and + type-mapping tables), and the model walks (`find_inbound`, `tph_plan`, + `instance_artifacts`). `template` is not ejectable: it has no emit logic of its own. Packaged + output is unchanged and gated to be. + - **`api-docs` is a generator you can own.** `metaobjects eject api-docs` copies the page + generator with its page model, builder, renderer and path math. The api reference is now a + generator (`docs_surface = "api"`) that `metaobjects docs` runs: an owned copy is listed + under the new `docs.generators` block of `metaobjects.config.yaml` and runs in place of the + packaged one. `gen` and `verify --codegen` skip a docs generator they find under a target + and say so. The reference asks `shared_predicates` which objects get a router (the new + `emits_writable_router` and the moved `is_read_only_routed`, beside `emits_router`), the + module your owned `routes` and `filter-allowlist` read, so one edit to the owned predicate + moves the router, the allowlist and the pages together. The cross-port manifest lists + `api-docs` for `python`. + - **Where things moved.** The emitters now live in `metaobjects.codegen.runtime` (a + module the helper-runtime copy already used): `extract_delegate_emitter`, + `fr010_field_mapping`, `output_format_spec_emitter`, `extract_schema_emitter`, + `fr019_shared_enum`, `m2m_codegen`, and the api reference as `api_docs_model`, + `api_docs_builder`, `api_docs_renderer` and `api_docs_paths`. `metaobjects.apidocs` keeps + `naming` and exports `PythonApiModelBuilder`, `PythonApiDocsRenderer` and the path + functions as before. New public API: `DocsRunFacts`, `GenContext.docs`, `DEFAULT_API_SUBDIR` + and `DOCS_SURFACE_API` in `metaobjects.codegen.generator`, `ProjectConfig.docs_generators`, + `GeneratorEntry.docs_surface`, and `metaobjects.codegen.docs_run`. + - **A copy you ejected before this release** keeps running: it imports the emitters from + their old paths, which are deprecated aliases (see **Deprecated**), so it calls the + packaged emitters as it always did. `metaobjects gen --list` marks it `DIFFERS: N behind`. + To own them, eject the same name into a scratch directory and copy the generator's + `runtime/` files over (see + [own-your-codegen](docs/features/own-your-codegen.md#the-emitters-a-generator-writes-through-come-with-it)). + - **A gate for the rule.** For every ejectable generator the registry lists, a test ejects it + alone into an empty project, fails on any import outside a written engine surface, runs it + with every packaged emit module made unimportable, and compares its output byte for byte + with the packaged generator's over a model that exercises each one (the router's own + runtime imports are the one documented difference). Another fails on a helper module no + registry entry hands over, or a generator module no registry entry names. + - **The stock `entity`, `queries`, `routes` and `barrel` generators are importable again (TypeScript).** `entityFile`, `queriesFile`, `routesFile` and `barrel` are exported from `@metaobjectsdev/codegen-ts/generators` once more, beside `namesFile`, `routesFileHono` @@ -805,6 +854,16 @@ until you regenerate. - **C#: `Fr010FieldMapping.CSharpStringLiteral(string)` is `[Obsolete]`.** It forwards to `CSharpNaming.CSharpStringLiteral(string)`, which every generator now calls (so an ejected `names` needs no emitter). Same signature and result; switch the call when you next touch it. +- **Python: the old import paths of the emitters that moved into `metaobjects.codegen.runtime`.** + `metaobjects.codegen.extract_delegate_emitter`, `.fr010_field_mapping`, + `.output_format_spec_emitter`, `.extract_schema_emitter`, `metaobjects.codegen.generators.fr019_shared_enum`, + `.m2m_codegen`, and `metaobjects.apidocs.api_model`, `.builder`, `.renderer`, `.paths` are + each one-line modules that become the new module and warn (`DeprecationWarning`) on import. + Same objects, so nothing else changes; import from `metaobjects.codegen.runtime` (the api + reference modules are `api_docs_model`, `api_docs_builder`, `api_docs_renderer` and + `api_docs_paths` there). The old paths will be removed in a later release. + `metaobjects.codegen.generators.router_generator.is_read_only_routed` moved to + `metaobjects.codegen.runtime.shared_predicates`; the router module still re-exports it. ### Removed diff --git a/docs/features/own-your-codegen.md b/docs/features/own-your-codegen.md index 73d932282..363bce4b5 100644 --- a/docs/features/own-your-codegen.md +++ b/docs/features/own-your-codegen.md @@ -688,9 +688,20 @@ targets: Eject prints the exact entry for each copy. It never overwrites an existing copy without `--force` and never edits your config. `metaobjects gen --list` marks each owned copy `identical` or `DIFFERS: N behind, M of your own` against the packaged reference, so you -can see when an upgrade changed the generator you copied. A copy imports the same -`metaobjects.codegen.*` modules the packaged one does, and those module paths are the -surface an owned generator builds on. +can see when an upgrade changed the generator you copied. Every generator `metaobjects +gen --list` shows is ejectable except `template`, the Mustache primitive, which has no emit +logic of its own to own. + +An ejected generator owns every line that writes its output. Eject copies the generator +with the emitters it writes through (below), and the copy imports only the engine from the +package: the generator base and file writer (`metaobjects.codegen.generator`), the +formatter, the naming and type-mapping tables (`metaobjects.apidocs.naming`, +`metaobjects.codegen.type_map`, `metaobjects.codegen.value_objects`), the model walks +(`find_inbound`, `tph_plan`, `instance_artifacts`) and the loader. Those module paths are +the surface an owned generator builds on. `tests/codegen/test_eject_alone.py` ejects each +generator alone into an empty project, rejects any import outside that surface, runs it with +the packaged emitters made unimportable, and compares its output byte for byte with the +packaged generator's. `requirement-tests` ejects the same way, to `codegen/generators/requirement_tests.py`, wired as `codegen.generators.requirement_tests:requirement_tests_generator`. The copy holds @@ -757,34 +768,98 @@ codegen/runtime/`). The same recipe covers the generator itself: `metaobjects ej routes` into a scratch directory gives you the new reference to diff your `codegen/generators/routes.py` against. -### The rules two generators share come with them +### The emitters a generator writes through come with it -Some emit rules are asked by more than one generator, and the answers have to match. -`routes` and `filter-allowlist` both ask `emits_router` (which objects get a router; the -generated router imports the allowlist module, so a mismatch is an import error at -startup). `entity` and `extractor` both ask `has_literal_default`. These live in one -module, `shared_predicates`, and ejecting any of those four generators copies it too: +A generator that calls a packaged emitter is a shell around it: a defect in the emitter +still waits for a release. So eject copies the emitters too, into `codegen/runtime/`, next +to the generator. The set is each registry entry's `runtime` list: + +| Ejecting | also copies | +|---|---| +| `entity` | `fr019_shared_enum`, `m2m_codegen`, `shared_predicates` | +| `routes` | `fr010_field_mapping`, `m2m_codegen`, `shared_predicates`, and the router's own runtime (above) | +| `filter-allowlist` | `m2m_codegen`, `shared_predicates` | +| `output-parser` | `extract_delegate_emitter`, `fr010_field_mapping` | +| `extractor` | `extract_delegate_emitter`, `fr010_field_mapping`, `shared_predicates` | +| `output-prompt` | `output_format_spec_emitter`, `fr010_field_mapping` | +| `trace-helper` | `extract_schema_emitter`, `fr010_field_mapping` | +| `api-docs` | `api_docs_model`, `api_docs_builder`, `api_docs_renderer`, `api_docs_paths`, `m2m_codegen`, `shared_predicates` | + +`names`, `render-helper` and `requirement-tests` call no packaged emitter. A file two +generators share is copied once and an existing copy is never overwritten, even with +`--force`; `metaobjects gen --list` marks each copy `identical` or `DIFFERS: N behind, M of +your own`. ```text codegen/ - generators/routes.py - generators/filter_allowlist.py - runtime/shared_predicates.py # the shared rules, now your code + generators/output_parser.py + generators/extractor.py + runtime/extract_delegate_emitter.py # the one copy both read + runtime/fr010_field_mapping.py ``` -Each generator imports it relatively (`from ..runtime.shared_predicates import ...`), so -the verbatim copy in `codegen/generators/` reads `codegen/runtime/shared_predicates.py` -and never the installed module. Two owned generators read the one copy: change -`emits_router` there and your `routes` and your `filter-allowlist` change together. No -generated code imports this module. It runs at `gen` time only, so nothing is emitted into -`_runtime/` for it. The rest works as for the runtime above: an existing copy is kept, and -`--list` marks it `identical` or `DIFFERS`. +Each generator imports its helpers relatively (`from ..runtime import +extract_delegate_emitter as rde`), so the verbatim copy in `codegen/generators/` reads +`codegen/runtime/` and never the installed package. Two owned generators read the one copy: +change it and both change. No generated code imports these modules; they run at `gen` time +only, so nothing is emitted into `_runtime/` for them. + +**The rules two generators share** live in the same place. `routes` and `filter-allowlist` +both ask `emits_router` (which objects get a router; the generated router imports the +allowlist module, so a mismatch is an import error at startup). `entity` and `extractor` +both ask `has_literal_default`. They are in `shared_predicates`, which each of those +generators lists, so editing `emits_router` there moves your `routes` and your +`filter-allowlist` together. -A packaged generator keeps reading the installed rules. If you own `routes` and change +A packaged generator keeps reading the installed copy. If you own `routes` and change `emits_router` but still run the packaged `filter-allowlist`, the two disagree. Eject `filter-allowlist` too before you change that rule; the same goes for `entity` and `extractor` with `has_literal_default`. +**What stays in the package.** The engine: the generator base, writer and formatter, the +naming and type-mapping tables, the model walks and the loader. A table that decides a name +or a type is a question about the model, and the generators, the docs and the runtime must +agree on it, so it stays one packaged, public copy. + +**A copy you ejected before this release** keeps running. It imports the emitters from the +paths they had (`metaobjects.codegen.fr010_field_mapping`, +`metaobjects.codegen.generators.m2m_codegen` and the others in the CHANGELOG), and each of +those is now a deprecated alias of the module in `metaobjects.codegen.runtime`, so your copy +calls the packaged emitter exactly as before and a `DeprecationWarning` names the new path. +`metaobjects gen --list` marks it `DIFFERS: N behind`. To own the emitters, eject the same name +into a scratch directory (`metaobjects eject --root `), copy the `runtime/` +files over into `codegen/runtime/`, and change your imports to the relative form the new copy +uses (`from ..runtime import fr010_field_mapping as fm`). + +### `api-docs`: the api reference as an owned generator + +`metaobjects docs` writes the Python api reference (`api/python`). `metaobjects eject +api-docs` copies the generator that writes it, with the page model (`api_docs_model.py`), the +builder that decides what to document (`api_docs_builder.py`), the renderer +(`api_docs_renderer.py`) and the page-path math (`api_docs_paths.py`). Change which nodes +get a page, how a page reads, or where it lands, by editing them. + +A docs generator is not a code generator, so it is wired in its own block, never in a +target's `generators`: + +```yaml +docs: + generators: [codegen.generators.api_docs:api_docs_generator] +``` + +`metaobjects docs` runs the copy in place of the packaged generator. Without the block it +runs the packaged one, and the positional `metaobjects docs ` form reads no +config, so it always does. An entry that is not a docs generator, or a second one for the +`api` surface, is refused by name. `gen` and `verify --codegen` skip a docs generator +they find under a target and say so (`'api-docs' writes the api reference, so gen skipped +it.`). + +**It describes the code your generators emit.** The page builder asks `shared_predicates` +which objects get a router, the same module your owned `routes` and `filter-allowlist` +read. Eject them together and edit the predicate once: the router, the allowlist and the +reference change together. The naming seam (`metaobjects.apidocs.naming`) is the engine, +the one set of names every generator and every page uses. + ## Java and Kotlin: `mvn metaobjects:eject` ```bash diff --git a/docs/ports/python.md b/docs/ports/python.md index f55000bb8..630a5be7a 100644 --- a/docs/ports/python.md +++ b/docs/ports/python.md @@ -660,6 +660,16 @@ standard-library module name (`types`, `json`) is refused there with an error na rename it. See [Own your codegen → Python](../features/own-your-codegen.md#python-metaobjects-eject). +Every generator `metaobjects gen --list` shows is ejectable except `template`, the Mustache +primitive, which has no emit logic of its own to own. An ejected generator comes with the +emitters it writes its output through (the shared-enum and M:N writers, the prompt tier's +field mapping, extract delegate, extract schema and format-spec writers), copied into +`codegen/runtime/` and read from there, so the copy owns the code that writes the file and +not a shell around the package's. The naming and type-mapping tables, the model walks and +the generator base stay in the package as the engine. `metaobjects eject api-docs` does the +same for the api reference `metaobjects docs` writes: the copy is wired under +`docs.generators` in `metaobjects.config.yaml`, never in a target's `generators`. + Ejecting `routes` also hands over the helper runtime the generated routers call: `filter_parser` and `constraint_errors` are copied into `codegen/runtime/`, and the owned generator emits them into the generated package as `_runtime/` and imports them from diff --git a/fixtures/generator-registry-conformance/registry.json b/fixtures/generator-registry-conformance/registry.json index cf9109d46..129a771bd 100644 --- a/fixtures/generator-registry-conformance/registry.json +++ b/fixtures/generator-registry-conformance/registry.json @@ -90,8 +90,8 @@ "concept": "Per-entity/template SDK API reference (generated-code API; human + agent forms).", "tier": "native", "layer": "docs", - "note": "Run by `meta docs` as its api surface (ADR-0025), not by `meta gen`: a `meta gen` config that lists it is warned about and skipped. On csharp it is run by `dotnet meta docs`, and `dotnet meta gen` skips it, warning only when `--generators` names it.", - "ports": ["typescript", "csharp"] + "note": "Run by `meta docs` as its api surface (ADR-0025), not by `meta gen`: a `meta gen` config that lists it is warned about and skipped. On csharp it is run by `dotnet meta docs`, and `dotnet meta gen` skips it, warning only when `--generators` names it. On python it is run by `metaobjects docs` (an owned copy is listed under `docs.generators` in `metaobjects.config.yaml`), and `gen` skips it with a warning.", + "ports": ["typescript", "csharp", "python"] }, "trace-helper": { "concept": "Per-entity typed record/call trace helpers (extract + buildLlmCallRow + persist; LlmCallBase-derived entities only).", diff --git a/server/python/src/metaobjects/apidocs/__init__.py b/server/python/src/metaobjects/apidocs/__init__.py index 37e06b83e..52a849e6a 100644 --- a/server/python/src/metaobjects/apidocs/__init__.py +++ b/server/python/src/metaobjects/apidocs/__init__.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any # Path math is leaf-level (no generator imports) — safe to expose eagerly. -from metaobjects.apidocs.paths import ( +from metaobjects.codegen.runtime.api_docs_paths import ( Layout, doc_page_output_path, model_cross_href, @@ -28,7 +28,7 @@ ) if TYPE_CHECKING: # pragma: no cover - import-time only for type checkers - from metaobjects.apidocs.api_model import ( + from metaobjects.codegen.runtime.api_docs_model import ( ApiModel, ApiSymbol, ApiSymbolKind, @@ -36,8 +36,8 @@ FieldShape, UnitExample, ) - from metaobjects.apidocs.builder import PythonApiModelBuilder, build_api_model - from metaobjects.apidocs.renderer import ( + from metaobjects.codegen.runtime.api_docs_builder import PythonApiModelBuilder, build_api_model + from metaobjects.codegen.runtime.api_docs_renderer import ( PythonApiDocsRenderer, render_agent_api, render_index, @@ -46,18 +46,18 @@ # Lazily-resolved exports → their defining submodule. _LAZY: dict[str, str] = { - "ApiModel": "api_model", - "ApiSymbol": "api_model", - "ApiSymbolKind": "api_model", - "ApiUnit": "api_model", - "FieldShape": "api_model", - "UnitExample": "api_model", - "PythonApiModelBuilder": "builder", - "build_api_model": "builder", - "PythonApiDocsRenderer": "renderer", - "render_unit_page": "renderer", - "render_index": "renderer", - "render_agent_api": "renderer", + "ApiModel": "api_docs_model", + "ApiSymbol": "api_docs_model", + "ApiSymbolKind": "api_docs_model", + "ApiUnit": "api_docs_model", + "FieldShape": "api_docs_model", + "UnitExample": "api_docs_model", + "PythonApiModelBuilder": "api_docs_builder", + "build_api_model": "api_docs_builder", + "PythonApiDocsRenderer": "api_docs_renderer", + "render_unit_page": "api_docs_renderer", + "render_index": "api_docs_renderer", + "render_agent_api": "api_docs_renderer", } __all__ = [ @@ -87,5 +87,5 @@ def __getattr__(name: str) -> Any: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") import importlib - mod = importlib.import_module(f"{__name__}.{submodule}") + mod = importlib.import_module(f"metaobjects.codegen.runtime.{submodule}") return getattr(mod, name) diff --git a/server/python/src/metaobjects/apidocs/api_model.py b/server/python/src/metaobjects/apidocs/api_model.py index 12766ddef..26b60c834 100644 --- a/server/python/src/metaobjects/apidocs/api_model.py +++ b/server/python/src/metaobjects/apidocs/api_model.py @@ -1,111 +1,4 @@ -"""The Python native SDK api-surface IR (intermediate representation). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.api_docs_model`.""" +from metaobjects.codegen._moved import alias -Mirrors the Java ``JavaApiModel`` IR (server/java/.../apidocs), the C# -``CSharpApiModel``, and the TS ``api-model.ts``, idiomatic to the Python -(Pydantic / FastAPI / ObjectManager) generated surface: Pydantic model / -``ObjectManager`` data-access / FastAPI routes / validation / extractor / -render helper / payload model / output-format prompt / output parser / filter -allowlist. - -NAMES on every symbol come from the :mod:`metaobjects.apidocs.naming` seam (never -re-concatenated in the builder), so what this model documents == what the real -generators emit. -""" -from __future__ import annotations - -from dataclasses import dataclass, field -from enum import Enum - - -class ApiSymbolKind(str, Enum): - """The generated-category axis a documented symbol belongs to (Python / - Pydantic + FastAPI flavored).""" - - #: The Pydantic v2 model class (entity) or value-object model. - MODEL = "model" - #: Data access — the consumer-implemented repository ``Protocol`` seam. - DATA_ACCESS = "data_access" - #: A FastAPI route (named ``"VERB path"``). - REST = "rest" - #: Pydantic field validation on the create/update shape. - VALIDATION = "validation" - #: The tolerant + strict extractor for a template payload. - EXTRACTOR = "extractor" - #: The typed render helper wrapping the render engine. - RENDER = "render" - #: The typed payload model bound to a template. - PAYLOAD = "payload" - #: The output-format prompt fragment. - PROMPT = "prompt" - #: The output parser (``parse_*``) back into the typed payload. - OUTPUT_PARSER = "output_parser" - #: The per-entity sort/filter allowlist. - FILTER = "filter" - - -@dataclass(frozen=True) -class FieldShape: - """A documented field: name + Python type + optionality + an optional note - (e.g. enum values).""" - - name: str - type: str - optional: bool - note: str | None = None - - -@dataclass(frozen=True) -class UnitExample: - """A worked example for a unit (reserved for later phases; carried for - cross-port IR parity).""" - - title: str - code: str - - -@dataclass(frozen=True) -class ApiSymbol: - """One documented symbol of the generated Python SDK surface. - - :param name: the emitted Python identifier (or ``"VERB path"`` for a REST route). - :param kind: which generated category this symbol belongs to. - :param module: the import line / module path the symbol lives in. - :param signature: a human-readable Python signature line. - :param usage: a one-line "what you use this for". - :param returns: the symbol's return surface, or ``None``. - :param fields: per-field shapes for model/validation/payload symbols (may be empty). - """ - - name: str - kind: ApiSymbolKind - module: str - signature: str - usage: str - returns: str | None = None - fields: list[FieldShape] = field(default_factory=list) - - -@dataclass(frozen=True) -class ApiUnit: - """One documented unit (an entity / value object, or a template) + its symbols. - - :param node: the unit's short name (the doc-page basename). - :param package: the unit's metadata package (e.g. ``acme::shop``). - :param kind: ``"entity"`` | ``"value"`` | ``"report"`` | ``"template"``. - :param symbols: the documented symbols, in canonical IR order. - :param example: an optional unit-level worked example (reserved). - """ - - node: str - package: str - kind: str - symbols: list[ApiSymbol] - example: UnitExample | None = None - - -@dataclass(frozen=True) -class ApiModel: - """The full per-project Python SDK api surface IR.""" - - project: str - units: list[ApiUnit] +alias(__name__, "metaobjects.codegen.runtime.api_docs_model") diff --git a/server/python/src/metaobjects/apidocs/builder.py b/server/python/src/metaobjects/apidocs/builder.py index 47e1b52a7..40fd35048 100644 --- a/server/python/src/metaobjects/apidocs/builder.py +++ b/server/python/src/metaobjects/apidocs/builder.py @@ -1,508 +1,4 @@ -"""Builds the Python SDK api-surface IR (:class:`ApiModel`) from a loaded MetaRoot. +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.api_docs_builder`.""" +from metaobjects.codegen._moved import alias -This is the ACCURATE-BY-CONSTRUCTION half of the api-docs pipeline: every -documented symbol name comes from the :mod:`metaobjects.apidocs.naming` seam (never -re-concatenated here — the SAME seam the real generators delegate to), and every -inclusion decision comes from an applies-predicate that REUSES the corresponding -generator's own gate helpers (``emits_instance_artifacts`` / ``primary_rdb_source`` -+ source kind / ``resolve_payload_vo`` / format check / ``is_tph_subtype``) rather -than re-implementing the gate. The result is that what this builder documents == -what the generators emit, by SHARING the single source of truth. - -Per-category enumeration (mirrors the Java ``JavaApiModelBuilder`` / C# -``CSharpApiModelBuilder``, Python/Pydantic-flavored): - -* Objects (``root.own_children()`` filtered to ``object.*``): each concrete object - → a MODEL symbol. A writable-table entity adds DATA_ACCESS (the repository - ``Protocol``) / REST (one per FastAPI route) / VALIDATION (Pydantic field - constraints on the create/update shape) / FILTER (the allowlist). A value object - → MODEL only. An abstract object / a TPH subtype yields no instance artifacts. -* Templates (EVERY subtype — ADR-0052): a resolvable ``@payloadRef`` gives the - PAYLOAD record it renders outbound, plus RENDER for a ``template.output`` (the - only subtype the render-helper generator emits for). A ``template.prompt`` - declaring ``@responseRef`` adds the INBOUND tier — the response PAYLOAD record, - PROMPT (the response-format fragment) and EXTRACTOR, plus OUTPUT_PARSER unless - the reply is XML, which gets no strict tier (ADR-0053). - -An object that yields no symbols (e.g. an abstract object) produces NO unit at all -(never an empty unit). -""" -from __future__ import annotations - -from metaobjects.apidocs import naming -from metaobjects.apidocs.api_model import ( - ApiModel, - ApiSymbol, - ApiSymbolKind, - ApiUnit, - FieldShape, -) -from metaobjects.codegen import type_map -from metaobjects.codegen.generators.m2m_codegen import ( - build_object_index, - resolve_m2m_descriptors, -) -from metaobjects.codegen.generators.find_inbound import is_xml, response_shape -from metaobjects.codegen.value_objects import is_field_required, pkg_of, resolve_payload_vo -from metaobjects.codegen.generators.tph_plan import is_tph_subtype -from metaobjects.codegen.generators.router_generator import is_read_only_routed -from metaobjects.codegen.instance_artifacts import ( - emits_instance_artifacts, - has_item_route, - is_abstract, - is_served_report, -) -from metaobjects.source_resolution import primary_rdb_source -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.core.field.meta_field import MetaField -from metaobjects.meta.core.object.meta_object import MetaObject -from metaobjects.meta.core.object.object_constants import ( - OBJECT_SUBTYPE_ENTITY, - OBJECT_SUBTYPE_REPORT, -) -from metaobjects.meta.core.reporting.report_read_model import report_read_model -from metaobjects.meta.meta_data import MetaData -from metaobjects.meta.persistence.source.source_constants import SOURCE_KIND_TABLE -from metaobjects.meta.template import template_constants as tc -from metaobjects.shared.base_types import TYPE_OBJECT, TYPE_TEMPLATE - - -# --------------------------------------------------------------------------- -# Applies-predicates — each REUSES the matching generator's own gate helpers, so -# inclusion can never drift from emission. (The Python generators gate inline; -# these mirror that gate exactly via the shared helpers.) -# --------------------------------------------------------------------------- - - -def _is_writable_table_entity(obj: MetaObject, object_index: dict[str, MetaObject]) -> bool: - """The shared gate for the router + filter-allowlist generators: a non-abstract - entity with a ``source.rdb`` ``@kind="table"`` child that is NOT a TPH subtype - (a TPH subtype is folded into its base's single table — it emits no standalone - router/allowlist).""" - if obj.sub_type != OBJECT_SUBTYPE_ENTITY: - return False - if not emits_instance_artifacts(obj): - return False - if is_tph_subtype(obj): - return False - src = primary_rdb_source(obj) - if src is None: - return False - return src.effective_kind() == SOURCE_KIND_TABLE - - -def _payload_resolves(tmpl: MetaData, root: MetaData) -> MetaObject | None: - """The payload VO a template resolves to (``@payloadRef`` → ``object.value``), - or ``None`` — the shared gate for payload / render / prompt / parser / extractor.""" - payload_ref = tmpl.get_meta_attr(tc.TEMPLATE_ATTR_PAYLOAD_REF) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) - if not isinstance(payload_ref, str) or not payload_ref: - return None - # ADR-0042 (#228): the referrer is THIS template — a bare @payloadRef resolves - # in ITS OWN package first. - return resolve_payload_vo(root, payload_ref, pkg_of(tmpl)) - - -def _is_email_kind(tmpl: MetaData) -> bool: - kind = tmpl.get_meta_attr(tc.TEMPLATE_ATTR_KIND) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) - return isinstance(kind, str) and kind.lower() == tc.TEMPLATE_KIND_EMAIL - - -# --------------------------------------------------------------------------- -# Builder. -# --------------------------------------------------------------------------- - - -class PythonApiModelBuilder: - """Builds the :class:`ApiModel` SDK-surface IR from a loaded ``MetaRoot``.""" - - def build(self, root: MetaData, project: str) -> ApiModel: - objects = [ - # ADR-0039 sanctioned own: top-level scan on the loader ROOT (never extended, own == effective) - c - for c in root.own_children() - if c.type == TYPE_OBJECT and isinstance(c, MetaObject) - ] - object_index = build_object_index(objects) - - units: list[ApiUnit] = [] - for obj in objects: - # FR-044: a served report (Table A) has a generated read API, documented from - # its read model (its derived fields); every other report has no output. - if obj.sub_type == OBJECT_SUBTYPE_REPORT: - unit = ( - self._build_report_unit(obj, root) if is_served_report(obj) else None - ) - if unit is not None: - units.append(unit) - continue - unit = self._build_object_unit(obj, root, object_index) - if unit is not None: - units.append(unit) - - # Templates: ADR-0052 — EVERY template subtype gets a unit, not just - # template.output. A template.prompt carries a @payloadRef record and, when it - # declares @responseRef, the whole inbound tier — all of it generated, so all - # of it documentable. - # ADR-0039 sanctioned own: top-level scan on the loader ROOT (never extended, own == effective) - for tmpl in root.own_children(): - if tmpl.type == TYPE_TEMPLATE: - units.append(self._build_template_unit(tmpl, root)) - - return ApiModel(project, units) - - # ----- objects ----------------------------------------------------------- - - def _build_object_unit( - self, obj: MetaObject, root: MetaData, object_index: dict[str, MetaObject] - ) -> ApiUnit | None: - entity = obj.sub_type == OBJECT_SUBTYPE_ENTITY - module = naming.snake_case(obj.name) - unit_kind = "entity" if entity else "value" - symbols: list[ApiSymbol] = [] - - # MODEL — only for concrete objects (an abstract object cannot be - # instantiated → documented ⊆ generated). A value object → MODEL only. - if not is_abstract(obj): - model = naming.model_class_name(obj) - symbols.append( - ApiSymbol( - name=model, - kind=ApiSymbolKind.MODEL, - module=naming.model_import(obj), - signature=f"class {model}(BaseModel)", - usage=( - "the Pydantic v2 entity model" - if entity - else "the Pydantic v2 value-object model" - ), - ) - ) - - if entity and _is_writable_table_entity(obj, object_index): - # VALIDATION — the Pydantic field constraints carried on the model - # (required / max-length / range / pattern). Names the model class. - model = naming.model_class_name(obj) - symbols.append( - ApiSymbol( - name=model, - kind=ApiSymbolKind.VALIDATION, - module=naming.model_import(obj), - signature=f"class {model}(BaseModel)", - usage="Pydantic field validation on the create/update shape", - fields=self._model_fields(obj), - ) - ) - - # DATA_ACCESS — the consumer-implemented repository ``Protocol`` seam. - repo = naming.repository_class_name(obj.name) - symbols.append( - ApiSymbol( - name=repo, - kind=ApiSymbolKind.DATA_ACCESS, - module=f"from .{naming.router_module_name(obj.name)} import {repo}", - signature=f"class {repo}(Protocol)", - usage="data access — the repository Protocol the consumer implements", - returns="list / Optional / bool", - ) - ) - - # REST — one symbol per FastAPI route the router registers. - self._add_rest_symbols(symbols, obj, module, root, object_index) - - # FILTER — the per-entity sort/filter allowlist. - fields_const = naming.filter_fields_const(obj.name) - symbols.append( - ApiSymbol( - name=fields_const, - kind=ApiSymbolKind.FILTER, - module=( - f"from .{naming.filter_allowlist_module_name(obj.name)} " - f"import {fields_const}" - ), - signature=f"{fields_const}: frozenset[str]", - usage="the filterable-field + filter-operator allowlist", - ) - ) - - elif is_read_only_routed(obj): - # A view-backed projection: the read-only router and its allowlist, no writes. - symbols.extend(self._read_only_symbols(obj)) - - if not symbols: - return None - return ApiUnit(obj.name, _package_of(obj), unit_kind, symbols) - - def _build_report_unit(self, obj: MetaObject, root: MetaData) -> ApiUnit: - """The unit for a SERVED report (FR-044 Table G): the row model plus the read-only - surface its router mounts (:meth:`_read_only_symbols`). Nothing else is generated - for it: no item route, no write verb, no validation model. Built over the - report's read model so the names are the generators'.""" - model_obj = report_read_model(obj, root) # type: ignore[arg-type] - model = naming.model_class_name(model_obj) - symbols = [ - ApiSymbol( - name=model, - kind=ApiSymbolKind.MODEL, - module=naming.model_import(model_obj), - signature=f"class {model}(BaseModel)", - usage="the Pydantic v2 report row model", - ), - *self._read_only_symbols(model_obj), - ] - return ApiUnit(obj.name, _package_of(obj), "report", symbols) - - def _read_only_symbols(self, obj: MetaObject) -> list[ApiSymbol]: - """What the READ-ONLY router (``is_read_only_routed``) actually mounts and emits, - and nothing more: the repository seam, ``GET`` list, ``GET`` by id ONLY when - :func:`has_item_route`, and the filter allowlist. The collection ``POST`` and the - item-verb refusals are 405s, not endpoints, so no write verb is documented; - a keyless object has no item route and its seam no ``find_by_id``.""" - router_module = naming.router_module_name(obj.name) - base_path = "/api/" + naming.route_path(obj.name) - repo = naming.repository_class_name(obj.name) - fields_const = naming.filter_fields_const(obj.name) - item = has_item_route(obj) - - def rest(verb_path: str, usage: str) -> ApiSymbol: - return ApiSymbol( - name=verb_path, - kind=ApiSymbolKind.REST, - module=f"# {router_module}.py — FastAPI APIRouter", - signature=verb_path, - usage=usage, - ) - - symbols = [ - ApiSymbol( - name=repo, - kind=ApiSymbolKind.DATA_ACCESS, - module=f"from .{router_module} import {repo}", - signature=f"class {repo}(Protocol)", - usage="data access — the read-only repository Protocol the consumer implements", - returns="list / count / Optional" if item else "list / count", - ), - rest("GET " + base_path, "list with pagination / sort / filters"), - ] - if item: - symbols.append( - rest("GET " + base_path + "/{" + naming.pk_param(obj.name) + "}", "fetch one by id") - ) - symbols.append( - ApiSymbol( - name=fields_const, - kind=ApiSymbolKind.FILTER, - module=( - f"from .{naming.filter_allowlist_module_name(obj.name)} " - f"import {fields_const}" - ), - signature=f"{fields_const}: frozenset[str]", - usage="the filterable-field + filter-operator allowlist", - ) - ) - return symbols - - def _add_rest_symbols( - self, - symbols: list[ApiSymbol], - entity: MetaObject, - module: str, - root: MetaData, - object_index: dict[str, MetaObject], - ) -> None: - router_module = naming.router_module_name(entity.name) - base_path = "/api/" + naming.route_path(entity.name) - # The router registers the item routes with a per-entity path param - # (`/{_id}`, e.g. `/{customer_id}`) — document that exact name, not - # a generic `{id}`, so the docs match the generated FastAPI routes. - pk = naming.pk_param(entity.name) - item = base_path + "/{" + pk + "}" - - def add(verb_path: str, usage: str) -> None: - symbols.append( - ApiSymbol( - name=verb_path, - kind=ApiSymbolKind.REST, - module=f"# {router_module}.py — FastAPI APIRouter", - signature=verb_path, - usage=usage, - ) - ) - - # The router registers GET list, GET item, POST, PATCH item, PUT item, - # DELETE item (PATCH + PUT share a handler but are two registered routes). - add("GET " + base_path, "list with pagination / sort / filters") - add("GET " + item, "fetch one by id") - add("POST " + base_path, "create") - add("PATCH " + item, "update") - add("PUT " + item, "update (PUT alias)") - add("DELETE " + item, "delete") - - # M:N traversal — GET //{id}/. `docs` has no - # `GenConfig`/`--column-naming` of its own (see `cli.py`'s docs handler) and - # this loop reads only `.target_entity` / `.relation_name` off the - # descriptor — never a physical column/table name — so resolving it under - # the port default here changes nothing observable; threading a naming - # strategy through would mean inventing a config seam this command doesn't - # have, for a value nothing here would read. - for d in resolve_m2m_descriptors(entity, object_index): - target = object_index.get(d.target_entity) - target_name = target.name if target is not None else d.target_entity - add( - "GET " + base_path + "/{" + pk + "}/" + d.relation_name, - f"M:N traversal — the related {target_name} rows", - ) - - # ----- templates --------------------------------------------------------- - - def _build_template_unit(self, tmpl: MetaData, root: MetaData) -> ApiUnit: - name = tmpl.name - module = naming.snake_case(name) - symbols: list[ApiSymbol] = [] - payload_vo = _payload_resolves(tmpl, root) - - if payload_vo is not None: - # PAYLOAD — the shape this template RENDERS. ADR-0056: it IS the @payloadRef - # value object's own model, emitted by the entity generator. - payload_class = naming.model_class_name(payload_vo) - symbols.append( - ApiSymbol( - name=payload_class, - kind=ApiSymbolKind.PAYLOAD, - module=naming.model_import(payload_vo), - signature=f"class {payload_class}(BaseModel)", - usage="the typed payload projection bound to the template", - fields=self._payload_fields(payload_vo), - ) - ) - - # RENDER — the typed render helper wrapping the render engine. OUTBOUND - # ONLY: render_helper_generator emits for a template.output alone, so - # documenting it for a prompt would name a function never generated. - if tmpl.sub_type == tc.TEMPLATE_SUBTYPE_OUTPUT: - render_fn = naming.render_helper_fn(name) - symbols.append( - ApiSymbol( - name=render_fn, - kind=ApiSymbolKind.RENDER, - module=f"from .{module}_render_helper import {render_fn}", - signature=f"def {render_fn}(payload: {payload_class}, provider)", - usage="renders the output template against a typed payload", - returns="EmailDocument" if _is_email_kind(tmpl) else "str", - ) - ) - - # ADR-0052 — the INBOUND symbols belong to a RESPONDING template.prompt, never - # to an output, and the gate is @responseRef PRESENCE rather than a format - # value. Shares `response_shape` with the generators, so the docs can never - # claim a symbol codegen suppressed. - inbound = response_shape(root, tmpl, pkg_of(tmpl)) - if inbound is not None: - # The RESPONSE shape the parser actually returns — the @responseRef value - # object's own model (ADR-0056). Documented once: when the prompt renders and - # parses the SAME value object, the model above already is it. - response_class = naming.model_class_name(inbound.vo) - if payload_vo is None or inbound.vo.resolution_key() != payload_vo.resolution_key(): - symbols.append( - ApiSymbol( - name=response_class, - kind=ApiSymbolKind.PAYLOAD, - module=naming.model_import(inbound.vo), - signature=f"class {response_class}(BaseModel)", - usage="the typed response shape a model reply is parsed into", - fields=self._payload_fields(inbound.vo), - ) - ) - - # OUTPUT_PARSER — the strict ``parse_*``. ADR-0053: JSON-only, so an XML - # reply gets no strict tier and documenting one would name a missing fn. - if not is_xml(inbound.format): - parse_fn = naming.output_parser_fn(name) - symbols.append( - ApiSymbol( - name=parse_fn, - kind=ApiSymbolKind.OUTPUT_PARSER, - module=f"from .{module}_response_parser import {parse_fn}", - signature=f"def {parse_fn}(text)", - usage="parses a model reply back into the typed response shape", - returns=response_class, - ) - ) - - prompt_fn = naming.output_prompt_fn(name) - symbols.append( - ApiSymbol( - name=prompt_fn, - kind=ApiSymbolKind.PROMPT, - module=f"from .{module}_response_format import {prompt_fn}", - signature=f"def {prompt_fn}(overrides=None)", - usage="builds the response-format prompt fragment", - returns="str", - ) - ) - extract_fn = naming.extractor_fn(name) - symbols.append( - ApiSymbol( - name=extract_fn, - kind=ApiSymbolKind.EXTRACTOR, - module=f"from .{module}_extractor import {extract_fn}", - signature=f"def {extract_fn}(root, text, opts=None)", - usage="extracts the strict typed response from dirty model output", - returns=response_class, - ) - ) - - return ApiUnit(name, _package_of(tmpl), "template", symbols) - - # ----- field shapes ------------------------------------------------------- - - def _model_fields(self, entity: MetaObject) -> list[FieldShape]: - """The entity's documented scalar/enum field shapes — one row per field the - Pydantic model maps (object-typed fields are owned-type navs, not a scalar - documented shape). Types via ``type_map.py_type_for``; required = own - ``@required`` is the boolean ``True`` (the same boundary the model uses).""" - rows: list[FieldShape] = [] - for f in entity.fields(): - if not isinstance(f, MetaField): - continue - if f.sub_type == fc.FIELD_SUBTYPE_OBJECT: - continue - type_expr = type_map.py_type_for(f).expr - note = None - if f.sub_type == fc.FIELD_SUBTYPE_ENUM: - values = type_map.effective_enum_values(f) - if values: - note = "allowed: " + " | ".join(values) - required = f.get_meta_attr(fc.FIELD_ATTR_REQUIRED) is True # ADR-0039: resolving (@required may be inherited) - rows.append(FieldShape(f.name, type_expr, optional=not required, note=note)) - return rows - - def _payload_fields(self, vo: MetaObject) -> list[FieldShape]: - """The payload model's documented field shapes — one row per field of the - value object, with optionality from ``is_field_required`` (the boundary the - value object's model draws between ``T`` and ``T | None``).""" - rows: list[FieldShape] = [] - for f in vo.fields(): - if not isinstance(f, MetaField): - continue - type_expr = type_map.py_type_for(f).expr - rows.append( - FieldShape(f.name, type_expr, optional=not is_field_required(f)) - ) - return rows - - -def _package_of(node: MetaData) -> str: - """The unit's metadata package (``"acme::shop"``) — drives the doc-page layout - path. Resolved from the nearest ancestor that carries one (objects inherit the - file/root package).""" - pkg = node.package - parent = node.parent - while pkg is None and parent is not None: - pkg = parent.package - parent = parent.parent - return pkg or "" - - -def build_api_model(root: MetaData, project: str) -> ApiModel: - """Module-level convenience: build the Python SDK api-surface IR.""" - return PythonApiModelBuilder().build(root, project) +alias(__name__, "metaobjects.codegen.runtime.api_docs_builder") diff --git a/server/python/src/metaobjects/apidocs/naming.py b/server/python/src/metaobjects/apidocs/naming.py index 76b60c3da..81fd1c3e7 100644 --- a/server/python/src/metaobjects/apidocs/naming.py +++ b/server/python/src/metaobjects/apidocs/naming.py @@ -3,7 +3,7 @@ This mirrors the Java ``SpringNaming`` / C# ``CSharpNaming`` pattern: the real generators delegate their name computation here, and the -:class:`~metaobjects.apidocs.builder.PythonApiModelBuilder` enumerates symbols by +:class:`~metaobjects.codegen.runtime.api_docs_builder.PythonApiModelBuilder` enumerates symbols by calling the SAME functions. So the documented name can never drift from the emitted name — they are computed once, in one place. diff --git a/server/python/src/metaobjects/apidocs/paths.py b/server/python/src/metaobjects/apidocs/paths.py index 6360d8d6a..4b5a853d6 100644 --- a/server/python/src/metaobjects/apidocs/paths.py +++ b/server/python/src/metaobjects/apidocs/paths.py @@ -1,75 +1,4 @@ -"""Doc-page path math for the Python native SDK-docs surface (``api/python``). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.api_docs_paths`.""" +from metaobjects.codegen._moved import alias -Byte-parity with the Java ``DocsPaths`` (server/java/.../apidocs/DocsPaths.java), -the C# ``DocsPaths`` (server/csharp/.../ApiDocs/DocsPaths.cs), and the TS -``docs-paths.ts`` contract: the same layout + relative-href math, so the shared -``fixtures/conformance/api-docs-cross-port/expected-paths.json`` resolves -identically across all ports. Nothing here is Python-specific — it is pure path -math. The api-docs renderer + the ``docs`` command call these helpers; they never -re-derive a path. -""" -from __future__ import annotations - -from enum import Enum - - -class Layout(str, Enum): - """Doc-page layout: one file per unit (:attr:`FLAT`) or foldered by package - (:attr:`PACKAGE`).""" - - #: One file per unit at the surface root (``.md``). - FLAT = "flat" - #: Foldered by package (``/.md``). - PACKAGE = "package" - - -def package_to_path(pkg: str | None) -> str: - """``"acme::shop"`` or ``"acme.shop"`` → ``"acme/shop"``; ``None``/"" → "".`""" - if not pkg: - return "" - return pkg.replace("::", "/").replace(".", "/") - - -def doc_page_output_path(layout: Layout, pkg: str | None, name: str) -> str: - """Flat → ``".md"``; Package → ``"/.md"``.""" - file = f"{name}.md" - if layout == Layout.FLAT: - return file - folder = package_to_path(pkg) - return file if not folder else f"{folder}/{file}" - - -def surface_cross_href(from_output_path: str, to_output_path: str) -> str: - """Relative posix href from *from_output_path*'s directory to *to_output_path* - (mirrors the TS ``surfaceCrossHref``).""" - slash = from_output_path.rfind("/") - from_dir = from_output_path[:slash] if slash >= 0 else "" - rel = _posix_relative(from_dir, to_output_path) - return rel if rel.startswith(".") else f"./{rel}" - - -def model_cross_href( - api_page_path: str, model_page_path: str, model_base_url: str | None = None -) -> str: - """From an api page to its model page: relative by default, absolute when - *model_base_url* is set (federated docs).""" - if model_base_url: - return f"{model_base_url.rstrip('/')}/{model_page_path}" - return surface_cross_href(api_page_path, model_page_path) - - -def _posix_relative(from_dir: str, to_path: str) -> str: - """``posixpath.relative(from_dir, to_path)``: drop the common prefix, ``..`` - per remaining *from_dir* segment, then the remaining *to_path* segments. - "" from_dir → to_path; identical → ".".""" - if not from_dir: - return to_path - frm = from_dir.split("/") - to = to_path.split("/") - common = 0 - cap = min(len(frm), len(to)) - while common < cap and frm[common] == to[common]: - common += 1 - parts: list[str] = [".."] * (len(frm) - common) - parts.extend(to[common:]) - return "/".join(parts) if parts else "." +alias(__name__, "metaobjects.codegen.runtime.api_docs_paths") diff --git a/server/python/src/metaobjects/apidocs/renderer.py b/server/python/src/metaobjects/apidocs/renderer.py index 3555fcb6a..791893e66 100644 --- a/server/python/src/metaobjects/apidocs/renderer.py +++ b/server/python/src/metaobjects/apidocs/renderer.py @@ -1,185 +1,4 @@ -"""Renders the :class:`ApiModel` IR into Python-idiomatic api-doc markdown. +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.api_docs_renderer`.""" +from metaobjects.codegen._moved import alias -One IR, three forms — all derived from the SAME model, never re-derived: - -* :func:`render_unit_page` — a per-unit HUMAN page (entity OR template), symbols - grouped into ordered sections + the ``**Model / metadata:**`` back-link. -* :func:`render_index` — the consolidated index (README.md), one bullet per unit. -* :func:`render_agent_api` — the token-frugal AGENT form (AGENT-API.md), symbols - grouped under their import line. - -PRESENTATION ONLY: every symbol name comes from the IR (which keys off the naming -seam) and every path comes from :mod:`metaobjects.apidocs.paths` — the renderer never -re-derives a name or a path. The section order + headings + the back-link literal -mirror the Java / C# renderers so the polyglot doc tree coheres. -""" -from __future__ import annotations - -from metaobjects.apidocs.api_model import ApiModel, ApiSymbolKind, ApiUnit -from metaobjects.apidocs.paths import Layout, doc_page_output_path, surface_cross_href - -# Canonical section order per kind (a unit renders only the kinds it carries). -_KIND_ORDER: tuple[ApiSymbolKind, ...] = ( - ApiSymbolKind.MODEL, - ApiSymbolKind.DATA_ACCESS, - ApiSymbolKind.REST, - ApiSymbolKind.VALIDATION, - ApiSymbolKind.EXTRACTOR, - ApiSymbolKind.RENDER, - ApiSymbolKind.PAYLOAD, - ApiSymbolKind.PROMPT, - ApiSymbolKind.OUTPUT_PARSER, - ApiSymbolKind.FILTER, -) - -_HEADINGS: dict[ApiSymbolKind, str] = { - ApiSymbolKind.MODEL: "Model", - ApiSymbolKind.DATA_ACCESS: "Data access", - ApiSymbolKind.REST: "REST", - ApiSymbolKind.VALIDATION: "Validation", - ApiSymbolKind.EXTRACTOR: "Extractor", - ApiSymbolKind.RENDER: "Render", - ApiSymbolKind.PAYLOAD: "Payload", - ApiSymbolKind.PROMPT: "Prompt", - ApiSymbolKind.OUTPUT_PARSER: "Output parser", - ApiSymbolKind.FILTER: "Filter", -} - -_SUMMARY_LABELS: dict[ApiSymbolKind, str] = { - ApiSymbolKind.MODEL: "model", - ApiSymbolKind.DATA_ACCESS: "data access", - ApiSymbolKind.REST: "REST", - ApiSymbolKind.VALIDATION: "validation", - ApiSymbolKind.EXTRACTOR: "extractor", - ApiSymbolKind.RENDER: "render", - ApiSymbolKind.PAYLOAD: "payload", - ApiSymbolKind.PROMPT: "prompt", - ApiSymbolKind.OUTPUT_PARSER: "output parser", - ApiSymbolKind.FILTER: "filter", -} - - -def _md_cell(text: str) -> str: - return text.replace("|", "\\|") - - -def render_unit_page(unit: ApiUnit, model_href: str | None) -> str: - """Render one per-unit human reference page. *model_href* (when non-empty) is a - pre-computed relative href back to this unit's model/metadata page — the caller - derives it via :func:`metaobjects.apidocs.paths.model_cross_href`; the renderer - only places it (as the contract ``**Model / metadata:**`` back-link).""" - out: list[str] = [f"# {unit.node} API\n"] - if model_href: - out.append(f"\n**Model / metadata:** [{unit.node}]({model_href})\n") - out.append( - "\n> Import paths are relative to the generated package; add it to your " - "Python path.\n" - ) - - for kind in _KIND_ORDER: - syms = [s for s in unit.symbols if s.kind == kind] - if not syms: - continue - out.append(f"\n## {_HEADINGS[kind]}\n") - for sym in syms: - out.append(f"\n### `{sym.signature}`\n") - out.append(f"\n{sym.usage}\n") - out.append(f"\n```python\n{sym.module}\n```\n") - if sym.returns: - out.append(f"\nReturns: {sym.returns}\n") - if sym.fields: - out.append("\n| Field | Type | Required | Notes |\n|---|---|---|---|\n") - for f in sym.fields: - required = "no" if f.optional else "yes" - out.append( - f"| `{f.name}` | `{_md_cell(f.type)}` | {required} | " - f"{_md_cell(f.note or '')} |\n" - ) - return "".join(out) - - -def render_index(model: ApiModel, layout: Layout) -> str: - """Render the consolidated api index (README.md): one bullet per unit, entities/ - values vs templates. Each unit's href is the relative link from the index to the - unit's page in the given *layout*.""" - entities = sorted( - (u for u in model.units if u.kind != "template"), key=lambda u: u.node - ) - templates = sorted( - (u for u in model.units if u.kind == "template"), key=lambda u: u.node - ) - - out: list[str] = [ - "# API Reference\n\nGenerated public API surface, one page per entity and " - "output template.\n" - ] - if entities: - out.append("\n## Entities\n\n") - for u in entities: - out.append(_index_row(u, layout)) - if templates: - out.append("\n## Templates\n\n") - for u in templates: - out.append(_index_row(u, layout)) - return "".join(out) - - -def _index_row(unit: ApiUnit, layout: Layout) -> str: - href = surface_cross_href( - "README.md", doc_page_output_path(layout, unit.package, unit.node) - ) - return ( - f"- [{unit.node}]({href}) — {_summary(unit)} " - f"({len(unit.symbols)} symbols)\n" - ) - - -def _summary(unit: ApiUnit) -> str: - parts: list[str] = [] - for kind in _KIND_ORDER: - n = sum(1 for s in unit.symbols if s.kind == kind) - if n == 0: - continue - label = _SUMMARY_LABELS[kind] - parts.append(label if n == 1 else f"{n} {label}") - return ", ".join(parts) if parts else "no public symbols" - - -def render_agent_api(model: ApiModel) -> str: - """Render the condensed agent/LLM form (AGENT-API.md): per unit, symbols grouped - under a single import-line header then one compact ``` `signature` — usage ``` line - each. NO prose / field-tables (token budget). Units + symbols keep their IR order.""" - out: list[str] = [ - f"# Agent API Reference\n\nGenerated Python API reference for " - f"{model.project}; call these exactly as written. Import paths are relative " - "to the generated package.\n" - ] - for unit in model.units: - if not unit.symbols: - continue - out.append(f"\n## {unit.node}\n") - groups: dict[str, list[str]] = {} - order: list[str] = [] - for s in unit.symbols: - if s.module not in groups: - groups[s.module] = [] - order.append(s.module) - groups[s.module].append(f"- `{s.signature}` — {s.usage}\n") - for module in order: - out.append(f"\n`{module}`\n") - out.extend(groups[module]) - return "".join(out) - - -class PythonApiDocsRenderer: - """Object wrapper around the module-level render functions (parity with the - Java / C# renderer classes; the functions are the canonical entry points).""" - - def render_unit_page(self, unit: ApiUnit, model_href: str | None) -> str: - return render_unit_page(unit, model_href) - - def render_index(self, model: ApiModel, layout: Layout) -> str: - return render_index(model, layout) - - def render_agent_api(self, model: ApiModel) -> str: - return render_agent_api(model) +alias(__name__, "metaobjects.codegen.runtime.api_docs_renderer") diff --git a/server/python/src/metaobjects/cli.py b/server/python/src/metaobjects/cli.py index 972779db3..0491fdef2 100644 --- a/server/python/src/metaobjects/cli.py +++ b/server/python/src/metaobjects/cli.py @@ -93,7 +93,8 @@ TargetConfig, load_project_config, ) -from metaobjects.codegen.generator import Generator +from metaobjects.codegen.docs_run import resolve_docs_generators, run_docs +from metaobjects.codegen.generator import DEFAULT_API_SUBDIR, DocsRunFacts, Generator from metaobjects.codegen.generators.entity_model import entity_model from metaobjects.codegen.generators.extractor_generator import extractor_generator from metaobjects.codegen.generators.filter_allowlist_generator import ( @@ -1007,33 +1008,25 @@ def _resolve_metadata_location_or_print_error( return None -#: The default api-surface subdir (the cross-port contract's ``api/python``). -_DOCS_DEFAULT_API_SUBDIR = "api/python" - - def _cmd_docs(args: argparse.Namespace) -> int: """``docs`` — emit the generated Python SDK api surface (the ``api/python`` surface of the cross-port SDK-docs contract) under ``--out``. - Wiring only: every symbol name comes from the ``PythonApiModelBuilder`` (which - keys off the naming seam + each generator's applies-predicate) and every path - comes from :mod:`metaobjects.apidocs.paths` — nothing is re-derived here. Output - paths are collision-checked (duplicate page path → failure) BEFORE anything is - written, mirroring the C# ``DocsCommand`` / the Java Mojo. Layout is ``package`` - (the cross-port contract). ``--model-base-url`` federates the model back-links - when set.""" - from metaobjects.apidocs import ( - Layout, - PythonApiDocsRenderer, - PythonApiModelBuilder, - doc_page_output_path, - model_cross_href, - ) - + Wiring only: the pages come from the ``api-docs`` generator, which builds them from + the naming seam plus each generator's applies-predicate and writes them under the + cross-port contract's ``package`` layout. A project that owns that generator + (``metaobjects eject api-docs``) lists it in the config's ``docs.generators`` and + ``docs`` runs the copy in its place. Output paths are collision-checked (duplicate + page path → failure) BEFORE anything is written, mirroring the C# ``DocsCommand`` / + the Java Mojo. ``--model-base-url`` federates the model back-links when set. The + positional ``metadata_dir`` form reads no config, so it always runs the packaged + generator.""" providers, providers_ok = _providers_from_args(args) if not providers_ok: return 1 + docs_generators: list[str] = [] + owned_root = Path.cwd() if args.metadata_dir is None: # Mirrors `gen`'s no-positional routing (#267): rung 2 (`metadata` in # `metaobjects.config.yaml`, if one is found), else rungs 3-4 via @@ -1067,6 +1060,8 @@ def _cmd_docs(args: argparse.Namespace) -> int: return 1 providers = [*providers, *config_providers] docs_libraries = config.libraries + docs_generators = config.docs_generators + owned_root = config.config_dir root, errors = _load_root_from_collection( collection, providers=providers, libraries=docs_libraries ) @@ -1081,43 +1076,28 @@ def _cmd_docs(args: argparse.Namespace) -> int: print(f" {msg}", file=sys.stderr) return 1 - api_subdir = args.api_subdir or _DOCS_DEFAULT_API_SUBDIR + api_subdir = args.api_subdir or DEFAULT_API_SUBDIR project = args.project or project_default model_base_url = getattr(args, "model_base_url", None) - model = PythonApiModelBuilder().build(root, project) - renderer = PythonApiDocsRenderer() - layout = Layout.PACKAGE - - # Collect rel-path → content first so a duplicate page path fails BEFORE any write. - emitted: dict[str, str] = {} - - def _put(path: str, content: str) -> None: - if path in emitted: - raise ValueError(f"docs — duplicate api page output path: {path}") - emitted[path] = content - - try: - for unit in model.units: - page_path = doc_page_output_path(layout, unit.package, unit.node) - api_page_path_from_docs_root = f"{api_subdir}/{page_path}" - href = model_cross_href(api_page_path_from_docs_root, page_path, model_base_url) - _put(page_path, renderer.render_unit_page(unit, href)) - _put("README.md", renderer.render_index(model, layout)) - _put("AGENT-API.md", renderer.render_agent_api(model)) - except ValueError as exc: + facts = DocsRunFacts(project=project, api_subdir=api_subdir, model_base_url=model_base_url) + generators, gen_err = resolve_docs_generators(docs_generators, owned_root) + if gen_err is not None: + print(f"error: {gen_err}", file=sys.stderr) + return 1 + outcome = run_docs(root, args.out, facts, generators) + if outcome.error is not None: # A duplicate output path (e.g. two units that resolve to the same page) is a # clean user-facing failure, not a stack trace. - print(f"error: {exc}", file=sys.stderr) + print(f"error: {outcome.error}", file=sys.stderr) return 1 - api_root = Path(args.out) / api_subdir written: list[str] = [] - for rel, content in emitted.items(): - dest = api_root / rel + for rel, content in outcome.files: + dest = Path(args.out) / rel dest.parent.mkdir(parents=True, exist_ok=True) dest.write_text(content, encoding="utf-8") - written.append(f"{api_subdir}/{rel}") + written.append(rel) for path in sorted(written): print(path) diff --git a/server/python/src/metaobjects/codegen/_moved.py b/server/python/src/metaobjects/codegen/_moved.py new file mode 100644 index 000000000..a9d0cfa88 --- /dev/null +++ b/server/python/src/metaobjects/codegen/_moved.py @@ -0,0 +1,23 @@ +"""Deprecated import paths of modules that moved into ``metaobjects.codegen.runtime``. + +``metaobjects eject`` hands the emitters a generator writes through to the adopter, and it +copies them from ``metaobjects/codegen/runtime/``, so that is where they live. A copy ejected +before they moved imports them from their old paths; each old path is a one-line module that +calls :func:`alias`, which makes the old name BE the new module (the same object, so private +names and monkeypatching behave as before) and warns once on import. Import from the new path. +""" +from __future__ import annotations + +import importlib +import sys +import warnings + + +def alias(old: str, new: str) -> None: + """Make module *old* the module *new*, warning that the old path is deprecated.""" + warnings.warn( + f"{old} moved to {new}; import it from there. The old path will be removed.", + DeprecationWarning, + stacklevel=3, + ) + sys.modules[old] = importlib.import_module(new) diff --git a/server/python/src/metaobjects/codegen/docs_run.py b/server/python/src/metaobjects/codegen/docs_run.py new file mode 100644 index 000000000..546f2c185 --- /dev/null +++ b/server/python/src/metaobjects/codegen/docs_run.py @@ -0,0 +1,91 @@ +"""``metaobjects docs`` — the docs run: the generators that write the reference pages, and +the collision-checked write. + +The api reference is a generator (``api-docs``, ``docs_surface = "api"``), so a project can +own it with ``metaobjects eject api-docs``. The owned copy is listed in the config's +``docs.generators`` — never the top-level ``generators`` ``gen`` runs — and ``docs`` runs it +in place of the packaged one. Engine code: an owned copy imports none of it. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from metaobjects.codegen.config import GenConfig +from metaobjects.codegen.eject import build_owned +from metaobjects.codegen.generator import ( + DOCS_SURFACE_API, + DocsRunFacts, + GenContext, + Generator, +) +from metaobjects.codegen.generator_registry import GENERATOR_REGISTRY, GeneratorBuildContext +from metaobjects.meta.meta_data import MetaData + + +@dataclass +class DocsOutcome: + #: ``(path relative to the docs out dir, content)`` in emission order. + files: list[tuple[str, str]] = field(default_factory=list) + #: A user-facing failure (duplicate page path, bad ``docs.generators`` entry), or ``None``. + error: str | None = None + + +def resolve_docs_generators( + specs: list[str], root: Path, +) -> tuple[list[Generator], str | None]: + """The generators ``docs`` runs: each owned entry in *specs* (``docs.generators``), then + the packaged generator for every surface no owned one takes over. + + An entry that is not a docs generator (no ``docs_surface``) or a second one for a + surface already taken is refused by name: either would write the same pages twice, or + write code into the docs tree.""" + ctx = GeneratorBuildContext() + owned: list[Generator] = [] + taken: dict[str, str] = {} + for spec in specs: + gen, err = build_owned(spec, root, ctx) + if err is not None or gen is None: + return [], err + surface = getattr(gen, "docs_surface", None) + if surface is None: + return [], ( + f"docs.generators entry {spec!r} is not a docs generator: it carries no " + "`docs_surface`. A generator that writes code belongs in a target's " + "`generators`." + ) + if surface in taken: + return [], ( + f"docs.generators lists two generators for the {surface!r} surface: " + f"{taken[surface]!r} and {spec!r}." + ) + taken[surface] = spec + owned.append(gen) + if DOCS_SURFACE_API not in taken: + owned.append(GENERATOR_REGISTRY["api-docs"].factory(ctx)) + return owned, None + + +def run_docs( + root: MetaData, out_dir: str, facts: DocsRunFacts, generators: list[Generator], +) -> DocsOutcome: + """Run *generators* over *root* and collect their pages, failing on a duplicate page path + BEFORE anything is written (the caller writes ``outcome.files``).""" + outcome = DocsOutcome() + seen: dict[str, str] = {} + for gen in generators: + ctx = GenContext( + entities=[], + loaded_root=root, + matches=lambda _e: True, + config=GenConfig(out_dir=out_dir), + warn=lambda _msg: None, + docs=facts, + ) + for f in gen.generate(ctx): + if f.path in seen: + outcome.error = f"docs — duplicate api page output path: {f.path}" + return outcome + seen[f.path] = gen.name + outcome.files.append((f.path, f.content)) + return outcome diff --git a/server/python/src/metaobjects/codegen/eject.py b/server/python/src/metaobjects/codegen/eject.py index c9afc5e5e..1ea0ab2c7 100644 --- a/server/python/src/metaobjects/codegen/eject.py +++ b/server/python/src/metaobjects/codegen/eject.py @@ -18,10 +18,19 @@ copy. Core modules (loader, registry, render, extract, ``ObjectManager``) stay package imports. An existing runtime copy is never overwritten, even with ``--force``. -The emit rules several generators share (``shared_predicates``) are listed and copied the -same way. A generator imports them relatively from its ``runtime`` sibling, so the verbatim -copy in ``codegen/generators/`` reads ``codegen/runtime/shared_predicates.py`` with no -edit, and two owned generators read the one copy. +Everything a generator writes its output THROUGH is handed over the same way: the emit +rules several generators share (``shared_predicates``), the fragment emitters behind a full +generator (the shared-enum and M:N writers), the prompt-tier emitters (the extract delegate, +the extract schema, the field mapping and the output-format spec) and, for ``api-docs``, the +page model, builder, renderer and path math. A generator imports each relatively from its +``runtime`` sibling, so the verbatim copy in ``codegen/generators/`` reads +``codegen/runtime/.py`` with no edit, and two owned generators read the one copy. +Only the engine stays a package import: the generator base and file writer, the naming and +type-mapping tables, the model walks and the loader. ``tests/codegen/test_eject_alone.py`` +ejects each generator alone and holds it to exactly that. + +A docs generator (``api-docs``) is not wired into ``generators``: ``metaobjects docs`` runs +it from the config's ``docs.generators``, and ``gen`` skips one it finds under a target. """ from __future__ import annotations @@ -138,7 +147,13 @@ def eject(names: list[str], root: Path, *, force: bool = False) -> EjectResult: target = owned_path(root, n) target.write_text(ejected_source(entry), encoding="utf-8") out.append(f"ejected {n} -> {target.relative_to(root)}") - out.append(f" wire it: replace \"{n}\" in `generators` with {owned_token(entry)}") + if entry.docs_surface is not None: + out.append( + f" wire it: list {owned_token(entry)} under `docs.generators` in " + "metaobjects.config.yaml — `metaobjects docs` runs it; `gen` never does" + ) + else: + out.append(f" wire it: replace \"{n}\" in `generators` with {owned_token(entry)}") for module in entry.runtime: rt = runtime_path(root, module) rel = rt.relative_to(root) @@ -152,7 +167,8 @@ def eject(names: list[str], root: Path, *, force: bool = False) -> EjectResult: out.append("") out.append( "The copies are yours: edit them freely. Keep only one of the packaged name and your " - "copy in `generators` — wiring both runs both, and they write the same files." + "copy in `generators` — wiring both runs both, and they write the same files. " + "A docs generator is the exception: `docs.generators` replaces the packaged one." ) return EjectResult(0, out, []) diff --git a/server/python/src/metaobjects/codegen/extract_delegate_emitter.py b/server/python/src/metaobjects/codegen/extract_delegate_emitter.py index d6cd88bcb..b0e4287e7 100644 --- a/server/python/src/metaobjects/codegen/extract_delegate_emitter.py +++ b/server/python/src/metaobjects/codegen/extract_delegate_emitter.py @@ -1,383 +1,4 @@ -"""FR-010 nested-codegen-gap — the runtime-DELEGATING extract emitter (Python). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.extract_delegate_emitter`.""" +from metaobjects.codegen._moved import alias -The self-contained ``extract_(text)`` path (``extract_schema_emitter`` + the -baked ``ExtractSchema``) covers scalars / enums / scalar-arrays but leaves -nested-object and array-of-object components ``None`` — the historical FR-010 codegen -gap. This module emits the additive *delegating* entry that CLOSES that gap by -wrapping the runtime extract: - - extract__with_loader(root, text, opts=None) -> ExtractionResult[Extracted] - -It resolves this payload's ``MetaObject`` from the supplied loaded ``MetaRoot`` by its -baked simple name (``PAYLOAD_NAME``), delegates to ``extract_object`` in -:mod:`metaobjects.meta.core.object.object_extract` (which assembles the FULL nested -object graph reflection-free via the Phase A object model — ``MetaObject.new_instance()`` -+ the ``MetaField`` set-by-name SPI), then maps the assembled ``ValueObject`` graph into -the typed nullable mirror graph via generated ``_from__extracted(o)`` mapper -functions (payload + every reachable nested VO, deduped). - -This is the codegen-wrapping-runtime pattern (a generated DAO calling the -dynamic-metadata runtime), mirroring the Java / Kotlin / TS pilots. The generated -mappers read the assembled graph through a tiny ``_read_prop`` helper that mirrors the -``MetaField`` get SPI (``ValueObject.get(name)`` else plain-attribute access), so the -emitted code stays self-sufficient and reflection-free. - -Bounded by the cross-port ``MAX_NEST_DEPTH`` via the runtime — codegen here only mirrors -the runtime's resolved object graph, so depth/cycle guarding lives in ``object_extract``. -The emitter dedupes mirrors/mappers by ``resolution_key()`` (the package-qualified FQN, -cycle-safe) — NOT the bare VO ``name``, which would silently collapse two same-short-name -value-objects from different packages into one (ADR-0044, #228). - -ADR-0056 rule 3: a mirror is keyed by the VALUE OBJECT, never the template. Every mirror -and mapper — the root's included — is named after the value object's emitted model name -(:func:`~metaobjects.codegen.value_objects.model_class_name`: ``Extracted`` / -``_from__extracted``), the same name the entity generator declares the model under, so -a cross-package short-name collision qualifies the mirror exactly as it qualifies the -model. Mirrors stay module-local (declared in each parser module that needs them), like -TypeScript's: a Python module scopes them, so two parsers over one response each carry -their own copy without clashing. -""" -from __future__ import annotations - -from metaobjects.codegen import fr010_field_mapping as fm -from metaobjects.codegen.value_objects import model_class_name, pkg_of -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.meta_data import MetaData -from metaobjects.naming_refs import resolve_object_ref - - -def ref_vo(field: MetaData, root: MetaData) -> MetaData | None: - """The ``@objectRef`` target VO for a nested-object field, or ``None`` when - unresolvable. - - ADR-0042 (#228) — resolves via the canonical `resolve_object_ref` package-local - contract: an FQN ref resolves EXACTLY; a bare ref resolves in the DECLARING - field's own package first, else a root-level object. NO bare-tail short-name - fallback — that pattern (matching an FQN ref by its trailing simple-name segment - against ANY same-named object root-wide) is the #219/ADR-0042-banned "wrong - node" bug: under a cross-package short-name collision it silently binds - whichever same-named object happens to load first, regardless of which package - the ref actually pointed at. *referrer_pkg* is the field's OWN declaring - package (which differs from the VO's when the field is inherited via `extends` - from an abstract VO in another package).""" - ref = field.attrs().get(fc.FIELD_ATTR_OBJECT_REF) - if not isinstance(ref, str) or not ref: - return None - referrer_pkg = pkg_of(field.parent) if field.parent is not None else "" - return resolve_object_ref(root, ref, referrer_pkg) - - -def _is_object_field(field: MetaData) -> bool: - """True iff the field is a nested object reference (``field.object`` — distinct - from the string-backed ``field.enum``, treated as a scalar).""" - return field.sub_type == fc.FIELD_SUBTYPE_OBJECT - - -def mirror_name(vo: MetaData) -> str: - """The extracted-mirror dataclass name for a value-object: ``Extracted``, where - ```` is the value object's emitted model name (collision-qualified when another - object shares its short name).""" - return f"{model_class_name(vo)}Extracted" - - -def mapper_name(vo: MetaData) -> str: - """The mapper function name for a value-object (``_from__extracted``).""" - return f"_from_{_snake(model_class_name(vo))}_extracted" - - -def _snake(name: str) -> str: - """``NpcResponseOutput`` → ``npc_response_output`` (matches the cross-generator - ``_snake_case`` convention; no acronym handling).""" - out: list[str] = [] - for i, ch in enumerate(name): - if ch.isupper() and i > 0: - out.append("_") - out.append(ch.lower()) - return "".join(out) - - -# ============================================================================= -# Nested-aware mirror type (recurses into nested mirror names) -# ============================================================================= - - -def _nested_mirror_type(field: MetaData, root: MetaData) -> str: - """The nullable mirror annotation for one field — nested-aware (nested objects - become ``Extracted``; array-of-objects become ``list[...]``).""" - if _is_object_field(field): - target = ref_vo(field, root) - base = f'"{mirror_name(target)}"' if target is not None else "object" - elem = f"{base} | None" - return f"list[{elem}] | None" if fm.is_array(field) else elem - if fm.is_array(field): - return "list[str | None] | None" - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - return "str | None" - kind = fm.scalar_kind(field.sub_type) - if kind in ("INT", "LONG"): - return "int | None" - if kind == "DOUBLE": - return "float | None" - if kind == "BOOLEAN": - return "bool | None" - return "str | None" - - -# ============================================================================= -# Reachability -# ============================================================================= - - -def reachable_vos(vo: MetaData, root: MetaData) -> list[MetaData]: - """``vo`` + every value-object reachable through nested ``@objectRef`` fields, in - stable BFS order, deduped by ``resolution_key()`` (cycle-safe). - - ADR-0044 (#228) — deduping by the bare ``name`` (pre-fix) silently DROPPED a - second cross-package value-object sharing the first one's bare short name (e.g. - ``acme::alpha::Note`` + ``acme::beta::Note``): once the first ``Note`` was seen, - the second's bare name matched ``seen`` and it was never queued/emitted — a - silent shape loss, not merely a naming cosmetic. ``resolution_key()`` is the - package-qualified FQN, so two same-short-name VOs from different packages are - two distinct keys and both survive the walk.""" - out: list[MetaData] = [] - seen: set[str] = set() - queue: list[MetaData] = [vo] - while queue: - cur = queue.pop(0) - key = cur.resolution_key() - if key in seen: - continue - seen.add(key) - out.append(cur) - for f in fm.fields(cur): - if _is_object_field(f): - target = ref_vo(f, root) - if target is not None and target.resolution_key() not in seen: - queue.append(target) - return out - - -def has_nested(vo: MetaData, root: MetaData) -> bool: - """True iff ``vo`` (or any reachable nested VO) has a nested-object field.""" - for cur in reachable_vos(vo, root): - if any(_is_object_field(f) for f in fm.fields(cur)): - return True - return False - - -# ============================================================================= -# Nested-aware mirror dataclasses -# ============================================================================= - - -def nested_mirror_dataclasses(vo: MetaData, root: MetaData) -> list[str]: - """Emit the nested-aware mirror dataclass for ``vo`` and every reachable nested VO - (deduped), each named :func:`mirror_name`. Returns source lines (blank-line - separated).""" - lines: list[str] = [] - for i, cur in enumerate(reachable_vos(vo, root)): - if i > 0: - lines.append("") - lines.append("") - lines.extend(_one_mirror(cur, root)) - return lines - - -def _one_mirror(vo: MetaData, root: MetaData) -> list[str]: - lines: list[str] = [ - "@dataclass(frozen=True, slots=True)", - f"class {mirror_name(vo)}:", - f' """Best-effort extracted twin of ``{model_class_name(vo)}`` — every field nullable', - ' (``None`` where the value was lost or malformed)."""', - ] - field_lines = [ - f" {f.name}: {_nested_mirror_type(f, root)} = None" - for f in fm.fields(vo) - ] - lines.extend(field_lines or [" pass"]) - return lines - - -# ============================================================================= -# Mapper functions (assembled ValueObject graph -> typed nullable mirror graph) -# ============================================================================= - - -def nested_mappers(vo: MetaData, root: MetaData) -> list[str]: - """Emit one ``_from__extracted(o)`` mapper per reachable VO (payload + - nested, deduped), each named :func:`mapper_name`. Returns source lines - (blank-line separated).""" - lines: list[str] = [] - for i, cur in enumerate(reachable_vos(vo, root)): - if i > 0: - lines.append("") - lines.append("") - lines.extend(_one_mapper(cur, root)) - return lines - - -def _one_mapper(vo: MetaData, root: MetaData) -> list[str]: - fn = mapper_name(vo) - mirror = mirror_name(vo) - lines: list[str] = [ - f'def {fn}(o: object | None) -> "{mirror} | None":', - f" \"\"\"Map an assembled ValueObject graph into a typed ``{mirror}`` mirror;" - " null-tolerant.\"\"\"", - " if o is None:", - " return None", - f" return {mirror}(", - ] - for f in fm.fields(vo): - lines.append(f" {f.name}={_mapper_arg(f, root)},") - lines.append(" )") - return lines - - -def _mapper_arg(field: MetaData, root: MetaData) -> str: - """The mirror-field initializer that reads ``field`` from the assembled object ``o``.""" - key = f'"{field.name}"' - if _is_object_field(field): - target = ref_vo(field, root) - if target is None: - return "None # unresolved @objectRef" - fn = mapper_name(target) - if fm.is_array(field): - return f"_map_object_list(_read_prop(o, {key}), {fn})" - return f"{fn}(_read_prop(o, {key}))" - - # Enum / scalar / scalar-array: the runtime already coerced; read + light-coerce to - # the mirror's nullable shape via the locally-defined _dlg_* readers. - # ARRAY is checked BEFORE the scalar-enum branch: an enum ARRAY must route through - # the string-LIST reader, not the scalar enum reader — otherwise the list collapses - # to a single stringified scalar (the cross-port "enum-before-isArray" ordering bug). - if fm.is_array(field): - return f"_dlg_str_list(_read_prop(o, {key}))" - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - return f"_dlg_str(_read_prop(o, {key}))" - kind = fm.scalar_kind(field.sub_type) - if kind in ("INT", "LONG"): - return f"_dlg_int(_read_prop(o, {key}))" - if kind == "DOUBLE": - return f"_dlg_float(_read_prop(o, {key}))" - if kind == "BOOLEAN": - return f"_dlg_bool(_read_prop(o, {key}))" - return f"_dlg_str(_read_prop(o, {key}))" - - -# ============================================================================= -# Used-helper scoping + the shared helper block -# ============================================================================= - - -def used_helpers(vo: MetaData, root: MetaData) -> set[str]: - """The generated-helper names the mappers for ``vo`` (+ reachable nested VOs) - actually reference. ``_read_prop`` is always present once any mapper is emitted.""" - used: set[str] = {"_read_prop"} - for cur in reachable_vos(vo, root): - for f in fm.fields(cur): - if _is_object_field(f): - if fm.is_array(f): - used.add("_map_object_list") - continue - if fm.is_array(f): - # ARRAY before scalar-enum — an enum array uses the string-LIST reader - # (mirrors the _mapper_arg ordering; avoids the collapse-to-scalar bug). - used.add("_dlg_str_list") - elif f.sub_type == fc.FIELD_SUBTYPE_ENUM: - used.add("_dlg_str") - else: - kind = fm.scalar_kind(f.sub_type) - if kind in ("INT", "LONG"): - used.add("_dlg_int") - elif kind == "DOUBLE": - used.add("_dlg_float") - elif kind == "BOOLEAN": - used.add("_dlg_bool") - else: - used.add("_dlg_str") - return used - - -# Each helper: name -> source block (lines). -_HELPER_BLOCKS: dict[str, list[str]] = { - "_read_prop": [ - "def _read_prop(o: object | None, name: str) -> object | None:", - ' """Read a property from an assembled backing object, mirroring the', - " MetaField get SPI (``ValueObject.get(name)`` else plain-attribute access).", - ' Keeps the mappers reflection-free + backing-agnostic."""', - " if o is None:", - " return None", - ' getter = getattr(o, "get", None)', - " if callable(getter):", - " return getter(name)", - " return getattr(o, name, None)", - ], - "_map_object_list": [ - "def _map_object_list(v: object | None, fn) -> list | None:", - ' """Map each element of an assembled array via ``fn``; non-list -> None."""', - " if not isinstance(v, list):", - " return None", - " return [fn(e) for e in v]", - ], - "_dlg_str": [ - "def _dlg_str(v: object | None) -> str | None:", - " return None if v is None else str(v)", - ], - "_dlg_int": [ - "def _dlg_int(v: object | None) -> int | None:", - " if v is None:", - " return None", - " try:", - " return int(v) # type: ignore[arg-type, call-overload]", - " except (TypeError, ValueError):", - " return None", - ], - "_dlg_float": [ - "def _dlg_float(v: object | None) -> float | None:", - " if v is None:", - " return None", - " try:", - " return float(v) # type: ignore[arg-type]", - " except (TypeError, ValueError):", - " return None", - ], - "_dlg_bool": [ - "def _dlg_bool(v: object | None) -> bool | None:", - " if v is None:", - " return None", - " if isinstance(v, bool):", - " return v", - ' return str(v).lower() == "true"', - ], - "_dlg_str_list": [ - "def _dlg_str_list(v: object | None) -> list | None:", - " if not isinstance(v, list):", - " return None", - " return [None if e is None else str(e) for e in v]", - ], -} - -# Stable emission order (deterministic output). -_HELPER_ORDER: tuple[str, ...] = ( - "_read_prop", - "_map_object_list", - "_dlg_str", - "_dlg_int", - "_dlg_float", - "_dlg_bool", - "_dlg_str_list", -) - - -def delegate_helpers(used: set[str]) -> list[str]: - """The shared helper block the generated mappers rely on, scoped to ``used`` and - emitted in stable order. Returns source lines (blank-line separated).""" - lines: list[str] = [ - "# ---- runtime-delegating extract helpers (generated) ----" - ] - for name in _HELPER_ORDER: - if name in used: - lines.append("") - lines.append("") - lines.extend(_HELPER_BLOCKS[name]) - return lines +alias(__name__, "metaobjects.codegen.runtime.extract_delegate_emitter") diff --git a/server/python/src/metaobjects/codegen/extract_schema_emitter.py b/server/python/src/metaobjects/codegen/extract_schema_emitter.py index a6c53d511..ee3c1d4f5 100644 --- a/server/python/src/metaobjects/codegen/extract_schema_emitter.py +++ b/server/python/src/metaobjects/codegen/extract_schema_emitter.py @@ -1,139 +1,4 @@ -"""Turns a payload value-object into Python source fragments for the FR-010 extract -codegen: +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.extract_schema_emitter`.""" +from metaobjects.codegen._moved import alias -* :func:`schema_literal` — a ``ExtractSchema(Format.X, "root", [FieldSpec(...), …])`` - baked descriptor for the emitted parser module. -* :func:`mirror_dataclass` — an all-nullable mirror dataclass ``Extracted`` - (every field ``X | None = None``; the extract entry - returns this nullable twin rather than the strict - Pydantic payload — same reasoning as the Kotlin / C# - nullable mirror). -* :func:`mirror_initializer` — ``Extracted(field=as_string(d, "field"), …)``. - -Mirrors the C# / Java ``ExtractSchemaEmitter`` adapted to Python syntax + the -nullable-mirror shape. Bounded scope: scalar / enum / scalar-array. Nested object + -array-of-enum deferred. -""" -from __future__ import annotations - -from metaobjects.codegen import fr010_field_mapping as fm -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.meta_data import MetaData - -# The extract_map accessors a generated module may import (sorted, deduped subset). -ALL_EXTRACT_MAP_HELPERS: tuple[str, ...] = ( - "as_bool", - "as_double", - "as_int", - "as_long", - "as_string", - "as_string_list", -) - - -def _format_enum(fmt: str) -> str: - return "Format.XML" if fmt.lower() == "xml" else "Format.JSON" - - -def _field_spec_literal(field: MetaData, owner: MetaData) -> str: - name = field.name - required = fm.is_required(field) - req = "True" if required else "False" - - # An enum ARRAY is checked BEFORE the scalar-enum branch below: it is baked as a - # string-list scalar slot (read via as_string_list), NOT a scalar enum_field — - # otherwise the runtime collapses the list to one stringified value (the cross-port - # "enum-before-isArray" ordering bug). The mirror type / extract_map helper already - # route arrays first; this keeps the baked schema spec consistent with them. Scoped - # to the enum case so NON-enum scalar arrays keep their real element FieldKind below - # (the self-contained baked path drops arrays as MALFORMED before kind is read, so - # the kind is inert either way — but emitting the true kind is correct + clearer). - if fm.is_array(field) and field.sub_type == fc.FIELD_SUBTYPE_ENUM: - return f'FieldSpec.scalar("{name}", FieldKind.STRING, {req})' - - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - values_lit = fm.string_list_literal(fm.enum_values(field)) - alias_lit = fm.properties_map_literal(field.attrs().get(fc.FIELD_ATTR_ENUM_ALIAS)) - # FR-011: resolve the three new enum args (field → object.value → "strip" for - # normalize). Keep the back-compat 4-arg form when nothing is set; otherwise - # emit the 7-arg form (..., coerce_default, normalize, default_value). - coerce_default = fm.coerce_default(field) - default_value = fm.default_value(field) - normalize = fm.resolve_normalize(field, owner) - if ( - coerce_default is None - and default_value is None - and normalize == fc.NORMALIZE_DEFAULT - ): - return f'FieldSpec.enum_field("{name}", {req}, {values_lit}, {alias_lit})' - cd_lit = "None" if coerce_default is None else fm.py_string_literal(coerce_default) - norm_lit = fm.py_string_literal(normalize) - dv_lit = "None" if default_value is None else fm.py_string_literal(default_value) - return ( - f'FieldSpec.enum_field("{name}", {req}, {values_lit}, {alias_lit}, ' - f"{cd_lit}, {norm_lit}, {dv_lit})" - ) - - if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: - # Self-contained path models a nested object as a plain (opaque) STRING scalar - # slot — it does not recurse. The runtime-delegating ``*_with_loader`` entry - # populates the real nested graph. (No trailing inline comment: this literal is - # joined into a ``[...]`` argument list, where a ``#`` comment would be illegal.) - return f'FieldSpec.scalar("{name}", FieldKind.STRING, {req})' - - kind = fm.scalar_kind(field.sub_type) or "STRING" - # @xmlText: a scalar field marked to receive its element's XML text content - # (JAXB @XmlValue / Jackson @JacksonXmlText / .NET [XmlText]). Mirrors the TS - # extract-schema-emitter textContentField branch. No effect for JSON. - if fm.xml_text(field): - return f'FieldSpec.text_content_field("{name}", FieldKind.{kind}, {req})' - return f'FieldSpec.scalar("{name}", FieldKind.{kind}, {req})' - - -def schema_literal(vo: MetaData, fmt: str, root_name: str) -> str: - """Emit ``ExtractSchema(Format.X, "rootName", [FieldSpec(...), …])``.""" - format_enum = _format_enum(fmt) - specs = [_field_spec_literal(f, vo) for f in fm.fields(vo)] - if not specs: - return f'ExtractSchema({format_enum}, "{root_name}", [])' - body = ", ".join(specs) - return f'ExtractSchema({format_enum}, "{root_name}", [{body}])' - - -def mirror_dataclass(vo: MetaData, record_name: str) -> list[str]: - """Emit the all-nullable mirror dataclass declaration (source lines).""" - base = ( - record_name[: -len("Extracted")] - if record_name.endswith("Extracted") - else record_name - ) - lines: list[str] = [ - "@dataclass(frozen=True, slots=True)", - f"class {record_name}:", - f' """Best-effort extracted twin of ``{base}`` — every field nullable', - ' (``None`` where the value was lost or malformed)."""', - ] - field_lines = [ - f" {f.name}: {fm.mirror_type(f)} = None" for f in fm.fields(vo) - ] - if field_lines: - lines.extend(field_lines) - else: - lines.append(" pass") - return lines - - -def mirror_initializer(vo: MetaData, record_name: str) -> str: - """Emit ``(field=as_string(d, "field"), …)``.""" - assigns = [f"{f.name}={fm.extract_map_call(f)}" for f in fm.fields(vo)] - return f"{record_name}({', '.join(assigns)})" - - -def extract_map_imports(vo: MetaData) -> list[str]: - """The sorted, deduped ``extract_map`` accessor names the mirror initializer needs.""" - used: set[str] = set() - for f in fm.fields(vo): - h = fm.extract_map_helper(f) - if h is not None: - used.add(h) - return [h for h in ALL_EXTRACT_MAP_HELPERS if h in used] +alias(__name__, "metaobjects.codegen.runtime.extract_schema_emitter") diff --git a/server/python/src/metaobjects/codegen/fr010_field_mapping.py b/server/python/src/metaobjects/codegen/fr010_field_mapping.py index ae2d7510b..6603103bb 100644 --- a/server/python/src/metaobjects/codegen/fr010_field_mapping.py +++ b/server/python/src/metaobjects/codegen/fr010_field_mapping.py @@ -1,228 +1,4 @@ -"""Shared field-kind mapping for the FR-010 codegen emitters (``extract_schema_emitter`` -+ ``output_format_spec_emitter``). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.fr010_field_mapping`.""" +from metaobjects.codegen._moved import alias -Maps a metadata field subtype onto the render engine's ``FieldKind`` member, the -idiomatic ``X | None`` Python type used by the extract mirror dataclass, and the -``extract_map.as_*`` accessor that reads it from the forgiving outcome map. - -Mirrors the C# ``Fr010FieldMapping`` and the Java ``ExtractSchemaEmitter`` ordering. -Bounded scope (parity with Java / Kotlin / C#): scalar / enum / scalar-array. Nested -object + array-of-enum are deferred (see ``render/extract/KNOWN_GAPS.md``). - -Python has a single ``int`` type, so the ``LONG`` and ``CURRENCY`` subtypes map to the -``LONG`` ``FieldKind`` (and ``extract_map.as_long`` — identical to ``as_int``), keeping -the cross-port ``FieldKind`` vocabulary intact even though the emitted accessor is the -same. The mirror annotation collapses ``INT``/``LONG`` to ``int`` (one numeric int type). -""" -from __future__ import annotations - -from collections.abc import Iterable - -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.meta_data import MetaData -from metaobjects.meta.template.template_constants import TEMPLATE_ATTR_XML_TEXT -from metaobjects.shared.base_types import TYPE_FIELD - - -def fields(vo: MetaData) -> list[MetaData]: - """The field children of a payload value-object, in declaration order - (effective children — own + inherited via ``extends``).""" - return [c for c in vo.children() if c.type == TYPE_FIELD] - - -def is_array(field: MetaData) -> bool: - """Effective array-ness (ADR-0039 resolving): the native ``is_array`` flag OR the - ``@isArray`` attr, resolved through the ``extends`` super chain so a concrete field - inheriting array-ness from an abstract parent is honored.""" - return field.resolved_is_array() - - -def is_required(field: MetaData) -> bool: - """``@required`` — accepts a bool ``True`` or the string ``"true"``.""" - v = field.attrs().get(fc.FIELD_ATTR_REQUIRED) - if v is True: - return True - return isinstance(v, str) and v.lower() == "true" - - -def xml_text(field: MetaData) -> bool: - """``@xmlText`` — the XML text-content extract marker (accepts a bool ``True`` or the - string ``"true"``). When set, codegen bakes a ``FieldSpec.text_content_field(...)``. - Mirrors the TS ``xmlText(field)`` helper.""" - v = field.attrs().get(TEMPLATE_ATTR_XML_TEXT) - if v is True: - return True - return isinstance(v, str) and v.lower() == "true" - - -def enum_values(field: MetaData) -> list[str]: - """The string members of an enum field's ``@values`` attr (empty when absent).""" - v = field.attrs().get(fc.FIELD_ATTR_VALUES) - if isinstance(v, (list, tuple)): - return [str(x) for x in v] - return [] - - -def _attr_string(node: MetaData, name: str) -> str | None: - """The RESOLVING string value of attr *name* (own + inherited via ``extends``), - or ``None``. - - ADR-0039 — effective read: a concrete enum field extending an abstract enum may - inherit ``@coerceDefault``/``@default``/``@normalize`` from the abstract parent, - so these resolve through the super chain (``get_meta_attr``), never own-only.""" - v = node.get_meta_attr(name) - return v if isinstance(v, str) else None - - -def coerce_default(field: MetaData) -> str | None: - """FR-011: the enum field's effective ``@coerceDefault`` member, or ``None``.""" - return _attr_string(field, fc.FIELD_ATTR_COERCE_DEFAULT) - - -def default_value(field: MetaData) -> str | None: - """FR-011: the enum field's effective ``@default`` absent-fill member, or ``None``.""" - return _attr_string(field, fc.FIELD_ATTR_DEFAULT) - - -def resolve_normalize(field: MetaData, owner: MetaData | None) -> str: - """FR-011: resolve the enum normalization mode — field-level ``@normalize``, else the - owning ``object.value``'s ``@normalize`` (the per-object default), else the global - default (``"strip"``). Mirrors the Java/Kotlin/C#/TS ``resolveNormalize``.""" - field_mode = _attr_string(field, fc.FIELD_ATTR_NORMALIZE) - if field_mode is not None: - return field_mode - if owner is not None: - owner_mode = _attr_string(owner, fc.FIELD_ATTR_NORMALIZE) - if owner_mode is not None: - return owner_mode - return fc.NORMALIZE_DEFAULT - - -def scalar_kind(sub_type: str) -> str | None: - """The render-engine ``FieldKind`` member name for a scalar field subtype, or - ``None`` if the subtype is non-scalar (enum / object).""" - if sub_type in ( - fc.FIELD_SUBTYPE_STRING, - fc.FIELD_SUBTYPE_DATE, - fc.FIELD_SUBTYPE_TIME, - fc.FIELD_SUBTYPE_TIMESTAMP, - # ADR-0036/0037 Wave 3: uri/inet are string scalars on the wire. - fc.FIELD_SUBTYPE_URI, - fc.FIELD_SUBTYPE_INET, - ): - return "STRING" - if sub_type == fc.FIELD_SUBTYPE_INT: - return "INT" - if sub_type in (fc.FIELD_SUBTYPE_LONG, fc.FIELD_SUBTYPE_CURRENCY): - return "LONG" - if sub_type in ( - fc.FIELD_SUBTYPE_DOUBLE, - fc.FIELD_SUBTYPE_FLOAT, - fc.FIELD_SUBTYPE_DECIMAL, - ): - return "DOUBLE" - if sub_type == fc.FIELD_SUBTYPE_BOOLEAN: - return "BOOLEAN" - return None - - -def mirror_type(field: MetaData) -> str: - """The ``X | None`` Python annotation for a field in the extract mirror dataclass. - - A extracted array can contain null elements where individual items were lost, so - the array type is ``list[str | None] | None`` (matches ``extract_map.as_string_list``). - """ - # Nested object (single OR array): the self-contained path defers to a typed null; - # the runtime-delegating path overrides this with the nested mirror type. Checked - # BEFORE the generic is_array branch so an array-of-objects is NOT mistyped as a - # string array. Mirrors the Java/TS fix. - if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: - return "object | None" # nested deferred (self-contained path) - if is_array(field): - return "list[str | None] | None" - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - return "str | None" # enum is string-backed - kind = scalar_kind(field.sub_type) - if kind in ("INT", "LONG"): - return "int | None" - if kind == "DOUBLE": - return "float | None" - if kind == "BOOLEAN": - return "bool | None" - return "str | None" - - -def extract_map_helper(field: MetaData) -> str | None: - """The ``extract_map`` accessor a field reads through, or ``None`` - (object/deferred fields read no helper). - - Single source of the field → accessor decision; ``extract_map_call`` builds - the ``as_*(d, "name")`` call on top of this.""" - # Nested object (single OR array) → no helper in the self-contained path. Checked - # BEFORE is_array so an array-of-objects is NOT read via as_string_list. Mirrors the - # Java/TS fix. - if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: - return None - if is_array(field): - return "as_string_list" - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - return "as_string" - return { - "INT": "as_int", - "LONG": "as_long", - "DOUBLE": "as_double", - "BOOLEAN": "as_bool", - }.get(scalar_kind(field.sub_type) or "", "as_string") - - -def extract_map_call(field: MetaData) -> str: - """The ``as_*(d, "name")`` call that reads this field from the forgiving map ``d``. - - The helpers are imported by name into the generated module (see - ``extract_schema_emitter.extract_map_imports``).""" - helper = extract_map_helper(field) - if helper is None: - # Nested object: the self-contained initializer leaves it None (the - # runtime-delegating path populates it). Bare ``None`` — no trailing inline - # comment: this expr is joined into a ``Name(field=..., …)`` call-args list. - return "None" - return f'{helper}(d, "{field.name}")' - - -def py_string_literal(value: str) -> str: - """A safe Python double-quoted string literal for embedding free text - (``@example`` / ``@instruction`` / ``@enumDoc`` values) into emitted source. - - Uses ``json.dumps`` — a JSON string is a valid Python string literal and escapes - backslashes / quotes / control chars correctly (NOT naive quoting).""" - import json - - return json.dumps(value, ensure_ascii=False) - - -def string_list_literal(values: Iterable[str]) -> str: - """A Python ``["a", "b"]`` list literal for enum members.""" - return "[" + ", ".join(py_string_literal(v) for v in values) + "]" - - -def properties_map_literal(attr: object) -> str: - """A Python ``{"k": "v", …}`` dict literal for a properties-shaped attr - (``@enumAlias`` / ``@enumDoc``), or ``"None"`` when absent/empty. - - Null values are dropped; keys are sorted to match the canonical-serializer - properties sort (deterministic output).""" - if not isinstance(attr, dict): - return "None" - entries = [ - (k, v) - for k, v in attr.items() - if v is not None - ] - if not entries: - return "None" - entries.sort(key=lambda kv: str(kv[0])) - parts = [ - f"{py_string_literal(str(k))}: {py_string_literal(str(v))}" - for k, v in entries - ] - return "{" + ", ".join(parts) + "}" +alias(__name__, "metaobjects.codegen.runtime.fr010_field_mapping") diff --git a/server/python/src/metaobjects/codegen/generator.py b/server/python/src/metaobjects/codegen/generator.py index d9a05ca38..eaf7ddc23 100644 --- a/server/python/src/metaobjects/codegen/generator.py +++ b/server/python/src/metaobjects/codegen/generator.py @@ -17,6 +17,26 @@ class EmittedFile: generated_by: str = "" # set by the runner from Generator.name +#: The docs surface the stock ``api-docs`` generator takes over: ``metaobjects docs`` runs a +#: generator carrying this value as ``docs_surface``, and ``gen`` skips it. +DOCS_SURFACE_API = "api" + + +#: The default api-surface subdirectory (the cross-port contract's ``api/python``). +DEFAULT_API_SUBDIR = "api/python" + + +@dataclass(frozen=True) +class DocsRunFacts: + """What a ``metaobjects docs`` run tells a docs generator beyond the model: the project + label shown in the pages, the subdirectory the pages land under, and the base URL the + model back-links federate to (``None`` = relative links).""" + + project: str = "project" + api_subdir: str = DEFAULT_API_SUBDIR + model_base_url: str | None = None + + @dataclass class GenContext: entities: list[MetaObject] @@ -24,6 +44,8 @@ class GenContext: matches: Callable[[MetaObject], bool] config: GenConfig warn: Callable[[str], None] + #: Set by ``metaobjects docs`` only; ``None`` in a ``gen`` run. + docs: DocsRunFacts | None = None class Generator(Protocol): diff --git a/server/python/src/metaobjects/codegen/generator_registry.py b/server/python/src/metaobjects/codegen/generator_registry.py index b8794ab0b..a8b1fb003 100644 --- a/server/python/src/metaobjects/codegen/generator_registry.py +++ b/server/python/src/metaobjects/codegen/generator_registry.py @@ -26,6 +26,7 @@ from metaobjects.codegen.generator import Generator from metaobjects.codegen.project_config import DEFAULT_REQUIREMENT_WITNESS_MODULE +from metaobjects.codegen.generators.api_docs_generator import api_docs_generator from metaobjects.codegen.generators.entity_model import entity_model from metaobjects.codegen.generators.extractor_generator import extractor_generator from metaobjects.codegen.generators.filter_allowlist_generator import ( @@ -157,6 +158,12 @@ def symbol(self) -> str | None: """The factory name an owned copy exports, or ``None``.""" return self.source.__name__ if self.source is not None else None + @property + def docs_surface(self) -> str | None: + """The docs surface this generator takes over (``metaobjects docs`` runs it, ``gen`` + skips it), or ``None`` for a generator ``gen`` runs.""" + return getattr(self.factory(GeneratorBuildContext()), "docs_surface", None) + def _template_primitive(_ctx: GeneratorBuildContext) -> Generator: """A no-op default for the ``template`` PRIMITIVE generator. @@ -195,7 +202,26 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: #: agree with each other (see ``runtime/shared_predicates.py``). _SHARED_PREDICATES: tuple[str, ...] = ("shared_predicates",) -#: Stable name -> GeneratorEntry. The 11 native generators whose manifest `ports` +#: The emitters a generator writes its output THROUGH, handed over with it: the helper modules +#: that return emitted text, which an owned copy therefore owns instead of calling the +#: packaged copy. Eject copies them next to the generator (``codegen/runtime/``) and the +#: generator imports them relatively. Naming and type-mapping tables, the model walks and the +#: generator base stay in the package as the engine. See ``tests/codegen/test_eject_alone.py``. +_FR010_FIELD_MAPPING = "fr010_field_mapping" +_M2M_CODEGEN = "m2m_codegen" +_FR019_SHARED_ENUM = "fr019_shared_enum" +_EXTRACT_DELEGATE = "extract_delegate_emitter" +_EXTRACT_SCHEMA = "extract_schema_emitter" +_OUTPUT_FORMAT_SPEC = "output_format_spec_emitter" +#: The api reference's page IR, builder, renderer and path math. +_API_DOCS_RUNTIME: tuple[str, ...] = ( + "api_docs_model", + "api_docs_builder", + "api_docs_renderer", + "api_docs_paths", +) + +#: Stable name -> GeneratorEntry. The 12 native generators whose manifest `ports` #: include `python` (ADR-0021 D3). Set equality, tier AND layer are conformance-tested #: against the manifest. GENERATOR_REGISTRY: dict[str, GeneratorEntry] = { @@ -211,7 +237,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: layer="model", factory=lambda _ctx: entity_model(), source=entity_model, - runtime=_SHARED_PREDICATES, + runtime=_SHARED_PREDICATES + (_FR019_SHARED_ENUM, _M2M_CODEGEN), ), "routes": GeneratorEntry( name="routes", @@ -224,7 +250,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: # generated package fails at import (ModuleNotFoundError). requires=("entity", "filter-allowlist"), source=router_generator, - runtime=_ROUTER_RUNTIME + _SHARED_PREDICATES, + runtime=_ROUTER_RUNTIME + _SHARED_PREDICATES + (_FR010_FIELD_MAPPING, _M2M_CODEGEN), ), "output-parser": GeneratorEntry( name="output-parser", @@ -234,6 +260,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: factory=lambda _ctx: output_parser_generator(), requires=("entity",), source=output_parser_generator, + runtime=(_EXTRACT_DELEGATE, _FR010_FIELD_MAPPING), ), "output-prompt": GeneratorEntry( name="output-prompt", @@ -242,6 +269,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: layer="capability", factory=lambda _ctx: output_prompt_generator(), source=output_prompt_generator, + runtime=(_OUTPUT_FORMAT_SPEC, _FR010_FIELD_MAPPING), ), "render-helper": GeneratorEntry( name="render-helper", @@ -260,7 +288,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: factory=lambda _ctx: extractor_generator(), requires=("entity",), source=extractor_generator, - runtime=_SHARED_PREDICATES, + runtime=_SHARED_PREDICATES + (_EXTRACT_DELEGATE, _FR010_FIELD_MAPPING), ), "requirement-tests": GeneratorEntry( name="requirement-tests", @@ -284,7 +312,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: layer="api", factory=lambda _ctx: filter_allowlist_generator(), source=filter_allowlist_generator, - runtime=_SHARED_PREDICATES, + runtime=_SHARED_PREDICATES + (_M2M_CODEGEN,), ), "names": GeneratorEntry( name="names", @@ -301,6 +329,19 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: layer="capability", factory=lambda _ctx: trace_helper_generator(), source=trace_helper_generator, + runtime=(_EXTRACT_SCHEMA, _FR010_FIELD_MAPPING), + ), + "api-docs": GeneratorEntry( + name="api-docs", + description=( + "Per-entity/template SDK API reference (generated-code API; human + agent forms). " + "Run by `metaobjects docs`, not `gen`." + ), + tier="native", + layer="docs", + factory=lambda _ctx: api_docs_generator(), + source=api_docs_generator, + runtime=_API_DOCS_RUNTIME + _SHARED_PREDICATES + (_M2M_CODEGEN,), ), } diff --git a/server/python/src/metaobjects/codegen/generators/api_docs_generator.py b/server/python/src/metaobjects/codegen/generators/api_docs_generator.py new file mode 100644 index 000000000..a3c86fae1 --- /dev/null +++ b/server/python/src/metaobjects/codegen/generators/api_docs_generator.py @@ -0,0 +1,68 @@ +"""``api-docs`` — the Python SDK api reference (the ``api/python`` surface of the cross-port +SDK-docs contract): one page per entity, value object, projection, served report and +template, an index, and the agent-facing ``AGENT-API.md`` page. + +Run by ``metaobjects docs``, not by ``metaobjects gen``: it carries ``docs_surface``, which +``gen`` skips. Its files land under the docs output directory, prefixed with the api +subdirectory the run names (``ctx.docs``). + +The page IR, the builder that decides which units and symbols get a page, the renderer and +the page-path math live beside this module in ``codegen/runtime/`` and are handed over with +it: ``metaobjects eject api-docs`` copies all of them, so the copy owns every line that +writes a page. What each page says a generator emits for comes from +``..runtime.shared_predicates`` — the module the router and filter-allowlist generators read +their gates from. An owned ``api-docs`` and an owned ``routes`` therefore read ONE owned copy +of those predicates, and the reference describes the code the project's own generators emit. +The naming seam (:mod:`metaobjects.apidocs.naming`) is the engine: the same names the +generators emit under. +""" +from __future__ import annotations + +from metaobjects.codegen.generator import ( + DOCS_SURFACE_API, + DocsRunFacts, + EmittedFile, + GenContext, + Generator, +) + +from ..runtime.api_docs_builder import PythonApiModelBuilder +from ..runtime.api_docs_paths import Layout, doc_page_output_path, model_cross_href +from ..runtime.api_docs_renderer import PythonApiDocsRenderer + + +class ApiDocsGenerator: + """Writes the api reference pages for the run's loaded model.""" + + name = "api-docs" + #: Which docs surface this generator takes over (see ``metaobjects docs``). + docs_surface = DOCS_SURFACE_API + + def generate(self, ctx: GenContext) -> list[EmittedFile]: + if ctx.loaded_root is None: + return [] + # A docs run supplies the facts. A caller that runs this generator directly, as a + # registry-wide probe does, gets the defaults and a neutral project label. + facts = ctx.docs or DocsRunFacts() + model = PythonApiModelBuilder().build(ctx.loaded_root, facts.project) + renderer = PythonApiDocsRenderer() + layout = Layout.PACKAGE + + def page(path: str, content: str) -> EmittedFile: + return EmittedFile(path=f"{facts.api_subdir}/{path}", content=content) + + files: list[EmittedFile] = [] + for unit in model.units: + page_path = doc_page_output_path(layout, unit.package, unit.node) + href = model_cross_href( + f"{facts.api_subdir}/{page_path}", page_path, facts.model_base_url + ) + files.append(page(page_path, renderer.render_unit_page(unit, href))) + files.append(page("README.md", renderer.render_index(model, layout))) + files.append(page("AGENT-API.md", renderer.render_agent_api(model))) + return files + + +def api_docs_generator() -> Generator: + """Generator factory: the api reference ``metaobjects docs`` writes.""" + return ApiDocsGenerator() diff --git a/server/python/src/metaobjects/codegen/generators/entity_model.py b/server/python/src/metaobjects/codegen/generators/entity_model.py index 476c0fc7c..367139cef 100644 --- a/server/python/src/metaobjects/codegen/generators/entity_model.py +++ b/server/python/src/metaobjects/codegen/generators/entity_model.py @@ -43,11 +43,6 @@ from metaobjects.codegen.type_map import field_is_array, py_type_for from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator, per_entity -from metaobjects.codegen.generators import fr019_shared_enum as fr019 -from metaobjects.codegen.generators.m2m_codegen import ( - build_object_index, - resolve_m2m_descriptors, -) from metaobjects.codegen.generators.tph_plan import tph_subtype_binding from metaobjects.codegen.value_objects import ( model_class_name, @@ -55,6 +50,8 @@ object_ref_class_name, ) +from ..runtime import fr019_shared_enum as fr019 +from ..runtime.m2m_codegen import build_object_index, resolve_m2m_descriptors from ..runtime.shared_predicates import has_literal_default, is_sql_expr_default diff --git a/server/python/src/metaobjects/codegen/generators/extractor_generator.py b/server/python/src/metaobjects/codegen/generators/extractor_generator.py index 2659ed6bc..792e57105 100644 --- a/server/python/src/metaobjects/codegen/generators/extractor_generator.py +++ b/server/python/src/metaobjects/codegen/generators/extractor_generator.py @@ -38,8 +38,6 @@ from collections.abc import Callable from metaobjects.apidocs.naming import snake_case as _snake_case -from metaobjects.codegen import fr010_field_mapping as fm -from metaobjects.codegen import extract_delegate_emitter as rde from metaobjects.codegen.constants import generated_header from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator @@ -52,6 +50,8 @@ from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.meta.meta_data import MetaData +from ..runtime import extract_delegate_emitter as rde +from ..runtime import fr010_field_mapping as fm from ..runtime.shared_predicates import has_literal_default _GENERATOR_NAME = "extractor-generator" diff --git a/server/python/src/metaobjects/codegen/generators/filter_allowlist_generator.py b/server/python/src/metaobjects/codegen/generators/filter_allowlist_generator.py index 6b67406ab..ca20de679 100644 --- a/server/python/src/metaobjects/codegen/generators/filter_allowlist_generator.py +++ b/server/python/src/metaobjects/codegen/generators/filter_allowlist_generator.py @@ -36,13 +36,13 @@ from metaobjects.codegen.constants import generated_header from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator, per_entity -from metaobjects.codegen.generators.m2m_codegen import build_object_index from metaobjects.codegen.generators.tph_plan import tph_plan_for from metaobjects.codegen.instance_artifacts import is_sourceless_entity from metaobjects.meta.core.field import field_constants as fc from metaobjects.meta.core.field.meta_field import MetaField from metaobjects.meta.core.object.meta_object import MetaObject +from ..runtime.m2m_codegen import build_object_index from ..runtime.shared_predicates import emits_router diff --git a/server/python/src/metaobjects/codegen/generators/fr019_shared_enum.py b/server/python/src/metaobjects/codegen/generators/fr019_shared_enum.py index 8dd24b7b9..b6280c9bf 100644 --- a/server/python/src/metaobjects/codegen/generators/fr019_shared_enum.py +++ b/server/python/src/metaobjects/codegen/generators/fr019_shared_enum.py @@ -1,154 +1,4 @@ -"""FR-019 — shared + externally-provided enum resolution (Python port of the TS reference -``enum-shared.ts`` and the C#/Java/Kotlin ``Fr019SharedEnum``). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.fr019_shared_enum`.""" +from metaobjects.codegen._moved import alias -A reusable enum is a ROOT/package-level abstract ``field.enum`` (a sibling of ``object.entity``) -that concrete entity fields ``extends``. Per ADR-0026 such a declaration is materialized ONCE per -port as a module-level ``class (str, Enum)`` and referenced, instead of redeclaring the -members inline (a per-entity ``Literal[...]``) in every consumer. - -Two cases on the abstract declaration: - -* non-``@provided`` → metaobjects MATERIALIZES the type once (a shared ``enums.py`` module), and - consuming fields reference it (``from .enums import ``). -* ``@provided: true`` → metaobjects emits NOTHING for the type; consuming fields import an existing - type from a per-port-configured module (``GenConfig.provided_enum_packages`` keyed by the enum's - declaring metadata package, then the ``provided_enum_namespace`` fallback). The module never lives - in metadata (ADR-0001); a missing config for a referenced provided enum is a codegen-time error - naming the enum + its package. - -The common inline case (a concrete ``field.enum`` with own ``@values``, no root-extends) is -UNCHANGED — it stays a per-entity ``Literal[...]`` (byte-identical default). -""" -from __future__ import annotations - -import warnings -from dataclasses import dataclass - -from metaobjects.meta.core.field.meta_field import ( - MetaField, - resolve_shared_enum_declaration, -) -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.meta_data import MetaData -from metaobjects.shared.separators import PACKAGE_SEP -from metaobjects.codegen import type_map -from metaobjects.codegen.config import GenConfig - - -@dataclass(frozen=True) -class SharedEnum: - """A shared (package-level abstract) enum declaration codegen reasons about.""" - - name: str #: materialized type name — the decl's PascalCase short name (cross-port id) - values: list[str] #: member symbols (the @values SSOT), verbatim - provided: bool #: True when the decl carries @provided: true (reference, don't materialize) - meta_package: str #: declaring metadata package in "::" form (the provided-config key) - - -def _pascal(name: str) -> str: - """``priority`` → ``Priority`` (leading char only; no snake-splitting — cross-port rule).""" - return name[:1].upper() + name[1:] if name else name - - -def resolve_shared_enum_decl(field: MetaField) -> MetaData | None: - """Deprecated name of - :func:`metaobjects.meta.core.field.meta_field.resolve_shared_enum_declaration`, - kept so a generator ejected before the resolver moved to the metadata layer - still imports. Same answer, including through a multi-hop ``extends``.""" - warnings.warn( - "resolve_shared_enum_decl is deprecated; call " - "metaobjects.meta.core.field.meta_field.resolve_shared_enum_declaration", - DeprecationWarning, - stacklevel=2, - ) - return resolve_shared_enum_declaration(field) - - -def is_provided(decl: MetaData) -> bool: - """``@provided`` truth of an enum DECLARATION. - - ADR-0039 sanctioned own (Python naming inversion: ``attr()`` is the OWN read, - ``get_meta_attr()`` resolves). ``@provided`` is a declaration-layer provenance - marker — "THIS type is supplied by hand-written/third-party code", like - ``is_abstract`` — and does not flow down an ``extends`` chain. - - This is only ever called on the resolved declaration (see - ``shared_enum_for_field``), never on the consuming field, so own and resolving - agree for a plain ``field extends @provided decl``. They diverge on a CHAINED - declaration (root abstract ``B extends`` root abstract ``@provided A``): a - resolving read reports B provided and emits a reference to a hand-written ``B`` - the adopter never declared, instead of materializing B. Matches the JVM ports. - """ - return decl.attr(fc.FIELD_ATTR_PROVIDED) is True - - -def _meta_package_of(decl: MetaData) -> str: - """The decl's metadata package in ``::`` form (its resolution key minus the trailing name).""" - key = decl.resolution_key() - idx = key.rfind(PACKAGE_SEP) - return key[:idx] if idx >= 0 else "" - - -def shared_enum_for_field(field: MetaField) -> SharedEnum | None: - """The shared-enum descriptor *field* resolves to, or ``None`` for inline enums. - - The member set is read off the DECLARATION *decl* (a resolving ``attrs()`` - read, never off *field*) so materialization cannot depend on which consuming - field a generator happens to visit first — every field resolving to the same - shared declaration must describe that declaration identically.""" - decl = resolve_shared_enum_declaration(field) - if decl is None: - return None - values = type_map.effective_enum_values(decl) - if not values: - return None - return SharedEnum( - name=_pascal(decl.name), - values=values, - provided=is_provided(decl), - meta_package=_meta_package_of(decl), - ) - - -def collect_shared_enums(entities: list) -> list[SharedEnum]: - """All shared (package-level abstract) enum decls CONSUMED by at least one entity field, - keyed by materialized type name, sorted by name (deterministic). A decl nobody extends is - not materialized (no dangling type).""" - out: dict[str, SharedEnum] = {} - for entity in entities: - # ADR-0039 — RESOLVING (fields()): mirrors the TS collectSharedEnums - # (``entity.fields()``); an inherited field consuming a shared enum counts. - for f in entity.fields(): - shared = shared_enum_for_field(f) - if shared is None: - continue - out.setdefault(shared.name, shared) - return sorted(out.values(), key=lambda e: e.name) - - -def materialized_shared_enums(entities: list) -> list[SharedEnum]: - """The shared enums metaobjects MATERIALIZES (non-``@provided``, consumed).""" - return [e for e in collect_shared_enums(entities) if not e.provided] - - -def enum_type_and_import(shared: SharedEnum, config: GenConfig) -> tuple[str, str]: - """The ``(type_name, import_line)`` a consuming field emits for a shared/provided enum: - - * materialized → ``("", "from .enums import ")`` (the shared module). - * ``@provided`` → ``("", "from import ")``, *module* resolved from - ``config.provided_enum_packages`` (keyed by the enum's metadata package) then the single - ``config.provided_enum_namespace`` fallback. A missing config is a codegen-time ``ValueError`` - naming the enum + its package (ADR-0026 / ADR-0001).""" - name = shared.name - if not shared.provided: - return name, f"from .enums import {name}" - module = config.provided_enum_packages.get(shared.meta_package) or config.provided_enum_namespace - if not module: - raise ValueError( - f'provided enum "{name}" (declared in package "{shared.meta_package}") is marked ' - f"@provided but no Python import module is configured to reference it from. Map its " - f'package in GenConfig.provided_enum_packages["{shared.meta_package}"] = ' - f'"your_app.enums", or set the single provided_enum_namespace fallback, so the ' - f'generated code can import "{name}".' - ) - return name, f"from {module} import {name}" +alias(__name__, "metaobjects.codegen.runtime.fr019_shared_enum") diff --git a/server/python/src/metaobjects/codegen/generators/m2m_codegen.py b/server/python/src/metaobjects/codegen/generators/m2m_codegen.py index 813794dbb..c1cc0bcb5 100644 --- a/server/python/src/metaobjects/codegen/generators/m2m_codegen.py +++ b/server/python/src/metaobjects/codegen/generators/m2m_codegen.py @@ -1,240 +1,4 @@ -"""Shared M:N (many-to-many) codegen descriptor — the build-time resolution of a -source entity's ``relationship.* @cardinality:"many" + @through`` children into -the concrete coordinates each Python generator needs (Pydantic nested collection, -FastAPI traversal route, and the physical junction/target column names the -repository seam joins on). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.m2m_codegen`.""" +from metaobjects.codegen._moved import alias -This mirrors the TS reference ``relation-resolver.ts`` (Unit 10): codegen derives -the descriptor at build time from the junction entity's two ``identity.reference`` -children (the cross-port SSOT for FK direction) via the shared -``derive_m2m_fields`` helper — the relationship never restates its FK columns. - -The descriptor carries: - * ``relation_name`` — the navigation member name (route segment + Pydantic field). - * ``target_entity`` — the related entity's bare name (Pydantic element type). - * ``source_plural`` / ``target_plural`` — pluralized entity names for the - REST contract: the source URL segment is the ENTITY name pluralized - (``Person`` → ``persons``), NOT the physical ``@table`` (cross-port grammar). - * ``source_column`` / ``target_column`` — the PHYSICAL junction FK columns - (e.g. ``follower_id`` / ``followee_id``); the repository seam joins on these. - * ``junction_table`` / ``target_table`` / ``target_pk_column`` — physical names - for the join. - * ``symmetric`` — selects the union-on-read traversal. -""" -from __future__ import annotations - -from dataclasses import dataclass - -from metaobjects.apidocs.naming import route_path -from metaobjects.source_resolution import resolve_table_name -from metaobjects.naming import ( - DEFAULT_COLUMN_NAMING, - apply_column_naming_strategy, - resolve_column_name, -) -from metaobjects.meta.core.field.meta_field import MetaField -from metaobjects.meta.meta_data import MetaData -from metaobjects.meta.core.identity.identity_constants import ( - IDENTITY_ATTR_FIELDS, - IDENTITY_SUBTYPE_PRIMARY, -) -from metaobjects.meta.core.object.meta_object import MetaObject -from metaobjects.meta.core.object.object_constants import OBJECT_ATTR_DISCRIMINATOR_VALUE -from metaobjects.codegen.generators.tph_plan import is_tph_subtype -from metaobjects.meta.core.relationship.derive_m2m_fields import ( - M2MDerivationError, - derive_m2m_fields, -) -from metaobjects.meta.core.relationship.meta_relationship import MetaRelationship -from metaobjects.meta.core.relationship.relationship_constants import CARDINALITY_MANY -from metaobjects.shared.base_types import TYPE_IDENTITY -from metaobjects.shared.separators import PACKAGE_SEP - - -@dataclass(frozen=True) -class M2mDescriptor: - """Build-time coordinates of one M:N navigation on a source entity.""" - - relation_name: str - target_entity: str - source_plural: str - target_plural: str - junction_table: str - target_table: str - source_column: str - target_column: str - target_pk_column: str - symmetric: bool - #: The target's OWN ``@discriminatorValue`` when ``@objectRef`` resolves to a - #: concrete TPH subtype (FW-8 follow-up; mirrors the TS reference's - #: ``targetDiscriminator``) — ``None`` for the overwhelming common case (a - #: vanilla, non-TPH target). The target's rows physically live in its - #: discriminator base's shared table, so a route generator threads this value - #: into the ``find_related_`` seam as an extra ``target_subtype`` - #: parameter, structurally handing the consumer's join what it needs to - #: filter to just that subtype rather than every row of the shared table. - #: Defaulted so existing positional construction (if any) stays valid. - target_discriminator: str | None = None - - -def _strip_package(name: str) -> str: - idx = name.rfind(PACKAGE_SEP) - return name[idx + len(PACKAGE_SEP):] if idx >= 0 else name - - -def _physical_table(entity: MetaObject, *, relation_context: str) -> str: - """The entity's physical SQL table name, via the shared - :func:`~metaobjects.source_resolution.resolve_table_name` resolver. - - Raises :class:`M2MDerivationError` — naming *entity* and *relation_context* - — when it declares (and inherits) no primary source at all. #248: DB - participation derives from a declared source, never from the object - subtype, so ``resolve_table_name`` returning ``None`` here means the - junction/target has no physical table to join through at all; a M:N - relationship's whole point is a real SQL join, so that is a metadata - authoring error, not a legitimate "doesn't participate in persistence" - shape. Coercing to the bare entity name — the pre-existing behaviour this - replaces — fabricated a table that was never created, precisely the defect - class the C# port's EF binding had to fix this same session (a - ``[Table(...)]`` attribute naming a class that was never generated).""" - table = resolve_table_name(entity) - if table is None: - raise M2MDerivationError( - f"{relation_context}: entity \"{entity.name}\" declares no primary " - f"source (source.rdb) — a M:N join needs a physical table on both " - f"the junction and the target." - ) - return table - - -def _column_of( - field: MetaField | None, fallback: str, strategy: str = DEFAULT_COLUMN_NAMING -) -> str: - """Physical column name for a junction FK. Delegates to the shared - :func:`metaobjects.naming.resolve_column_name` — the SAME resolver the runtime uses, - so generated junction SQL and runtime SQL cannot disagree about a column name. - - *fallback* covers the case where the field could not be resolved at all; it is a - bare metadata name, so it goes through the strategy too.""" - if field is None: - return apply_column_naming_strategy(fallback, strategy) - return resolve_column_name(field, strategy) - - -def _field_named(entity: MetaObject, field_name: str) -> MetaField | None: - for f in entity.fields(): - if f.name == field_name: - return f - return None - - -def pk_field_name(entity: MetaObject) -> str: - """The single PK field name (``identity.primary @fields``), default ``id``.""" - for c in entity.children(): - if c.type == TYPE_IDENTITY and c.sub_type == IDENTITY_SUBTYPE_PRIMARY: - fields = c.get_meta_attr(IDENTITY_ATTR_FIELDS) # ADR-0039: resolving (identity attr) - if isinstance(fields, (list, tuple)) and fields: - first = fields[0] - if isinstance(first, str): - return first - if isinstance(fields, str) and fields: - return fields.split(",")[0].strip() or "id" - return "id" - - -def m2m_relationships(entity: MetaObject) -> list[MetaRelationship]: - """The entity's ``relationship.* @cardinality:"many" + @through`` children - (declaration order). 1:N relationships (no ``@through``) are excluded. - - ADR-0039 — RESOLVING (children()): mirrors the TS ``relationships()`` (built on - the effective ``children()``); a relationship inherited via ``extends`` counts. - """ - out: list[MetaRelationship] = [] - for c in entity.children(): - if not isinstance(c, MetaRelationship): - continue - if c.cardinality() != CARDINALITY_MANY: - continue - if c.through() is None: - continue - out.append(c) - return out - - -def resolve_m2m_descriptors( - entity: MetaObject, - object_index: dict[str, MetaObject], - column_naming: str = DEFAULT_COLUMN_NAMING, -) -> list[M2mDescriptor]: - """Resolve every M:N navigation on *entity* into a build-time descriptor. - - *object_index* is a bare-name → entity map of the loaded model. A relationship - that fails derivation (missing/malformed junction, ambiguous self-join) raises - :class:`M2MDerivationError` — the loader validates this; reaching it here is a - metadata error, surfaced loudly (never silently skipped). - """ - descriptors: list[M2mDescriptor] = [] - # The source is *entity* for every relation, so its route segment is - # loop-invariant; only the target varies per relation. - source_plural = route_path(entity.name) - for rel in m2m_relationships(entity): - target_name = _strip_package(rel.object_ref() or "") - junction_name = _strip_package(rel.through() or "") - target = object_index.get(target_name) - junction = object_index.get(junction_name) - if target is None or junction is None: - raise M2MDerivationError( - f'M:N relationship "{entity.name}.{rel.name}": @objectRef ' - f'"{target_name}" / @through "{junction_name}" must resolve to ' - f"loaded entities" - ) - - # Widen to dict[str, MetaData] for the shared derivation helper (dict is - # invariant; MetaObject is a MetaData subtype). - meta_index: dict[str, MetaData] = dict(object_index) - fields = derive_m2m_fields(rel, entity, meta_index) - source_fk = _field_named(junction, fields.source_field) - target_fk = _field_named(junction, fields.target_field) - # FW-8 follow-up: the target's rows live in its discriminator base's shared - # table when @objectRef resolves to a concrete TPH subtype — @discriminatorValue - # is the subtype's OWN value (ADR-0039 sanctioned own, same idiom as tph_plan.py). - target_discriminator = ( - target.attr(OBJECT_ATTR_DISCRIMINATOR_VALUE) if is_tph_subtype(target) else None - ) - - descriptors.append( - M2mDescriptor( - relation_name=rel.name, - target_entity=target.name, - source_plural=source_plural, - target_plural=route_path(target.name), - junction_table=_physical_table( - junction, - relation_context=( - f'M:N relationship "{entity.name}.{rel.name}" @through junction' - ), - ), - target_table=_physical_table( - target, - relation_context=( - f'M:N relationship "{entity.name}.{rel.name}" @objectRef target' - ), - ), - source_column=_column_of(source_fk, fields.source_field, column_naming), - target_column=_column_of(target_fk, fields.target_field, column_naming), - target_pk_column=_column_of( - _field_named(target, pk_field_name(target)), - pk_field_name(target), - column_naming, - ), - symmetric=rel.symmetric(), - target_discriminator=target_discriminator, - ) - ) - return descriptors - - -def build_object_index(entities: list[MetaObject]) -> dict[str, MetaObject]: - """Bare-name → entity map (the codegen resolution surface, mirroring the - runtime resolver's ``object_index``).""" - return {e.name: e for e in entities} +alias(__name__, "metaobjects.codegen.runtime.m2m_codegen") diff --git a/server/python/src/metaobjects/codegen/generators/output_parser_generator.py b/server/python/src/metaobjects/codegen/generators/output_parser_generator.py index 00ee12a55..4aee3894e 100644 --- a/server/python/src/metaobjects/codegen/generators/output_parser_generator.py +++ b/server/python/src/metaobjects/codegen/generators/output_parser_generator.py @@ -31,7 +31,6 @@ from collections.abc import Callable from metaobjects.apidocs.naming import snake_case as _snake_case -from metaobjects.codegen import extract_delegate_emitter as rde from metaobjects.codegen.constants import generated_header from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator @@ -44,6 +43,8 @@ from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.meta.meta_data import MetaData +from ..runtime import extract_delegate_emitter as rde + _GENERATOR_NAME = "output-parser-generator" diff --git a/server/python/src/metaobjects/codegen/generators/output_prompt_generator.py b/server/python/src/metaobjects/codegen/generators/output_prompt_generator.py index a7f615d68..002ef037d 100644 --- a/server/python/src/metaobjects/codegen/generators/output_prompt_generator.py +++ b/server/python/src/metaobjects/codegen/generators/output_prompt_generator.py @@ -24,7 +24,6 @@ from collections.abc import Callable from metaobjects.apidocs.naming import snake_case as _snake_case -from metaobjects.codegen import output_format_spec_emitter as ofs from metaobjects.codegen.constants import generated_header from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator @@ -36,6 +35,8 @@ from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.meta.meta_data import MetaData +from ..runtime import output_format_spec_emitter as ofs + _GENERATOR_NAME = "output-prompt-generator" diff --git a/server/python/src/metaobjects/codegen/generators/router_generator.py b/server/python/src/metaobjects/codegen/generators/router_generator.py index e25c7a348..d2ada70d8 100644 --- a/server/python/src/metaobjects/codegen/generators/router_generator.py +++ b/server/python/src/metaobjects/codegen/generators/router_generator.py @@ -46,14 +46,7 @@ from metaobjects.apidocs.naming import snake_case as _snake_case from metaobjects.codegen.constants import GENERATED_MARKER, generated_header from metaobjects.codegen.format import ruff_format -from metaobjects.codegen.fr010_field_mapping import is_required from metaobjects.codegen.generator import EmittedFile, GenContext, Generator, per_entity -from metaobjects.codegen.generators.m2m_codegen import ( - M2mDescriptor, - build_object_index, - pk_field_name, - resolve_m2m_descriptors, -) from metaobjects.codegen.generators.tph_plan import TphPlan, is_tph_subtype, tph_plan_for from metaobjects.codegen.instance_artifacts import has_item_route from metaobjects.source_resolution import primary_rdb_source @@ -81,7 +74,14 @@ from metaobjects.shared.base_types import TYPE_IDENTITY from metaobjects.shared.separators import PACKAGE_SEP -from ..runtime.shared_predicates import READ_ONLY_ROUTED_KINDS, emits_router +from ..runtime.fr010_field_mapping import is_required +from ..runtime.m2m_codegen import ( + M2mDescriptor, + build_object_index, + pk_field_name, + resolve_m2m_descriptors, +) +from ..runtime.shared_predicates import emits_router, is_read_only_routed #: Where the emitted router imports its helper runtime (``filter_parser``, #: ``constraint_errors``) from. ``False`` — the packaged generator — imports the copy @@ -412,19 +412,6 @@ def reverse_fks_for(entity: MetaObject) -> list[ReverseFk]: return out -def is_read_only_routed(entity: MetaObject) -> bool: - """Whether :meth:`RouterGenerator.render_router` chooses the READ-ONLY router for - this object (a view / materialized-view source that is not write-through): GET list, - GET by id when it has an item route, and the 405 refusals, with no write surface. - - THE predicate behind that choice, shared with the API-docs builder so a page can never - present write verbs for an object whose router refuses them.""" - if not emits_router(entity) or entity.is_write_through(): - return False - src = primary_rdb_source(entity) - return src is not None and src.effective_kind() in READ_ONLY_ROUTED_KINDS - - def _sort_type_and_tables_lines(sort_field_nodes: list[MetaField]) -> list[str]: """The `_SortClause` model + the two sort lookup tables. diff --git a/server/python/src/metaobjects/codegen/generators/trace_helper_generator.py b/server/python/src/metaobjects/codegen/generators/trace_helper_generator.py index 453530943..500e47a2f 100644 --- a/server/python/src/metaobjects/codegen/generators/trace_helper_generator.py +++ b/server/python/src/metaobjects/codegen/generators/trace_helper_generator.py @@ -42,7 +42,6 @@ from collections.abc import Callable -from metaobjects.codegen import extract_schema_emitter as rse from metaobjects.codegen.constants import generated_header from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator @@ -56,6 +55,8 @@ from metaobjects.shared.base_types import TYPE_TEMPLATE from metaobjects.library.library_sources import LIBRARY_MANIFESTS +from ..runtime import extract_schema_emitter as rse + _GENERATOR_NAME = "trace-helper" def _anchor_fqns() -> frozenset[str]: diff --git a/server/python/src/metaobjects/codegen/metaobjects-config.schema.json b/server/python/src/metaobjects/codegen/metaobjects-config.schema.json index dfd8aa68a..a2db91104 100644 --- a/server/python/src/metaobjects/codegen/metaobjects-config.schema.json +++ b/server/python/src/metaobjects/codegen/metaobjects-config.schema.json @@ -55,6 +55,18 @@ } } }, + "docs": { + "type": "object", + "additionalProperties": false, + "description": "Options of `metaobjects docs`.", + "properties": { + "generators": { + "type": "array", + "items": { "type": "string", "pattern": "^[^:\\s]+:[^:\\s]+$" }, + "description": "Owned docs generators as 'module:symbol' (`metaobjects eject api-docs` prints the entry), resolved relative to this config file's directory. `metaobjects docs` runs one in place of the packaged generator for the surface it takes over; `gen` never runs it." + } + } + }, "targets": { "type": "object", "minProperties": 1, diff --git a/server/python/src/metaobjects/codegen/output_format_spec_emitter.py b/server/python/src/metaobjects/codegen/output_format_spec_emitter.py index cbd018d65..6c389b56c 100644 --- a/server/python/src/metaobjects/codegen/output_format_spec_emitter.py +++ b/server/python/src/metaobjects/codegen/output_format_spec_emitter.py @@ -1,83 +1,4 @@ -"""Turns a response value-object + its ``template.prompt`` node into a Python source -literal for an :class:`~metaobjects.render.OutputFormatSpec` — the artifact-1 prompt -descriptor used by the FR-010 response-format codegen. +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.output_format_spec_emitter`.""" +from metaobjects.codegen._moved import alias -Emits ``OutputFormatSpec(Format.X, "rootName", PromptStyle.X, [PromptField(...), …])``. -Mirrors the C# / Java ``OutputFormatSpecEmitter`` adapted to Python (keyword args for -the long ``PromptField`` ctor to keep the emitted source readable + order-robust). -Bounded scope: scalar / enum. Nested object → ``FieldKind.OBJECT`` placeholder. -""" -from __future__ import annotations - -from metaobjects.codegen import fr010_field_mapping as fm -from metaobjects.codegen.generators.find_inbound import is_xml, response_format_of -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.meta_data import MetaData -from metaobjects.meta.template import template_constants as tc - - -def _format_enum(template: MetaData) -> str: - """ADR-0053 — the fragment describes the REPLY, so its syntax is ``@responseFormat``, - NOT ``@format``, which is the syntax of the rendered prompt BODY. Reading ``@format`` - here typed the instruction "produce your answer like this" off the format of the - QUESTION.""" - return "Format.XML" if is_xml(response_format_of(template)) else "Format.JSON" - - -def _prompt_style_enum(template: MetaData) -> str: - style = template.get_meta_attr(tc.TEMPLATE_ATTR_PROMPT_STYLE) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) - if style == tc.PROMPT_STYLE_INLINE: - return "PromptStyle.INLINE" - if style == tc.PROMPT_STYLE_EXAMPLE_ONLY: - return "PromptStyle.EXAMPLE_ONLY" - return "PromptStyle.GUIDE" - - -def _opt_string_attr(field: MetaData, attr_name: str) -> str: - v = field.attrs().get(attr_name) - return fm.py_string_literal(v) if isinstance(v, str) else "None" - - -def _prompt_field_literal(field: MetaData) -> str: - name = field.name - req = "True" if fm.is_required(field) else "False" - array = "True" if fm.is_array(field) else "False" - - if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: - # FR-010: nested prompt deferred — emit a FieldKind.OBJECT placeholder (nested - # defaults to None on PromptField). The note lives HERE, not in the returned - # literal: spec_literal joins every field onto ONE line, and a Python '#' inline - # comment would swallow the rest of that line — including the closing `])` — so a - # nested-object payload emitted invalid Python (unterminated list) and crashed - # `gen` + `verify --codegen`. The other ports use `/* */` block comments, which - # are inline-safe; Python has none, so the literal must carry no comment. - return f'PromptField("{name}", FieldKind.OBJECT, {req}, array={array})' - - example = _opt_string_attr(field, fc.FIELD_ATTR_EXAMPLE) - instruction = _opt_string_attr(field, fc.FIELD_ATTR_INSTRUCTION) - - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - values_lit = fm.string_list_literal(fm.enum_values(field)) - enum_doc_lit = fm.properties_map_literal(field.attrs().get(fc.FIELD_ATTR_ENUM_DOC)) - return ( - f'PromptField("{name}", FieldKind.ENUM, {req}, array={array}, ' - f"enum_values={values_lit}, enum_doc={enum_doc_lit}, " - f"example={example}, instruction={instruction})" - ) - - kind = fm.scalar_kind(field.sub_type) or "STRING" - return ( - f'PromptField("{name}", FieldKind.{kind}, {req}, array={array}, ' - f"example={example}, instruction={instruction})" - ) - - -def spec_literal(vo: MetaData, template: MetaData, root_name: str) -> str: - """Emit ``OutputFormatSpec(Format.X, "rootName", PromptStyle.X, [PromptField(...), …])``.""" - format_enum = _format_enum(template) - style_enum = _prompt_style_enum(template) - field_lits = [_prompt_field_literal(f) for f in fm.fields(vo)] - body = ", ".join(field_lits) - return ( - f'OutputFormatSpec({format_enum}, "{root_name}", {style_enum}, [{body}])' - ) +alias(__name__, "metaobjects.codegen.runtime.output_format_spec_emitter") diff --git a/server/python/src/metaobjects/codegen/project_config.py b/server/python/src/metaobjects/codegen/project_config.py index f3ddf7ffa..15f931310 100644 --- a/server/python/src/metaobjects/codegen/project_config.py +++ b/server/python/src/metaobjects/codegen/project_config.py @@ -22,7 +22,7 @@ from __future__ import annotations import re -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path import yaml # type: ignore[import-untyped] # PyYAML ships no type stubs @@ -38,12 +38,17 @@ #: shipped schema has always said an unknown key is invalid; naming the set here is what #: lets the loader — the thing that actually runs — say the same. Kept in step with the #: schema by ``test_schema_and_loader_accept_EXACTLY_the_same_keys``. -TOP_LEVEL_KEYS: tuple[str, ...] = ("metadata", "providers", "libraries", "targets", "requirementTests") +TOP_LEVEL_KEYS: tuple[str, ...] = ( + "metadata", "providers", "libraries", "targets", "requirementTests", "docs", +) #: Every key a ``targets.`` mapping may carry. Same contract as above. TARGET_KEYS: tuple[str, ...] = ("outDir", "generators", "entities") +#: Every key the ``docs`` block may carry. Same contract as above. +DOCS_KEYS: tuple[str, ...] = ("generators",) + #: Every key the ``requirementTests`` block may carry (the ``requirement-tests`` generator's #: options). Same contract as above. REQUIREMENT_TESTS_KEYS: tuple[str, ...] = ( @@ -137,6 +142,9 @@ class ProjectConfig: targets: list[TargetConfig] #: Options of the ``requirement-tests`` generator; ``None`` => its defaults. requirement_tests: RequirementTestsConfig | None = None + #: The ``docs.generators`` block: owned docs generators (``module:symbol``) that + #: ``metaobjects docs`` runs in place of the packaged ones. ``[]`` => the packaged ones. + docs_generators: list[str] = field(default_factory=list) def metadata_dir(self) -> str: """The metadata dir resolved against ``config_dir`` (absolute path string).""" @@ -227,6 +235,20 @@ def symbol(key: str) -> str | None: ) +def _parse_docs(raw: object, ctx: str) -> list[str]: + if not isinstance(raw, dict): + raise ConfigError(f"{ctx} must be a mapping.") + _reject_unknown_keys(raw, DOCS_KEYS, ctx) + generators = _require_str_list(raw.get("generators", []), f"{ctx} 'generators'") + bad = [g for g in generators if not re.fullmatch(MODULE_SYMBOL_PATTERN, g)] + if bad: + raise ConfigError( + f"{ctx}: 'generators' takes owned generators in 'module:symbol' form " + f"(`metaobjects eject api-docs` prints the entry), got {bad}." + ) + return generators + + def load_project_config(path: Path) -> ProjectConfig: """Parse + validate ``metaobjects.config.yaml`` at ``path``. @@ -306,6 +328,8 @@ def load_project_config(path: Path) -> ProjectConfig: else None ) + docs_generators = _parse_docs(raw["docs"], f"{path}: 'docs'") if "docs" in raw else [] + return ProjectConfig( config_dir=path.parent.resolve(), metadata=metadata, @@ -313,4 +337,5 @@ def load_project_config(path: Path) -> ProjectConfig: libraries=libraries, targets=targets, requirement_tests=requirement_tests, + docs_generators=docs_generators, ) diff --git a/server/python/src/metaobjects/codegen/runner.py b/server/python/src/metaobjects/codegen/runner.py index 395402871..288b366eb 100644 --- a/server/python/src/metaobjects/codegen/runner.py +++ b/server/python/src/metaobjects/codegen/runner.py @@ -133,6 +133,17 @@ def run_gen( emitted: dict[str, tuple[str, str]] = {} # full_path -> (content, generated_by) for gen in generators: + docs_surface = getattr(gen, "docs_surface", None) + if docs_surface is not None: + # A docs generator writes pages under the docs output directory, and + # `metaobjects docs` runs it from `docs.generators`: here it would write them + # into the code tree, so it is skipped and the miswiring is named. + result.warnings.append( + f"{gen.name!r} writes the {docs_surface} reference, so gen skipped it. " + "List it under `docs.generators` in metaobjects.config.yaml: " + "`metaobjects docs` runs it from there." + ) + continue ctx = GenContext( entities=safe, loaded_root=metadata, diff --git a/server/python/src/metaobjects/codegen/runtime/__init__.py b/server/python/src/metaobjects/codegen/runtime/__init__.py index beabbcd3d..995a56f52 100644 --- a/server/python/src/metaobjects/codegen/runtime/__init__.py +++ b/server/python/src/metaobjects/codegen/runtime/__init__.py @@ -9,6 +9,20 @@ stay self-contained — standard library only, no ``metaobjects`` import and no FastAPI / SQLAlchemy / pg8000 dependency — or the copy would not run on its own. -``shared_predicates`` is what several GENERATORS import, at ``gen`` time: the emit rules -they have to agree on. No generated code imports it, so it may import ``metaobjects``. +Everything else is what GENERATORS import, at ``gen`` time. No generated code imports it, so +it may import ``metaobjects``. Each generator lists the ones it uses on its registry entry's +``runtime``, and an owned copy reads the copies eject placed beside it: + +* ``shared_predicates`` — the emit rules several generators have to agree on. +* ``fr019_shared_enum``, ``m2m_codegen`` — the fragment writers behind the entity, routes and + filter-allowlist generators (the shared-enum module, M:N navigation and the object index). +* ``fr010_field_mapping``, ``extract_delegate_emitter``, ``extract_schema_emitter``, + ``output_format_spec_emitter`` — the prompt-tier emitters behind the output-parser, + extractor, output-prompt and trace-helper generators. +* ``api_docs_model``, ``api_docs_builder``, ``api_docs_renderer``, ``api_docs_paths`` — the + page IR, builder, renderer and path math behind ``api-docs``. + +What stays a package import is the engine: the generator base and file writer, the naming +and type-mapping tables (``metaobjects.apidocs.naming``, ``type_map``, ``value_objects``), +the model walks (``find_inbound``, ``tph_plan``, ``instance_artifacts``) and the loader. """ diff --git a/server/python/src/metaobjects/codegen/runtime/api_docs_builder.py b/server/python/src/metaobjects/codegen/runtime/api_docs_builder.py new file mode 100644 index 000000000..0f3754c18 --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/api_docs_builder.py @@ -0,0 +1,488 @@ +"""Builds the Python SDK api-surface IR (:class:`ApiModel`) from a loaded MetaRoot. + +This is the ACCURATE-BY-CONSTRUCTION half of the api-docs pipeline: every +documented symbol name comes from the :mod:`metaobjects.apidocs.naming` seam (never +re-concatenated here — the SAME seam the real generators delegate to), and every +inclusion decision comes from an applies-predicate that REUSES the corresponding +generator's own gate helpers (``emits_instance_artifacts`` / ``primary_rdb_source`` ++ source kind / ``resolve_payload_vo`` / format check / ``is_tph_subtype``) rather +than re-implementing the gate. The result is that what this builder documents == +what the generators emit, by SHARING the single source of truth. + +Per-category enumeration (mirrors the Java ``JavaApiModelBuilder`` / C# +``CSharpApiModelBuilder``, Python/Pydantic-flavored): + +* Objects (``root.own_children()`` filtered to ``object.*``): each concrete object + → a MODEL symbol. A writable-table entity adds DATA_ACCESS (the repository + ``Protocol``) / REST (one per FastAPI route) / VALIDATION (Pydantic field + constraints on the create/update shape) / FILTER (the allowlist). A value object + → MODEL only. An abstract object / a TPH subtype yields no instance artifacts. +* Templates (EVERY subtype — ADR-0052): a resolvable ``@payloadRef`` gives the + PAYLOAD record it renders outbound, plus RENDER for a ``template.output`` (the + only subtype the render-helper generator emits for). A ``template.prompt`` + declaring ``@responseRef`` adds the INBOUND tier — the response PAYLOAD record, + PROMPT (the response-format fragment) and EXTRACTOR, plus OUTPUT_PARSER unless + the reply is XML, which gets no strict tier (ADR-0053). + +An object that yields no symbols (e.g. an abstract object) produces NO unit at all +(never an empty unit). +""" +from __future__ import annotations + +from metaobjects.apidocs import naming +from .api_docs_model import ( + ApiModel, + ApiSymbol, + ApiSymbolKind, + ApiUnit, + FieldShape, +) +from metaobjects.codegen import type_map +from .m2m_codegen import ( + build_object_index, + resolve_m2m_descriptors, +) +from metaobjects.codegen.generators.find_inbound import is_xml, response_shape +from metaobjects.codegen.value_objects import is_field_required, pkg_of, resolve_payload_vo +from metaobjects.codegen.instance_artifacts import ( + has_item_route, + is_abstract, + is_served_report, +) +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.core.field.meta_field import MetaField +from metaobjects.meta.core.object.meta_object import MetaObject +from metaobjects.meta.core.object.object_constants import ( + OBJECT_SUBTYPE_ENTITY, + OBJECT_SUBTYPE_REPORT, +) +from metaobjects.meta.core.reporting.report_read_model import report_read_model +from metaobjects.meta.meta_data import MetaData +from metaobjects.meta.template import template_constants as tc +from metaobjects.shared.base_types import TYPE_OBJECT, TYPE_TEMPLATE + +from .shared_predicates import emits_writable_router, is_read_only_routed + + +# --------------------------------------------------------------------------- +# Applies-predicates — each REUSES the matching generator's own gate helpers, so +# inclusion can never drift from emission. (The Python generators gate inline; +# these mirror that gate exactly via the shared helpers.) +# --------------------------------------------------------------------------- + + +def _payload_resolves(tmpl: MetaData, root: MetaData) -> MetaObject | None: + """The payload VO a template resolves to (``@payloadRef`` → ``object.value``), + or ``None`` — the shared gate for payload / render / prompt / parser / extractor.""" + payload_ref = tmpl.get_meta_attr(tc.TEMPLATE_ATTR_PAYLOAD_REF) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) + if not isinstance(payload_ref, str) or not payload_ref: + return None + # ADR-0042 (#228): the referrer is THIS template — a bare @payloadRef resolves + # in ITS OWN package first. + return resolve_payload_vo(root, payload_ref, pkg_of(tmpl)) + + +def _is_email_kind(tmpl: MetaData) -> bool: + kind = tmpl.get_meta_attr(tc.TEMPLATE_ATTR_KIND) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) + return isinstance(kind, str) and kind.lower() == tc.TEMPLATE_KIND_EMAIL + + +# --------------------------------------------------------------------------- +# Builder. +# --------------------------------------------------------------------------- + + +class PythonApiModelBuilder: + """Builds the :class:`ApiModel` SDK-surface IR from a loaded ``MetaRoot``.""" + + def build(self, root: MetaData, project: str) -> ApiModel: + objects = [ + # ADR-0039 sanctioned own: top-level scan on the loader ROOT (never extended, own == effective) + c + for c in root.own_children() + if c.type == TYPE_OBJECT and isinstance(c, MetaObject) + ] + object_index = build_object_index(objects) + + units: list[ApiUnit] = [] + for obj in objects: + # FR-044: a served report (Table A) has a generated read API, documented from + # its read model (its derived fields); every other report has no output. + if obj.sub_type == OBJECT_SUBTYPE_REPORT: + unit = ( + self._build_report_unit(obj, root) if is_served_report(obj) else None + ) + if unit is not None: + units.append(unit) + continue + unit = self._build_object_unit(obj, root, object_index) + if unit is not None: + units.append(unit) + + # Templates: ADR-0052 — EVERY template subtype gets a unit, not just + # template.output. A template.prompt carries a @payloadRef record and, when it + # declares @responseRef, the whole inbound tier — all of it generated, so all + # of it documentable. + # ADR-0039 sanctioned own: top-level scan on the loader ROOT (never extended, own == effective) + for tmpl in root.own_children(): + if tmpl.type == TYPE_TEMPLATE: + units.append(self._build_template_unit(tmpl, root)) + + return ApiModel(project, units) + + # ----- objects ----------------------------------------------------------- + + def _build_object_unit( + self, obj: MetaObject, root: MetaData, object_index: dict[str, MetaObject] + ) -> ApiUnit | None: + entity = obj.sub_type == OBJECT_SUBTYPE_ENTITY + module = naming.snake_case(obj.name) + unit_kind = "entity" if entity else "value" + symbols: list[ApiSymbol] = [] + + # MODEL — only for concrete objects (an abstract object cannot be + # instantiated → documented ⊆ generated). A value object → MODEL only. + if not is_abstract(obj): + model = naming.model_class_name(obj) + symbols.append( + ApiSymbol( + name=model, + kind=ApiSymbolKind.MODEL, + module=naming.model_import(obj), + signature=f"class {model}(BaseModel)", + usage=( + "the Pydantic v2 entity model" + if entity + else "the Pydantic v2 value-object model" + ), + ) + ) + + if entity and emits_writable_router(obj): + # VALIDATION — the Pydantic field constraints carried on the model + # (required / max-length / range / pattern). Names the model class. + model = naming.model_class_name(obj) + symbols.append( + ApiSymbol( + name=model, + kind=ApiSymbolKind.VALIDATION, + module=naming.model_import(obj), + signature=f"class {model}(BaseModel)", + usage="Pydantic field validation on the create/update shape", + fields=self._model_fields(obj), + ) + ) + + # DATA_ACCESS — the consumer-implemented repository ``Protocol`` seam. + repo = naming.repository_class_name(obj.name) + symbols.append( + ApiSymbol( + name=repo, + kind=ApiSymbolKind.DATA_ACCESS, + module=f"from .{naming.router_module_name(obj.name)} import {repo}", + signature=f"class {repo}(Protocol)", + usage="data access — the repository Protocol the consumer implements", + returns="list / Optional / bool", + ) + ) + + # REST — one symbol per FastAPI route the router registers. + self._add_rest_symbols(symbols, obj, module, root, object_index) + + # FILTER — the per-entity sort/filter allowlist. + fields_const = naming.filter_fields_const(obj.name) + symbols.append( + ApiSymbol( + name=fields_const, + kind=ApiSymbolKind.FILTER, + module=( + f"from .{naming.filter_allowlist_module_name(obj.name)} " + f"import {fields_const}" + ), + signature=f"{fields_const}: frozenset[str]", + usage="the filterable-field + filter-operator allowlist", + ) + ) + + elif is_read_only_routed(obj): + # A view-backed projection: the read-only router and its allowlist, no writes. + symbols.extend(self._read_only_symbols(obj)) + + if not symbols: + return None + return ApiUnit(obj.name, _package_of(obj), unit_kind, symbols) + + def _build_report_unit(self, obj: MetaObject, root: MetaData) -> ApiUnit: + """The unit for a SERVED report (FR-044 Table G): the row model plus the read-only + surface its router mounts (:meth:`_read_only_symbols`). Nothing else is generated + for it: no item route, no write verb, no validation model. Built over the + report's read model so the names are the generators'.""" + model_obj = report_read_model(obj, root) # type: ignore[arg-type] + model = naming.model_class_name(model_obj) + symbols = [ + ApiSymbol( + name=model, + kind=ApiSymbolKind.MODEL, + module=naming.model_import(model_obj), + signature=f"class {model}(BaseModel)", + usage="the Pydantic v2 report row model", + ), + *self._read_only_symbols(model_obj), + ] + return ApiUnit(obj.name, _package_of(obj), "report", symbols) + + def _read_only_symbols(self, obj: MetaObject) -> list[ApiSymbol]: + """What the READ-ONLY router (``is_read_only_routed``) actually mounts and emits, + and nothing more: the repository seam, ``GET`` list, ``GET`` by id ONLY when + :func:`has_item_route`, and the filter allowlist. The collection ``POST`` and the + item-verb refusals are 405s, not endpoints, so no write verb is documented; + a keyless object has no item route and its seam no ``find_by_id``.""" + router_module = naming.router_module_name(obj.name) + base_path = "/api/" + naming.route_path(obj.name) + repo = naming.repository_class_name(obj.name) + fields_const = naming.filter_fields_const(obj.name) + item = has_item_route(obj) + + def rest(verb_path: str, usage: str) -> ApiSymbol: + return ApiSymbol( + name=verb_path, + kind=ApiSymbolKind.REST, + module=f"# {router_module}.py — FastAPI APIRouter", + signature=verb_path, + usage=usage, + ) + + symbols = [ + ApiSymbol( + name=repo, + kind=ApiSymbolKind.DATA_ACCESS, + module=f"from .{router_module} import {repo}", + signature=f"class {repo}(Protocol)", + usage="data access — the read-only repository Protocol the consumer implements", + returns="list / count / Optional" if item else "list / count", + ), + rest("GET " + base_path, "list with pagination / sort / filters"), + ] + if item: + symbols.append( + rest("GET " + base_path + "/{" + naming.pk_param(obj.name) + "}", "fetch one by id") + ) + symbols.append( + ApiSymbol( + name=fields_const, + kind=ApiSymbolKind.FILTER, + module=( + f"from .{naming.filter_allowlist_module_name(obj.name)} " + f"import {fields_const}" + ), + signature=f"{fields_const}: frozenset[str]", + usage="the filterable-field + filter-operator allowlist", + ) + ) + return symbols + + def _add_rest_symbols( + self, + symbols: list[ApiSymbol], + entity: MetaObject, + module: str, + root: MetaData, + object_index: dict[str, MetaObject], + ) -> None: + router_module = naming.router_module_name(entity.name) + base_path = "/api/" + naming.route_path(entity.name) + # The router registers the item routes with a per-entity path param + # (`/{_id}`, e.g. `/{customer_id}`) — document that exact name, not + # a generic `{id}`, so the docs match the generated FastAPI routes. + pk = naming.pk_param(entity.name) + item = base_path + "/{" + pk + "}" + + def add(verb_path: str, usage: str) -> None: + symbols.append( + ApiSymbol( + name=verb_path, + kind=ApiSymbolKind.REST, + module=f"# {router_module}.py — FastAPI APIRouter", + signature=verb_path, + usage=usage, + ) + ) + + # The router registers GET list, GET item, POST, PATCH item, PUT item, + # DELETE item (PATCH + PUT share a handler but are two registered routes). + add("GET " + base_path, "list with pagination / sort / filters") + add("GET " + item, "fetch one by id") + add("POST " + base_path, "create") + add("PATCH " + item, "update") + add("PUT " + item, "update (PUT alias)") + add("DELETE " + item, "delete") + + # M:N traversal — GET //{id}/. `docs` has no + # `GenConfig`/`--column-naming` of its own (see `cli.py`'s docs handler) and + # this loop reads only `.target_entity` / `.relation_name` off the + # descriptor — never a physical column/table name — so resolving it under + # the port default here changes nothing observable; threading a naming + # strategy through would mean inventing a config seam this command doesn't + # have, for a value nothing here would read. + for d in resolve_m2m_descriptors(entity, object_index): + target = object_index.get(d.target_entity) + target_name = target.name if target is not None else d.target_entity + add( + "GET " + base_path + "/{" + pk + "}/" + d.relation_name, + f"M:N traversal — the related {target_name} rows", + ) + + # ----- templates --------------------------------------------------------- + + def _build_template_unit(self, tmpl: MetaData, root: MetaData) -> ApiUnit: + name = tmpl.name + module = naming.snake_case(name) + symbols: list[ApiSymbol] = [] + payload_vo = _payload_resolves(tmpl, root) + + if payload_vo is not None: + # PAYLOAD — the shape this template RENDERS. ADR-0056: it IS the @payloadRef + # value object's own model, emitted by the entity generator. + payload_class = naming.model_class_name(payload_vo) + symbols.append( + ApiSymbol( + name=payload_class, + kind=ApiSymbolKind.PAYLOAD, + module=naming.model_import(payload_vo), + signature=f"class {payload_class}(BaseModel)", + usage="the typed payload projection bound to the template", + fields=self._payload_fields(payload_vo), + ) + ) + + # RENDER — the typed render helper wrapping the render engine. OUTBOUND + # ONLY: render_helper_generator emits for a template.output alone, so + # documenting it for a prompt would name a function never generated. + if tmpl.sub_type == tc.TEMPLATE_SUBTYPE_OUTPUT: + render_fn = naming.render_helper_fn(name) + symbols.append( + ApiSymbol( + name=render_fn, + kind=ApiSymbolKind.RENDER, + module=f"from .{module}_render_helper import {render_fn}", + signature=f"def {render_fn}(payload: {payload_class}, provider)", + usage="renders the output template against a typed payload", + returns="EmailDocument" if _is_email_kind(tmpl) else "str", + ) + ) + + # ADR-0052 — the INBOUND symbols belong to a RESPONDING template.prompt, never + # to an output, and the gate is @responseRef PRESENCE rather than a format + # value. Shares `response_shape` with the generators, so the docs can never + # claim a symbol codegen suppressed. + inbound = response_shape(root, tmpl, pkg_of(tmpl)) + if inbound is not None: + # The RESPONSE shape the parser actually returns — the @responseRef value + # object's own model (ADR-0056). Documented once: when the prompt renders and + # parses the SAME value object, the model above already is it. + response_class = naming.model_class_name(inbound.vo) + if payload_vo is None or inbound.vo.resolution_key() != payload_vo.resolution_key(): + symbols.append( + ApiSymbol( + name=response_class, + kind=ApiSymbolKind.PAYLOAD, + module=naming.model_import(inbound.vo), + signature=f"class {response_class}(BaseModel)", + usage="the typed response shape a model reply is parsed into", + fields=self._payload_fields(inbound.vo), + ) + ) + + # OUTPUT_PARSER — the strict ``parse_*``. ADR-0053: JSON-only, so an XML + # reply gets no strict tier and documenting one would name a missing fn. + if not is_xml(inbound.format): + parse_fn = naming.output_parser_fn(name) + symbols.append( + ApiSymbol( + name=parse_fn, + kind=ApiSymbolKind.OUTPUT_PARSER, + module=f"from .{module}_response_parser import {parse_fn}", + signature=f"def {parse_fn}(text)", + usage="parses a model reply back into the typed response shape", + returns=response_class, + ) + ) + + prompt_fn = naming.output_prompt_fn(name) + symbols.append( + ApiSymbol( + name=prompt_fn, + kind=ApiSymbolKind.PROMPT, + module=f"from .{module}_response_format import {prompt_fn}", + signature=f"def {prompt_fn}(overrides=None)", + usage="builds the response-format prompt fragment", + returns="str", + ) + ) + extract_fn = naming.extractor_fn(name) + symbols.append( + ApiSymbol( + name=extract_fn, + kind=ApiSymbolKind.EXTRACTOR, + module=f"from .{module}_extractor import {extract_fn}", + signature=f"def {extract_fn}(root, text, opts=None)", + usage="extracts the strict typed response from dirty model output", + returns=response_class, + ) + ) + + return ApiUnit(name, _package_of(tmpl), "template", symbols) + + # ----- field shapes ------------------------------------------------------- + + def _model_fields(self, entity: MetaObject) -> list[FieldShape]: + """The entity's documented scalar/enum field shapes — one row per field the + Pydantic model maps (object-typed fields are owned-type navs, not a scalar + documented shape). Types via ``type_map.py_type_for``; required = own + ``@required`` is the boolean ``True`` (the same boundary the model uses).""" + rows: list[FieldShape] = [] + for f in entity.fields(): + if not isinstance(f, MetaField): + continue + if f.sub_type == fc.FIELD_SUBTYPE_OBJECT: + continue + type_expr = type_map.py_type_for(f).expr + note = None + if f.sub_type == fc.FIELD_SUBTYPE_ENUM: + values = type_map.effective_enum_values(f) + if values: + note = "allowed: " + " | ".join(values) + required = f.get_meta_attr(fc.FIELD_ATTR_REQUIRED) is True # ADR-0039: resolving (@required may be inherited) + rows.append(FieldShape(f.name, type_expr, optional=not required, note=note)) + return rows + + def _payload_fields(self, vo: MetaObject) -> list[FieldShape]: + """The payload model's documented field shapes — one row per field of the + value object, with optionality from ``is_field_required`` (the boundary the + value object's model draws between ``T`` and ``T | None``).""" + rows: list[FieldShape] = [] + for f in vo.fields(): + if not isinstance(f, MetaField): + continue + type_expr = type_map.py_type_for(f).expr + rows.append( + FieldShape(f.name, type_expr, optional=not is_field_required(f)) + ) + return rows + + +def _package_of(node: MetaData) -> str: + """The unit's metadata package (``"acme::shop"``) — drives the doc-page layout + path. Resolved from the nearest ancestor that carries one (objects inherit the + file/root package).""" + pkg = node.package + parent = node.parent + while pkg is None and parent is not None: + pkg = parent.package + parent = parent.parent + return pkg or "" + + +def build_api_model(root: MetaData, project: str) -> ApiModel: + """Module-level convenience: build the Python SDK api-surface IR.""" + return PythonApiModelBuilder().build(root, project) diff --git a/server/python/src/metaobjects/codegen/runtime/api_docs_model.py b/server/python/src/metaobjects/codegen/runtime/api_docs_model.py new file mode 100644 index 000000000..12766ddef --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/api_docs_model.py @@ -0,0 +1,111 @@ +"""The Python native SDK api-surface IR (intermediate representation). + +Mirrors the Java ``JavaApiModel`` IR (server/java/.../apidocs), the C# +``CSharpApiModel``, and the TS ``api-model.ts``, idiomatic to the Python +(Pydantic / FastAPI / ObjectManager) generated surface: Pydantic model / +``ObjectManager`` data-access / FastAPI routes / validation / extractor / +render helper / payload model / output-format prompt / output parser / filter +allowlist. + +NAMES on every symbol come from the :mod:`metaobjects.apidocs.naming` seam (never +re-concatenated in the builder), so what this model documents == what the real +generators emit. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + + +class ApiSymbolKind(str, Enum): + """The generated-category axis a documented symbol belongs to (Python / + Pydantic + FastAPI flavored).""" + + #: The Pydantic v2 model class (entity) or value-object model. + MODEL = "model" + #: Data access — the consumer-implemented repository ``Protocol`` seam. + DATA_ACCESS = "data_access" + #: A FastAPI route (named ``"VERB path"``). + REST = "rest" + #: Pydantic field validation on the create/update shape. + VALIDATION = "validation" + #: The tolerant + strict extractor for a template payload. + EXTRACTOR = "extractor" + #: The typed render helper wrapping the render engine. + RENDER = "render" + #: The typed payload model bound to a template. + PAYLOAD = "payload" + #: The output-format prompt fragment. + PROMPT = "prompt" + #: The output parser (``parse_*``) back into the typed payload. + OUTPUT_PARSER = "output_parser" + #: The per-entity sort/filter allowlist. + FILTER = "filter" + + +@dataclass(frozen=True) +class FieldShape: + """A documented field: name + Python type + optionality + an optional note + (e.g. enum values).""" + + name: str + type: str + optional: bool + note: str | None = None + + +@dataclass(frozen=True) +class UnitExample: + """A worked example for a unit (reserved for later phases; carried for + cross-port IR parity).""" + + title: str + code: str + + +@dataclass(frozen=True) +class ApiSymbol: + """One documented symbol of the generated Python SDK surface. + + :param name: the emitted Python identifier (or ``"VERB path"`` for a REST route). + :param kind: which generated category this symbol belongs to. + :param module: the import line / module path the symbol lives in. + :param signature: a human-readable Python signature line. + :param usage: a one-line "what you use this for". + :param returns: the symbol's return surface, or ``None``. + :param fields: per-field shapes for model/validation/payload symbols (may be empty). + """ + + name: str + kind: ApiSymbolKind + module: str + signature: str + usage: str + returns: str | None = None + fields: list[FieldShape] = field(default_factory=list) + + +@dataclass(frozen=True) +class ApiUnit: + """One documented unit (an entity / value object, or a template) + its symbols. + + :param node: the unit's short name (the doc-page basename). + :param package: the unit's metadata package (e.g. ``acme::shop``). + :param kind: ``"entity"`` | ``"value"`` | ``"report"`` | ``"template"``. + :param symbols: the documented symbols, in canonical IR order. + :param example: an optional unit-level worked example (reserved). + """ + + node: str + package: str + kind: str + symbols: list[ApiSymbol] + example: UnitExample | None = None + + +@dataclass(frozen=True) +class ApiModel: + """The full per-project Python SDK api surface IR.""" + + project: str + units: list[ApiUnit] diff --git a/server/python/src/metaobjects/codegen/runtime/api_docs_paths.py b/server/python/src/metaobjects/codegen/runtime/api_docs_paths.py new file mode 100644 index 000000000..6360d8d6a --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/api_docs_paths.py @@ -0,0 +1,75 @@ +"""Doc-page path math for the Python native SDK-docs surface (``api/python``). + +Byte-parity with the Java ``DocsPaths`` (server/java/.../apidocs/DocsPaths.java), +the C# ``DocsPaths`` (server/csharp/.../ApiDocs/DocsPaths.cs), and the TS +``docs-paths.ts`` contract: the same layout + relative-href math, so the shared +``fixtures/conformance/api-docs-cross-port/expected-paths.json`` resolves +identically across all ports. Nothing here is Python-specific — it is pure path +math. The api-docs renderer + the ``docs`` command call these helpers; they never +re-derive a path. +""" +from __future__ import annotations + +from enum import Enum + + +class Layout(str, Enum): + """Doc-page layout: one file per unit (:attr:`FLAT`) or foldered by package + (:attr:`PACKAGE`).""" + + #: One file per unit at the surface root (``.md``). + FLAT = "flat" + #: Foldered by package (``/.md``). + PACKAGE = "package" + + +def package_to_path(pkg: str | None) -> str: + """``"acme::shop"`` or ``"acme.shop"`` → ``"acme/shop"``; ``None``/"" → "".`""" + if not pkg: + return "" + return pkg.replace("::", "/").replace(".", "/") + + +def doc_page_output_path(layout: Layout, pkg: str | None, name: str) -> str: + """Flat → ``".md"``; Package → ``"/.md"``.""" + file = f"{name}.md" + if layout == Layout.FLAT: + return file + folder = package_to_path(pkg) + return file if not folder else f"{folder}/{file}" + + +def surface_cross_href(from_output_path: str, to_output_path: str) -> str: + """Relative posix href from *from_output_path*'s directory to *to_output_path* + (mirrors the TS ``surfaceCrossHref``).""" + slash = from_output_path.rfind("/") + from_dir = from_output_path[:slash] if slash >= 0 else "" + rel = _posix_relative(from_dir, to_output_path) + return rel if rel.startswith(".") else f"./{rel}" + + +def model_cross_href( + api_page_path: str, model_page_path: str, model_base_url: str | None = None +) -> str: + """From an api page to its model page: relative by default, absolute when + *model_base_url* is set (federated docs).""" + if model_base_url: + return f"{model_base_url.rstrip('/')}/{model_page_path}" + return surface_cross_href(api_page_path, model_page_path) + + +def _posix_relative(from_dir: str, to_path: str) -> str: + """``posixpath.relative(from_dir, to_path)``: drop the common prefix, ``..`` + per remaining *from_dir* segment, then the remaining *to_path* segments. + "" from_dir → to_path; identical → ".".""" + if not from_dir: + return to_path + frm = from_dir.split("/") + to = to_path.split("/") + common = 0 + cap = min(len(frm), len(to)) + while common < cap and frm[common] == to[common]: + common += 1 + parts: list[str] = [".."] * (len(frm) - common) + parts.extend(to[common:]) + return "/".join(parts) if parts else "." diff --git a/server/python/src/metaobjects/codegen/runtime/api_docs_renderer.py b/server/python/src/metaobjects/codegen/runtime/api_docs_renderer.py new file mode 100644 index 000000000..ce4c709af --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/api_docs_renderer.py @@ -0,0 +1,185 @@ +"""Renders the :class:`ApiModel` IR into Python-idiomatic api-doc markdown. + +One IR, three forms — all derived from the SAME model, never re-derived: + +* :func:`render_unit_page` — a per-unit HUMAN page (entity OR template), symbols + grouped into ordered sections + the ``**Model / metadata:**`` back-link. +* :func:`render_index` — the consolidated index (README.md), one bullet per unit. +* :func:`render_agent_api` — the token-frugal AGENT form (AGENT-API.md), symbols + grouped under their import line. + +PRESENTATION ONLY: every symbol name comes from the IR (which keys off the naming +seam) and every path comes from :mod:`metaobjects.codegen.runtime.api_docs_paths` — the renderer never +re-derives a name or a path. The section order + headings + the back-link literal +mirror the Java / C# renderers so the polyglot doc tree coheres. +""" +from __future__ import annotations + +from .api_docs_model import ApiModel, ApiSymbolKind, ApiUnit +from .api_docs_paths import Layout, doc_page_output_path, surface_cross_href + +# Canonical section order per kind (a unit renders only the kinds it carries). +_KIND_ORDER: tuple[ApiSymbolKind, ...] = ( + ApiSymbolKind.MODEL, + ApiSymbolKind.DATA_ACCESS, + ApiSymbolKind.REST, + ApiSymbolKind.VALIDATION, + ApiSymbolKind.EXTRACTOR, + ApiSymbolKind.RENDER, + ApiSymbolKind.PAYLOAD, + ApiSymbolKind.PROMPT, + ApiSymbolKind.OUTPUT_PARSER, + ApiSymbolKind.FILTER, +) + +_HEADINGS: dict[ApiSymbolKind, str] = { + ApiSymbolKind.MODEL: "Model", + ApiSymbolKind.DATA_ACCESS: "Data access", + ApiSymbolKind.REST: "REST", + ApiSymbolKind.VALIDATION: "Validation", + ApiSymbolKind.EXTRACTOR: "Extractor", + ApiSymbolKind.RENDER: "Render", + ApiSymbolKind.PAYLOAD: "Payload", + ApiSymbolKind.PROMPT: "Prompt", + ApiSymbolKind.OUTPUT_PARSER: "Output parser", + ApiSymbolKind.FILTER: "Filter", +} + +_SUMMARY_LABELS: dict[ApiSymbolKind, str] = { + ApiSymbolKind.MODEL: "model", + ApiSymbolKind.DATA_ACCESS: "data access", + ApiSymbolKind.REST: "REST", + ApiSymbolKind.VALIDATION: "validation", + ApiSymbolKind.EXTRACTOR: "extractor", + ApiSymbolKind.RENDER: "render", + ApiSymbolKind.PAYLOAD: "payload", + ApiSymbolKind.PROMPT: "prompt", + ApiSymbolKind.OUTPUT_PARSER: "output parser", + ApiSymbolKind.FILTER: "filter", +} + + +def _md_cell(text: str) -> str: + return text.replace("|", "\\|") + + +def render_unit_page(unit: ApiUnit, model_href: str | None) -> str: + """Render one per-unit human reference page. *model_href* (when non-empty) is a + pre-computed relative href back to this unit's model/metadata page — the caller + derives it via :func:`metaobjects.codegen.runtime.api_docs_paths.model_cross_href`; the renderer + only places it (as the contract ``**Model / metadata:**`` back-link).""" + out: list[str] = [f"# {unit.node} API\n"] + if model_href: + out.append(f"\n**Model / metadata:** [{unit.node}]({model_href})\n") + out.append( + "\n> Import paths are relative to the generated package; add it to your " + "Python path.\n" + ) + + for kind in _KIND_ORDER: + syms = [s for s in unit.symbols if s.kind == kind] + if not syms: + continue + out.append(f"\n## {_HEADINGS[kind]}\n") + for sym in syms: + out.append(f"\n### `{sym.signature}`\n") + out.append(f"\n{sym.usage}\n") + out.append(f"\n```python\n{sym.module}\n```\n") + if sym.returns: + out.append(f"\nReturns: {sym.returns}\n") + if sym.fields: + out.append("\n| Field | Type | Required | Notes |\n|---|---|---|---|\n") + for f in sym.fields: + required = "no" if f.optional else "yes" + out.append( + f"| `{f.name}` | `{_md_cell(f.type)}` | {required} | " + f"{_md_cell(f.note or '')} |\n" + ) + return "".join(out) + + +def render_index(model: ApiModel, layout: Layout) -> str: + """Render the consolidated api index (README.md): one bullet per unit, entities/ + values vs templates. Each unit's href is the relative link from the index to the + unit's page in the given *layout*.""" + entities = sorted( + (u for u in model.units if u.kind != "template"), key=lambda u: u.node + ) + templates = sorted( + (u for u in model.units if u.kind == "template"), key=lambda u: u.node + ) + + out: list[str] = [ + "# API Reference\n\nGenerated public API surface, one page per entity and " + "output template.\n" + ] + if entities: + out.append("\n## Entities\n\n") + for u in entities: + out.append(_index_row(u, layout)) + if templates: + out.append("\n## Templates\n\n") + for u in templates: + out.append(_index_row(u, layout)) + return "".join(out) + + +def _index_row(unit: ApiUnit, layout: Layout) -> str: + href = surface_cross_href( + "README.md", doc_page_output_path(layout, unit.package, unit.node) + ) + return ( + f"- [{unit.node}]({href}) — {_summary(unit)} " + f"({len(unit.symbols)} symbols)\n" + ) + + +def _summary(unit: ApiUnit) -> str: + parts: list[str] = [] + for kind in _KIND_ORDER: + n = sum(1 for s in unit.symbols if s.kind == kind) + if n == 0: + continue + label = _SUMMARY_LABELS[kind] + parts.append(label if n == 1 else f"{n} {label}") + return ", ".join(parts) if parts else "no public symbols" + + +def render_agent_api(model: ApiModel) -> str: + """Render the condensed agent/LLM form (AGENT-API.md): per unit, symbols grouped + under a single import-line header then one compact ``` `signature` — usage ``` line + each. NO prose / field-tables (token budget). Units + symbols keep their IR order.""" + out: list[str] = [ + f"# Agent API Reference\n\nGenerated Python API reference for " + f"{model.project}; call these exactly as written. Import paths are relative " + "to the generated package.\n" + ] + for unit in model.units: + if not unit.symbols: + continue + out.append(f"\n## {unit.node}\n") + groups: dict[str, list[str]] = {} + order: list[str] = [] + for s in unit.symbols: + if s.module not in groups: + groups[s.module] = [] + order.append(s.module) + groups[s.module].append(f"- `{s.signature}` — {s.usage}\n") + for module in order: + out.append(f"\n`{module}`\n") + out.extend(groups[module]) + return "".join(out) + + +class PythonApiDocsRenderer: + """Object wrapper around the module-level render functions (parity with the + Java / C# renderer classes; the functions are the canonical entry points).""" + + def render_unit_page(self, unit: ApiUnit, model_href: str | None) -> str: + return render_unit_page(unit, model_href) + + def render_index(self, model: ApiModel, layout: Layout) -> str: + return render_index(model, layout) + + def render_agent_api(self, model: ApiModel) -> str: + return render_agent_api(model) diff --git a/server/python/src/metaobjects/codegen/runtime/extract_delegate_emitter.py b/server/python/src/metaobjects/codegen/runtime/extract_delegate_emitter.py new file mode 100644 index 000000000..258011557 --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/extract_delegate_emitter.py @@ -0,0 +1,383 @@ +"""FR-010 nested-codegen-gap — the runtime-DELEGATING extract emitter (Python). + +The self-contained ``extract_(text)`` path (``extract_schema_emitter`` + the +baked ``ExtractSchema``) covers scalars / enums / scalar-arrays but leaves +nested-object and array-of-object components ``None`` — the historical FR-010 codegen +gap. This module emits the additive *delegating* entry that CLOSES that gap by +wrapping the runtime extract: + + extract__with_loader(root, text, opts=None) -> ExtractionResult[Extracted] + +It resolves this payload's ``MetaObject`` from the supplied loaded ``MetaRoot`` by its +baked simple name (``PAYLOAD_NAME``), delegates to ``extract_object`` in +:mod:`metaobjects.meta.core.object.object_extract` (which assembles the FULL nested +object graph reflection-free via the Phase A object model — ``MetaObject.new_instance()`` ++ the ``MetaField`` set-by-name SPI), then maps the assembled ``ValueObject`` graph into +the typed nullable mirror graph via generated ``_from__extracted(o)`` mapper +functions (payload + every reachable nested VO, deduped). + +This is the codegen-wrapping-runtime pattern (a generated DAO calling the +dynamic-metadata runtime), mirroring the Java / Kotlin / TS pilots. The generated +mappers read the assembled graph through a tiny ``_read_prop`` helper that mirrors the +``MetaField`` get SPI (``ValueObject.get(name)`` else plain-attribute access), so the +emitted code stays self-sufficient and reflection-free. + +Bounded by the cross-port ``MAX_NEST_DEPTH`` via the runtime — codegen here only mirrors +the runtime's resolved object graph, so depth/cycle guarding lives in ``object_extract``. +The emitter dedupes mirrors/mappers by ``resolution_key()`` (the package-qualified FQN, +cycle-safe) — NOT the bare VO ``name``, which would silently collapse two same-short-name +value-objects from different packages into one (ADR-0044, #228). + +ADR-0056 rule 3: a mirror is keyed by the VALUE OBJECT, never the template. Every mirror +and mapper — the root's included — is named after the value object's emitted model name +(:func:`~metaobjects.codegen.value_objects.model_class_name`: ``Extracted`` / +``_from__extracted``), the same name the entity generator declares the model under, so +a cross-package short-name collision qualifies the mirror exactly as it qualifies the +model. Mirrors stay module-local (declared in each parser module that needs them), like +TypeScript's: a Python module scopes them, so two parsers over one response each carry +their own copy without clashing. +""" +from __future__ import annotations + +from . import fr010_field_mapping as fm +from metaobjects.codegen.value_objects import model_class_name, pkg_of +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.meta_data import MetaData +from metaobjects.naming_refs import resolve_object_ref + + +def ref_vo(field: MetaData, root: MetaData) -> MetaData | None: + """The ``@objectRef`` target VO for a nested-object field, or ``None`` when + unresolvable. + + ADR-0042 (#228) — resolves via the canonical `resolve_object_ref` package-local + contract: an FQN ref resolves EXACTLY; a bare ref resolves in the DECLARING + field's own package first, else a root-level object. NO bare-tail short-name + fallback — that pattern (matching an FQN ref by its trailing simple-name segment + against ANY same-named object root-wide) is the #219/ADR-0042-banned "wrong + node" bug: under a cross-package short-name collision it silently binds + whichever same-named object happens to load first, regardless of which package + the ref actually pointed at. *referrer_pkg* is the field's OWN declaring + package (which differs from the VO's when the field is inherited via `extends` + from an abstract VO in another package).""" + ref = field.attrs().get(fc.FIELD_ATTR_OBJECT_REF) + if not isinstance(ref, str) or not ref: + return None + referrer_pkg = pkg_of(field.parent) if field.parent is not None else "" + return resolve_object_ref(root, ref, referrer_pkg) + + +def _is_object_field(field: MetaData) -> bool: + """True iff the field is a nested object reference (``field.object`` — distinct + from the string-backed ``field.enum``, treated as a scalar).""" + return field.sub_type == fc.FIELD_SUBTYPE_OBJECT + + +def mirror_name(vo: MetaData) -> str: + """The extracted-mirror dataclass name for a value-object: ``Extracted``, where + ```` is the value object's emitted model name (collision-qualified when another + object shares its short name).""" + return f"{model_class_name(vo)}Extracted" + + +def mapper_name(vo: MetaData) -> str: + """The mapper function name for a value-object (``_from__extracted``).""" + return f"_from_{_snake(model_class_name(vo))}_extracted" + + +def _snake(name: str) -> str: + """``NpcResponseOutput`` → ``npc_response_output`` (matches the cross-generator + ``_snake_case`` convention; no acronym handling).""" + out: list[str] = [] + for i, ch in enumerate(name): + if ch.isupper() and i > 0: + out.append("_") + out.append(ch.lower()) + return "".join(out) + + +# ============================================================================= +# Nested-aware mirror type (recurses into nested mirror names) +# ============================================================================= + + +def _nested_mirror_type(field: MetaData, root: MetaData) -> str: + """The nullable mirror annotation for one field — nested-aware (nested objects + become ``Extracted``; array-of-objects become ``list[...]``).""" + if _is_object_field(field): + target = ref_vo(field, root) + base = f'"{mirror_name(target)}"' if target is not None else "object" + elem = f"{base} | None" + return f"list[{elem}] | None" if fm.is_array(field) else elem + if fm.is_array(field): + return "list[str | None] | None" + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + return "str | None" + kind = fm.scalar_kind(field.sub_type) + if kind in ("INT", "LONG"): + return "int | None" + if kind == "DOUBLE": + return "float | None" + if kind == "BOOLEAN": + return "bool | None" + return "str | None" + + +# ============================================================================= +# Reachability +# ============================================================================= + + +def reachable_vos(vo: MetaData, root: MetaData) -> list[MetaData]: + """``vo`` + every value-object reachable through nested ``@objectRef`` fields, in + stable BFS order, deduped by ``resolution_key()`` (cycle-safe). + + ADR-0044 (#228) — deduping by the bare ``name`` (pre-fix) silently DROPPED a + second cross-package value-object sharing the first one's bare short name (e.g. + ``acme::alpha::Note`` + ``acme::beta::Note``): once the first ``Note`` was seen, + the second's bare name matched ``seen`` and it was never queued/emitted — a + silent shape loss, not merely a naming cosmetic. ``resolution_key()`` is the + package-qualified FQN, so two same-short-name VOs from different packages are + two distinct keys and both survive the walk.""" + out: list[MetaData] = [] + seen: set[str] = set() + queue: list[MetaData] = [vo] + while queue: + cur = queue.pop(0) + key = cur.resolution_key() + if key in seen: + continue + seen.add(key) + out.append(cur) + for f in fm.fields(cur): + if _is_object_field(f): + target = ref_vo(f, root) + if target is not None and target.resolution_key() not in seen: + queue.append(target) + return out + + +def has_nested(vo: MetaData, root: MetaData) -> bool: + """True iff ``vo`` (or any reachable nested VO) has a nested-object field.""" + for cur in reachable_vos(vo, root): + if any(_is_object_field(f) for f in fm.fields(cur)): + return True + return False + + +# ============================================================================= +# Nested-aware mirror dataclasses +# ============================================================================= + + +def nested_mirror_dataclasses(vo: MetaData, root: MetaData) -> list[str]: + """Emit the nested-aware mirror dataclass for ``vo`` and every reachable nested VO + (deduped), each named :func:`mirror_name`. Returns source lines (blank-line + separated).""" + lines: list[str] = [] + for i, cur in enumerate(reachable_vos(vo, root)): + if i > 0: + lines.append("") + lines.append("") + lines.extend(_one_mirror(cur, root)) + return lines + + +def _one_mirror(vo: MetaData, root: MetaData) -> list[str]: + lines: list[str] = [ + "@dataclass(frozen=True, slots=True)", + f"class {mirror_name(vo)}:", + f' """Best-effort extracted twin of ``{model_class_name(vo)}`` — every field nullable', + ' (``None`` where the value was lost or malformed)."""', + ] + field_lines = [ + f" {f.name}: {_nested_mirror_type(f, root)} = None" + for f in fm.fields(vo) + ] + lines.extend(field_lines or [" pass"]) + return lines + + +# ============================================================================= +# Mapper functions (assembled ValueObject graph -> typed nullable mirror graph) +# ============================================================================= + + +def nested_mappers(vo: MetaData, root: MetaData) -> list[str]: + """Emit one ``_from__extracted(o)`` mapper per reachable VO (payload + + nested, deduped), each named :func:`mapper_name`. Returns source lines + (blank-line separated).""" + lines: list[str] = [] + for i, cur in enumerate(reachable_vos(vo, root)): + if i > 0: + lines.append("") + lines.append("") + lines.extend(_one_mapper(cur, root)) + return lines + + +def _one_mapper(vo: MetaData, root: MetaData) -> list[str]: + fn = mapper_name(vo) + mirror = mirror_name(vo) + lines: list[str] = [ + f'def {fn}(o: object | None) -> "{mirror} | None":', + f" \"\"\"Map an assembled ValueObject graph into a typed ``{mirror}`` mirror;" + " null-tolerant.\"\"\"", + " if o is None:", + " return None", + f" return {mirror}(", + ] + for f in fm.fields(vo): + lines.append(f" {f.name}={_mapper_arg(f, root)},") + lines.append(" )") + return lines + + +def _mapper_arg(field: MetaData, root: MetaData) -> str: + """The mirror-field initializer that reads ``field`` from the assembled object ``o``.""" + key = f'"{field.name}"' + if _is_object_field(field): + target = ref_vo(field, root) + if target is None: + return "None # unresolved @objectRef" + fn = mapper_name(target) + if fm.is_array(field): + return f"_map_object_list(_read_prop(o, {key}), {fn})" + return f"{fn}(_read_prop(o, {key}))" + + # Enum / scalar / scalar-array: the runtime already coerced; read + light-coerce to + # the mirror's nullable shape via the locally-defined _dlg_* readers. + # ARRAY is checked BEFORE the scalar-enum branch: an enum ARRAY must route through + # the string-LIST reader, not the scalar enum reader — otherwise the list collapses + # to a single stringified scalar (the cross-port "enum-before-isArray" ordering bug). + if fm.is_array(field): + return f"_dlg_str_list(_read_prop(o, {key}))" + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + return f"_dlg_str(_read_prop(o, {key}))" + kind = fm.scalar_kind(field.sub_type) + if kind in ("INT", "LONG"): + return f"_dlg_int(_read_prop(o, {key}))" + if kind == "DOUBLE": + return f"_dlg_float(_read_prop(o, {key}))" + if kind == "BOOLEAN": + return f"_dlg_bool(_read_prop(o, {key}))" + return f"_dlg_str(_read_prop(o, {key}))" + + +# ============================================================================= +# Used-helper scoping + the shared helper block +# ============================================================================= + + +def used_helpers(vo: MetaData, root: MetaData) -> set[str]: + """The generated-helper names the mappers for ``vo`` (+ reachable nested VOs) + actually reference. ``_read_prop`` is always present once any mapper is emitted.""" + used: set[str] = {"_read_prop"} + for cur in reachable_vos(vo, root): + for f in fm.fields(cur): + if _is_object_field(f): + if fm.is_array(f): + used.add("_map_object_list") + continue + if fm.is_array(f): + # ARRAY before scalar-enum — an enum array uses the string-LIST reader + # (mirrors the _mapper_arg ordering; avoids the collapse-to-scalar bug). + used.add("_dlg_str_list") + elif f.sub_type == fc.FIELD_SUBTYPE_ENUM: + used.add("_dlg_str") + else: + kind = fm.scalar_kind(f.sub_type) + if kind in ("INT", "LONG"): + used.add("_dlg_int") + elif kind == "DOUBLE": + used.add("_dlg_float") + elif kind == "BOOLEAN": + used.add("_dlg_bool") + else: + used.add("_dlg_str") + return used + + +# Each helper: name -> source block (lines). +_HELPER_BLOCKS: dict[str, list[str]] = { + "_read_prop": [ + "def _read_prop(o: object | None, name: str) -> object | None:", + ' """Read a property from an assembled backing object, mirroring the', + " MetaField get SPI (``ValueObject.get(name)`` else plain-attribute access).", + ' Keeps the mappers reflection-free + backing-agnostic."""', + " if o is None:", + " return None", + ' getter = getattr(o, "get", None)', + " if callable(getter):", + " return getter(name)", + " return getattr(o, name, None)", + ], + "_map_object_list": [ + "def _map_object_list(v: object | None, fn) -> list | None:", + ' """Map each element of an assembled array via ``fn``; non-list -> None."""', + " if not isinstance(v, list):", + " return None", + " return [fn(e) for e in v]", + ], + "_dlg_str": [ + "def _dlg_str(v: object | None) -> str | None:", + " return None if v is None else str(v)", + ], + "_dlg_int": [ + "def _dlg_int(v: object | None) -> int | None:", + " if v is None:", + " return None", + " try:", + " return int(v) # type: ignore[arg-type, call-overload]", + " except (TypeError, ValueError):", + " return None", + ], + "_dlg_float": [ + "def _dlg_float(v: object | None) -> float | None:", + " if v is None:", + " return None", + " try:", + " return float(v) # type: ignore[arg-type]", + " except (TypeError, ValueError):", + " return None", + ], + "_dlg_bool": [ + "def _dlg_bool(v: object | None) -> bool | None:", + " if v is None:", + " return None", + " if isinstance(v, bool):", + " return v", + ' return str(v).lower() == "true"', + ], + "_dlg_str_list": [ + "def _dlg_str_list(v: object | None) -> list | None:", + " if not isinstance(v, list):", + " return None", + " return [None if e is None else str(e) for e in v]", + ], +} + +# Stable emission order (deterministic output). +_HELPER_ORDER: tuple[str, ...] = ( + "_read_prop", + "_map_object_list", + "_dlg_str", + "_dlg_int", + "_dlg_float", + "_dlg_bool", + "_dlg_str_list", +) + + +def delegate_helpers(used: set[str]) -> list[str]: + """The shared helper block the generated mappers rely on, scoped to ``used`` and + emitted in stable order. Returns source lines (blank-line separated).""" + lines: list[str] = [ + "# ---- runtime-delegating extract helpers (generated) ----" + ] + for name in _HELPER_ORDER: + if name in used: + lines.append("") + lines.append("") + lines.extend(_HELPER_BLOCKS[name]) + return lines diff --git a/server/python/src/metaobjects/codegen/runtime/extract_schema_emitter.py b/server/python/src/metaobjects/codegen/runtime/extract_schema_emitter.py new file mode 100644 index 000000000..d13306129 --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/extract_schema_emitter.py @@ -0,0 +1,139 @@ +"""Turns a payload value-object into Python source fragments for the FR-010 extract +codegen: + +* :func:`schema_literal` — a ``ExtractSchema(Format.X, "root", [FieldSpec(...), …])`` + baked descriptor for the emitted parser module. +* :func:`mirror_dataclass` — an all-nullable mirror dataclass ``Extracted`` + (every field ``X | None = None``; the extract entry + returns this nullable twin rather than the strict + Pydantic payload — same reasoning as the Kotlin / C# + nullable mirror). +* :func:`mirror_initializer` — ``Extracted(field=as_string(d, "field"), …)``. + +Mirrors the C# / Java ``ExtractSchemaEmitter`` adapted to Python syntax + the +nullable-mirror shape. Bounded scope: scalar / enum / scalar-array. Nested object + +array-of-enum deferred. +""" +from __future__ import annotations + +from . import fr010_field_mapping as fm +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.meta_data import MetaData + +# The extract_map accessors a generated module may import (sorted, deduped subset). +ALL_EXTRACT_MAP_HELPERS: tuple[str, ...] = ( + "as_bool", + "as_double", + "as_int", + "as_long", + "as_string", + "as_string_list", +) + + +def _format_enum(fmt: str) -> str: + return "Format.XML" if fmt.lower() == "xml" else "Format.JSON" + + +def _field_spec_literal(field: MetaData, owner: MetaData) -> str: + name = field.name + required = fm.is_required(field) + req = "True" if required else "False" + + # An enum ARRAY is checked BEFORE the scalar-enum branch below: it is baked as a + # string-list scalar slot (read via as_string_list), NOT a scalar enum_field — + # otherwise the runtime collapses the list to one stringified value (the cross-port + # "enum-before-isArray" ordering bug). The mirror type / extract_map helper already + # route arrays first; this keeps the baked schema spec consistent with them. Scoped + # to the enum case so NON-enum scalar arrays keep their real element FieldKind below + # (the self-contained baked path drops arrays as MALFORMED before kind is read, so + # the kind is inert either way — but emitting the true kind is correct + clearer). + if fm.is_array(field) and field.sub_type == fc.FIELD_SUBTYPE_ENUM: + return f'FieldSpec.scalar("{name}", FieldKind.STRING, {req})' + + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + values_lit = fm.string_list_literal(fm.enum_values(field)) + alias_lit = fm.properties_map_literal(field.attrs().get(fc.FIELD_ATTR_ENUM_ALIAS)) + # FR-011: resolve the three new enum args (field → object.value → "strip" for + # normalize). Keep the back-compat 4-arg form when nothing is set; otherwise + # emit the 7-arg form (..., coerce_default, normalize, default_value). + coerce_default = fm.coerce_default(field) + default_value = fm.default_value(field) + normalize = fm.resolve_normalize(field, owner) + if ( + coerce_default is None + and default_value is None + and normalize == fc.NORMALIZE_DEFAULT + ): + return f'FieldSpec.enum_field("{name}", {req}, {values_lit}, {alias_lit})' + cd_lit = "None" if coerce_default is None else fm.py_string_literal(coerce_default) + norm_lit = fm.py_string_literal(normalize) + dv_lit = "None" if default_value is None else fm.py_string_literal(default_value) + return ( + f'FieldSpec.enum_field("{name}", {req}, {values_lit}, {alias_lit}, ' + f"{cd_lit}, {norm_lit}, {dv_lit})" + ) + + if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: + # Self-contained path models a nested object as a plain (opaque) STRING scalar + # slot — it does not recurse. The runtime-delegating ``*_with_loader`` entry + # populates the real nested graph. (No trailing inline comment: this literal is + # joined into a ``[...]`` argument list, where a ``#`` comment would be illegal.) + return f'FieldSpec.scalar("{name}", FieldKind.STRING, {req})' + + kind = fm.scalar_kind(field.sub_type) or "STRING" + # @xmlText: a scalar field marked to receive its element's XML text content + # (JAXB @XmlValue / Jackson @JacksonXmlText / .NET [XmlText]). Mirrors the TS + # extract-schema-emitter textContentField branch. No effect for JSON. + if fm.xml_text(field): + return f'FieldSpec.text_content_field("{name}", FieldKind.{kind}, {req})' + return f'FieldSpec.scalar("{name}", FieldKind.{kind}, {req})' + + +def schema_literal(vo: MetaData, fmt: str, root_name: str) -> str: + """Emit ``ExtractSchema(Format.X, "rootName", [FieldSpec(...), …])``.""" + format_enum = _format_enum(fmt) + specs = [_field_spec_literal(f, vo) for f in fm.fields(vo)] + if not specs: + return f'ExtractSchema({format_enum}, "{root_name}", [])' + body = ", ".join(specs) + return f'ExtractSchema({format_enum}, "{root_name}", [{body}])' + + +def mirror_dataclass(vo: MetaData, record_name: str) -> list[str]: + """Emit the all-nullable mirror dataclass declaration (source lines).""" + base = ( + record_name[: -len("Extracted")] + if record_name.endswith("Extracted") + else record_name + ) + lines: list[str] = [ + "@dataclass(frozen=True, slots=True)", + f"class {record_name}:", + f' """Best-effort extracted twin of ``{base}`` — every field nullable', + ' (``None`` where the value was lost or malformed)."""', + ] + field_lines = [ + f" {f.name}: {fm.mirror_type(f)} = None" for f in fm.fields(vo) + ] + if field_lines: + lines.extend(field_lines) + else: + lines.append(" pass") + return lines + + +def mirror_initializer(vo: MetaData, record_name: str) -> str: + """Emit ``(field=as_string(d, "field"), …)``.""" + assigns = [f"{f.name}={fm.extract_map_call(f)}" for f in fm.fields(vo)] + return f"{record_name}({', '.join(assigns)})" + + +def extract_map_imports(vo: MetaData) -> list[str]: + """The sorted, deduped ``extract_map`` accessor names the mirror initializer needs.""" + used: set[str] = set() + for f in fm.fields(vo): + h = fm.extract_map_helper(f) + if h is not None: + used.add(h) + return [h for h in ALL_EXTRACT_MAP_HELPERS if h in used] diff --git a/server/python/src/metaobjects/codegen/runtime/fr010_field_mapping.py b/server/python/src/metaobjects/codegen/runtime/fr010_field_mapping.py new file mode 100644 index 000000000..ae2d7510b --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/fr010_field_mapping.py @@ -0,0 +1,228 @@ +"""Shared field-kind mapping for the FR-010 codegen emitters (``extract_schema_emitter`` ++ ``output_format_spec_emitter``). + +Maps a metadata field subtype onto the render engine's ``FieldKind`` member, the +idiomatic ``X | None`` Python type used by the extract mirror dataclass, and the +``extract_map.as_*`` accessor that reads it from the forgiving outcome map. + +Mirrors the C# ``Fr010FieldMapping`` and the Java ``ExtractSchemaEmitter`` ordering. +Bounded scope (parity with Java / Kotlin / C#): scalar / enum / scalar-array. Nested +object + array-of-enum are deferred (see ``render/extract/KNOWN_GAPS.md``). + +Python has a single ``int`` type, so the ``LONG`` and ``CURRENCY`` subtypes map to the +``LONG`` ``FieldKind`` (and ``extract_map.as_long`` — identical to ``as_int``), keeping +the cross-port ``FieldKind`` vocabulary intact even though the emitted accessor is the +same. The mirror annotation collapses ``INT``/``LONG`` to ``int`` (one numeric int type). +""" +from __future__ import annotations + +from collections.abc import Iterable + +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.meta_data import MetaData +from metaobjects.meta.template.template_constants import TEMPLATE_ATTR_XML_TEXT +from metaobjects.shared.base_types import TYPE_FIELD + + +def fields(vo: MetaData) -> list[MetaData]: + """The field children of a payload value-object, in declaration order + (effective children — own + inherited via ``extends``).""" + return [c for c in vo.children() if c.type == TYPE_FIELD] + + +def is_array(field: MetaData) -> bool: + """Effective array-ness (ADR-0039 resolving): the native ``is_array`` flag OR the + ``@isArray`` attr, resolved through the ``extends`` super chain so a concrete field + inheriting array-ness from an abstract parent is honored.""" + return field.resolved_is_array() + + +def is_required(field: MetaData) -> bool: + """``@required`` — accepts a bool ``True`` or the string ``"true"``.""" + v = field.attrs().get(fc.FIELD_ATTR_REQUIRED) + if v is True: + return True + return isinstance(v, str) and v.lower() == "true" + + +def xml_text(field: MetaData) -> bool: + """``@xmlText`` — the XML text-content extract marker (accepts a bool ``True`` or the + string ``"true"``). When set, codegen bakes a ``FieldSpec.text_content_field(...)``. + Mirrors the TS ``xmlText(field)`` helper.""" + v = field.attrs().get(TEMPLATE_ATTR_XML_TEXT) + if v is True: + return True + return isinstance(v, str) and v.lower() == "true" + + +def enum_values(field: MetaData) -> list[str]: + """The string members of an enum field's ``@values`` attr (empty when absent).""" + v = field.attrs().get(fc.FIELD_ATTR_VALUES) + if isinstance(v, (list, tuple)): + return [str(x) for x in v] + return [] + + +def _attr_string(node: MetaData, name: str) -> str | None: + """The RESOLVING string value of attr *name* (own + inherited via ``extends``), + or ``None``. + + ADR-0039 — effective read: a concrete enum field extending an abstract enum may + inherit ``@coerceDefault``/``@default``/``@normalize`` from the abstract parent, + so these resolve through the super chain (``get_meta_attr``), never own-only.""" + v = node.get_meta_attr(name) + return v if isinstance(v, str) else None + + +def coerce_default(field: MetaData) -> str | None: + """FR-011: the enum field's effective ``@coerceDefault`` member, or ``None``.""" + return _attr_string(field, fc.FIELD_ATTR_COERCE_DEFAULT) + + +def default_value(field: MetaData) -> str | None: + """FR-011: the enum field's effective ``@default`` absent-fill member, or ``None``.""" + return _attr_string(field, fc.FIELD_ATTR_DEFAULT) + + +def resolve_normalize(field: MetaData, owner: MetaData | None) -> str: + """FR-011: resolve the enum normalization mode — field-level ``@normalize``, else the + owning ``object.value``'s ``@normalize`` (the per-object default), else the global + default (``"strip"``). Mirrors the Java/Kotlin/C#/TS ``resolveNormalize``.""" + field_mode = _attr_string(field, fc.FIELD_ATTR_NORMALIZE) + if field_mode is not None: + return field_mode + if owner is not None: + owner_mode = _attr_string(owner, fc.FIELD_ATTR_NORMALIZE) + if owner_mode is not None: + return owner_mode + return fc.NORMALIZE_DEFAULT + + +def scalar_kind(sub_type: str) -> str | None: + """The render-engine ``FieldKind`` member name for a scalar field subtype, or + ``None`` if the subtype is non-scalar (enum / object).""" + if sub_type in ( + fc.FIELD_SUBTYPE_STRING, + fc.FIELD_SUBTYPE_DATE, + fc.FIELD_SUBTYPE_TIME, + fc.FIELD_SUBTYPE_TIMESTAMP, + # ADR-0036/0037 Wave 3: uri/inet are string scalars on the wire. + fc.FIELD_SUBTYPE_URI, + fc.FIELD_SUBTYPE_INET, + ): + return "STRING" + if sub_type == fc.FIELD_SUBTYPE_INT: + return "INT" + if sub_type in (fc.FIELD_SUBTYPE_LONG, fc.FIELD_SUBTYPE_CURRENCY): + return "LONG" + if sub_type in ( + fc.FIELD_SUBTYPE_DOUBLE, + fc.FIELD_SUBTYPE_FLOAT, + fc.FIELD_SUBTYPE_DECIMAL, + ): + return "DOUBLE" + if sub_type == fc.FIELD_SUBTYPE_BOOLEAN: + return "BOOLEAN" + return None + + +def mirror_type(field: MetaData) -> str: + """The ``X | None`` Python annotation for a field in the extract mirror dataclass. + + A extracted array can contain null elements where individual items were lost, so + the array type is ``list[str | None] | None`` (matches ``extract_map.as_string_list``). + """ + # Nested object (single OR array): the self-contained path defers to a typed null; + # the runtime-delegating path overrides this with the nested mirror type. Checked + # BEFORE the generic is_array branch so an array-of-objects is NOT mistyped as a + # string array. Mirrors the Java/TS fix. + if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: + return "object | None" # nested deferred (self-contained path) + if is_array(field): + return "list[str | None] | None" + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + return "str | None" # enum is string-backed + kind = scalar_kind(field.sub_type) + if kind in ("INT", "LONG"): + return "int | None" + if kind == "DOUBLE": + return "float | None" + if kind == "BOOLEAN": + return "bool | None" + return "str | None" + + +def extract_map_helper(field: MetaData) -> str | None: + """The ``extract_map`` accessor a field reads through, or ``None`` + (object/deferred fields read no helper). + + Single source of the field → accessor decision; ``extract_map_call`` builds + the ``as_*(d, "name")`` call on top of this.""" + # Nested object (single OR array) → no helper in the self-contained path. Checked + # BEFORE is_array so an array-of-objects is NOT read via as_string_list. Mirrors the + # Java/TS fix. + if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: + return None + if is_array(field): + return "as_string_list" + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + return "as_string" + return { + "INT": "as_int", + "LONG": "as_long", + "DOUBLE": "as_double", + "BOOLEAN": "as_bool", + }.get(scalar_kind(field.sub_type) or "", "as_string") + + +def extract_map_call(field: MetaData) -> str: + """The ``as_*(d, "name")`` call that reads this field from the forgiving map ``d``. + + The helpers are imported by name into the generated module (see + ``extract_schema_emitter.extract_map_imports``).""" + helper = extract_map_helper(field) + if helper is None: + # Nested object: the self-contained initializer leaves it None (the + # runtime-delegating path populates it). Bare ``None`` — no trailing inline + # comment: this expr is joined into a ``Name(field=..., …)`` call-args list. + return "None" + return f'{helper}(d, "{field.name}")' + + +def py_string_literal(value: str) -> str: + """A safe Python double-quoted string literal for embedding free text + (``@example`` / ``@instruction`` / ``@enumDoc`` values) into emitted source. + + Uses ``json.dumps`` — a JSON string is a valid Python string literal and escapes + backslashes / quotes / control chars correctly (NOT naive quoting).""" + import json + + return json.dumps(value, ensure_ascii=False) + + +def string_list_literal(values: Iterable[str]) -> str: + """A Python ``["a", "b"]`` list literal for enum members.""" + return "[" + ", ".join(py_string_literal(v) for v in values) + "]" + + +def properties_map_literal(attr: object) -> str: + """A Python ``{"k": "v", …}`` dict literal for a properties-shaped attr + (``@enumAlias`` / ``@enumDoc``), or ``"None"`` when absent/empty. + + Null values are dropped; keys are sorted to match the canonical-serializer + properties sort (deterministic output).""" + if not isinstance(attr, dict): + return "None" + entries = [ + (k, v) + for k, v in attr.items() + if v is not None + ] + if not entries: + return "None" + entries.sort(key=lambda kv: str(kv[0])) + parts = [ + f"{py_string_literal(str(k))}: {py_string_literal(str(v))}" + for k, v in entries + ] + return "{" + ", ".join(parts) + "}" diff --git a/server/python/src/metaobjects/codegen/runtime/fr019_shared_enum.py b/server/python/src/metaobjects/codegen/runtime/fr019_shared_enum.py new file mode 100644 index 000000000..8dd24b7b9 --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/fr019_shared_enum.py @@ -0,0 +1,154 @@ +"""FR-019 — shared + externally-provided enum resolution (Python port of the TS reference +``enum-shared.ts`` and the C#/Java/Kotlin ``Fr019SharedEnum``). + +A reusable enum is a ROOT/package-level abstract ``field.enum`` (a sibling of ``object.entity``) +that concrete entity fields ``extends``. Per ADR-0026 such a declaration is materialized ONCE per +port as a module-level ``class (str, Enum)`` and referenced, instead of redeclaring the +members inline (a per-entity ``Literal[...]``) in every consumer. + +Two cases on the abstract declaration: + +* non-``@provided`` → metaobjects MATERIALIZES the type once (a shared ``enums.py`` module), and + consuming fields reference it (``from .enums import ``). +* ``@provided: true`` → metaobjects emits NOTHING for the type; consuming fields import an existing + type from a per-port-configured module (``GenConfig.provided_enum_packages`` keyed by the enum's + declaring metadata package, then the ``provided_enum_namespace`` fallback). The module never lives + in metadata (ADR-0001); a missing config for a referenced provided enum is a codegen-time error + naming the enum + its package. + +The common inline case (a concrete ``field.enum`` with own ``@values``, no root-extends) is +UNCHANGED — it stays a per-entity ``Literal[...]`` (byte-identical default). +""" +from __future__ import annotations + +import warnings +from dataclasses import dataclass + +from metaobjects.meta.core.field.meta_field import ( + MetaField, + resolve_shared_enum_declaration, +) +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.meta_data import MetaData +from metaobjects.shared.separators import PACKAGE_SEP +from metaobjects.codegen import type_map +from metaobjects.codegen.config import GenConfig + + +@dataclass(frozen=True) +class SharedEnum: + """A shared (package-level abstract) enum declaration codegen reasons about.""" + + name: str #: materialized type name — the decl's PascalCase short name (cross-port id) + values: list[str] #: member symbols (the @values SSOT), verbatim + provided: bool #: True when the decl carries @provided: true (reference, don't materialize) + meta_package: str #: declaring metadata package in "::" form (the provided-config key) + + +def _pascal(name: str) -> str: + """``priority`` → ``Priority`` (leading char only; no snake-splitting — cross-port rule).""" + return name[:1].upper() + name[1:] if name else name + + +def resolve_shared_enum_decl(field: MetaField) -> MetaData | None: + """Deprecated name of + :func:`metaobjects.meta.core.field.meta_field.resolve_shared_enum_declaration`, + kept so a generator ejected before the resolver moved to the metadata layer + still imports. Same answer, including through a multi-hop ``extends``.""" + warnings.warn( + "resolve_shared_enum_decl is deprecated; call " + "metaobjects.meta.core.field.meta_field.resolve_shared_enum_declaration", + DeprecationWarning, + stacklevel=2, + ) + return resolve_shared_enum_declaration(field) + + +def is_provided(decl: MetaData) -> bool: + """``@provided`` truth of an enum DECLARATION. + + ADR-0039 sanctioned own (Python naming inversion: ``attr()`` is the OWN read, + ``get_meta_attr()`` resolves). ``@provided`` is a declaration-layer provenance + marker — "THIS type is supplied by hand-written/third-party code", like + ``is_abstract`` — and does not flow down an ``extends`` chain. + + This is only ever called on the resolved declaration (see + ``shared_enum_for_field``), never on the consuming field, so own and resolving + agree for a plain ``field extends @provided decl``. They diverge on a CHAINED + declaration (root abstract ``B extends`` root abstract ``@provided A``): a + resolving read reports B provided and emits a reference to a hand-written ``B`` + the adopter never declared, instead of materializing B. Matches the JVM ports. + """ + return decl.attr(fc.FIELD_ATTR_PROVIDED) is True + + +def _meta_package_of(decl: MetaData) -> str: + """The decl's metadata package in ``::`` form (its resolution key minus the trailing name).""" + key = decl.resolution_key() + idx = key.rfind(PACKAGE_SEP) + return key[:idx] if idx >= 0 else "" + + +def shared_enum_for_field(field: MetaField) -> SharedEnum | None: + """The shared-enum descriptor *field* resolves to, or ``None`` for inline enums. + + The member set is read off the DECLARATION *decl* (a resolving ``attrs()`` + read, never off *field*) so materialization cannot depend on which consuming + field a generator happens to visit first — every field resolving to the same + shared declaration must describe that declaration identically.""" + decl = resolve_shared_enum_declaration(field) + if decl is None: + return None + values = type_map.effective_enum_values(decl) + if not values: + return None + return SharedEnum( + name=_pascal(decl.name), + values=values, + provided=is_provided(decl), + meta_package=_meta_package_of(decl), + ) + + +def collect_shared_enums(entities: list) -> list[SharedEnum]: + """All shared (package-level abstract) enum decls CONSUMED by at least one entity field, + keyed by materialized type name, sorted by name (deterministic). A decl nobody extends is + not materialized (no dangling type).""" + out: dict[str, SharedEnum] = {} + for entity in entities: + # ADR-0039 — RESOLVING (fields()): mirrors the TS collectSharedEnums + # (``entity.fields()``); an inherited field consuming a shared enum counts. + for f in entity.fields(): + shared = shared_enum_for_field(f) + if shared is None: + continue + out.setdefault(shared.name, shared) + return sorted(out.values(), key=lambda e: e.name) + + +def materialized_shared_enums(entities: list) -> list[SharedEnum]: + """The shared enums metaobjects MATERIALIZES (non-``@provided``, consumed).""" + return [e for e in collect_shared_enums(entities) if not e.provided] + + +def enum_type_and_import(shared: SharedEnum, config: GenConfig) -> tuple[str, str]: + """The ``(type_name, import_line)`` a consuming field emits for a shared/provided enum: + + * materialized → ``("", "from .enums import ")`` (the shared module). + * ``@provided`` → ``("", "from import ")``, *module* resolved from + ``config.provided_enum_packages`` (keyed by the enum's metadata package) then the single + ``config.provided_enum_namespace`` fallback. A missing config is a codegen-time ``ValueError`` + naming the enum + its package (ADR-0026 / ADR-0001).""" + name = shared.name + if not shared.provided: + return name, f"from .enums import {name}" + module = config.provided_enum_packages.get(shared.meta_package) or config.provided_enum_namespace + if not module: + raise ValueError( + f'provided enum "{name}" (declared in package "{shared.meta_package}") is marked ' + f"@provided but no Python import module is configured to reference it from. Map its " + f'package in GenConfig.provided_enum_packages["{shared.meta_package}"] = ' + f'"your_app.enums", or set the single provided_enum_namespace fallback, so the ' + f'generated code can import "{name}".' + ) + return name, f"from {module} import {name}" diff --git a/server/python/src/metaobjects/codegen/runtime/m2m_codegen.py b/server/python/src/metaobjects/codegen/runtime/m2m_codegen.py new file mode 100644 index 000000000..813794dbb --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/m2m_codegen.py @@ -0,0 +1,240 @@ +"""Shared M:N (many-to-many) codegen descriptor — the build-time resolution of a +source entity's ``relationship.* @cardinality:"many" + @through`` children into +the concrete coordinates each Python generator needs (Pydantic nested collection, +FastAPI traversal route, and the physical junction/target column names the +repository seam joins on). + +This mirrors the TS reference ``relation-resolver.ts`` (Unit 10): codegen derives +the descriptor at build time from the junction entity's two ``identity.reference`` +children (the cross-port SSOT for FK direction) via the shared +``derive_m2m_fields`` helper — the relationship never restates its FK columns. + +The descriptor carries: + * ``relation_name`` — the navigation member name (route segment + Pydantic field). + * ``target_entity`` — the related entity's bare name (Pydantic element type). + * ``source_plural`` / ``target_plural`` — pluralized entity names for the + REST contract: the source URL segment is the ENTITY name pluralized + (``Person`` → ``persons``), NOT the physical ``@table`` (cross-port grammar). + * ``source_column`` / ``target_column`` — the PHYSICAL junction FK columns + (e.g. ``follower_id`` / ``followee_id``); the repository seam joins on these. + * ``junction_table`` / ``target_table`` / ``target_pk_column`` — physical names + for the join. + * ``symmetric`` — selects the union-on-read traversal. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from metaobjects.apidocs.naming import route_path +from metaobjects.source_resolution import resolve_table_name +from metaobjects.naming import ( + DEFAULT_COLUMN_NAMING, + apply_column_naming_strategy, + resolve_column_name, +) +from metaobjects.meta.core.field.meta_field import MetaField +from metaobjects.meta.meta_data import MetaData +from metaobjects.meta.core.identity.identity_constants import ( + IDENTITY_ATTR_FIELDS, + IDENTITY_SUBTYPE_PRIMARY, +) +from metaobjects.meta.core.object.meta_object import MetaObject +from metaobjects.meta.core.object.object_constants import OBJECT_ATTR_DISCRIMINATOR_VALUE +from metaobjects.codegen.generators.tph_plan import is_tph_subtype +from metaobjects.meta.core.relationship.derive_m2m_fields import ( + M2MDerivationError, + derive_m2m_fields, +) +from metaobjects.meta.core.relationship.meta_relationship import MetaRelationship +from metaobjects.meta.core.relationship.relationship_constants import CARDINALITY_MANY +from metaobjects.shared.base_types import TYPE_IDENTITY +from metaobjects.shared.separators import PACKAGE_SEP + + +@dataclass(frozen=True) +class M2mDescriptor: + """Build-time coordinates of one M:N navigation on a source entity.""" + + relation_name: str + target_entity: str + source_plural: str + target_plural: str + junction_table: str + target_table: str + source_column: str + target_column: str + target_pk_column: str + symmetric: bool + #: The target's OWN ``@discriminatorValue`` when ``@objectRef`` resolves to a + #: concrete TPH subtype (FW-8 follow-up; mirrors the TS reference's + #: ``targetDiscriminator``) — ``None`` for the overwhelming common case (a + #: vanilla, non-TPH target). The target's rows physically live in its + #: discriminator base's shared table, so a route generator threads this value + #: into the ``find_related_`` seam as an extra ``target_subtype`` + #: parameter, structurally handing the consumer's join what it needs to + #: filter to just that subtype rather than every row of the shared table. + #: Defaulted so existing positional construction (if any) stays valid. + target_discriminator: str | None = None + + +def _strip_package(name: str) -> str: + idx = name.rfind(PACKAGE_SEP) + return name[idx + len(PACKAGE_SEP):] if idx >= 0 else name + + +def _physical_table(entity: MetaObject, *, relation_context: str) -> str: + """The entity's physical SQL table name, via the shared + :func:`~metaobjects.source_resolution.resolve_table_name` resolver. + + Raises :class:`M2MDerivationError` — naming *entity* and *relation_context* + — when it declares (and inherits) no primary source at all. #248: DB + participation derives from a declared source, never from the object + subtype, so ``resolve_table_name`` returning ``None`` here means the + junction/target has no physical table to join through at all; a M:N + relationship's whole point is a real SQL join, so that is a metadata + authoring error, not a legitimate "doesn't participate in persistence" + shape. Coercing to the bare entity name — the pre-existing behaviour this + replaces — fabricated a table that was never created, precisely the defect + class the C# port's EF binding had to fix this same session (a + ``[Table(...)]`` attribute naming a class that was never generated).""" + table = resolve_table_name(entity) + if table is None: + raise M2MDerivationError( + f"{relation_context}: entity \"{entity.name}\" declares no primary " + f"source (source.rdb) — a M:N join needs a physical table on both " + f"the junction and the target." + ) + return table + + +def _column_of( + field: MetaField | None, fallback: str, strategy: str = DEFAULT_COLUMN_NAMING +) -> str: + """Physical column name for a junction FK. Delegates to the shared + :func:`metaobjects.naming.resolve_column_name` — the SAME resolver the runtime uses, + so generated junction SQL and runtime SQL cannot disagree about a column name. + + *fallback* covers the case where the field could not be resolved at all; it is a + bare metadata name, so it goes through the strategy too.""" + if field is None: + return apply_column_naming_strategy(fallback, strategy) + return resolve_column_name(field, strategy) + + +def _field_named(entity: MetaObject, field_name: str) -> MetaField | None: + for f in entity.fields(): + if f.name == field_name: + return f + return None + + +def pk_field_name(entity: MetaObject) -> str: + """The single PK field name (``identity.primary @fields``), default ``id``.""" + for c in entity.children(): + if c.type == TYPE_IDENTITY and c.sub_type == IDENTITY_SUBTYPE_PRIMARY: + fields = c.get_meta_attr(IDENTITY_ATTR_FIELDS) # ADR-0039: resolving (identity attr) + if isinstance(fields, (list, tuple)) and fields: + first = fields[0] + if isinstance(first, str): + return first + if isinstance(fields, str) and fields: + return fields.split(",")[0].strip() or "id" + return "id" + + +def m2m_relationships(entity: MetaObject) -> list[MetaRelationship]: + """The entity's ``relationship.* @cardinality:"many" + @through`` children + (declaration order). 1:N relationships (no ``@through``) are excluded. + + ADR-0039 — RESOLVING (children()): mirrors the TS ``relationships()`` (built on + the effective ``children()``); a relationship inherited via ``extends`` counts. + """ + out: list[MetaRelationship] = [] + for c in entity.children(): + if not isinstance(c, MetaRelationship): + continue + if c.cardinality() != CARDINALITY_MANY: + continue + if c.through() is None: + continue + out.append(c) + return out + + +def resolve_m2m_descriptors( + entity: MetaObject, + object_index: dict[str, MetaObject], + column_naming: str = DEFAULT_COLUMN_NAMING, +) -> list[M2mDescriptor]: + """Resolve every M:N navigation on *entity* into a build-time descriptor. + + *object_index* is a bare-name → entity map of the loaded model. A relationship + that fails derivation (missing/malformed junction, ambiguous self-join) raises + :class:`M2MDerivationError` — the loader validates this; reaching it here is a + metadata error, surfaced loudly (never silently skipped). + """ + descriptors: list[M2mDescriptor] = [] + # The source is *entity* for every relation, so its route segment is + # loop-invariant; only the target varies per relation. + source_plural = route_path(entity.name) + for rel in m2m_relationships(entity): + target_name = _strip_package(rel.object_ref() or "") + junction_name = _strip_package(rel.through() or "") + target = object_index.get(target_name) + junction = object_index.get(junction_name) + if target is None or junction is None: + raise M2MDerivationError( + f'M:N relationship "{entity.name}.{rel.name}": @objectRef ' + f'"{target_name}" / @through "{junction_name}" must resolve to ' + f"loaded entities" + ) + + # Widen to dict[str, MetaData] for the shared derivation helper (dict is + # invariant; MetaObject is a MetaData subtype). + meta_index: dict[str, MetaData] = dict(object_index) + fields = derive_m2m_fields(rel, entity, meta_index) + source_fk = _field_named(junction, fields.source_field) + target_fk = _field_named(junction, fields.target_field) + # FW-8 follow-up: the target's rows live in its discriminator base's shared + # table when @objectRef resolves to a concrete TPH subtype — @discriminatorValue + # is the subtype's OWN value (ADR-0039 sanctioned own, same idiom as tph_plan.py). + target_discriminator = ( + target.attr(OBJECT_ATTR_DISCRIMINATOR_VALUE) if is_tph_subtype(target) else None + ) + + descriptors.append( + M2mDescriptor( + relation_name=rel.name, + target_entity=target.name, + source_plural=source_plural, + target_plural=route_path(target.name), + junction_table=_physical_table( + junction, + relation_context=( + f'M:N relationship "{entity.name}.{rel.name}" @through junction' + ), + ), + target_table=_physical_table( + target, + relation_context=( + f'M:N relationship "{entity.name}.{rel.name}" @objectRef target' + ), + ), + source_column=_column_of(source_fk, fields.source_field, column_naming), + target_column=_column_of(target_fk, fields.target_field, column_naming), + target_pk_column=_column_of( + _field_named(target, pk_field_name(target)), + pk_field_name(target), + column_naming, + ), + symmetric=rel.symmetric(), + target_discriminator=target_discriminator, + ) + ) + return descriptors + + +def build_object_index(entities: list[MetaObject]) -> dict[str, MetaObject]: + """Bare-name → entity map (the codegen resolution surface, mirroring the + runtime resolver's ``object_index``).""" + return {e.name: e for e in entities} diff --git a/server/python/src/metaobjects/codegen/runtime/output_format_spec_emitter.py b/server/python/src/metaobjects/codegen/runtime/output_format_spec_emitter.py new file mode 100644 index 000000000..be0193de5 --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/output_format_spec_emitter.py @@ -0,0 +1,83 @@ +"""Turns a response value-object + its ``template.prompt`` node into a Python source +literal for an :class:`~metaobjects.render.OutputFormatSpec` — the artifact-1 prompt +descriptor used by the FR-010 response-format codegen. + +Emits ``OutputFormatSpec(Format.X, "rootName", PromptStyle.X, [PromptField(...), …])``. +Mirrors the C# / Java ``OutputFormatSpecEmitter`` adapted to Python (keyword args for +the long ``PromptField`` ctor to keep the emitted source readable + order-robust). +Bounded scope: scalar / enum. Nested object → ``FieldKind.OBJECT`` placeholder. +""" +from __future__ import annotations + +from . import fr010_field_mapping as fm +from metaobjects.codegen.generators.find_inbound import is_xml, response_format_of +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.meta_data import MetaData +from metaobjects.meta.template import template_constants as tc + + +def _format_enum(template: MetaData) -> str: + """ADR-0053 — the fragment describes the REPLY, so its syntax is ``@responseFormat``, + NOT ``@format``, which is the syntax of the rendered prompt BODY. Reading ``@format`` + here typed the instruction "produce your answer like this" off the format of the + QUESTION.""" + return "Format.XML" if is_xml(response_format_of(template)) else "Format.JSON" + + +def _prompt_style_enum(template: MetaData) -> str: + style = template.get_meta_attr(tc.TEMPLATE_ATTR_PROMPT_STYLE) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) + if style == tc.PROMPT_STYLE_INLINE: + return "PromptStyle.INLINE" + if style == tc.PROMPT_STYLE_EXAMPLE_ONLY: + return "PromptStyle.EXAMPLE_ONLY" + return "PromptStyle.GUIDE" + + +def _opt_string_attr(field: MetaData, attr_name: str) -> str: + v = field.attrs().get(attr_name) + return fm.py_string_literal(v) if isinstance(v, str) else "None" + + +def _prompt_field_literal(field: MetaData) -> str: + name = field.name + req = "True" if fm.is_required(field) else "False" + array = "True" if fm.is_array(field) else "False" + + if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: + # FR-010: nested prompt deferred — emit a FieldKind.OBJECT placeholder (nested + # defaults to None on PromptField). The note lives HERE, not in the returned + # literal: spec_literal joins every field onto ONE line, and a Python '#' inline + # comment would swallow the rest of that line — including the closing `])` — so a + # nested-object payload emitted invalid Python (unterminated list) and crashed + # `gen` + `verify --codegen`. The other ports use `/* */` block comments, which + # are inline-safe; Python has none, so the literal must carry no comment. + return f'PromptField("{name}", FieldKind.OBJECT, {req}, array={array})' + + example = _opt_string_attr(field, fc.FIELD_ATTR_EXAMPLE) + instruction = _opt_string_attr(field, fc.FIELD_ATTR_INSTRUCTION) + + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + values_lit = fm.string_list_literal(fm.enum_values(field)) + enum_doc_lit = fm.properties_map_literal(field.attrs().get(fc.FIELD_ATTR_ENUM_DOC)) + return ( + f'PromptField("{name}", FieldKind.ENUM, {req}, array={array}, ' + f"enum_values={values_lit}, enum_doc={enum_doc_lit}, " + f"example={example}, instruction={instruction})" + ) + + kind = fm.scalar_kind(field.sub_type) or "STRING" + return ( + f'PromptField("{name}", FieldKind.{kind}, {req}, array={array}, ' + f"example={example}, instruction={instruction})" + ) + + +def spec_literal(vo: MetaData, template: MetaData, root_name: str) -> str: + """Emit ``OutputFormatSpec(Format.X, "rootName", PromptStyle.X, [PromptField(...), …])``.""" + format_enum = _format_enum(template) + style_enum = _prompt_style_enum(template) + field_lits = [_prompt_field_literal(f) for f in fm.fields(vo)] + body = ", ".join(field_lits) + return ( + f'OutputFormatSpec({format_enum}, "{root_name}", {style_enum}, [{body}])' + ) diff --git a/server/python/src/metaobjects/codegen/runtime/shared_predicates.py b/server/python/src/metaobjects/codegen/runtime/shared_predicates.py index 1ed052e6c..0a0cdded2 100644 --- a/server/python/src/metaobjects/codegen/runtime/shared_predicates.py +++ b/server/python/src/metaobjects/codegen/runtime/shared_predicates.py @@ -20,6 +20,7 @@ import re +from metaobjects.codegen.generators.tph_plan import is_tph_subtype from metaobjects.codegen.instance_artifacts import emits_instance_artifacts from metaobjects.meta.core.field import field_constants as fc from metaobjects.meta.core.field.meta_field import MetaField @@ -93,3 +94,30 @@ def emits_router(entity: MetaObject) -> bool: or kind == SOURCE_KIND_TABLE or kind in READ_ONLY_ROUTED_KINDS ) + + +def is_read_only_routed(entity: MetaObject) -> bool: + """Whether the router generator chooses the READ-ONLY router for this object (a view / + materialized-view source that is not write-through): GET list, GET by id when it has an + item route, and the 405 refusals, with no write surface. + + THE predicate behind that choice, shared with the API-docs generator so a page can never + present write verbs for an object whose router refuses them.""" + if not emits_router(entity) or entity.is_write_through(): + return False + src = primary_rdb_source(entity) + return src is not None and src.effective_kind() in READ_ONLY_ROUTED_KINDS + + +def emits_writable_router(entity: MetaObject) -> bool: + """Whether the router generator emits the full CRUD router for this object: it gets a + router, that router is not the read-only one, and it is not a table-per-hierarchy subtype + (a subtype is folded into its base's single table, so its CRUD lives under the base). + + THE predicate behind that choice, shared with the API-docs generator so a page can never + document write routes for an object the router does not give them.""" + return ( + emits_router(entity) + and not is_read_only_routed(entity) + and not is_tph_subtype(entity) + ) diff --git a/server/python/tests/codegen/test_api_docs_paths.py b/server/python/tests/codegen/test_api_docs_paths.py index 0c4e6683b..96f824227 100644 --- a/server/python/tests/codegen/test_api_docs_paths.py +++ b/server/python/tests/codegen/test_api_docs_paths.py @@ -11,7 +11,7 @@ import json from pathlib import Path -from metaobjects.apidocs.paths import ( +from metaobjects.codegen.runtime.api_docs_paths import ( Layout, doc_page_output_path, model_cross_href, diff --git a/server/python/tests/codegen/test_cli_registry.py b/server/python/tests/codegen/test_cli_registry.py index 55377eb3a..0ead45d7e 100644 --- a/server/python/tests/codegen/test_cli_registry.py +++ b/server/python/tests/codegen/test_cli_registry.py @@ -39,8 +39,8 @@ def test_gen_list_prints_all_and_exits_zero(capsys, tmp_path: Path) -> None: # Every registered stable name appears in the listing. for name in GENERATOR_REGISTRY: assert name in out, f"--list omitted {name!r}" - # Exactly the 11 python-slice names are registered (ADR-0056 removed `payload`). - assert len(GENERATOR_REGISTRY) == 11 + # Exactly the 12 python-slice names are registered (ADR-0056 removed `payload`). + assert len(GENERATOR_REGISTRY) == 12 assert "payload" not in GENERATOR_REGISTRY # A template-tier generator says it needs the entity generator's models. assert "output-parser — " in out diff --git a/server/python/tests/codegen/test_docs_generators.py b/server/python/tests/codegen/test_docs_generators.py new file mode 100644 index 000000000..6bdcd8109 --- /dev/null +++ b/server/python/tests/codegen/test_docs_generators.py @@ -0,0 +1,205 @@ +"""`api-docs` is a generator `metaobjects docs` runs, and a project can own it. + +An owned copy is listed under ``docs.generators`` in ``metaobjects.config.yaml`` and ``docs`` +runs it in place of the packaged one. ``gen`` never runs a docs generator: it skips one found +under a target and says so. What the reference documents comes from the same +``shared_predicates`` module the owned ``routes`` and ``filter-allowlist`` read, so an edit +to the owned predicate moves the router, the allowlist and the reference together. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from metaobjects.cli import main +from metaobjects.codegen.project_config import ConfigError, load_project_config + +REPO = Path(__file__).parents[4] +FITNESS = REPO / "fixtures" / "persistence-conformance" / "canonical" / "meta.fitness.json" +OWNED_DOCS = "codegen.generators.api_docs:api_docs_generator" + + +def _project(root: Path, extra: str = "", generators: str = "entity") -> Path: + (root / "metaobjects").mkdir(parents=True, exist_ok=True) + (root / "metaobjects" / FITNESS.name).write_text(FITNESS.read_text(encoding="utf-8")) + cfg = root / "metaobjects.config.yaml" + cfg.write_text( + f"targets:\n main:\n outDir: gen\n generators: [{generators}]\n{extra}", + encoding="utf-8", + ) + return cfg + + +@pytest.fixture(autouse=True) +def _fresh_owned_modules(): + """A real run is a fresh process; here one process runs several projects, and an edit to + an owned module must be seen by the next run, so each run starts with no `codegen` loaded.""" + _forget_owned() + yield + _forget_owned() + + +def _forget_owned() -> None: + for name in [k for k in sys.modules if k == "codegen" or k.startswith("codegen.")]: + del sys.modules[name] + + +def _docs(root: Path, monkeypatch, *args: str) -> int: + _forget_owned() + monkeypatch.chdir(root) + return main(["docs", "--out", "docs", "--project", "model", *args]) + + +def _pages(root: Path) -> dict[str, str]: + base = root / "docs" + return {p.relative_to(base).as_posix(): p.read_text(encoding="utf-8") + for p in sorted(base.rglob("*.md"))} + + +# --- config ----------------------------------------------------------------------------- + + +def test_the_docs_block_is_read(tmp_path: Path) -> None: + cfg = _project(tmp_path, f"docs:\n generators: [{OWNED_DOCS}]\n") + assert load_project_config(cfg).docs_generators == [OWNED_DOCS] + assert load_project_config(_project(tmp_path / "plain")).docs_generators == [] + + +@pytest.mark.parametrize("block, message", [ + ("docs:\n generators: [api-docs]\n", "module:symbol"), + ("docs:\n generators: nope\n", "list of strings"), + ("docs:\n generator: [a:b]\n", "unknown key"), + ("docs: []\n", "must be a mapping"), +]) +def test_a_bad_docs_block_is_refused_by_name(tmp_path: Path, block: str, message: str) -> None: + with pytest.raises(ConfigError, match=message): + load_project_config(_project(tmp_path, block)) + + +# --- docs runs the owned copy ----------------------------------------------------------- + + +def test_docs_runs_an_owned_api_docs_in_place_of_the_packaged_one(tmp_path: Path, monkeypatch, capsys) -> None: + _project(tmp_path, f"docs:\n generators: [{OWNED_DOCS}]\n") + monkeypatch.chdir(tmp_path) + assert main(["eject", "api-docs"]) == 0 + out = capsys.readouterr().out + assert "docs.generators" in out and OWNED_DOCS in out and "`gen` never does" in out + assert _docs(tmp_path, monkeypatch) == 0 + owned_pages = _pages(tmp_path) + + packaged = tmp_path / "packaged" + _project(packaged) + assert _docs(packaged, monkeypatch) == 0 + # Unchanged, the copy writes what the packaged generator writes. + assert owned_pages == _pages(packaged) and owned_pages + + # An edit to the owned renderer reaches the page: the copy, not the package, ran. + renderer = tmp_path / "codegen" / "runtime" / "api_docs_renderer.py" + source = renderer.read_text(encoding="utf-8") + assert source.count("# Agent API Reference") == 1 + renderer.write_text(source.replace("# Agent API Reference", "# Owned Agent API Reference"), + encoding="utf-8") + assert _docs(tmp_path, monkeypatch) == 0 + assert _pages(tmp_path)["api/python/AGENT-API.md"].startswith("# Owned Agent API Reference") + + +def test_docs_without_a_docs_block_runs_the_packaged_generator_even_when_one_is_owned( + tmp_path: Path, monkeypatch, +) -> None: + _project(tmp_path) + monkeypatch.chdir(tmp_path) + assert main(["eject", "api-docs"]) == 0 + renderer = tmp_path / "codegen" / "runtime" / "api_docs_renderer.py" + renderer.write_text(renderer.read_text(encoding="utf-8").replace( + "# Agent API Reference", "# Owned Agent API Reference"), encoding="utf-8") + assert _docs(tmp_path, monkeypatch) == 0 + assert _pages(tmp_path)["api/python/AGENT-API.md"].startswith("# Agent API Reference") + + +def test_the_positional_docs_form_reads_no_config(tmp_path: Path, monkeypatch) -> None: + _project(tmp_path, f"docs:\n generators: [{OWNED_DOCS}]\n") + monkeypatch.chdir(tmp_path) + assert main(["eject", "api-docs"]) == 0 + renderer = tmp_path / "codegen" / "runtime" / "api_docs_renderer.py" + renderer.write_text(renderer.read_text(encoding="utf-8").replace( + "# Agent API Reference", "# Owned Agent API Reference"), encoding="utf-8") + assert _docs(tmp_path, monkeypatch, "metaobjects") == 0 + assert _pages(tmp_path)["api/python/AGENT-API.md"].startswith("# Agent API Reference") + + +def test_a_docs_entry_that_is_not_a_docs_generator_is_refused_by_name( + tmp_path: Path, monkeypatch, capsys, +) -> None: + _project(tmp_path, "docs:\n generators: [codegen.generators.entity:entity_model]\n") + monkeypatch.chdir(tmp_path) + assert main(["eject", "entity"]) == 0 + assert _docs(tmp_path, monkeypatch) == 1 + err = capsys.readouterr().err + assert "codegen.generators.entity:entity_model" in err and "not a docs generator" in err + assert not (tmp_path / "docs").exists() + + +def test_two_owned_generators_for_the_api_surface_are_refused(tmp_path: Path, monkeypatch, capsys) -> None: + _project(tmp_path, f"docs:\n generators: [{OWNED_DOCS}, {OWNED_DOCS}]\n") + monkeypatch.chdir(tmp_path) + assert main(["eject", "api-docs"]) == 0 + assert _docs(tmp_path, monkeypatch) == 1 + assert "two generators for the 'api' surface" in capsys.readouterr().err + + +# --- gen leaves a docs generator out ---------------------------------------------------- + + +def test_gen_skips_a_docs_generator_under_a_target_and_says_so(tmp_path: Path, monkeypatch, capsys) -> None: + _project(tmp_path, generators="entity, api-docs") + monkeypatch.chdir(tmp_path) + assert main(["gen"]) == 0 + out = capsys.readouterr().out + assert "warning: 'api-docs' writes the api reference, so gen skipped it" in out + assert "docs.generators" in out + written = {p.relative_to(tmp_path / "gen").as_posix() for p in (tmp_path / "gen").rglob("*") + if p.is_file()} + assert written and not any(w.startswith("api/") or w.endswith("AGENT-API.md") for w in written) + assert not (tmp_path / "gen" / "api").exists() + + +# --- the reference describes the code the owned generators emit -------------------------- + + +def test_one_owned_predicate_moves_the_router_the_allowlist_and_the_reference( + tmp_path: Path, monkeypatch, +) -> None: + owned = tmp_path / "owned" + _project(owned, f"docs:\n generators: [{OWNED_DOCS}]\n", + generators="entity, codegen.generators.filter_allowlist:filter_allowlist_generator, " + "codegen.generators.routes:router_generator") + monkeypatch.chdir(owned) + assert main(["eject", "api-docs", "filter-allowlist", "routes"]) == 0 + assert main(["gen"]) == 0 + assert _docs(owned, monkeypatch) == 0 + before = _pages(owned) + assert any(f.name == "post_router.py" for f in (owned / "gen").glob("*_router.py")) + assert any("GET /api/posts" in page for page in before.values()) + + rule = owned / "codegen" / "runtime" / "shared_predicates.py" + rule.write_text( + rule.read_text(encoding="utf-8") + + "\n\n_reference_emits_router = emits_router\n\n\n" + "def emits_router(entity):\n" + " return entity.name != 'Post' and _reference_emits_router(entity)\n", + encoding="utf-8", + ) + for stale in (owned / "gen").glob("*"): + if stale.is_file(): + stale.unlink() + _forget_owned() + assert main(["gen"]) == 0 + assert _docs(owned, monkeypatch) == 0 + after = _pages(owned) + assert not any(f.name == "post_router.py" for f in (owned / "gen").glob("*_router.py")) + assert not any(f.name == "post_filter_allowlist.py" for f in (owned / "gen").glob("*.py")) + assert not any("GET /api/posts" in page for page in after.values()) + assert after != before diff --git a/server/python/tests/codegen/test_eject_alone.py b/server/python/tests/codegen/test_eject_alone.py new file mode 100644 index 000000000..ab473c435 --- /dev/null +++ b/server/python/tests/codegen/test_eject_alone.py @@ -0,0 +1,323 @@ +"""Each ejectable generator, ejected ALONE, owns everything that writes its output. + +ADR-0034 Amendment 3: the adopter owns the generator and the code that code calls, when +that code is a helper. So for every generator the registry makes ejectable, derived from the +registry and not from a list kept beside it, an empty project that ejects only that generator +must get a copy that + +* imports no packaged emit module: the copy and the helper source eject placed beside it + import only the engine surface written below (the generator base, the file writer and + formatter, the naming and type-mapping tables, the model walks and the loader); +* loads and runs with every packaged emit module made unimportable; and +* writes output byte-identical to the packaged generator over the same model. + +The one documented difference is the router: an owned ``routes`` emits its runtime into the +generated package and imports it from there, so its router modules differ from the packaged +ones in those import lines and in the ``_runtime/`` files (``test_eject_runtime.py``). + +The census at the bottom keeps the guard honest: a helper module no registry entry hands over, +or a generator module no registry entry names, fails here instead of staying packaged. +""" +from __future__ import annotations + +import ast +import json +import os +import shutil +import subprocess +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +import pytest + +import metaobjects +from metaobjects.codegen import eject as owned +from metaobjects.codegen.generator_registry import GENERATOR_REGISTRY + +REPO = Path(__file__).parents[4] +FIXTURES = REPO / "fixtures" +CODEGEN = Path(metaobjects.__file__).parent / "codegen" + +#: The packaged ``metaobjects.codegen`` modules an owned copy may import: the engine. Everything +#: that returns emitted text is handed over instead, and everything outside ``metaobjects.codegen`` +#: (the loader, the metamodel, naming, render, extract, the runtime) is core by ruling. +ENGINE_CODEGEN = frozenset({ + "metaobjects.codegen.config", # GenConfig + "metaobjects.codegen.constants", # the @generated marker and header + "metaobjects.codegen.format", # the formatter + "metaobjects.codegen.generator", # the generator base, EmittedFile, GenContext + "metaobjects.codegen.instance_artifacts", # model questions: what an object emits + "metaobjects.codegen.model_walk", + "metaobjects.codegen.project_config", # the requirementTests options + "metaobjects.codegen.requirement_hooks", + "metaobjects.codegen.requirement_walk", # shared with `verify` + "metaobjects.codegen.type_map", # the type-mapping table + "metaobjects.codegen.value_objects", # naming and value-object resolution + "metaobjects.codegen.generators.find_inbound", # model walk: the inbound tier + "metaobjects.codegen.generators.tph_plan", # model walk: table-per-hierarchy +}) +#: Of ``metaobjects.apidocs``, only the naming seam: the names the generators emit under. +ENGINE_APIDOCS = frozenset({"metaobjects.apidocs.naming"}) + +EJECTABLE = sorted(e.name for e in owned.ejectable()) + + +def _runtime_imports(node: ast.AST) -> list[ast.stmt]: + """The import statements under *node* that run: an ``if TYPE_CHECKING:`` block is only an + annotation for a type checker, so it is not walked.""" + found: list[ast.stmt] = [] + for child in ast.iter_child_nodes(node): + if isinstance(child, ast.If) and isinstance(child.test, ast.Name) \ + and child.test.id == "TYPE_CHECKING": + found.extend(_runtime_imports_of(child.orelse)) + elif isinstance(child, (ast.Import, ast.ImportFrom)): + found.append(child) + else: + found.extend(_runtime_imports(child)) + return found + + +def _runtime_imports_of(body: list[ast.stmt]) -> list[ast.stmt]: + found: list[ast.stmt] = [] + for stmt in body: + if isinstance(stmt, (ast.Import, ast.ImportFrom)): + found.append(stmt) + else: + found.extend(_runtime_imports(stmt)) + return found + + +def _imports(source: str) -> list[tuple[int, str | None, list[str]]]: + """``(level, module, imported names)`` for each import statement in *source* that runs.""" + out: list[tuple[int, str | None, list[str]]] = [] + for node in _runtime_imports(ast.parse(source)): + if isinstance(node, ast.ImportFrom): + out.append((node.level, node.module, [a.name for a in node.names])) + elif isinstance(node, ast.Import): + out.extend((0, a.name, []) for a in node.names) + return out + + +def _outside_the_engine(source: str, handed_over: set[str]) -> list[str]: + """What *source* imports that is neither the engine surface nor a helper handed over.""" + offenders: list[str] = [] + for level, module, names in _imports(source): + if level: + # A relative import is the handed-over helper: `..runtime.` from a generator, + # `.` or `from . import ` from a helper. + if module is None: + targets = names + elif module.startswith("runtime."): + targets = [module.split(".", 1)[1]] + elif module == "runtime": + targets = names + else: + targets = [module] + offenders.extend( + f"relative import of {t!r}, which eject does not hand over" + for t in targets if t not in handed_over + ) + continue + if module is None or not module.startswith("metaobjects."): + continue + # `from metaobjects.codegen import type_map` names a module through the package. + packages = ("metaobjects.codegen", "metaobjects.codegen.generators", + "metaobjects.codegen.runtime", "metaobjects.apidocs") + candidates = [f"{module}.{n}" for n in names] if module in packages else [module] + for c in candidates: + if c.startswith("metaobjects.codegen"): + if c not in ENGINE_CODEGEN: + offenders.append(c) + elif c.startswith("metaobjects.apidocs"): + if c not in ENGINE_APIDOCS: + offenders.append(c) + return offenders + + +@pytest.mark.parametrize("name", EJECTABLE) +def test_an_ejected_generator_imports_only_the_engine(name: str, tmp_path: Path) -> None: + entry = GENERATOR_REGISTRY[name] + assert owned.eject([name], tmp_path).code == 0 + handed_over = set(entry.runtime) + copied = [owned.owned_path(tmp_path, name)] + [ + owned.runtime_path(tmp_path, m) for m in entry.runtime + ] + offenders: list[str] = [] + for path in copied: + assert path.is_file(), f"eject did not write {path.relative_to(tmp_path)}" + offenders += [ + f"{path.relative_to(tmp_path)}: {o}" + for o in _outside_the_engine(path.read_text(encoding="utf-8"), handed_over) + ] + assert not offenders, "\n".join(offenders) + # Nothing else was written: the copy is the generator and its helpers. + written = {p.relative_to(tmp_path).as_posix() for p in (tmp_path / "codegen").rglob("*.py") + if p.name != "__init__.py"} + assert written == {p.relative_to(tmp_path).as_posix() for p in copied} + + +# --- the same project, run twice ------------------------------------------------------- + +_PROMPTS = { + "metadata.root": { + "package": "acme::prompts", + "children": [ + {"object.value": {"name": "ClassifyRequest", "children": [ + {"field.string": {"name": "text", "@required": True}}]}}, + {"object.value": {"name": "ClassifyResponse", "children": [ + {"field.string": {"name": "label", "@required": True}}, + {"field.int": {"name": "score"}}, + {"field.string": {"name": "tags", "isArray": True}}]}}, + {"template.prompt": { + "name": "ClassifyPrompt", "@payloadRef": "ClassifyRequest", + "@responseRef": "ClassifyResponse", "@textRef": "p/classify", "@format": "json"}}, + {"template.prompt": { + "name": "ClassifyXmlPrompt", "@payloadRef": "ClassifyRequest", + "@responseRef": "ClassifyResponse", "@textRef": "p/classify", "@format": "xml"}}, + {"object.entity": { + "name": "ClassifyCall", "extends": "metaobjects::ai::LlmCallBase", + "children": [ + {"source.rdb": {"@table": "classify_call", "@role": "primary"}}, + {"identity.primary": {"name": "id", "@fields": ["traceId"]}}, + {"template.prompt": { + "name": "ClassifyTraced", "@payloadRef": "ClassifyRequest", + "@responseRef": "ClassifyResponse", "@textRef": "p/classify", + "@format": "json"}}]}}, + ], + } +} + + +def _model_project(root: Path, generators: str, docs: str = "") -> Path: + """A project over every metadata shape the ejectable generators read, with one target + that runs *generators* (and, when given, a ``docs.generators`` block).""" + meta = root / "metaobjects" + meta.mkdir(parents=True) + canonical = FIXTURES / "persistence-conformance" / "canonical" / "meta.fitness.json" + (meta / canonical.name).write_text(canonical.read_text(encoding="utf-8"), encoding="utf-8") + render = FIXTURES / "template-output-render-conformance" + (meta / "meta.render.json").write_text((render / "meta.json").read_text(encoding="utf-8")) + shutil.copytree(render / "templates", root / "templates") + shutil.copytree(canonical.parent / "prompts", root / "templates", dirs_exist_ok=True) + (root / "templates" / "p").mkdir() + (root / "templates" / "p" / "classify.mustache").write_text("Classify {{text}}\n") + (meta / "meta.prompts.json").write_text(json.dumps(_PROMPTS), encoding="utf-8") + shop = FIXTURES / "requirement-test-identity-conformance" / "worked-example" / "input" + for doc in shop.iterdir(): + (meta / doc.name).write_text(doc.read_text(encoding="utf-8"), encoding="utf-8") + config = f"libraries: [ai]\ntargets:\n main:\n outDir: gen\n generators: [{generators}]\n" + if docs: + config += f"docs:\n generators: [{docs}]\n" + (root / "metaobjects.config.yaml").write_text(config, encoding="utf-8") + return root + + +#: Run in a fresh interpreter: load the CLI, then (for an owned run) make every packaged emit +#: module unimportable, run the command, and print the tree it wrote. +_SCRIPT = """ +import json, os, sys +from pathlib import Path +from metaobjects.cli import main +block = json.loads(sys.argv[3]) +for name in list(sys.modules): + if any(name == b or name.startswith(b + ".") for b in block): + sys.modules[name] = None +sys.path.insert(0, os.getcwd()) +rc = main(json.loads(sys.argv[2])) +out = Path(sys.argv[1]) +tree = {p.relative_to(out).as_posix(): p.read_text(encoding="utf-8") + for p in sorted(out.rglob("*")) if p.is_file() and "__pycache__" not in p.parts} +print(json.dumps({"rc": rc, "tree": tree})) +""" + + +def _run(root: Path, argv: list[str], out: str, block: list[str]) -> dict[str, str]: + src = str(Path(metaobjects.__file__).resolve().parents[1]) + env = {**os.environ, "PYTHONPATH": os.pathsep.join( + p for p in (src, os.environ.get("PYTHONPATH", "")) if p)} + proc = subprocess.run( + [sys.executable, "-c", _SCRIPT, str(root / out), json.dumps(argv), json.dumps(block)], + cwd=root, env=env, capture_output=True, text=True, + ) + assert proc.returncode == 0, proc.stderr + result: dict[str, Any] = json.loads(proc.stdout.splitlines()[-1]) + assert result["rc"] == 0, proc.stderr + return result["tree"] # type: ignore[no-any-return] + + +def _packaged_modules() -> list[str]: + """Every packaged module an owned copy must not reach: each ejectable generator's own and + the helper runtime beside it.""" + return sorted({"metaobjects.codegen.runtime"} + | {e.source_module for e in owned.ejectable() if e.source_module}) + + +def _lines(text: str) -> Counter[str]: + return Counter(ln for ln in text.splitlines() if ln.strip()) + + +@pytest.mark.parametrize("name", EJECTABLE) +def test_an_ejected_generator_writes_what_the_packaged_one_writes(name: str, tmp_path: Path) -> None: + entry = GENERATOR_REGISTRY[name] + docs = entry.docs_surface is not None + # The docs label defaults to the directory's name, which differs between the two projects. + argv = ["docs", "--out", "docs", "--project", "model"] if docs else ["gen"] + out = "docs" if docs else "gen" + + packaged_root = _model_project(tmp_path / "packaged", "" if docs else name) + packaged = _run(packaged_root, argv, out, block=[]) + assert packaged, f"{name} wrote nothing over the test model, so it proves nothing" + + owned_root = _model_project( + tmp_path / "owned", + "" if docs else owned.owned_token(entry), + owned.owned_token(entry) if docs else "", + ) + assert owned.eject([name], owned_root).code == 0 + mine = _run(owned_root, argv, out, block=_packaged_modules()) + + if entry.runtime and {"constraint_errors", "filter_parser"} <= set(entry.runtime): + # An owned router emits its runtime into the generated package and imports it there. + runtime_files = {k for k in mine if k.startswith("_runtime/")} + assert runtime_files and runtime_files == { + f"_runtime/{m}.py" for m in ("__init__", "constraint_errors", "filter_parser") + } + mine = {k: v for k, v in mine.items() if k not in runtime_files} + packaged = {k: v.replace("metaobjects.codegen.runtime.", "._runtime.") + for k, v in packaged.items()} + assert set(mine) == set(packaged) + # Import sorting moves the now-relative imports, so compare lines, not positions. + assert {k: _lines(v) for k, v in mine.items()} == {k: _lines(v) for k, v in packaged.items()} + else: + assert mine == packaged + + +# --- the census ------------------------------------------------------------------------- + + +def test_every_helper_module_is_handed_over_by_some_generator() -> None: + """A module in ``codegen/runtime/`` no registry entry lists is one eject would leave + packaged behind an owned generator.""" + on_disk = {p.stem for p in (CODEGEN / "runtime").glob("*.py") if p.stem != "__init__"} + listed = {m for e in GENERATOR_REGISTRY.values() for m in e.runtime} + assert on_disk == listed + + +def test_every_generator_module_is_a_registry_entry_or_a_model_walk() -> None: + """The census for Python: every module in ``codegen/generators/`` is a registered + generator or one of the two model walks the engine surface names.""" + registered = {Path(e.source_module.replace(".", "/")).name + for e in GENERATOR_REGISTRY.values() if e.source_module} + registered |= {"template_generator"} # the primitive: exempt, it has no emit logic of its own + walks = {Path(m.replace(".", "/")).name for m in ENGINE_CODEGEN + if m.startswith("metaobjects.codegen.generators.")} + # A module that moved into `runtime/` leaves a deprecated alias at its old path. + on_disk = {p.stem for p in (CODEGEN / "generators").glob("*.py") + if p.stem != "__init__" and "codegen._moved" not in p.read_text(encoding="utf-8")} + assert on_disk == registered | walks + + +def test_template_is_the_only_registered_generator_without_an_eject_source() -> None: + assert {e.name for e in GENERATOR_REGISTRY.values() if e.source is None} == {"template"} diff --git a/server/python/tests/codegen/test_eject_independence.py b/server/python/tests/codegen/test_eject_independence.py index 420d4f79b..de13c7121 100644 --- a/server/python/tests/codegen/test_eject_independence.py +++ b/server/python/tests/codegen/test_eject_independence.py @@ -71,9 +71,14 @@ def test_loading_one_ejectable_generator_loads_no_other_generators_module(tmp_pa assert not offenders, "\n".join(offenders) +#: The docs generator reads the router gates (`is_read_only_routed`, `emits_writable_router`) from +#: the same module, so the reference describes the routers an owned `routes` emits. +DOCS_READERS = {"api-docs"} + + def test_the_generators_that_share_a_predicate_hand_it_over() -> None: users = {n for n, e in GENERATOR_REGISTRY.items() if SHARED in e.runtime} - assert users == set(SHARED_USERS) + assert users == set(SHARED_USERS) | DOCS_READERS def test_each_owned_generator_ejected_alone_reads_the_owned_predicates(tmp_path, monkeypatch) -> None: diff --git a/server/python/tests/codegen/test_eject_runtime.py b/server/python/tests/codegen/test_eject_runtime.py index 5e98e47bd..9872687cf 100644 --- a/server/python/tests/codegen/test_eject_runtime.py +++ b/server/python/tests/codegen/test_eject_runtime.py @@ -85,10 +85,11 @@ def _block_installed_runtime(monkeypatch) -> None: def test_routes_entry_lists_the_runtime_its_output_imports_and_nothing_else_does() -> None: - """``shared_predicates`` is handed over through the same field but no generated code - imports it (``test_eject_independence.py`` covers it), so it is set aside here.""" + """The emitters and predicates a generator reads at ``gen`` time are handed over through + the same field, but no generated code imports them (``test_eject_independence.py`` and + ``test_eject_alone.py`` cover them), so only the two modules a router imports count.""" output_runtime = { - n: tuple(m for m in e.runtime if m != "shared_predicates") + n: tuple(m for m in e.runtime if m in RUNTIME_MODULES) for n, e in GENERATOR_REGISTRY.items() } assert output_runtime.pop("routes") == RUNTIME_MODULES @@ -151,7 +152,7 @@ def test_eject_never_overwrites_an_owned_runtime_copy(tmp_path, monkeypatch, cap def test_eject_of_a_generator_without_runtime_writes_no_runtime(tmp_path, monkeypatch) -> None: monkeypatch.chdir(tmp_path) - assert main(["eject", "names", "output-prompt", "trace-helper"]) == 0 + assert main(["eject", "names", "render-helper", "requirement-tests"]) == 0 assert not (tmp_path / "codegen" / "runtime").exists() diff --git a/server/python/tests/codegen/test_fr010_nested_object_prompt.py b/server/python/tests/codegen/test_fr010_nested_object_prompt.py index 9e446d5ee..52336423c 100644 --- a/server/python/tests/codegen/test_fr010_nested_object_prompt.py +++ b/server/python/tests/codegen/test_fr010_nested_object_prompt.py @@ -14,7 +14,7 @@ import ast import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes -from metaobjects.codegen import output_format_spec_emitter as ofs +from metaobjects.codegen.runtime import output_format_spec_emitter as ofs from metaobjects.meta.core.field import field_constants as fc from metaobjects.meta.core.field.meta_field import MetaField from metaobjects.meta.core.object.meta_object import MetaObject diff --git a/server/python/tests/codegen/test_fr010_output_codegen.py b/server/python/tests/codegen/test_fr010_output_codegen.py index eee02ea5a..448569de0 100644 --- a/server/python/tests/codegen/test_fr010_output_codegen.py +++ b/server/python/tests/codegen/test_fr010_output_codegen.py @@ -15,7 +15,7 @@ import sys import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes -from metaobjects.codegen import ( +from metaobjects.codegen.runtime import ( output_format_spec_emitter as ofs, extract_schema_emitter as rse, ) diff --git a/server/python/tests/codegen/test_fr019_shared_provided_conformance.py b/server/python/tests/codegen/test_fr019_shared_provided_conformance.py index f1f037d83..05ba7d5a8 100644 --- a/server/python/tests/codegen/test_fr019_shared_provided_conformance.py +++ b/server/python/tests/codegen/test_fr019_shared_provided_conformance.py @@ -27,7 +27,7 @@ from metaobjects.codegen.config import GenConfig from metaobjects.codegen.generator import GenContext from metaobjects.codegen.generators.entity_model import EntityModelGenerator, entity_model -from metaobjects.codegen.generators.fr019_shared_enum import ( +from metaobjects.codegen.runtime.fr019_shared_enum import ( is_provided, materialized_shared_enums, resolve_shared_enum_decl, diff --git a/server/python/tests/codegen/test_m2m_codegen.py b/server/python/tests/codegen/test_m2m_codegen.py index 408edb3f7..ace6ca006 100644 --- a/server/python/tests/codegen/test_m2m_codegen.py +++ b/server/python/tests/codegen/test_m2m_codegen.py @@ -28,7 +28,7 @@ EntityModelGenerator, render_entity_model, ) -from metaobjects.codegen.generators.m2m_codegen import ( +from metaobjects.codegen.runtime.m2m_codegen import ( M2mDescriptor, build_object_index, resolve_m2m_descriptors, diff --git a/server/python/tests/codegen/test_moved_modules.py b/server/python/tests/codegen/test_moved_modules.py new file mode 100644 index 000000000..10fe687c8 --- /dev/null +++ b/server/python/tests/codegen/test_moved_modules.py @@ -0,0 +1,55 @@ +"""The emitters that moved into ``metaobjects.codegen.runtime`` keep answering at their old paths. + +``metaobjects eject`` copies the emitters a generator writes through from +``metaobjects/codegen/runtime/``, so they moved there. A generator ejected before they moved +imports them from the old paths; each old path is a deprecated alias that IS the new module +(same object, so private names and monkeypatching behave as before) and warns once. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +import metaobjects + +MOVED = { + "metaobjects.codegen.extract_delegate_emitter": "metaobjects.codegen.runtime.extract_delegate_emitter", + "metaobjects.codegen.fr010_field_mapping": "metaobjects.codegen.runtime.fr010_field_mapping", + "metaobjects.codegen.output_format_spec_emitter": "metaobjects.codegen.runtime.output_format_spec_emitter", + "metaobjects.codegen.extract_schema_emitter": "metaobjects.codegen.runtime.extract_schema_emitter", + "metaobjects.codegen.generators.fr019_shared_enum": "metaobjects.codegen.runtime.fr019_shared_enum", + "metaobjects.codegen.generators.m2m_codegen": "metaobjects.codegen.runtime.m2m_codegen", + "metaobjects.apidocs.api_model": "metaobjects.codegen.runtime.api_docs_model", + "metaobjects.apidocs.builder": "metaobjects.codegen.runtime.api_docs_builder", + "metaobjects.apidocs.renderer": "metaobjects.codegen.runtime.api_docs_renderer", + "metaobjects.apidocs.paths": "metaobjects.codegen.runtime.api_docs_paths", +} + + +@pytest.mark.parametrize("old, new", sorted(MOVED.items())) +def test_an_old_path_is_the_new_module_and_warns(old: str, new: str, tmp_path: Path) -> None: + # A fresh interpreter: in this process the old path may already be imported, which warns once. + src = str(Path(metaobjects.__file__).resolve().parents[1]) + code = ( + "import importlib, json, warnings\n" + "with warnings.catch_warnings(record=True) as w:\n" + " warnings.simplefilter('always')\n" + f" old = importlib.import_module({old!r})\n" + f"new = importlib.import_module({new!r})\n" + "print(json.dumps({'same': old is new, " + "'warned': [str(x.message) for x in w if issubclass(x.category, DeprecationWarning)]}))\n" + ) + proc = subprocess.run( + [sys.executable, "-c", code], cwd=tmp_path, capture_output=True, text=True, + env={**os.environ, "PYTHONPATH": os.pathsep.join( + p for p in (src, os.environ.get("PYTHONPATH", "")) if p)}, + ) + assert proc.returncode == 0, proc.stderr + result = json.loads(proc.stdout) + assert result["same"] is True + assert len(result["warned"]) == 1 and old in result["warned"][0] and new in result["warned"][0] diff --git a/server/python/tests/codegen/test_reverse_finders.py b/server/python/tests/codegen/test_reverse_finders.py index e7b7ce39a..a6fb2e25d 100644 --- a/server/python/tests/codegen/test_reverse_finders.py +++ b/server/python/tests/codegen/test_reverse_finders.py @@ -32,7 +32,7 @@ reverse_finder_fn, reverse_finder_in_fn, ) -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import ( render_router, reverse_fks_for, diff --git a/server/python/tests/codegen/test_router_autoset.py b/server/python/tests/codegen/test_router_autoset.py index 69557480d..0e9c852ca 100644 --- a/server/python/tests/codegen/test_router_autoset.py +++ b/server/python/tests/codegen/test_router_autoset.py @@ -11,7 +11,7 @@ from pathlib import Path from metaobjects import MetaDataLoader -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.shared.base_types import TYPE_OBJECT diff --git a/server/python/tests/codegen/test_router_generator.py b/server/python/tests/codegen/test_router_generator.py index b50c93d99..53b6acfa1 100644 --- a/server/python/tests/codegen/test_router_generator.py +++ b/server/python/tests/codegen/test_router_generator.py @@ -12,7 +12,7 @@ import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes from metaobjects import MetaDataLoader -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.codegen.runtime.filter_parser import parse_filter from metaobjects.meta.core.field.meta_field import MetaField diff --git a/server/python/tests/codegen/test_router_generator_m2m_target_tph.py b/server/python/tests/codegen/test_router_generator_m2m_target_tph.py index 5416acc96..3fa29813f 100644 --- a/server/python/tests/codegen/test_router_generator_m2m_target_tph.py +++ b/server/python/tests/codegen/test_router_generator_m2m_target_tph.py @@ -30,7 +30,7 @@ import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes from metaobjects import MetaDataLoader -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.shared.base_types import TYPE_OBJECT diff --git a/server/python/tests/codegen/test_router_generator_tph_m2m.py b/server/python/tests/codegen/test_router_generator_tph_m2m.py index 2ffe574dc..f13af41f9 100644 --- a/server/python/tests/codegen/test_router_generator_tph_m2m.py +++ b/server/python/tests/codegen/test_router_generator_tph_m2m.py @@ -41,7 +41,7 @@ import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes from metaobjects import MetaDataLoader -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.shared.base_types import TYPE_OBJECT diff --git a/server/python/tests/codegen/test_router_generator_tph_m2m_shadowed.py b/server/python/tests/codegen/test_router_generator_tph_m2m_shadowed.py index 7196098cf..9bad9710b 100644 --- a/server/python/tests/codegen/test_router_generator_tph_m2m_shadowed.py +++ b/server/python/tests/codegen/test_router_generator_tph_m2m_shadowed.py @@ -30,7 +30,7 @@ import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes from metaobjects import MetaDataLoader -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.shared.base_types import TYPE_OBJECT diff --git a/server/python/tests/codegen/test_tph_codegen.py b/server/python/tests/codegen/test_tph_codegen.py index fa44a9892..9b405a9ac 100644 --- a/server/python/tests/codegen/test_tph_codegen.py +++ b/server/python/tests/codegen/test_tph_codegen.py @@ -19,7 +19,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.codegen.generators.tph_plan import ( is_tph_base, diff --git a/server/python/tests/integration/generated_m2m_app.py b/server/python/tests/integration/generated_m2m_app.py index c88baa038..d9359afbd 100644 --- a/server/python/tests/integration/generated_m2m_app.py +++ b/server/python/tests/integration/generated_m2m_app.py @@ -53,7 +53,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import ( +from metaobjects.codegen.runtime.m2m_codegen import ( M2mDescriptor, build_object_index, pk_field_name, diff --git a/server/python/tests/integration/generated_tph_app.py b/server/python/tests/integration/generated_tph_app.py index c0070a1a5..e87ddddcb 100644 --- a/server/python/tests/integration/generated_tph_app.py +++ b/server/python/tests/integration/generated_tph_app.py @@ -31,7 +31,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.codegen.generators.tph_plan import tph_plan_for from metaobjects.meta.core.object.meta_object import MetaObject diff --git a/server/python/tests/integration/test_m2m_target_tph_generated.py b/server/python/tests/integration/test_m2m_target_tph_generated.py index eb5bffaba..c4427c067 100644 --- a/server/python/tests/integration/test_m2m_target_tph_generated.py +++ b/server/python/tests/integration/test_m2m_target_tph_generated.py @@ -36,7 +36,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.shared.base_types import TYPE_OBJECT diff --git a/server/python/tests/integration/test_tph_m2m_generated.py b/server/python/tests/integration/test_tph_m2m_generated.py index 852d57b9f..3b28ffb84 100644 --- a/server/python/tests/integration/test_tph_m2m_generated.py +++ b/server/python/tests/integration/test_tph_m2m_generated.py @@ -35,7 +35,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.codegen.generators.tph_plan import tph_plan_for from metaobjects.meta.core.object.meta_object import MetaObject diff --git a/server/python/tests/integration/test_tph_m2m_shadowed_generated.py b/server/python/tests/integration/test_tph_m2m_shadowed_generated.py index d12d6d224..7427d4d9a 100644 --- a/server/python/tests/integration/test_tph_m2m_shadowed_generated.py +++ b/server/python/tests/integration/test_tph_m2m_shadowed_generated.py @@ -30,7 +30,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.codegen.generators.tph_plan import tph_plan_for from metaobjects.meta.core.object.meta_object import MetaObject diff --git a/server/python/tests/test_reporting_inert.py b/server/python/tests/test_reporting_inert.py index ab334df5c..f40eaea2b 100644 --- a/server/python/tests/test_reporting_inert.py +++ b/server/python/tests/test_reporting_inert.py @@ -26,9 +26,9 @@ import pytest from metaobjects import load_uris -from metaobjects.apidocs.builder import PythonApiModelBuilder -from metaobjects.apidocs.paths import Layout, doc_page_output_path -from metaobjects.apidocs.renderer import render_agent_api, render_index, render_unit_page +from metaobjects.codegen.runtime.api_docs_builder import PythonApiModelBuilder +from metaobjects.codegen.runtime.api_docs_paths import Layout, doc_page_output_path +from metaobjects.codegen.runtime.api_docs_renderer import render_agent_api, render_index, render_unit_page from metaobjects.codegen.config import GenConfig from metaobjects.codegen.generator import Generator from metaobjects.codegen.generator_registry import ( From 9bf13caec61c67f27880212dd2d8def79859a355 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Sun, 11 Oct 2026 21:28:19 -0400 Subject: [PATCH 2/8] feat(python): eject hands over the emitters a generator writes through; api-docs is ejectable An ejected Python generator used to be one file over packaged emitters, so a defect in the shared-enum, M:N, prompt-tier or api-reference writers still waited for a release. Eject now copies those emitters with the generator into codegen/runtime/, through each registry entry's existing runtime list, and the copy reads them from there. The emitters moved into metaobjects.codegen.runtime; their old import paths stay as deprecated aliases so a copy ejected earlier keeps running. Naming and type-mapping tables, the model walks and the generator base stay in the package as the engine. api-docs is registered for Python and ejectable. The api reference is a generator (docs_surface = "api") that `metaobjects docs` runs; an owned copy is listed under the new docs.generators config block. `gen` and `verify --codegen` skip a docs generator and say so. The page builder reads the router gates from shared_predicates (new emits_writable_router, moved is_read_only_routed), the module an owned routes and filter-allowlist read, so the reference describes an owned copy. test_eject_alone derives its list from the registry: each ejectable generator is ejected alone into an empty project, checked against a written engine surface, run with every packaged emit module unimportable, and compared byte for byte with the packaged generator. Packaged output is unchanged. --- CHANGELOG.md | 61 +++ docs/features/own-your-codegen.md | 115 +++- docs/ports/python.md | 10 + .../registry.json | 21 - .../src/metaobjects/apidocs/__init__.py | 34 +- .../src/metaobjects/apidocs/api_model.py | 113 +--- .../python/src/metaobjects/apidocs/builder.py | 510 +----------------- .../python/src/metaobjects/apidocs/naming.py | 2 +- .../python/src/metaobjects/apidocs/paths.py | 77 +-- .../src/metaobjects/apidocs/renderer.py | 187 +------ server/python/src/metaobjects/cli.py | 74 +-- .../python/src/metaobjects/codegen/_moved.py | 23 + .../src/metaobjects/codegen/docs_run.py | 91 ++++ .../python/src/metaobjects/codegen/eject.py | 28 +- .../codegen/extract_delegate_emitter.py | 385 +------------ .../codegen/extract_schema_emitter.py | 141 +---- .../codegen/fr010_field_mapping.py | 230 +------- .../src/metaobjects/codegen/generator.py | 22 + .../metaobjects/codegen/generator_registry.py | 51 +- .../codegen/generators/api_docs_generator.py | 68 +++ .../codegen/generators/entity_model.py | 7 +- .../codegen/generators/extractor_generator.py | 4 +- .../generators/filter_allowlist_generator.py | 2 +- .../codegen/generators/fr019_shared_enum.py | 156 +----- .../codegen/generators/m2m_codegen.py | 242 +-------- .../generators/output_parser_generator.py | 3 +- .../generators/output_prompt_generator.py | 3 +- .../codegen/generators/router_generator.py | 29 +- .../generators/trace_helper_generator.py | 3 +- .../codegen/metaobjects-config.schema.json | 12 + .../codegen/output_format_spec_emitter.py | 85 +-- .../src/metaobjects/codegen/project_config.py | 29 +- .../python/src/metaobjects/codegen/runner.py | 11 + .../metaobjects/codegen/runtime/__init__.py | 18 +- .../codegen/runtime/api_docs_builder.py | 488 +++++++++++++++++ .../codegen/runtime/api_docs_model.py | 111 ++++ .../codegen/runtime/api_docs_paths.py | 75 +++ .../codegen/runtime/api_docs_renderer.py | 185 +++++++ .../runtime/extract_delegate_emitter.py | 383 +++++++++++++ .../codegen/runtime/extract_schema_emitter.py | 139 +++++ .../codegen/runtime/fr010_field_mapping.py | 228 ++++++++ .../codegen/runtime/fr019_shared_enum.py | 154 ++++++ .../codegen/runtime/m2m_codegen.py | 240 +++++++++ .../runtime/output_format_spec_emitter.py | 83 +++ .../codegen/runtime/shared_predicates.py | 28 + .../tests/codegen/test_api_docs_paths.py | 2 +- .../python/tests/codegen/test_cli_registry.py | 4 +- .../tests/codegen/test_docs_generators.py | 205 +++++++ .../python/tests/codegen/test_eject_alone.py | 323 +++++++++++ .../tests/codegen/test_eject_independence.py | 7 +- .../tests/codegen/test_eject_runtime.py | 9 +- .../test_fr010_nested_object_prompt.py | 2 +- .../codegen/test_fr010_output_codegen.py | 2 +- .../test_fr019_shared_provided_conformance.py | 2 +- .../python/tests/codegen/test_m2m_codegen.py | 2 +- .../tests/codegen/test_moved_modules.py | 55 ++ .../tests/codegen/test_reverse_finders.py | 2 +- .../tests/codegen/test_router_autoset.py | 2 +- .../tests/codegen/test_router_generator.py | 2 +- .../test_router_generator_m2m_target_tph.py | 2 +- .../codegen/test_router_generator_tph_m2m.py | 2 +- .../test_router_generator_tph_m2m_shadowed.py | 2 +- .../python/tests/codegen/test_tph_codegen.py | 2 +- .../tests/conformance/test_eject_contract.py | 8 + .../tests/integration/generated_m2m_app.py | 2 +- .../tests/integration/generated_tph_app.py | 2 +- .../test_m2m_target_tph_generated.py | 2 +- .../integration/test_tph_m2m_generated.py | 2 +- .../test_tph_m2m_shadowed_generated.py | 2 +- server/python/tests/test_reporting_inert.py | 6 +- 70 files changed, 3336 insertions(+), 2276 deletions(-) create mode 100644 server/python/src/metaobjects/codegen/_moved.py create mode 100644 server/python/src/metaobjects/codegen/docs_run.py create mode 100644 server/python/src/metaobjects/codegen/generators/api_docs_generator.py create mode 100644 server/python/src/metaobjects/codegen/runtime/api_docs_builder.py create mode 100644 server/python/src/metaobjects/codegen/runtime/api_docs_model.py create mode 100644 server/python/src/metaobjects/codegen/runtime/api_docs_paths.py create mode 100644 server/python/src/metaobjects/codegen/runtime/api_docs_renderer.py create mode 100644 server/python/src/metaobjects/codegen/runtime/extract_delegate_emitter.py create mode 100644 server/python/src/metaobjects/codegen/runtime/extract_schema_emitter.py create mode 100644 server/python/src/metaobjects/codegen/runtime/fr010_field_mapping.py create mode 100644 server/python/src/metaobjects/codegen/runtime/fr019_shared_enum.py create mode 100644 server/python/src/metaobjects/codegen/runtime/m2m_codegen.py create mode 100644 server/python/src/metaobjects/codegen/runtime/output_format_spec_emitter.py create mode 100644 server/python/tests/codegen/test_docs_generators.py create mode 100644 server/python/tests/codegen/test_eject_alone.py create mode 100644 server/python/tests/codegen/test_moved_modules.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 863f9300e..c9a76eea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -217,6 +217,57 @@ generator catalog and the Atlas evaluation are 1.2 work. calls one. The cross-port manifest lists `api-docs` for `csharp`. +- **Python: an ejected generator owns the emitters it writes through, and `api-docs` can be + ejected.** `metaobjects eject` used to copy one file per generator, so an owned `entity` + still called the shared-enum and M:N writers, and an owned `output-parser`, `extractor`, + `output-prompt` or `trace-helper` still called the prompt tier's field mapping, extract + delegate, extract schema and format-spec writers. A defect in any of them waited for a + release. Eject now copies those emitters with the generator, into `codegen/runtime/`, and the + copy reads them from there: `fr019_shared_enum` and `m2m_codegen` with `entity`; + `m2m_codegen` with `filter-allowlist`; `fr010_field_mapping` and `m2m_codegen` with `routes`; + `extract_delegate_emitter` and `fr010_field_mapping` with `output-parser` and `extractor`; + `output_format_spec_emitter` and `fr010_field_mapping` with `output-prompt`; + `extract_schema_emitter` and `fr010_field_mapping` with `trace-helper`. A file two generators + share is copied once and never overwritten, and `metaobjects gen --list` marks each copy + `identical` or `DIFFERS`. What stays in the package is the engine: the generator base and + file writer, `metaobjects.apidocs.naming`, `type_map` and `value_objects` (the naming and + type-mapping tables), and the model walks (`find_inbound`, `tph_plan`, + `instance_artifacts`). `template` is not ejectable: it has no emit logic of its own. Packaged + output is unchanged and gated to be. + - **`api-docs` is a generator you can own.** `metaobjects eject api-docs` copies the page + generator with its page model, builder, renderer and path math. The api reference is now a + generator (`docs_surface = "api"`) that `metaobjects docs` runs: an owned copy is listed + under the new `docs.generators` block of `metaobjects.config.yaml` and runs in place of the + packaged one. `gen` and `verify --codegen` skip a docs generator they find under a target + and say so. The reference asks `shared_predicates` which objects get a router (the new + `emits_writable_router` and the moved `is_read_only_routed`, beside `emits_router`), the + module your owned `routes` and `filter-allowlist` read, so one edit to the owned predicate + moves the router, the allowlist and the pages together. The cross-port manifest lists + `api-docs` for `python`. + - **Where things moved.** The emitters now live in `metaobjects.codegen.runtime` (a + module the helper-runtime copy already used): `extract_delegate_emitter`, + `fr010_field_mapping`, `output_format_spec_emitter`, `extract_schema_emitter`, + `fr019_shared_enum`, `m2m_codegen`, and the api reference as `api_docs_model`, + `api_docs_builder`, `api_docs_renderer` and `api_docs_paths`. `metaobjects.apidocs` keeps + `naming` and exports `PythonApiModelBuilder`, `PythonApiDocsRenderer` and the path + functions as before. New public API: `DocsRunFacts`, `GenContext.docs`, `DEFAULT_API_SUBDIR` + and `DOCS_SURFACE_API` in `metaobjects.codegen.generator`, `ProjectConfig.docs_generators`, + `GeneratorEntry.docs_surface`, and `metaobjects.codegen.docs_run`. + - **A copy you ejected before this release** keeps running: it imports the emitters from + their old paths, which are deprecated aliases (see **Deprecated**), so it calls the + packaged emitters as it always did. `metaobjects gen --list` marks it `DIFFERS: N behind`. + To own them, eject the same name into a scratch directory and copy the generator's + `runtime/` files over (see + [own-your-codegen](docs/features/own-your-codegen.md#the-emitters-a-generator-writes-through-come-with-it)). + - **A gate for the rule.** For every ejectable generator the registry lists, a test ejects it + alone into an empty project, fails on any import outside a written engine surface, runs it + with every packaged emit module made unimportable, and compares its output byte for byte + with the packaged generator's over a model that exercises each one (the router's own + runtime imports are the one documented difference). Another fails on a helper module no + registry entry hands over, or a generator module no registry entry names. + The nine Python entries of the shared eject contract's `eject.pending` list are closed, and the + contract test runs `api-docs` through `metaobjects docs`, the way it is run. + - **The stock `entity`, `queries`, `routes` and `barrel` generators are importable again (TypeScript).** `entityFile`, `queriesFile`, `routesFile` and `barrel` are exported from `@metaobjectsdev/codegen-ts/generators` once more, beside `namesFile`, `routesFileHono` @@ -871,6 +922,16 @@ until you regenerate. - **C#: `Fr010FieldMapping.CSharpStringLiteral(string)` is `[Obsolete]`.** It forwards to `CSharpNaming.CSharpStringLiteral(string)`, which every generator now calls (so an ejected `names` needs no emitter). Same signature and result; switch the call when you next touch it. +- **Python: the old import paths of the emitters that moved into `metaobjects.codegen.runtime`.** + `metaobjects.codegen.extract_delegate_emitter`, `.fr010_field_mapping`, + `.output_format_spec_emitter`, `.extract_schema_emitter`, `metaobjects.codegen.generators.fr019_shared_enum`, + `.m2m_codegen`, and `metaobjects.apidocs.api_model`, `.builder`, `.renderer`, `.paths` are + each one-line modules that become the new module and warn (`DeprecationWarning`) on import. + Same objects, so nothing else changes; import from `metaobjects.codegen.runtime` (the api + reference modules are `api_docs_model`, `api_docs_builder`, `api_docs_renderer` and + `api_docs_paths` there). The old paths will be removed in a later release. + `metaobjects.codegen.generators.router_generator.is_read_only_routed` moved to + `metaobjects.codegen.runtime.shared_predicates`; the router module still re-exports it. ### Removed diff --git a/docs/features/own-your-codegen.md b/docs/features/own-your-codegen.md index efae4176c..3659a6f06 100644 --- a/docs/features/own-your-codegen.md +++ b/docs/features/own-your-codegen.md @@ -695,9 +695,20 @@ targets: Eject prints the exact entry for each copy. It never overwrites an existing copy without `--force` and never edits your config. `metaobjects gen --list` marks each owned copy `identical` or `DIFFERS: N behind, M of your own` against the packaged reference, so you -can see when an upgrade changed the generator you copied. A copy imports the same -`metaobjects.codegen.*` modules the packaged one does, and those module paths are the -surface an owned generator builds on. +can see when an upgrade changed the generator you copied. Every generator `metaobjects +gen --list` shows is ejectable except `template`, the Mustache primitive, which has no emit +logic of its own to own. + +An ejected generator owns every line that writes its output. Eject copies the generator +with the emitters it writes through (below), and the copy imports only the engine from the +package: the generator base and file writer (`metaobjects.codegen.generator`), the +formatter, the naming and type-mapping tables (`metaobjects.apidocs.naming`, +`metaobjects.codegen.type_map`, `metaobjects.codegen.value_objects`), the model walks +(`find_inbound`, `tph_plan`, `instance_artifacts`) and the loader. Those module paths are +the surface an owned generator builds on. `tests/codegen/test_eject_alone.py` ejects each +generator alone into an empty project, rejects any import outside that surface, runs it with +the packaged emitters made unimportable, and compares its output byte for byte with the +packaged generator's. `requirement-tests` ejects the same way, to `codegen/generators/requirement_tests.py`, wired as `codegen.generators.requirement_tests:requirement_tests_generator`. The copy holds @@ -764,34 +775,98 @@ codegen/runtime/`). The same recipe covers the generator itself: `metaobjects ej routes` into a scratch directory gives you the new reference to diff your `codegen/generators/routes.py` against. -### The rules two generators share come with them +### The emitters a generator writes through come with it -Some emit rules are asked by more than one generator, and the answers have to match. -`routes` and `filter-allowlist` both ask `emits_router` (which objects get a router; the -generated router imports the allowlist module, so a mismatch is an import error at -startup). `entity` and `extractor` both ask `has_literal_default`. These live in one -module, `shared_predicates`, and ejecting any of those four generators copies it too: +A generator that calls a packaged emitter is a shell around it: a defect in the emitter +still waits for a release. So eject copies the emitters too, into `codegen/runtime/`, next +to the generator. The set is each registry entry's `runtime` list: + +| Ejecting | also copies | +|---|---| +| `entity` | `fr019_shared_enum`, `m2m_codegen`, `shared_predicates` | +| `routes` | `fr010_field_mapping`, `m2m_codegen`, `shared_predicates`, and the router's own runtime (above) | +| `filter-allowlist` | `m2m_codegen`, `shared_predicates` | +| `output-parser` | `extract_delegate_emitter`, `fr010_field_mapping` | +| `extractor` | `extract_delegate_emitter`, `fr010_field_mapping`, `shared_predicates` | +| `output-prompt` | `output_format_spec_emitter`, `fr010_field_mapping` | +| `trace-helper` | `extract_schema_emitter`, `fr010_field_mapping` | +| `api-docs` | `api_docs_model`, `api_docs_builder`, `api_docs_renderer`, `api_docs_paths`, `m2m_codegen`, `shared_predicates` | + +`names`, `render-helper` and `requirement-tests` call no packaged emitter. A file two +generators share is copied once and an existing copy is never overwritten, even with +`--force`; `metaobjects gen --list` marks each copy `identical` or `DIFFERS: N behind, M of +your own`. ```text codegen/ - generators/routes.py - generators/filter_allowlist.py - runtime/shared_predicates.py # the shared rules, now your code + generators/output_parser.py + generators/extractor.py + runtime/extract_delegate_emitter.py # the one copy both read + runtime/fr010_field_mapping.py ``` -Each generator imports it relatively (`from ..runtime.shared_predicates import ...`), so -the verbatim copy in `codegen/generators/` reads `codegen/runtime/shared_predicates.py` -and never the installed module. Two owned generators read the one copy: change -`emits_router` there and your `routes` and your `filter-allowlist` change together. No -generated code imports this module. It runs at `gen` time only, so nothing is emitted into -`_runtime/` for it. The rest works as for the runtime above: an existing copy is kept, and -`--list` marks it `identical` or `DIFFERS`. +Each generator imports its helpers relatively (`from ..runtime import +extract_delegate_emitter as rde`), so the verbatim copy in `codegen/generators/` reads +`codegen/runtime/` and never the installed package. Two owned generators read the one copy: +change it and both change. No generated code imports these modules; they run at `gen` time +only, so nothing is emitted into `_runtime/` for them. + +**The rules two generators share** live in the same place. `routes` and `filter-allowlist` +both ask `emits_router` (which objects get a router; the generated router imports the +allowlist module, so a mismatch is an import error at startup). `entity` and `extractor` +both ask `has_literal_default`. They are in `shared_predicates`, which each of those +generators lists, so editing `emits_router` there moves your `routes` and your +`filter-allowlist` together. -A packaged generator keeps reading the installed rules. If you own `routes` and change +A packaged generator keeps reading the installed copy. If you own `routes` and change `emits_router` but still run the packaged `filter-allowlist`, the two disagree. Eject `filter-allowlist` too before you change that rule; the same goes for `entity` and `extractor` with `has_literal_default`. +**What stays in the package.** The engine: the generator base, writer and formatter, the +naming and type-mapping tables, the model walks and the loader. A table that decides a name +or a type is a question about the model, and the generators, the docs and the runtime must +agree on it, so it stays one packaged, public copy. + +**A copy you ejected before this release** keeps running. It imports the emitters from the +paths they had (`metaobjects.codegen.fr010_field_mapping`, +`metaobjects.codegen.generators.m2m_codegen` and the others in the CHANGELOG), and each of +those is now a deprecated alias of the module in `metaobjects.codegen.runtime`, so your copy +calls the packaged emitter exactly as before and a `DeprecationWarning` names the new path. +`metaobjects gen --list` marks it `DIFFERS: N behind`. To own the emitters, eject the same name +into a scratch directory (`metaobjects eject --root `), copy the `runtime/` +files over into `codegen/runtime/`, and change your imports to the relative form the new copy +uses (`from ..runtime import fr010_field_mapping as fm`). + +### `api-docs`: the api reference as an owned generator + +`metaobjects docs` writes the Python api reference (`api/python`). `metaobjects eject +api-docs` copies the generator that writes it, with the page model (`api_docs_model.py`), the +builder that decides what to document (`api_docs_builder.py`), the renderer +(`api_docs_renderer.py`) and the page-path math (`api_docs_paths.py`). Change which nodes +get a page, how a page reads, or where it lands, by editing them. + +A docs generator is not a code generator, so it is wired in its own block, never in a +target's `generators`: + +```yaml +docs: + generators: [codegen.generators.api_docs:api_docs_generator] +``` + +`metaobjects docs` runs the copy in place of the packaged generator. Without the block it +runs the packaged one, and the positional `metaobjects docs ` form reads no +config, so it always does. An entry that is not a docs generator, or a second one for the +`api` surface, is refused by name. `gen` and `verify --codegen` skip a docs generator +they find under a target and say so (`'api-docs' writes the api reference, so gen skipped +it.`). + +**It describes the code your generators emit.** The page builder asks `shared_predicates` +which objects get a router, the same module your owned `routes` and `filter-allowlist` +read. Eject them together and edit the predicate once: the router, the allowlist and the +reference change together. The naming seam (`metaobjects.apidocs.naming`) is the engine, +the one set of names every generator and every page uses. + ## Java and Kotlin: `mvn metaobjects:eject` ```bash diff --git a/docs/ports/python.md b/docs/ports/python.md index f55000bb8..630a5be7a 100644 --- a/docs/ports/python.md +++ b/docs/ports/python.md @@ -660,6 +660,16 @@ standard-library module name (`types`, `json`) is refused there with an error na rename it. See [Own your codegen → Python](../features/own-your-codegen.md#python-metaobjects-eject). +Every generator `metaobjects gen --list` shows is ejectable except `template`, the Mustache +primitive, which has no emit logic of its own to own. An ejected generator comes with the +emitters it writes its output through (the shared-enum and M:N writers, the prompt tier's +field mapping, extract delegate, extract schema and format-spec writers), copied into +`codegen/runtime/` and read from there, so the copy owns the code that writes the file and +not a shell around the package's. The naming and type-mapping tables, the model walks and +the generator base stay in the package as the engine. `metaobjects eject api-docs` does the +same for the api reference `metaobjects docs` writes: the copy is wired under +`docs.generators` in `metaobjects.config.yaml`, never in a target's `generators`. + Ejecting `routes` also hands over the helper runtime the generated routers call: `filter_parser` and `constraint_errors` are copied into `codegen/runtime/`, and the owned generator emits them into the generated package as `_runtime/` and imports them from diff --git a/fixtures/generator-registry-conformance/registry.json b/fixtures/generator-registry-conformance/registry.json index ab2cd2306..d5e49a450 100644 --- a/fixtures/generator-registry-conformance/registry.json +++ b/fixtures/generator-registry-conformance/registry.json @@ -452,27 +452,6 @@ "pending": [ {"generator": "api-docs", "port": "java", "assertion": "census", "gap": 6}, {"generator": "api-docs", "port": "kotlin", "assertion": "census", "gap": 6}, - {"generator": "api-docs", "port": "python", "assertion": "census", "gap": 6}, - { - "generator": "entity", - "port": "python", - "assertion": "depth", - "gap": 5, - "outside": ["metaobjects.codegen.generators.fr019_shared_enum", "metaobjects.codegen.generators.m2m_codegen"] - }, - { - "generator": "extractor", - "port": "python", - "assertion": "depth", - "gap": 4, - "outside": ["metaobjects.codegen.extract_delegate_emitter", "metaobjects.codegen.fr010_field_mapping"] - }, - {"generator": "filter-allowlist", "port": "python", "assertion": "depth", "gap": 5, "outside": ["metaobjects.codegen.generators.m2m_codegen"]}, - {"generator": "output-parser", "port": "python", "assertion": "depth", "gap": 4, "outside": ["metaobjects.codegen.extract_delegate_emitter"]}, - {"generator": "output-prompt", "port": "python", "assertion": "depth", "gap": 4, "outside": ["metaobjects.codegen.output_format_spec_emitter"]}, - {"generator": "routes", "port": "python", "assertion": "depth", "gap": 4, "outside": ["metaobjects.codegen.fr010_field_mapping"]}, - {"generator": "routes", "port": "python", "assertion": "depth", "gap": 5, "outside": ["metaobjects.codegen.generators.m2m_codegen"]}, - {"generator": "trace-helper", "port": "python", "assertion": "depth", "gap": 4, "outside": ["metaobjects.codegen.extract_schema_emitter"]}, {"generator": "ai-docs", "port": "java", "assertion": "completeness", "gap": 7}, {"generator": "html-docs", "port": "java", "assertion": "completeness", "gap": 7}, {"generator": "json-schema", "port": "java", "assertion": "completeness", "gap": 7}, diff --git a/server/python/src/metaobjects/apidocs/__init__.py b/server/python/src/metaobjects/apidocs/__init__.py index 37e06b83e..52a849e6a 100644 --- a/server/python/src/metaobjects/apidocs/__init__.py +++ b/server/python/src/metaobjects/apidocs/__init__.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any # Path math is leaf-level (no generator imports) — safe to expose eagerly. -from metaobjects.apidocs.paths import ( +from metaobjects.codegen.runtime.api_docs_paths import ( Layout, doc_page_output_path, model_cross_href, @@ -28,7 +28,7 @@ ) if TYPE_CHECKING: # pragma: no cover - import-time only for type checkers - from metaobjects.apidocs.api_model import ( + from metaobjects.codegen.runtime.api_docs_model import ( ApiModel, ApiSymbol, ApiSymbolKind, @@ -36,8 +36,8 @@ FieldShape, UnitExample, ) - from metaobjects.apidocs.builder import PythonApiModelBuilder, build_api_model - from metaobjects.apidocs.renderer import ( + from metaobjects.codegen.runtime.api_docs_builder import PythonApiModelBuilder, build_api_model + from metaobjects.codegen.runtime.api_docs_renderer import ( PythonApiDocsRenderer, render_agent_api, render_index, @@ -46,18 +46,18 @@ # Lazily-resolved exports → their defining submodule. _LAZY: dict[str, str] = { - "ApiModel": "api_model", - "ApiSymbol": "api_model", - "ApiSymbolKind": "api_model", - "ApiUnit": "api_model", - "FieldShape": "api_model", - "UnitExample": "api_model", - "PythonApiModelBuilder": "builder", - "build_api_model": "builder", - "PythonApiDocsRenderer": "renderer", - "render_unit_page": "renderer", - "render_index": "renderer", - "render_agent_api": "renderer", + "ApiModel": "api_docs_model", + "ApiSymbol": "api_docs_model", + "ApiSymbolKind": "api_docs_model", + "ApiUnit": "api_docs_model", + "FieldShape": "api_docs_model", + "UnitExample": "api_docs_model", + "PythonApiModelBuilder": "api_docs_builder", + "build_api_model": "api_docs_builder", + "PythonApiDocsRenderer": "api_docs_renderer", + "render_unit_page": "api_docs_renderer", + "render_index": "api_docs_renderer", + "render_agent_api": "api_docs_renderer", } __all__ = [ @@ -87,5 +87,5 @@ def __getattr__(name: str) -> Any: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") import importlib - mod = importlib.import_module(f"{__name__}.{submodule}") + mod = importlib.import_module(f"metaobjects.codegen.runtime.{submodule}") return getattr(mod, name) diff --git a/server/python/src/metaobjects/apidocs/api_model.py b/server/python/src/metaobjects/apidocs/api_model.py index 12766ddef..26b60c834 100644 --- a/server/python/src/metaobjects/apidocs/api_model.py +++ b/server/python/src/metaobjects/apidocs/api_model.py @@ -1,111 +1,4 @@ -"""The Python native SDK api-surface IR (intermediate representation). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.api_docs_model`.""" +from metaobjects.codegen._moved import alias -Mirrors the Java ``JavaApiModel`` IR (server/java/.../apidocs), the C# -``CSharpApiModel``, and the TS ``api-model.ts``, idiomatic to the Python -(Pydantic / FastAPI / ObjectManager) generated surface: Pydantic model / -``ObjectManager`` data-access / FastAPI routes / validation / extractor / -render helper / payload model / output-format prompt / output parser / filter -allowlist. - -NAMES on every symbol come from the :mod:`metaobjects.apidocs.naming` seam (never -re-concatenated in the builder), so what this model documents == what the real -generators emit. -""" -from __future__ import annotations - -from dataclasses import dataclass, field -from enum import Enum - - -class ApiSymbolKind(str, Enum): - """The generated-category axis a documented symbol belongs to (Python / - Pydantic + FastAPI flavored).""" - - #: The Pydantic v2 model class (entity) or value-object model. - MODEL = "model" - #: Data access — the consumer-implemented repository ``Protocol`` seam. - DATA_ACCESS = "data_access" - #: A FastAPI route (named ``"VERB path"``). - REST = "rest" - #: Pydantic field validation on the create/update shape. - VALIDATION = "validation" - #: The tolerant + strict extractor for a template payload. - EXTRACTOR = "extractor" - #: The typed render helper wrapping the render engine. - RENDER = "render" - #: The typed payload model bound to a template. - PAYLOAD = "payload" - #: The output-format prompt fragment. - PROMPT = "prompt" - #: The output parser (``parse_*``) back into the typed payload. - OUTPUT_PARSER = "output_parser" - #: The per-entity sort/filter allowlist. - FILTER = "filter" - - -@dataclass(frozen=True) -class FieldShape: - """A documented field: name + Python type + optionality + an optional note - (e.g. enum values).""" - - name: str - type: str - optional: bool - note: str | None = None - - -@dataclass(frozen=True) -class UnitExample: - """A worked example for a unit (reserved for later phases; carried for - cross-port IR parity).""" - - title: str - code: str - - -@dataclass(frozen=True) -class ApiSymbol: - """One documented symbol of the generated Python SDK surface. - - :param name: the emitted Python identifier (or ``"VERB path"`` for a REST route). - :param kind: which generated category this symbol belongs to. - :param module: the import line / module path the symbol lives in. - :param signature: a human-readable Python signature line. - :param usage: a one-line "what you use this for". - :param returns: the symbol's return surface, or ``None``. - :param fields: per-field shapes for model/validation/payload symbols (may be empty). - """ - - name: str - kind: ApiSymbolKind - module: str - signature: str - usage: str - returns: str | None = None - fields: list[FieldShape] = field(default_factory=list) - - -@dataclass(frozen=True) -class ApiUnit: - """One documented unit (an entity / value object, or a template) + its symbols. - - :param node: the unit's short name (the doc-page basename). - :param package: the unit's metadata package (e.g. ``acme::shop``). - :param kind: ``"entity"`` | ``"value"`` | ``"report"`` | ``"template"``. - :param symbols: the documented symbols, in canonical IR order. - :param example: an optional unit-level worked example (reserved). - """ - - node: str - package: str - kind: str - symbols: list[ApiSymbol] - example: UnitExample | None = None - - -@dataclass(frozen=True) -class ApiModel: - """The full per-project Python SDK api surface IR.""" - - project: str - units: list[ApiUnit] +alias(__name__, "metaobjects.codegen.runtime.api_docs_model") diff --git a/server/python/src/metaobjects/apidocs/builder.py b/server/python/src/metaobjects/apidocs/builder.py index 47e1b52a7..40fd35048 100644 --- a/server/python/src/metaobjects/apidocs/builder.py +++ b/server/python/src/metaobjects/apidocs/builder.py @@ -1,508 +1,4 @@ -"""Builds the Python SDK api-surface IR (:class:`ApiModel`) from a loaded MetaRoot. +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.api_docs_builder`.""" +from metaobjects.codegen._moved import alias -This is the ACCURATE-BY-CONSTRUCTION half of the api-docs pipeline: every -documented symbol name comes from the :mod:`metaobjects.apidocs.naming` seam (never -re-concatenated here — the SAME seam the real generators delegate to), and every -inclusion decision comes from an applies-predicate that REUSES the corresponding -generator's own gate helpers (``emits_instance_artifacts`` / ``primary_rdb_source`` -+ source kind / ``resolve_payload_vo`` / format check / ``is_tph_subtype``) rather -than re-implementing the gate. The result is that what this builder documents == -what the generators emit, by SHARING the single source of truth. - -Per-category enumeration (mirrors the Java ``JavaApiModelBuilder`` / C# -``CSharpApiModelBuilder``, Python/Pydantic-flavored): - -* Objects (``root.own_children()`` filtered to ``object.*``): each concrete object - → a MODEL symbol. A writable-table entity adds DATA_ACCESS (the repository - ``Protocol``) / REST (one per FastAPI route) / VALIDATION (Pydantic field - constraints on the create/update shape) / FILTER (the allowlist). A value object - → MODEL only. An abstract object / a TPH subtype yields no instance artifacts. -* Templates (EVERY subtype — ADR-0052): a resolvable ``@payloadRef`` gives the - PAYLOAD record it renders outbound, plus RENDER for a ``template.output`` (the - only subtype the render-helper generator emits for). A ``template.prompt`` - declaring ``@responseRef`` adds the INBOUND tier — the response PAYLOAD record, - PROMPT (the response-format fragment) and EXTRACTOR, plus OUTPUT_PARSER unless - the reply is XML, which gets no strict tier (ADR-0053). - -An object that yields no symbols (e.g. an abstract object) produces NO unit at all -(never an empty unit). -""" -from __future__ import annotations - -from metaobjects.apidocs import naming -from metaobjects.apidocs.api_model import ( - ApiModel, - ApiSymbol, - ApiSymbolKind, - ApiUnit, - FieldShape, -) -from metaobjects.codegen import type_map -from metaobjects.codegen.generators.m2m_codegen import ( - build_object_index, - resolve_m2m_descriptors, -) -from metaobjects.codegen.generators.find_inbound import is_xml, response_shape -from metaobjects.codegen.value_objects import is_field_required, pkg_of, resolve_payload_vo -from metaobjects.codegen.generators.tph_plan import is_tph_subtype -from metaobjects.codegen.generators.router_generator import is_read_only_routed -from metaobjects.codegen.instance_artifacts import ( - emits_instance_artifacts, - has_item_route, - is_abstract, - is_served_report, -) -from metaobjects.source_resolution import primary_rdb_source -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.core.field.meta_field import MetaField -from metaobjects.meta.core.object.meta_object import MetaObject -from metaobjects.meta.core.object.object_constants import ( - OBJECT_SUBTYPE_ENTITY, - OBJECT_SUBTYPE_REPORT, -) -from metaobjects.meta.core.reporting.report_read_model import report_read_model -from metaobjects.meta.meta_data import MetaData -from metaobjects.meta.persistence.source.source_constants import SOURCE_KIND_TABLE -from metaobjects.meta.template import template_constants as tc -from metaobjects.shared.base_types import TYPE_OBJECT, TYPE_TEMPLATE - - -# --------------------------------------------------------------------------- -# Applies-predicates — each REUSES the matching generator's own gate helpers, so -# inclusion can never drift from emission. (The Python generators gate inline; -# these mirror that gate exactly via the shared helpers.) -# --------------------------------------------------------------------------- - - -def _is_writable_table_entity(obj: MetaObject, object_index: dict[str, MetaObject]) -> bool: - """The shared gate for the router + filter-allowlist generators: a non-abstract - entity with a ``source.rdb`` ``@kind="table"`` child that is NOT a TPH subtype - (a TPH subtype is folded into its base's single table — it emits no standalone - router/allowlist).""" - if obj.sub_type != OBJECT_SUBTYPE_ENTITY: - return False - if not emits_instance_artifacts(obj): - return False - if is_tph_subtype(obj): - return False - src = primary_rdb_source(obj) - if src is None: - return False - return src.effective_kind() == SOURCE_KIND_TABLE - - -def _payload_resolves(tmpl: MetaData, root: MetaData) -> MetaObject | None: - """The payload VO a template resolves to (``@payloadRef`` → ``object.value``), - or ``None`` — the shared gate for payload / render / prompt / parser / extractor.""" - payload_ref = tmpl.get_meta_attr(tc.TEMPLATE_ATTR_PAYLOAD_REF) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) - if not isinstance(payload_ref, str) or not payload_ref: - return None - # ADR-0042 (#228): the referrer is THIS template — a bare @payloadRef resolves - # in ITS OWN package first. - return resolve_payload_vo(root, payload_ref, pkg_of(tmpl)) - - -def _is_email_kind(tmpl: MetaData) -> bool: - kind = tmpl.get_meta_attr(tc.TEMPLATE_ATTR_KIND) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) - return isinstance(kind, str) and kind.lower() == tc.TEMPLATE_KIND_EMAIL - - -# --------------------------------------------------------------------------- -# Builder. -# --------------------------------------------------------------------------- - - -class PythonApiModelBuilder: - """Builds the :class:`ApiModel` SDK-surface IR from a loaded ``MetaRoot``.""" - - def build(self, root: MetaData, project: str) -> ApiModel: - objects = [ - # ADR-0039 sanctioned own: top-level scan on the loader ROOT (never extended, own == effective) - c - for c in root.own_children() - if c.type == TYPE_OBJECT and isinstance(c, MetaObject) - ] - object_index = build_object_index(objects) - - units: list[ApiUnit] = [] - for obj in objects: - # FR-044: a served report (Table A) has a generated read API, documented from - # its read model (its derived fields); every other report has no output. - if obj.sub_type == OBJECT_SUBTYPE_REPORT: - unit = ( - self._build_report_unit(obj, root) if is_served_report(obj) else None - ) - if unit is not None: - units.append(unit) - continue - unit = self._build_object_unit(obj, root, object_index) - if unit is not None: - units.append(unit) - - # Templates: ADR-0052 — EVERY template subtype gets a unit, not just - # template.output. A template.prompt carries a @payloadRef record and, when it - # declares @responseRef, the whole inbound tier — all of it generated, so all - # of it documentable. - # ADR-0039 sanctioned own: top-level scan on the loader ROOT (never extended, own == effective) - for tmpl in root.own_children(): - if tmpl.type == TYPE_TEMPLATE: - units.append(self._build_template_unit(tmpl, root)) - - return ApiModel(project, units) - - # ----- objects ----------------------------------------------------------- - - def _build_object_unit( - self, obj: MetaObject, root: MetaData, object_index: dict[str, MetaObject] - ) -> ApiUnit | None: - entity = obj.sub_type == OBJECT_SUBTYPE_ENTITY - module = naming.snake_case(obj.name) - unit_kind = "entity" if entity else "value" - symbols: list[ApiSymbol] = [] - - # MODEL — only for concrete objects (an abstract object cannot be - # instantiated → documented ⊆ generated). A value object → MODEL only. - if not is_abstract(obj): - model = naming.model_class_name(obj) - symbols.append( - ApiSymbol( - name=model, - kind=ApiSymbolKind.MODEL, - module=naming.model_import(obj), - signature=f"class {model}(BaseModel)", - usage=( - "the Pydantic v2 entity model" - if entity - else "the Pydantic v2 value-object model" - ), - ) - ) - - if entity and _is_writable_table_entity(obj, object_index): - # VALIDATION — the Pydantic field constraints carried on the model - # (required / max-length / range / pattern). Names the model class. - model = naming.model_class_name(obj) - symbols.append( - ApiSymbol( - name=model, - kind=ApiSymbolKind.VALIDATION, - module=naming.model_import(obj), - signature=f"class {model}(BaseModel)", - usage="Pydantic field validation on the create/update shape", - fields=self._model_fields(obj), - ) - ) - - # DATA_ACCESS — the consumer-implemented repository ``Protocol`` seam. - repo = naming.repository_class_name(obj.name) - symbols.append( - ApiSymbol( - name=repo, - kind=ApiSymbolKind.DATA_ACCESS, - module=f"from .{naming.router_module_name(obj.name)} import {repo}", - signature=f"class {repo}(Protocol)", - usage="data access — the repository Protocol the consumer implements", - returns="list / Optional / bool", - ) - ) - - # REST — one symbol per FastAPI route the router registers. - self._add_rest_symbols(symbols, obj, module, root, object_index) - - # FILTER — the per-entity sort/filter allowlist. - fields_const = naming.filter_fields_const(obj.name) - symbols.append( - ApiSymbol( - name=fields_const, - kind=ApiSymbolKind.FILTER, - module=( - f"from .{naming.filter_allowlist_module_name(obj.name)} " - f"import {fields_const}" - ), - signature=f"{fields_const}: frozenset[str]", - usage="the filterable-field + filter-operator allowlist", - ) - ) - - elif is_read_only_routed(obj): - # A view-backed projection: the read-only router and its allowlist, no writes. - symbols.extend(self._read_only_symbols(obj)) - - if not symbols: - return None - return ApiUnit(obj.name, _package_of(obj), unit_kind, symbols) - - def _build_report_unit(self, obj: MetaObject, root: MetaData) -> ApiUnit: - """The unit for a SERVED report (FR-044 Table G): the row model plus the read-only - surface its router mounts (:meth:`_read_only_symbols`). Nothing else is generated - for it: no item route, no write verb, no validation model. Built over the - report's read model so the names are the generators'.""" - model_obj = report_read_model(obj, root) # type: ignore[arg-type] - model = naming.model_class_name(model_obj) - symbols = [ - ApiSymbol( - name=model, - kind=ApiSymbolKind.MODEL, - module=naming.model_import(model_obj), - signature=f"class {model}(BaseModel)", - usage="the Pydantic v2 report row model", - ), - *self._read_only_symbols(model_obj), - ] - return ApiUnit(obj.name, _package_of(obj), "report", symbols) - - def _read_only_symbols(self, obj: MetaObject) -> list[ApiSymbol]: - """What the READ-ONLY router (``is_read_only_routed``) actually mounts and emits, - and nothing more: the repository seam, ``GET`` list, ``GET`` by id ONLY when - :func:`has_item_route`, and the filter allowlist. The collection ``POST`` and the - item-verb refusals are 405s, not endpoints, so no write verb is documented; - a keyless object has no item route and its seam no ``find_by_id``.""" - router_module = naming.router_module_name(obj.name) - base_path = "/api/" + naming.route_path(obj.name) - repo = naming.repository_class_name(obj.name) - fields_const = naming.filter_fields_const(obj.name) - item = has_item_route(obj) - - def rest(verb_path: str, usage: str) -> ApiSymbol: - return ApiSymbol( - name=verb_path, - kind=ApiSymbolKind.REST, - module=f"# {router_module}.py — FastAPI APIRouter", - signature=verb_path, - usage=usage, - ) - - symbols = [ - ApiSymbol( - name=repo, - kind=ApiSymbolKind.DATA_ACCESS, - module=f"from .{router_module} import {repo}", - signature=f"class {repo}(Protocol)", - usage="data access — the read-only repository Protocol the consumer implements", - returns="list / count / Optional" if item else "list / count", - ), - rest("GET " + base_path, "list with pagination / sort / filters"), - ] - if item: - symbols.append( - rest("GET " + base_path + "/{" + naming.pk_param(obj.name) + "}", "fetch one by id") - ) - symbols.append( - ApiSymbol( - name=fields_const, - kind=ApiSymbolKind.FILTER, - module=( - f"from .{naming.filter_allowlist_module_name(obj.name)} " - f"import {fields_const}" - ), - signature=f"{fields_const}: frozenset[str]", - usage="the filterable-field + filter-operator allowlist", - ) - ) - return symbols - - def _add_rest_symbols( - self, - symbols: list[ApiSymbol], - entity: MetaObject, - module: str, - root: MetaData, - object_index: dict[str, MetaObject], - ) -> None: - router_module = naming.router_module_name(entity.name) - base_path = "/api/" + naming.route_path(entity.name) - # The router registers the item routes with a per-entity path param - # (`/{_id}`, e.g. `/{customer_id}`) — document that exact name, not - # a generic `{id}`, so the docs match the generated FastAPI routes. - pk = naming.pk_param(entity.name) - item = base_path + "/{" + pk + "}" - - def add(verb_path: str, usage: str) -> None: - symbols.append( - ApiSymbol( - name=verb_path, - kind=ApiSymbolKind.REST, - module=f"# {router_module}.py — FastAPI APIRouter", - signature=verb_path, - usage=usage, - ) - ) - - # The router registers GET list, GET item, POST, PATCH item, PUT item, - # DELETE item (PATCH + PUT share a handler but are two registered routes). - add("GET " + base_path, "list with pagination / sort / filters") - add("GET " + item, "fetch one by id") - add("POST " + base_path, "create") - add("PATCH " + item, "update") - add("PUT " + item, "update (PUT alias)") - add("DELETE " + item, "delete") - - # M:N traversal — GET //{id}/. `docs` has no - # `GenConfig`/`--column-naming` of its own (see `cli.py`'s docs handler) and - # this loop reads only `.target_entity` / `.relation_name` off the - # descriptor — never a physical column/table name — so resolving it under - # the port default here changes nothing observable; threading a naming - # strategy through would mean inventing a config seam this command doesn't - # have, for a value nothing here would read. - for d in resolve_m2m_descriptors(entity, object_index): - target = object_index.get(d.target_entity) - target_name = target.name if target is not None else d.target_entity - add( - "GET " + base_path + "/{" + pk + "}/" + d.relation_name, - f"M:N traversal — the related {target_name} rows", - ) - - # ----- templates --------------------------------------------------------- - - def _build_template_unit(self, tmpl: MetaData, root: MetaData) -> ApiUnit: - name = tmpl.name - module = naming.snake_case(name) - symbols: list[ApiSymbol] = [] - payload_vo = _payload_resolves(tmpl, root) - - if payload_vo is not None: - # PAYLOAD — the shape this template RENDERS. ADR-0056: it IS the @payloadRef - # value object's own model, emitted by the entity generator. - payload_class = naming.model_class_name(payload_vo) - symbols.append( - ApiSymbol( - name=payload_class, - kind=ApiSymbolKind.PAYLOAD, - module=naming.model_import(payload_vo), - signature=f"class {payload_class}(BaseModel)", - usage="the typed payload projection bound to the template", - fields=self._payload_fields(payload_vo), - ) - ) - - # RENDER — the typed render helper wrapping the render engine. OUTBOUND - # ONLY: render_helper_generator emits for a template.output alone, so - # documenting it for a prompt would name a function never generated. - if tmpl.sub_type == tc.TEMPLATE_SUBTYPE_OUTPUT: - render_fn = naming.render_helper_fn(name) - symbols.append( - ApiSymbol( - name=render_fn, - kind=ApiSymbolKind.RENDER, - module=f"from .{module}_render_helper import {render_fn}", - signature=f"def {render_fn}(payload: {payload_class}, provider)", - usage="renders the output template against a typed payload", - returns="EmailDocument" if _is_email_kind(tmpl) else "str", - ) - ) - - # ADR-0052 — the INBOUND symbols belong to a RESPONDING template.prompt, never - # to an output, and the gate is @responseRef PRESENCE rather than a format - # value. Shares `response_shape` with the generators, so the docs can never - # claim a symbol codegen suppressed. - inbound = response_shape(root, tmpl, pkg_of(tmpl)) - if inbound is not None: - # The RESPONSE shape the parser actually returns — the @responseRef value - # object's own model (ADR-0056). Documented once: when the prompt renders and - # parses the SAME value object, the model above already is it. - response_class = naming.model_class_name(inbound.vo) - if payload_vo is None or inbound.vo.resolution_key() != payload_vo.resolution_key(): - symbols.append( - ApiSymbol( - name=response_class, - kind=ApiSymbolKind.PAYLOAD, - module=naming.model_import(inbound.vo), - signature=f"class {response_class}(BaseModel)", - usage="the typed response shape a model reply is parsed into", - fields=self._payload_fields(inbound.vo), - ) - ) - - # OUTPUT_PARSER — the strict ``parse_*``. ADR-0053: JSON-only, so an XML - # reply gets no strict tier and documenting one would name a missing fn. - if not is_xml(inbound.format): - parse_fn = naming.output_parser_fn(name) - symbols.append( - ApiSymbol( - name=parse_fn, - kind=ApiSymbolKind.OUTPUT_PARSER, - module=f"from .{module}_response_parser import {parse_fn}", - signature=f"def {parse_fn}(text)", - usage="parses a model reply back into the typed response shape", - returns=response_class, - ) - ) - - prompt_fn = naming.output_prompt_fn(name) - symbols.append( - ApiSymbol( - name=prompt_fn, - kind=ApiSymbolKind.PROMPT, - module=f"from .{module}_response_format import {prompt_fn}", - signature=f"def {prompt_fn}(overrides=None)", - usage="builds the response-format prompt fragment", - returns="str", - ) - ) - extract_fn = naming.extractor_fn(name) - symbols.append( - ApiSymbol( - name=extract_fn, - kind=ApiSymbolKind.EXTRACTOR, - module=f"from .{module}_extractor import {extract_fn}", - signature=f"def {extract_fn}(root, text, opts=None)", - usage="extracts the strict typed response from dirty model output", - returns=response_class, - ) - ) - - return ApiUnit(name, _package_of(tmpl), "template", symbols) - - # ----- field shapes ------------------------------------------------------- - - def _model_fields(self, entity: MetaObject) -> list[FieldShape]: - """The entity's documented scalar/enum field shapes — one row per field the - Pydantic model maps (object-typed fields are owned-type navs, not a scalar - documented shape). Types via ``type_map.py_type_for``; required = own - ``@required`` is the boolean ``True`` (the same boundary the model uses).""" - rows: list[FieldShape] = [] - for f in entity.fields(): - if not isinstance(f, MetaField): - continue - if f.sub_type == fc.FIELD_SUBTYPE_OBJECT: - continue - type_expr = type_map.py_type_for(f).expr - note = None - if f.sub_type == fc.FIELD_SUBTYPE_ENUM: - values = type_map.effective_enum_values(f) - if values: - note = "allowed: " + " | ".join(values) - required = f.get_meta_attr(fc.FIELD_ATTR_REQUIRED) is True # ADR-0039: resolving (@required may be inherited) - rows.append(FieldShape(f.name, type_expr, optional=not required, note=note)) - return rows - - def _payload_fields(self, vo: MetaObject) -> list[FieldShape]: - """The payload model's documented field shapes — one row per field of the - value object, with optionality from ``is_field_required`` (the boundary the - value object's model draws between ``T`` and ``T | None``).""" - rows: list[FieldShape] = [] - for f in vo.fields(): - if not isinstance(f, MetaField): - continue - type_expr = type_map.py_type_for(f).expr - rows.append( - FieldShape(f.name, type_expr, optional=not is_field_required(f)) - ) - return rows - - -def _package_of(node: MetaData) -> str: - """The unit's metadata package (``"acme::shop"``) — drives the doc-page layout - path. Resolved from the nearest ancestor that carries one (objects inherit the - file/root package).""" - pkg = node.package - parent = node.parent - while pkg is None and parent is not None: - pkg = parent.package - parent = parent.parent - return pkg or "" - - -def build_api_model(root: MetaData, project: str) -> ApiModel: - """Module-level convenience: build the Python SDK api-surface IR.""" - return PythonApiModelBuilder().build(root, project) +alias(__name__, "metaobjects.codegen.runtime.api_docs_builder") diff --git a/server/python/src/metaobjects/apidocs/naming.py b/server/python/src/metaobjects/apidocs/naming.py index 76b60c3da..81fd1c3e7 100644 --- a/server/python/src/metaobjects/apidocs/naming.py +++ b/server/python/src/metaobjects/apidocs/naming.py @@ -3,7 +3,7 @@ This mirrors the Java ``SpringNaming`` / C# ``CSharpNaming`` pattern: the real generators delegate their name computation here, and the -:class:`~metaobjects.apidocs.builder.PythonApiModelBuilder` enumerates symbols by +:class:`~metaobjects.codegen.runtime.api_docs_builder.PythonApiModelBuilder` enumerates symbols by calling the SAME functions. So the documented name can never drift from the emitted name — they are computed once, in one place. diff --git a/server/python/src/metaobjects/apidocs/paths.py b/server/python/src/metaobjects/apidocs/paths.py index 6360d8d6a..4b5a853d6 100644 --- a/server/python/src/metaobjects/apidocs/paths.py +++ b/server/python/src/metaobjects/apidocs/paths.py @@ -1,75 +1,4 @@ -"""Doc-page path math for the Python native SDK-docs surface (``api/python``). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.api_docs_paths`.""" +from metaobjects.codegen._moved import alias -Byte-parity with the Java ``DocsPaths`` (server/java/.../apidocs/DocsPaths.java), -the C# ``DocsPaths`` (server/csharp/.../ApiDocs/DocsPaths.cs), and the TS -``docs-paths.ts`` contract: the same layout + relative-href math, so the shared -``fixtures/conformance/api-docs-cross-port/expected-paths.json`` resolves -identically across all ports. Nothing here is Python-specific — it is pure path -math. The api-docs renderer + the ``docs`` command call these helpers; they never -re-derive a path. -""" -from __future__ import annotations - -from enum import Enum - - -class Layout(str, Enum): - """Doc-page layout: one file per unit (:attr:`FLAT`) or foldered by package - (:attr:`PACKAGE`).""" - - #: One file per unit at the surface root (``.md``). - FLAT = "flat" - #: Foldered by package (``/.md``). - PACKAGE = "package" - - -def package_to_path(pkg: str | None) -> str: - """``"acme::shop"`` or ``"acme.shop"`` → ``"acme/shop"``; ``None``/"" → "".`""" - if not pkg: - return "" - return pkg.replace("::", "/").replace(".", "/") - - -def doc_page_output_path(layout: Layout, pkg: str | None, name: str) -> str: - """Flat → ``".md"``; Package → ``"/.md"``.""" - file = f"{name}.md" - if layout == Layout.FLAT: - return file - folder = package_to_path(pkg) - return file if not folder else f"{folder}/{file}" - - -def surface_cross_href(from_output_path: str, to_output_path: str) -> str: - """Relative posix href from *from_output_path*'s directory to *to_output_path* - (mirrors the TS ``surfaceCrossHref``).""" - slash = from_output_path.rfind("/") - from_dir = from_output_path[:slash] if slash >= 0 else "" - rel = _posix_relative(from_dir, to_output_path) - return rel if rel.startswith(".") else f"./{rel}" - - -def model_cross_href( - api_page_path: str, model_page_path: str, model_base_url: str | None = None -) -> str: - """From an api page to its model page: relative by default, absolute when - *model_base_url* is set (federated docs).""" - if model_base_url: - return f"{model_base_url.rstrip('/')}/{model_page_path}" - return surface_cross_href(api_page_path, model_page_path) - - -def _posix_relative(from_dir: str, to_path: str) -> str: - """``posixpath.relative(from_dir, to_path)``: drop the common prefix, ``..`` - per remaining *from_dir* segment, then the remaining *to_path* segments. - "" from_dir → to_path; identical → ".".""" - if not from_dir: - return to_path - frm = from_dir.split("/") - to = to_path.split("/") - common = 0 - cap = min(len(frm), len(to)) - while common < cap and frm[common] == to[common]: - common += 1 - parts: list[str] = [".."] * (len(frm) - common) - parts.extend(to[common:]) - return "/".join(parts) if parts else "." +alias(__name__, "metaobjects.codegen.runtime.api_docs_paths") diff --git a/server/python/src/metaobjects/apidocs/renderer.py b/server/python/src/metaobjects/apidocs/renderer.py index 3555fcb6a..791893e66 100644 --- a/server/python/src/metaobjects/apidocs/renderer.py +++ b/server/python/src/metaobjects/apidocs/renderer.py @@ -1,185 +1,4 @@ -"""Renders the :class:`ApiModel` IR into Python-idiomatic api-doc markdown. +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.api_docs_renderer`.""" +from metaobjects.codegen._moved import alias -One IR, three forms — all derived from the SAME model, never re-derived: - -* :func:`render_unit_page` — a per-unit HUMAN page (entity OR template), symbols - grouped into ordered sections + the ``**Model / metadata:**`` back-link. -* :func:`render_index` — the consolidated index (README.md), one bullet per unit. -* :func:`render_agent_api` — the token-frugal AGENT form (AGENT-API.md), symbols - grouped under their import line. - -PRESENTATION ONLY: every symbol name comes from the IR (which keys off the naming -seam) and every path comes from :mod:`metaobjects.apidocs.paths` — the renderer never -re-derives a name or a path. The section order + headings + the back-link literal -mirror the Java / C# renderers so the polyglot doc tree coheres. -""" -from __future__ import annotations - -from metaobjects.apidocs.api_model import ApiModel, ApiSymbolKind, ApiUnit -from metaobjects.apidocs.paths import Layout, doc_page_output_path, surface_cross_href - -# Canonical section order per kind (a unit renders only the kinds it carries). -_KIND_ORDER: tuple[ApiSymbolKind, ...] = ( - ApiSymbolKind.MODEL, - ApiSymbolKind.DATA_ACCESS, - ApiSymbolKind.REST, - ApiSymbolKind.VALIDATION, - ApiSymbolKind.EXTRACTOR, - ApiSymbolKind.RENDER, - ApiSymbolKind.PAYLOAD, - ApiSymbolKind.PROMPT, - ApiSymbolKind.OUTPUT_PARSER, - ApiSymbolKind.FILTER, -) - -_HEADINGS: dict[ApiSymbolKind, str] = { - ApiSymbolKind.MODEL: "Model", - ApiSymbolKind.DATA_ACCESS: "Data access", - ApiSymbolKind.REST: "REST", - ApiSymbolKind.VALIDATION: "Validation", - ApiSymbolKind.EXTRACTOR: "Extractor", - ApiSymbolKind.RENDER: "Render", - ApiSymbolKind.PAYLOAD: "Payload", - ApiSymbolKind.PROMPT: "Prompt", - ApiSymbolKind.OUTPUT_PARSER: "Output parser", - ApiSymbolKind.FILTER: "Filter", -} - -_SUMMARY_LABELS: dict[ApiSymbolKind, str] = { - ApiSymbolKind.MODEL: "model", - ApiSymbolKind.DATA_ACCESS: "data access", - ApiSymbolKind.REST: "REST", - ApiSymbolKind.VALIDATION: "validation", - ApiSymbolKind.EXTRACTOR: "extractor", - ApiSymbolKind.RENDER: "render", - ApiSymbolKind.PAYLOAD: "payload", - ApiSymbolKind.PROMPT: "prompt", - ApiSymbolKind.OUTPUT_PARSER: "output parser", - ApiSymbolKind.FILTER: "filter", -} - - -def _md_cell(text: str) -> str: - return text.replace("|", "\\|") - - -def render_unit_page(unit: ApiUnit, model_href: str | None) -> str: - """Render one per-unit human reference page. *model_href* (when non-empty) is a - pre-computed relative href back to this unit's model/metadata page — the caller - derives it via :func:`metaobjects.apidocs.paths.model_cross_href`; the renderer - only places it (as the contract ``**Model / metadata:**`` back-link).""" - out: list[str] = [f"# {unit.node} API\n"] - if model_href: - out.append(f"\n**Model / metadata:** [{unit.node}]({model_href})\n") - out.append( - "\n> Import paths are relative to the generated package; add it to your " - "Python path.\n" - ) - - for kind in _KIND_ORDER: - syms = [s for s in unit.symbols if s.kind == kind] - if not syms: - continue - out.append(f"\n## {_HEADINGS[kind]}\n") - for sym in syms: - out.append(f"\n### `{sym.signature}`\n") - out.append(f"\n{sym.usage}\n") - out.append(f"\n```python\n{sym.module}\n```\n") - if sym.returns: - out.append(f"\nReturns: {sym.returns}\n") - if sym.fields: - out.append("\n| Field | Type | Required | Notes |\n|---|---|---|---|\n") - for f in sym.fields: - required = "no" if f.optional else "yes" - out.append( - f"| `{f.name}` | `{_md_cell(f.type)}` | {required} | " - f"{_md_cell(f.note or '')} |\n" - ) - return "".join(out) - - -def render_index(model: ApiModel, layout: Layout) -> str: - """Render the consolidated api index (README.md): one bullet per unit, entities/ - values vs templates. Each unit's href is the relative link from the index to the - unit's page in the given *layout*.""" - entities = sorted( - (u for u in model.units if u.kind != "template"), key=lambda u: u.node - ) - templates = sorted( - (u for u in model.units if u.kind == "template"), key=lambda u: u.node - ) - - out: list[str] = [ - "# API Reference\n\nGenerated public API surface, one page per entity and " - "output template.\n" - ] - if entities: - out.append("\n## Entities\n\n") - for u in entities: - out.append(_index_row(u, layout)) - if templates: - out.append("\n## Templates\n\n") - for u in templates: - out.append(_index_row(u, layout)) - return "".join(out) - - -def _index_row(unit: ApiUnit, layout: Layout) -> str: - href = surface_cross_href( - "README.md", doc_page_output_path(layout, unit.package, unit.node) - ) - return ( - f"- [{unit.node}]({href}) — {_summary(unit)} " - f"({len(unit.symbols)} symbols)\n" - ) - - -def _summary(unit: ApiUnit) -> str: - parts: list[str] = [] - for kind in _KIND_ORDER: - n = sum(1 for s in unit.symbols if s.kind == kind) - if n == 0: - continue - label = _SUMMARY_LABELS[kind] - parts.append(label if n == 1 else f"{n} {label}") - return ", ".join(parts) if parts else "no public symbols" - - -def render_agent_api(model: ApiModel) -> str: - """Render the condensed agent/LLM form (AGENT-API.md): per unit, symbols grouped - under a single import-line header then one compact ``` `signature` — usage ``` line - each. NO prose / field-tables (token budget). Units + symbols keep their IR order.""" - out: list[str] = [ - f"# Agent API Reference\n\nGenerated Python API reference for " - f"{model.project}; call these exactly as written. Import paths are relative " - "to the generated package.\n" - ] - for unit in model.units: - if not unit.symbols: - continue - out.append(f"\n## {unit.node}\n") - groups: dict[str, list[str]] = {} - order: list[str] = [] - for s in unit.symbols: - if s.module not in groups: - groups[s.module] = [] - order.append(s.module) - groups[s.module].append(f"- `{s.signature}` — {s.usage}\n") - for module in order: - out.append(f"\n`{module}`\n") - out.extend(groups[module]) - return "".join(out) - - -class PythonApiDocsRenderer: - """Object wrapper around the module-level render functions (parity with the - Java / C# renderer classes; the functions are the canonical entry points).""" - - def render_unit_page(self, unit: ApiUnit, model_href: str | None) -> str: - return render_unit_page(unit, model_href) - - def render_index(self, model: ApiModel, layout: Layout) -> str: - return render_index(model, layout) - - def render_agent_api(self, model: ApiModel) -> str: - return render_agent_api(model) +alias(__name__, "metaobjects.codegen.runtime.api_docs_renderer") diff --git a/server/python/src/metaobjects/cli.py b/server/python/src/metaobjects/cli.py index 972779db3..0491fdef2 100644 --- a/server/python/src/metaobjects/cli.py +++ b/server/python/src/metaobjects/cli.py @@ -93,7 +93,8 @@ TargetConfig, load_project_config, ) -from metaobjects.codegen.generator import Generator +from metaobjects.codegen.docs_run import resolve_docs_generators, run_docs +from metaobjects.codegen.generator import DEFAULT_API_SUBDIR, DocsRunFacts, Generator from metaobjects.codegen.generators.entity_model import entity_model from metaobjects.codegen.generators.extractor_generator import extractor_generator from metaobjects.codegen.generators.filter_allowlist_generator import ( @@ -1007,33 +1008,25 @@ def _resolve_metadata_location_or_print_error( return None -#: The default api-surface subdir (the cross-port contract's ``api/python``). -_DOCS_DEFAULT_API_SUBDIR = "api/python" - - def _cmd_docs(args: argparse.Namespace) -> int: """``docs`` — emit the generated Python SDK api surface (the ``api/python`` surface of the cross-port SDK-docs contract) under ``--out``. - Wiring only: every symbol name comes from the ``PythonApiModelBuilder`` (which - keys off the naming seam + each generator's applies-predicate) and every path - comes from :mod:`metaobjects.apidocs.paths` — nothing is re-derived here. Output - paths are collision-checked (duplicate page path → failure) BEFORE anything is - written, mirroring the C# ``DocsCommand`` / the Java Mojo. Layout is ``package`` - (the cross-port contract). ``--model-base-url`` federates the model back-links - when set.""" - from metaobjects.apidocs import ( - Layout, - PythonApiDocsRenderer, - PythonApiModelBuilder, - doc_page_output_path, - model_cross_href, - ) - + Wiring only: the pages come from the ``api-docs`` generator, which builds them from + the naming seam plus each generator's applies-predicate and writes them under the + cross-port contract's ``package`` layout. A project that owns that generator + (``metaobjects eject api-docs``) lists it in the config's ``docs.generators`` and + ``docs`` runs the copy in its place. Output paths are collision-checked (duplicate + page path → failure) BEFORE anything is written, mirroring the C# ``DocsCommand`` / + the Java Mojo. ``--model-base-url`` federates the model back-links when set. The + positional ``metadata_dir`` form reads no config, so it always runs the packaged + generator.""" providers, providers_ok = _providers_from_args(args) if not providers_ok: return 1 + docs_generators: list[str] = [] + owned_root = Path.cwd() if args.metadata_dir is None: # Mirrors `gen`'s no-positional routing (#267): rung 2 (`metadata` in # `metaobjects.config.yaml`, if one is found), else rungs 3-4 via @@ -1067,6 +1060,8 @@ def _cmd_docs(args: argparse.Namespace) -> int: return 1 providers = [*providers, *config_providers] docs_libraries = config.libraries + docs_generators = config.docs_generators + owned_root = config.config_dir root, errors = _load_root_from_collection( collection, providers=providers, libraries=docs_libraries ) @@ -1081,43 +1076,28 @@ def _cmd_docs(args: argparse.Namespace) -> int: print(f" {msg}", file=sys.stderr) return 1 - api_subdir = args.api_subdir or _DOCS_DEFAULT_API_SUBDIR + api_subdir = args.api_subdir or DEFAULT_API_SUBDIR project = args.project or project_default model_base_url = getattr(args, "model_base_url", None) - model = PythonApiModelBuilder().build(root, project) - renderer = PythonApiDocsRenderer() - layout = Layout.PACKAGE - - # Collect rel-path → content first so a duplicate page path fails BEFORE any write. - emitted: dict[str, str] = {} - - def _put(path: str, content: str) -> None: - if path in emitted: - raise ValueError(f"docs — duplicate api page output path: {path}") - emitted[path] = content - - try: - for unit in model.units: - page_path = doc_page_output_path(layout, unit.package, unit.node) - api_page_path_from_docs_root = f"{api_subdir}/{page_path}" - href = model_cross_href(api_page_path_from_docs_root, page_path, model_base_url) - _put(page_path, renderer.render_unit_page(unit, href)) - _put("README.md", renderer.render_index(model, layout)) - _put("AGENT-API.md", renderer.render_agent_api(model)) - except ValueError as exc: + facts = DocsRunFacts(project=project, api_subdir=api_subdir, model_base_url=model_base_url) + generators, gen_err = resolve_docs_generators(docs_generators, owned_root) + if gen_err is not None: + print(f"error: {gen_err}", file=sys.stderr) + return 1 + outcome = run_docs(root, args.out, facts, generators) + if outcome.error is not None: # A duplicate output path (e.g. two units that resolve to the same page) is a # clean user-facing failure, not a stack trace. - print(f"error: {exc}", file=sys.stderr) + print(f"error: {outcome.error}", file=sys.stderr) return 1 - api_root = Path(args.out) / api_subdir written: list[str] = [] - for rel, content in emitted.items(): - dest = api_root / rel + for rel, content in outcome.files: + dest = Path(args.out) / rel dest.parent.mkdir(parents=True, exist_ok=True) dest.write_text(content, encoding="utf-8") - written.append(f"{api_subdir}/{rel}") + written.append(rel) for path in sorted(written): print(path) diff --git a/server/python/src/metaobjects/codegen/_moved.py b/server/python/src/metaobjects/codegen/_moved.py new file mode 100644 index 000000000..a9d0cfa88 --- /dev/null +++ b/server/python/src/metaobjects/codegen/_moved.py @@ -0,0 +1,23 @@ +"""Deprecated import paths of modules that moved into ``metaobjects.codegen.runtime``. + +``metaobjects eject`` hands the emitters a generator writes through to the adopter, and it +copies them from ``metaobjects/codegen/runtime/``, so that is where they live. A copy ejected +before they moved imports them from their old paths; each old path is a one-line module that +calls :func:`alias`, which makes the old name BE the new module (the same object, so private +names and monkeypatching behave as before) and warns once on import. Import from the new path. +""" +from __future__ import annotations + +import importlib +import sys +import warnings + + +def alias(old: str, new: str) -> None: + """Make module *old* the module *new*, warning that the old path is deprecated.""" + warnings.warn( + f"{old} moved to {new}; import it from there. The old path will be removed.", + DeprecationWarning, + stacklevel=3, + ) + sys.modules[old] = importlib.import_module(new) diff --git a/server/python/src/metaobjects/codegen/docs_run.py b/server/python/src/metaobjects/codegen/docs_run.py new file mode 100644 index 000000000..546f2c185 --- /dev/null +++ b/server/python/src/metaobjects/codegen/docs_run.py @@ -0,0 +1,91 @@ +"""``metaobjects docs`` — the docs run: the generators that write the reference pages, and +the collision-checked write. + +The api reference is a generator (``api-docs``, ``docs_surface = "api"``), so a project can +own it with ``metaobjects eject api-docs``. The owned copy is listed in the config's +``docs.generators`` — never the top-level ``generators`` ``gen`` runs — and ``docs`` runs it +in place of the packaged one. Engine code: an owned copy imports none of it. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from metaobjects.codegen.config import GenConfig +from metaobjects.codegen.eject import build_owned +from metaobjects.codegen.generator import ( + DOCS_SURFACE_API, + DocsRunFacts, + GenContext, + Generator, +) +from metaobjects.codegen.generator_registry import GENERATOR_REGISTRY, GeneratorBuildContext +from metaobjects.meta.meta_data import MetaData + + +@dataclass +class DocsOutcome: + #: ``(path relative to the docs out dir, content)`` in emission order. + files: list[tuple[str, str]] = field(default_factory=list) + #: A user-facing failure (duplicate page path, bad ``docs.generators`` entry), or ``None``. + error: str | None = None + + +def resolve_docs_generators( + specs: list[str], root: Path, +) -> tuple[list[Generator], str | None]: + """The generators ``docs`` runs: each owned entry in *specs* (``docs.generators``), then + the packaged generator for every surface no owned one takes over. + + An entry that is not a docs generator (no ``docs_surface``) or a second one for a + surface already taken is refused by name: either would write the same pages twice, or + write code into the docs tree.""" + ctx = GeneratorBuildContext() + owned: list[Generator] = [] + taken: dict[str, str] = {} + for spec in specs: + gen, err = build_owned(spec, root, ctx) + if err is not None or gen is None: + return [], err + surface = getattr(gen, "docs_surface", None) + if surface is None: + return [], ( + f"docs.generators entry {spec!r} is not a docs generator: it carries no " + "`docs_surface`. A generator that writes code belongs in a target's " + "`generators`." + ) + if surface in taken: + return [], ( + f"docs.generators lists two generators for the {surface!r} surface: " + f"{taken[surface]!r} and {spec!r}." + ) + taken[surface] = spec + owned.append(gen) + if DOCS_SURFACE_API not in taken: + owned.append(GENERATOR_REGISTRY["api-docs"].factory(ctx)) + return owned, None + + +def run_docs( + root: MetaData, out_dir: str, facts: DocsRunFacts, generators: list[Generator], +) -> DocsOutcome: + """Run *generators* over *root* and collect their pages, failing on a duplicate page path + BEFORE anything is written (the caller writes ``outcome.files``).""" + outcome = DocsOutcome() + seen: dict[str, str] = {} + for gen in generators: + ctx = GenContext( + entities=[], + loaded_root=root, + matches=lambda _e: True, + config=GenConfig(out_dir=out_dir), + warn=lambda _msg: None, + docs=facts, + ) + for f in gen.generate(ctx): + if f.path in seen: + outcome.error = f"docs — duplicate api page output path: {f.path}" + return outcome + seen[f.path] = gen.name + outcome.files.append((f.path, f.content)) + return outcome diff --git a/server/python/src/metaobjects/codegen/eject.py b/server/python/src/metaobjects/codegen/eject.py index c9afc5e5e..1ea0ab2c7 100644 --- a/server/python/src/metaobjects/codegen/eject.py +++ b/server/python/src/metaobjects/codegen/eject.py @@ -18,10 +18,19 @@ copy. Core modules (loader, registry, render, extract, ``ObjectManager``) stay package imports. An existing runtime copy is never overwritten, even with ``--force``. -The emit rules several generators share (``shared_predicates``) are listed and copied the -same way. A generator imports them relatively from its ``runtime`` sibling, so the verbatim -copy in ``codegen/generators/`` reads ``codegen/runtime/shared_predicates.py`` with no -edit, and two owned generators read the one copy. +Everything a generator writes its output THROUGH is handed over the same way: the emit +rules several generators share (``shared_predicates``), the fragment emitters behind a full +generator (the shared-enum and M:N writers), the prompt-tier emitters (the extract delegate, +the extract schema, the field mapping and the output-format spec) and, for ``api-docs``, the +page model, builder, renderer and path math. A generator imports each relatively from its +``runtime`` sibling, so the verbatim copy in ``codegen/generators/`` reads +``codegen/runtime/.py`` with no edit, and two owned generators read the one copy. +Only the engine stays a package import: the generator base and file writer, the naming and +type-mapping tables, the model walks and the loader. ``tests/codegen/test_eject_alone.py`` +ejects each generator alone and holds it to exactly that. + +A docs generator (``api-docs``) is not wired into ``generators``: ``metaobjects docs`` runs +it from the config's ``docs.generators``, and ``gen`` skips one it finds under a target. """ from __future__ import annotations @@ -138,7 +147,13 @@ def eject(names: list[str], root: Path, *, force: bool = False) -> EjectResult: target = owned_path(root, n) target.write_text(ejected_source(entry), encoding="utf-8") out.append(f"ejected {n} -> {target.relative_to(root)}") - out.append(f" wire it: replace \"{n}\" in `generators` with {owned_token(entry)}") + if entry.docs_surface is not None: + out.append( + f" wire it: list {owned_token(entry)} under `docs.generators` in " + "metaobjects.config.yaml — `metaobjects docs` runs it; `gen` never does" + ) + else: + out.append(f" wire it: replace \"{n}\" in `generators` with {owned_token(entry)}") for module in entry.runtime: rt = runtime_path(root, module) rel = rt.relative_to(root) @@ -152,7 +167,8 @@ def eject(names: list[str], root: Path, *, force: bool = False) -> EjectResult: out.append("") out.append( "The copies are yours: edit them freely. Keep only one of the packaged name and your " - "copy in `generators` — wiring both runs both, and they write the same files." + "copy in `generators` — wiring both runs both, and they write the same files. " + "A docs generator is the exception: `docs.generators` replaces the packaged one." ) return EjectResult(0, out, []) diff --git a/server/python/src/metaobjects/codegen/extract_delegate_emitter.py b/server/python/src/metaobjects/codegen/extract_delegate_emitter.py index d6cd88bcb..b0e4287e7 100644 --- a/server/python/src/metaobjects/codegen/extract_delegate_emitter.py +++ b/server/python/src/metaobjects/codegen/extract_delegate_emitter.py @@ -1,383 +1,4 @@ -"""FR-010 nested-codegen-gap — the runtime-DELEGATING extract emitter (Python). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.extract_delegate_emitter`.""" +from metaobjects.codegen._moved import alias -The self-contained ``extract_(text)`` path (``extract_schema_emitter`` + the -baked ``ExtractSchema``) covers scalars / enums / scalar-arrays but leaves -nested-object and array-of-object components ``None`` — the historical FR-010 codegen -gap. This module emits the additive *delegating* entry that CLOSES that gap by -wrapping the runtime extract: - - extract__with_loader(root, text, opts=None) -> ExtractionResult[Extracted] - -It resolves this payload's ``MetaObject`` from the supplied loaded ``MetaRoot`` by its -baked simple name (``PAYLOAD_NAME``), delegates to ``extract_object`` in -:mod:`metaobjects.meta.core.object.object_extract` (which assembles the FULL nested -object graph reflection-free via the Phase A object model — ``MetaObject.new_instance()`` -+ the ``MetaField`` set-by-name SPI), then maps the assembled ``ValueObject`` graph into -the typed nullable mirror graph via generated ``_from__extracted(o)`` mapper -functions (payload + every reachable nested VO, deduped). - -This is the codegen-wrapping-runtime pattern (a generated DAO calling the -dynamic-metadata runtime), mirroring the Java / Kotlin / TS pilots. The generated -mappers read the assembled graph through a tiny ``_read_prop`` helper that mirrors the -``MetaField`` get SPI (``ValueObject.get(name)`` else plain-attribute access), so the -emitted code stays self-sufficient and reflection-free. - -Bounded by the cross-port ``MAX_NEST_DEPTH`` via the runtime — codegen here only mirrors -the runtime's resolved object graph, so depth/cycle guarding lives in ``object_extract``. -The emitter dedupes mirrors/mappers by ``resolution_key()`` (the package-qualified FQN, -cycle-safe) — NOT the bare VO ``name``, which would silently collapse two same-short-name -value-objects from different packages into one (ADR-0044, #228). - -ADR-0056 rule 3: a mirror is keyed by the VALUE OBJECT, never the template. Every mirror -and mapper — the root's included — is named after the value object's emitted model name -(:func:`~metaobjects.codegen.value_objects.model_class_name`: ``Extracted`` / -``_from__extracted``), the same name the entity generator declares the model under, so -a cross-package short-name collision qualifies the mirror exactly as it qualifies the -model. Mirrors stay module-local (declared in each parser module that needs them), like -TypeScript's: a Python module scopes them, so two parsers over one response each carry -their own copy without clashing. -""" -from __future__ import annotations - -from metaobjects.codegen import fr010_field_mapping as fm -from metaobjects.codegen.value_objects import model_class_name, pkg_of -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.meta_data import MetaData -from metaobjects.naming_refs import resolve_object_ref - - -def ref_vo(field: MetaData, root: MetaData) -> MetaData | None: - """The ``@objectRef`` target VO for a nested-object field, or ``None`` when - unresolvable. - - ADR-0042 (#228) — resolves via the canonical `resolve_object_ref` package-local - contract: an FQN ref resolves EXACTLY; a bare ref resolves in the DECLARING - field's own package first, else a root-level object. NO bare-tail short-name - fallback — that pattern (matching an FQN ref by its trailing simple-name segment - against ANY same-named object root-wide) is the #219/ADR-0042-banned "wrong - node" bug: under a cross-package short-name collision it silently binds - whichever same-named object happens to load first, regardless of which package - the ref actually pointed at. *referrer_pkg* is the field's OWN declaring - package (which differs from the VO's when the field is inherited via `extends` - from an abstract VO in another package).""" - ref = field.attrs().get(fc.FIELD_ATTR_OBJECT_REF) - if not isinstance(ref, str) or not ref: - return None - referrer_pkg = pkg_of(field.parent) if field.parent is not None else "" - return resolve_object_ref(root, ref, referrer_pkg) - - -def _is_object_field(field: MetaData) -> bool: - """True iff the field is a nested object reference (``field.object`` — distinct - from the string-backed ``field.enum``, treated as a scalar).""" - return field.sub_type == fc.FIELD_SUBTYPE_OBJECT - - -def mirror_name(vo: MetaData) -> str: - """The extracted-mirror dataclass name for a value-object: ``Extracted``, where - ```` is the value object's emitted model name (collision-qualified when another - object shares its short name).""" - return f"{model_class_name(vo)}Extracted" - - -def mapper_name(vo: MetaData) -> str: - """The mapper function name for a value-object (``_from__extracted``).""" - return f"_from_{_snake(model_class_name(vo))}_extracted" - - -def _snake(name: str) -> str: - """``NpcResponseOutput`` → ``npc_response_output`` (matches the cross-generator - ``_snake_case`` convention; no acronym handling).""" - out: list[str] = [] - for i, ch in enumerate(name): - if ch.isupper() and i > 0: - out.append("_") - out.append(ch.lower()) - return "".join(out) - - -# ============================================================================= -# Nested-aware mirror type (recurses into nested mirror names) -# ============================================================================= - - -def _nested_mirror_type(field: MetaData, root: MetaData) -> str: - """The nullable mirror annotation for one field — nested-aware (nested objects - become ``Extracted``; array-of-objects become ``list[...]``).""" - if _is_object_field(field): - target = ref_vo(field, root) - base = f'"{mirror_name(target)}"' if target is not None else "object" - elem = f"{base} | None" - return f"list[{elem}] | None" if fm.is_array(field) else elem - if fm.is_array(field): - return "list[str | None] | None" - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - return "str | None" - kind = fm.scalar_kind(field.sub_type) - if kind in ("INT", "LONG"): - return "int | None" - if kind == "DOUBLE": - return "float | None" - if kind == "BOOLEAN": - return "bool | None" - return "str | None" - - -# ============================================================================= -# Reachability -# ============================================================================= - - -def reachable_vos(vo: MetaData, root: MetaData) -> list[MetaData]: - """``vo`` + every value-object reachable through nested ``@objectRef`` fields, in - stable BFS order, deduped by ``resolution_key()`` (cycle-safe). - - ADR-0044 (#228) — deduping by the bare ``name`` (pre-fix) silently DROPPED a - second cross-package value-object sharing the first one's bare short name (e.g. - ``acme::alpha::Note`` + ``acme::beta::Note``): once the first ``Note`` was seen, - the second's bare name matched ``seen`` and it was never queued/emitted — a - silent shape loss, not merely a naming cosmetic. ``resolution_key()`` is the - package-qualified FQN, so two same-short-name VOs from different packages are - two distinct keys and both survive the walk.""" - out: list[MetaData] = [] - seen: set[str] = set() - queue: list[MetaData] = [vo] - while queue: - cur = queue.pop(0) - key = cur.resolution_key() - if key in seen: - continue - seen.add(key) - out.append(cur) - for f in fm.fields(cur): - if _is_object_field(f): - target = ref_vo(f, root) - if target is not None and target.resolution_key() not in seen: - queue.append(target) - return out - - -def has_nested(vo: MetaData, root: MetaData) -> bool: - """True iff ``vo`` (or any reachable nested VO) has a nested-object field.""" - for cur in reachable_vos(vo, root): - if any(_is_object_field(f) for f in fm.fields(cur)): - return True - return False - - -# ============================================================================= -# Nested-aware mirror dataclasses -# ============================================================================= - - -def nested_mirror_dataclasses(vo: MetaData, root: MetaData) -> list[str]: - """Emit the nested-aware mirror dataclass for ``vo`` and every reachable nested VO - (deduped), each named :func:`mirror_name`. Returns source lines (blank-line - separated).""" - lines: list[str] = [] - for i, cur in enumerate(reachable_vos(vo, root)): - if i > 0: - lines.append("") - lines.append("") - lines.extend(_one_mirror(cur, root)) - return lines - - -def _one_mirror(vo: MetaData, root: MetaData) -> list[str]: - lines: list[str] = [ - "@dataclass(frozen=True, slots=True)", - f"class {mirror_name(vo)}:", - f' """Best-effort extracted twin of ``{model_class_name(vo)}`` — every field nullable', - ' (``None`` where the value was lost or malformed)."""', - ] - field_lines = [ - f" {f.name}: {_nested_mirror_type(f, root)} = None" - for f in fm.fields(vo) - ] - lines.extend(field_lines or [" pass"]) - return lines - - -# ============================================================================= -# Mapper functions (assembled ValueObject graph -> typed nullable mirror graph) -# ============================================================================= - - -def nested_mappers(vo: MetaData, root: MetaData) -> list[str]: - """Emit one ``_from__extracted(o)`` mapper per reachable VO (payload + - nested, deduped), each named :func:`mapper_name`. Returns source lines - (blank-line separated).""" - lines: list[str] = [] - for i, cur in enumerate(reachable_vos(vo, root)): - if i > 0: - lines.append("") - lines.append("") - lines.extend(_one_mapper(cur, root)) - return lines - - -def _one_mapper(vo: MetaData, root: MetaData) -> list[str]: - fn = mapper_name(vo) - mirror = mirror_name(vo) - lines: list[str] = [ - f'def {fn}(o: object | None) -> "{mirror} | None":', - f" \"\"\"Map an assembled ValueObject graph into a typed ``{mirror}`` mirror;" - " null-tolerant.\"\"\"", - " if o is None:", - " return None", - f" return {mirror}(", - ] - for f in fm.fields(vo): - lines.append(f" {f.name}={_mapper_arg(f, root)},") - lines.append(" )") - return lines - - -def _mapper_arg(field: MetaData, root: MetaData) -> str: - """The mirror-field initializer that reads ``field`` from the assembled object ``o``.""" - key = f'"{field.name}"' - if _is_object_field(field): - target = ref_vo(field, root) - if target is None: - return "None # unresolved @objectRef" - fn = mapper_name(target) - if fm.is_array(field): - return f"_map_object_list(_read_prop(o, {key}), {fn})" - return f"{fn}(_read_prop(o, {key}))" - - # Enum / scalar / scalar-array: the runtime already coerced; read + light-coerce to - # the mirror's nullable shape via the locally-defined _dlg_* readers. - # ARRAY is checked BEFORE the scalar-enum branch: an enum ARRAY must route through - # the string-LIST reader, not the scalar enum reader — otherwise the list collapses - # to a single stringified scalar (the cross-port "enum-before-isArray" ordering bug). - if fm.is_array(field): - return f"_dlg_str_list(_read_prop(o, {key}))" - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - return f"_dlg_str(_read_prop(o, {key}))" - kind = fm.scalar_kind(field.sub_type) - if kind in ("INT", "LONG"): - return f"_dlg_int(_read_prop(o, {key}))" - if kind == "DOUBLE": - return f"_dlg_float(_read_prop(o, {key}))" - if kind == "BOOLEAN": - return f"_dlg_bool(_read_prop(o, {key}))" - return f"_dlg_str(_read_prop(o, {key}))" - - -# ============================================================================= -# Used-helper scoping + the shared helper block -# ============================================================================= - - -def used_helpers(vo: MetaData, root: MetaData) -> set[str]: - """The generated-helper names the mappers for ``vo`` (+ reachable nested VOs) - actually reference. ``_read_prop`` is always present once any mapper is emitted.""" - used: set[str] = {"_read_prop"} - for cur in reachable_vos(vo, root): - for f in fm.fields(cur): - if _is_object_field(f): - if fm.is_array(f): - used.add("_map_object_list") - continue - if fm.is_array(f): - # ARRAY before scalar-enum — an enum array uses the string-LIST reader - # (mirrors the _mapper_arg ordering; avoids the collapse-to-scalar bug). - used.add("_dlg_str_list") - elif f.sub_type == fc.FIELD_SUBTYPE_ENUM: - used.add("_dlg_str") - else: - kind = fm.scalar_kind(f.sub_type) - if kind in ("INT", "LONG"): - used.add("_dlg_int") - elif kind == "DOUBLE": - used.add("_dlg_float") - elif kind == "BOOLEAN": - used.add("_dlg_bool") - else: - used.add("_dlg_str") - return used - - -# Each helper: name -> source block (lines). -_HELPER_BLOCKS: dict[str, list[str]] = { - "_read_prop": [ - "def _read_prop(o: object | None, name: str) -> object | None:", - ' """Read a property from an assembled backing object, mirroring the', - " MetaField get SPI (``ValueObject.get(name)`` else plain-attribute access).", - ' Keeps the mappers reflection-free + backing-agnostic."""', - " if o is None:", - " return None", - ' getter = getattr(o, "get", None)', - " if callable(getter):", - " return getter(name)", - " return getattr(o, name, None)", - ], - "_map_object_list": [ - "def _map_object_list(v: object | None, fn) -> list | None:", - ' """Map each element of an assembled array via ``fn``; non-list -> None."""', - " if not isinstance(v, list):", - " return None", - " return [fn(e) for e in v]", - ], - "_dlg_str": [ - "def _dlg_str(v: object | None) -> str | None:", - " return None if v is None else str(v)", - ], - "_dlg_int": [ - "def _dlg_int(v: object | None) -> int | None:", - " if v is None:", - " return None", - " try:", - " return int(v) # type: ignore[arg-type, call-overload]", - " except (TypeError, ValueError):", - " return None", - ], - "_dlg_float": [ - "def _dlg_float(v: object | None) -> float | None:", - " if v is None:", - " return None", - " try:", - " return float(v) # type: ignore[arg-type]", - " except (TypeError, ValueError):", - " return None", - ], - "_dlg_bool": [ - "def _dlg_bool(v: object | None) -> bool | None:", - " if v is None:", - " return None", - " if isinstance(v, bool):", - " return v", - ' return str(v).lower() == "true"', - ], - "_dlg_str_list": [ - "def _dlg_str_list(v: object | None) -> list | None:", - " if not isinstance(v, list):", - " return None", - " return [None if e is None else str(e) for e in v]", - ], -} - -# Stable emission order (deterministic output). -_HELPER_ORDER: tuple[str, ...] = ( - "_read_prop", - "_map_object_list", - "_dlg_str", - "_dlg_int", - "_dlg_float", - "_dlg_bool", - "_dlg_str_list", -) - - -def delegate_helpers(used: set[str]) -> list[str]: - """The shared helper block the generated mappers rely on, scoped to ``used`` and - emitted in stable order. Returns source lines (blank-line separated).""" - lines: list[str] = [ - "# ---- runtime-delegating extract helpers (generated) ----" - ] - for name in _HELPER_ORDER: - if name in used: - lines.append("") - lines.append("") - lines.extend(_HELPER_BLOCKS[name]) - return lines +alias(__name__, "metaobjects.codegen.runtime.extract_delegate_emitter") diff --git a/server/python/src/metaobjects/codegen/extract_schema_emitter.py b/server/python/src/metaobjects/codegen/extract_schema_emitter.py index a6c53d511..ee3c1d4f5 100644 --- a/server/python/src/metaobjects/codegen/extract_schema_emitter.py +++ b/server/python/src/metaobjects/codegen/extract_schema_emitter.py @@ -1,139 +1,4 @@ -"""Turns a payload value-object into Python source fragments for the FR-010 extract -codegen: +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.extract_schema_emitter`.""" +from metaobjects.codegen._moved import alias -* :func:`schema_literal` — a ``ExtractSchema(Format.X, "root", [FieldSpec(...), …])`` - baked descriptor for the emitted parser module. -* :func:`mirror_dataclass` — an all-nullable mirror dataclass ``Extracted`` - (every field ``X | None = None``; the extract entry - returns this nullable twin rather than the strict - Pydantic payload — same reasoning as the Kotlin / C# - nullable mirror). -* :func:`mirror_initializer` — ``Extracted(field=as_string(d, "field"), …)``. - -Mirrors the C# / Java ``ExtractSchemaEmitter`` adapted to Python syntax + the -nullable-mirror shape. Bounded scope: scalar / enum / scalar-array. Nested object + -array-of-enum deferred. -""" -from __future__ import annotations - -from metaobjects.codegen import fr010_field_mapping as fm -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.meta_data import MetaData - -# The extract_map accessors a generated module may import (sorted, deduped subset). -ALL_EXTRACT_MAP_HELPERS: tuple[str, ...] = ( - "as_bool", - "as_double", - "as_int", - "as_long", - "as_string", - "as_string_list", -) - - -def _format_enum(fmt: str) -> str: - return "Format.XML" if fmt.lower() == "xml" else "Format.JSON" - - -def _field_spec_literal(field: MetaData, owner: MetaData) -> str: - name = field.name - required = fm.is_required(field) - req = "True" if required else "False" - - # An enum ARRAY is checked BEFORE the scalar-enum branch below: it is baked as a - # string-list scalar slot (read via as_string_list), NOT a scalar enum_field — - # otherwise the runtime collapses the list to one stringified value (the cross-port - # "enum-before-isArray" ordering bug). The mirror type / extract_map helper already - # route arrays first; this keeps the baked schema spec consistent with them. Scoped - # to the enum case so NON-enum scalar arrays keep their real element FieldKind below - # (the self-contained baked path drops arrays as MALFORMED before kind is read, so - # the kind is inert either way — but emitting the true kind is correct + clearer). - if fm.is_array(field) and field.sub_type == fc.FIELD_SUBTYPE_ENUM: - return f'FieldSpec.scalar("{name}", FieldKind.STRING, {req})' - - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - values_lit = fm.string_list_literal(fm.enum_values(field)) - alias_lit = fm.properties_map_literal(field.attrs().get(fc.FIELD_ATTR_ENUM_ALIAS)) - # FR-011: resolve the three new enum args (field → object.value → "strip" for - # normalize). Keep the back-compat 4-arg form when nothing is set; otherwise - # emit the 7-arg form (..., coerce_default, normalize, default_value). - coerce_default = fm.coerce_default(field) - default_value = fm.default_value(field) - normalize = fm.resolve_normalize(field, owner) - if ( - coerce_default is None - and default_value is None - and normalize == fc.NORMALIZE_DEFAULT - ): - return f'FieldSpec.enum_field("{name}", {req}, {values_lit}, {alias_lit})' - cd_lit = "None" if coerce_default is None else fm.py_string_literal(coerce_default) - norm_lit = fm.py_string_literal(normalize) - dv_lit = "None" if default_value is None else fm.py_string_literal(default_value) - return ( - f'FieldSpec.enum_field("{name}", {req}, {values_lit}, {alias_lit}, ' - f"{cd_lit}, {norm_lit}, {dv_lit})" - ) - - if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: - # Self-contained path models a nested object as a plain (opaque) STRING scalar - # slot — it does not recurse. The runtime-delegating ``*_with_loader`` entry - # populates the real nested graph. (No trailing inline comment: this literal is - # joined into a ``[...]`` argument list, where a ``#`` comment would be illegal.) - return f'FieldSpec.scalar("{name}", FieldKind.STRING, {req})' - - kind = fm.scalar_kind(field.sub_type) or "STRING" - # @xmlText: a scalar field marked to receive its element's XML text content - # (JAXB @XmlValue / Jackson @JacksonXmlText / .NET [XmlText]). Mirrors the TS - # extract-schema-emitter textContentField branch. No effect for JSON. - if fm.xml_text(field): - return f'FieldSpec.text_content_field("{name}", FieldKind.{kind}, {req})' - return f'FieldSpec.scalar("{name}", FieldKind.{kind}, {req})' - - -def schema_literal(vo: MetaData, fmt: str, root_name: str) -> str: - """Emit ``ExtractSchema(Format.X, "rootName", [FieldSpec(...), …])``.""" - format_enum = _format_enum(fmt) - specs = [_field_spec_literal(f, vo) for f in fm.fields(vo)] - if not specs: - return f'ExtractSchema({format_enum}, "{root_name}", [])' - body = ", ".join(specs) - return f'ExtractSchema({format_enum}, "{root_name}", [{body}])' - - -def mirror_dataclass(vo: MetaData, record_name: str) -> list[str]: - """Emit the all-nullable mirror dataclass declaration (source lines).""" - base = ( - record_name[: -len("Extracted")] - if record_name.endswith("Extracted") - else record_name - ) - lines: list[str] = [ - "@dataclass(frozen=True, slots=True)", - f"class {record_name}:", - f' """Best-effort extracted twin of ``{base}`` — every field nullable', - ' (``None`` where the value was lost or malformed)."""', - ] - field_lines = [ - f" {f.name}: {fm.mirror_type(f)} = None" for f in fm.fields(vo) - ] - if field_lines: - lines.extend(field_lines) - else: - lines.append(" pass") - return lines - - -def mirror_initializer(vo: MetaData, record_name: str) -> str: - """Emit ``(field=as_string(d, "field"), …)``.""" - assigns = [f"{f.name}={fm.extract_map_call(f)}" for f in fm.fields(vo)] - return f"{record_name}({', '.join(assigns)})" - - -def extract_map_imports(vo: MetaData) -> list[str]: - """The sorted, deduped ``extract_map`` accessor names the mirror initializer needs.""" - used: set[str] = set() - for f in fm.fields(vo): - h = fm.extract_map_helper(f) - if h is not None: - used.add(h) - return [h for h in ALL_EXTRACT_MAP_HELPERS if h in used] +alias(__name__, "metaobjects.codegen.runtime.extract_schema_emitter") diff --git a/server/python/src/metaobjects/codegen/fr010_field_mapping.py b/server/python/src/metaobjects/codegen/fr010_field_mapping.py index ae2d7510b..6603103bb 100644 --- a/server/python/src/metaobjects/codegen/fr010_field_mapping.py +++ b/server/python/src/metaobjects/codegen/fr010_field_mapping.py @@ -1,228 +1,4 @@ -"""Shared field-kind mapping for the FR-010 codegen emitters (``extract_schema_emitter`` -+ ``output_format_spec_emitter``). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.fr010_field_mapping`.""" +from metaobjects.codegen._moved import alias -Maps a metadata field subtype onto the render engine's ``FieldKind`` member, the -idiomatic ``X | None`` Python type used by the extract mirror dataclass, and the -``extract_map.as_*`` accessor that reads it from the forgiving outcome map. - -Mirrors the C# ``Fr010FieldMapping`` and the Java ``ExtractSchemaEmitter`` ordering. -Bounded scope (parity with Java / Kotlin / C#): scalar / enum / scalar-array. Nested -object + array-of-enum are deferred (see ``render/extract/KNOWN_GAPS.md``). - -Python has a single ``int`` type, so the ``LONG`` and ``CURRENCY`` subtypes map to the -``LONG`` ``FieldKind`` (and ``extract_map.as_long`` — identical to ``as_int``), keeping -the cross-port ``FieldKind`` vocabulary intact even though the emitted accessor is the -same. The mirror annotation collapses ``INT``/``LONG`` to ``int`` (one numeric int type). -""" -from __future__ import annotations - -from collections.abc import Iterable - -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.meta_data import MetaData -from metaobjects.meta.template.template_constants import TEMPLATE_ATTR_XML_TEXT -from metaobjects.shared.base_types import TYPE_FIELD - - -def fields(vo: MetaData) -> list[MetaData]: - """The field children of a payload value-object, in declaration order - (effective children — own + inherited via ``extends``).""" - return [c for c in vo.children() if c.type == TYPE_FIELD] - - -def is_array(field: MetaData) -> bool: - """Effective array-ness (ADR-0039 resolving): the native ``is_array`` flag OR the - ``@isArray`` attr, resolved through the ``extends`` super chain so a concrete field - inheriting array-ness from an abstract parent is honored.""" - return field.resolved_is_array() - - -def is_required(field: MetaData) -> bool: - """``@required`` — accepts a bool ``True`` or the string ``"true"``.""" - v = field.attrs().get(fc.FIELD_ATTR_REQUIRED) - if v is True: - return True - return isinstance(v, str) and v.lower() == "true" - - -def xml_text(field: MetaData) -> bool: - """``@xmlText`` — the XML text-content extract marker (accepts a bool ``True`` or the - string ``"true"``). When set, codegen bakes a ``FieldSpec.text_content_field(...)``. - Mirrors the TS ``xmlText(field)`` helper.""" - v = field.attrs().get(TEMPLATE_ATTR_XML_TEXT) - if v is True: - return True - return isinstance(v, str) and v.lower() == "true" - - -def enum_values(field: MetaData) -> list[str]: - """The string members of an enum field's ``@values`` attr (empty when absent).""" - v = field.attrs().get(fc.FIELD_ATTR_VALUES) - if isinstance(v, (list, tuple)): - return [str(x) for x in v] - return [] - - -def _attr_string(node: MetaData, name: str) -> str | None: - """The RESOLVING string value of attr *name* (own + inherited via ``extends``), - or ``None``. - - ADR-0039 — effective read: a concrete enum field extending an abstract enum may - inherit ``@coerceDefault``/``@default``/``@normalize`` from the abstract parent, - so these resolve through the super chain (``get_meta_attr``), never own-only.""" - v = node.get_meta_attr(name) - return v if isinstance(v, str) else None - - -def coerce_default(field: MetaData) -> str | None: - """FR-011: the enum field's effective ``@coerceDefault`` member, or ``None``.""" - return _attr_string(field, fc.FIELD_ATTR_COERCE_DEFAULT) - - -def default_value(field: MetaData) -> str | None: - """FR-011: the enum field's effective ``@default`` absent-fill member, or ``None``.""" - return _attr_string(field, fc.FIELD_ATTR_DEFAULT) - - -def resolve_normalize(field: MetaData, owner: MetaData | None) -> str: - """FR-011: resolve the enum normalization mode — field-level ``@normalize``, else the - owning ``object.value``'s ``@normalize`` (the per-object default), else the global - default (``"strip"``). Mirrors the Java/Kotlin/C#/TS ``resolveNormalize``.""" - field_mode = _attr_string(field, fc.FIELD_ATTR_NORMALIZE) - if field_mode is not None: - return field_mode - if owner is not None: - owner_mode = _attr_string(owner, fc.FIELD_ATTR_NORMALIZE) - if owner_mode is not None: - return owner_mode - return fc.NORMALIZE_DEFAULT - - -def scalar_kind(sub_type: str) -> str | None: - """The render-engine ``FieldKind`` member name for a scalar field subtype, or - ``None`` if the subtype is non-scalar (enum / object).""" - if sub_type in ( - fc.FIELD_SUBTYPE_STRING, - fc.FIELD_SUBTYPE_DATE, - fc.FIELD_SUBTYPE_TIME, - fc.FIELD_SUBTYPE_TIMESTAMP, - # ADR-0036/0037 Wave 3: uri/inet are string scalars on the wire. - fc.FIELD_SUBTYPE_URI, - fc.FIELD_SUBTYPE_INET, - ): - return "STRING" - if sub_type == fc.FIELD_SUBTYPE_INT: - return "INT" - if sub_type in (fc.FIELD_SUBTYPE_LONG, fc.FIELD_SUBTYPE_CURRENCY): - return "LONG" - if sub_type in ( - fc.FIELD_SUBTYPE_DOUBLE, - fc.FIELD_SUBTYPE_FLOAT, - fc.FIELD_SUBTYPE_DECIMAL, - ): - return "DOUBLE" - if sub_type == fc.FIELD_SUBTYPE_BOOLEAN: - return "BOOLEAN" - return None - - -def mirror_type(field: MetaData) -> str: - """The ``X | None`` Python annotation for a field in the extract mirror dataclass. - - A extracted array can contain null elements where individual items were lost, so - the array type is ``list[str | None] | None`` (matches ``extract_map.as_string_list``). - """ - # Nested object (single OR array): the self-contained path defers to a typed null; - # the runtime-delegating path overrides this with the nested mirror type. Checked - # BEFORE the generic is_array branch so an array-of-objects is NOT mistyped as a - # string array. Mirrors the Java/TS fix. - if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: - return "object | None" # nested deferred (self-contained path) - if is_array(field): - return "list[str | None] | None" - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - return "str | None" # enum is string-backed - kind = scalar_kind(field.sub_type) - if kind in ("INT", "LONG"): - return "int | None" - if kind == "DOUBLE": - return "float | None" - if kind == "BOOLEAN": - return "bool | None" - return "str | None" - - -def extract_map_helper(field: MetaData) -> str | None: - """The ``extract_map`` accessor a field reads through, or ``None`` - (object/deferred fields read no helper). - - Single source of the field → accessor decision; ``extract_map_call`` builds - the ``as_*(d, "name")`` call on top of this.""" - # Nested object (single OR array) → no helper in the self-contained path. Checked - # BEFORE is_array so an array-of-objects is NOT read via as_string_list. Mirrors the - # Java/TS fix. - if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: - return None - if is_array(field): - return "as_string_list" - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - return "as_string" - return { - "INT": "as_int", - "LONG": "as_long", - "DOUBLE": "as_double", - "BOOLEAN": "as_bool", - }.get(scalar_kind(field.sub_type) or "", "as_string") - - -def extract_map_call(field: MetaData) -> str: - """The ``as_*(d, "name")`` call that reads this field from the forgiving map ``d``. - - The helpers are imported by name into the generated module (see - ``extract_schema_emitter.extract_map_imports``).""" - helper = extract_map_helper(field) - if helper is None: - # Nested object: the self-contained initializer leaves it None (the - # runtime-delegating path populates it). Bare ``None`` — no trailing inline - # comment: this expr is joined into a ``Name(field=..., …)`` call-args list. - return "None" - return f'{helper}(d, "{field.name}")' - - -def py_string_literal(value: str) -> str: - """A safe Python double-quoted string literal for embedding free text - (``@example`` / ``@instruction`` / ``@enumDoc`` values) into emitted source. - - Uses ``json.dumps`` — a JSON string is a valid Python string literal and escapes - backslashes / quotes / control chars correctly (NOT naive quoting).""" - import json - - return json.dumps(value, ensure_ascii=False) - - -def string_list_literal(values: Iterable[str]) -> str: - """A Python ``["a", "b"]`` list literal for enum members.""" - return "[" + ", ".join(py_string_literal(v) for v in values) + "]" - - -def properties_map_literal(attr: object) -> str: - """A Python ``{"k": "v", …}`` dict literal for a properties-shaped attr - (``@enumAlias`` / ``@enumDoc``), or ``"None"`` when absent/empty. - - Null values are dropped; keys are sorted to match the canonical-serializer - properties sort (deterministic output).""" - if not isinstance(attr, dict): - return "None" - entries = [ - (k, v) - for k, v in attr.items() - if v is not None - ] - if not entries: - return "None" - entries.sort(key=lambda kv: str(kv[0])) - parts = [ - f"{py_string_literal(str(k))}: {py_string_literal(str(v))}" - for k, v in entries - ] - return "{" + ", ".join(parts) + "}" +alias(__name__, "metaobjects.codegen.runtime.fr010_field_mapping") diff --git a/server/python/src/metaobjects/codegen/generator.py b/server/python/src/metaobjects/codegen/generator.py index d9a05ca38..eaf7ddc23 100644 --- a/server/python/src/metaobjects/codegen/generator.py +++ b/server/python/src/metaobjects/codegen/generator.py @@ -17,6 +17,26 @@ class EmittedFile: generated_by: str = "" # set by the runner from Generator.name +#: The docs surface the stock ``api-docs`` generator takes over: ``metaobjects docs`` runs a +#: generator carrying this value as ``docs_surface``, and ``gen`` skips it. +DOCS_SURFACE_API = "api" + + +#: The default api-surface subdirectory (the cross-port contract's ``api/python``). +DEFAULT_API_SUBDIR = "api/python" + + +@dataclass(frozen=True) +class DocsRunFacts: + """What a ``metaobjects docs`` run tells a docs generator beyond the model: the project + label shown in the pages, the subdirectory the pages land under, and the base URL the + model back-links federate to (``None`` = relative links).""" + + project: str = "project" + api_subdir: str = DEFAULT_API_SUBDIR + model_base_url: str | None = None + + @dataclass class GenContext: entities: list[MetaObject] @@ -24,6 +44,8 @@ class GenContext: matches: Callable[[MetaObject], bool] config: GenConfig warn: Callable[[str], None] + #: Set by ``metaobjects docs`` only; ``None`` in a ``gen`` run. + docs: DocsRunFacts | None = None class Generator(Protocol): diff --git a/server/python/src/metaobjects/codegen/generator_registry.py b/server/python/src/metaobjects/codegen/generator_registry.py index b8794ab0b..a8b1fb003 100644 --- a/server/python/src/metaobjects/codegen/generator_registry.py +++ b/server/python/src/metaobjects/codegen/generator_registry.py @@ -26,6 +26,7 @@ from metaobjects.codegen.generator import Generator from metaobjects.codegen.project_config import DEFAULT_REQUIREMENT_WITNESS_MODULE +from metaobjects.codegen.generators.api_docs_generator import api_docs_generator from metaobjects.codegen.generators.entity_model import entity_model from metaobjects.codegen.generators.extractor_generator import extractor_generator from metaobjects.codegen.generators.filter_allowlist_generator import ( @@ -157,6 +158,12 @@ def symbol(self) -> str | None: """The factory name an owned copy exports, or ``None``.""" return self.source.__name__ if self.source is not None else None + @property + def docs_surface(self) -> str | None: + """The docs surface this generator takes over (``metaobjects docs`` runs it, ``gen`` + skips it), or ``None`` for a generator ``gen`` runs.""" + return getattr(self.factory(GeneratorBuildContext()), "docs_surface", None) + def _template_primitive(_ctx: GeneratorBuildContext) -> Generator: """A no-op default for the ``template`` PRIMITIVE generator. @@ -195,7 +202,26 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: #: agree with each other (see ``runtime/shared_predicates.py``). _SHARED_PREDICATES: tuple[str, ...] = ("shared_predicates",) -#: Stable name -> GeneratorEntry. The 11 native generators whose manifest `ports` +#: The emitters a generator writes its output THROUGH, handed over with it: the helper modules +#: that return emitted text, which an owned copy therefore owns instead of calling the +#: packaged copy. Eject copies them next to the generator (``codegen/runtime/``) and the +#: generator imports them relatively. Naming and type-mapping tables, the model walks and the +#: generator base stay in the package as the engine. See ``tests/codegen/test_eject_alone.py``. +_FR010_FIELD_MAPPING = "fr010_field_mapping" +_M2M_CODEGEN = "m2m_codegen" +_FR019_SHARED_ENUM = "fr019_shared_enum" +_EXTRACT_DELEGATE = "extract_delegate_emitter" +_EXTRACT_SCHEMA = "extract_schema_emitter" +_OUTPUT_FORMAT_SPEC = "output_format_spec_emitter" +#: The api reference's page IR, builder, renderer and path math. +_API_DOCS_RUNTIME: tuple[str, ...] = ( + "api_docs_model", + "api_docs_builder", + "api_docs_renderer", + "api_docs_paths", +) + +#: Stable name -> GeneratorEntry. The 12 native generators whose manifest `ports` #: include `python` (ADR-0021 D3). Set equality, tier AND layer are conformance-tested #: against the manifest. GENERATOR_REGISTRY: dict[str, GeneratorEntry] = { @@ -211,7 +237,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: layer="model", factory=lambda _ctx: entity_model(), source=entity_model, - runtime=_SHARED_PREDICATES, + runtime=_SHARED_PREDICATES + (_FR019_SHARED_ENUM, _M2M_CODEGEN), ), "routes": GeneratorEntry( name="routes", @@ -224,7 +250,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: # generated package fails at import (ModuleNotFoundError). requires=("entity", "filter-allowlist"), source=router_generator, - runtime=_ROUTER_RUNTIME + _SHARED_PREDICATES, + runtime=_ROUTER_RUNTIME + _SHARED_PREDICATES + (_FR010_FIELD_MAPPING, _M2M_CODEGEN), ), "output-parser": GeneratorEntry( name="output-parser", @@ -234,6 +260,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: factory=lambda _ctx: output_parser_generator(), requires=("entity",), source=output_parser_generator, + runtime=(_EXTRACT_DELEGATE, _FR010_FIELD_MAPPING), ), "output-prompt": GeneratorEntry( name="output-prompt", @@ -242,6 +269,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: layer="capability", factory=lambda _ctx: output_prompt_generator(), source=output_prompt_generator, + runtime=(_OUTPUT_FORMAT_SPEC, _FR010_FIELD_MAPPING), ), "render-helper": GeneratorEntry( name="render-helper", @@ -260,7 +288,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: factory=lambda _ctx: extractor_generator(), requires=("entity",), source=extractor_generator, - runtime=_SHARED_PREDICATES, + runtime=_SHARED_PREDICATES + (_EXTRACT_DELEGATE, _FR010_FIELD_MAPPING), ), "requirement-tests": GeneratorEntry( name="requirement-tests", @@ -284,7 +312,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: layer="api", factory=lambda _ctx: filter_allowlist_generator(), source=filter_allowlist_generator, - runtime=_SHARED_PREDICATES, + runtime=_SHARED_PREDICATES + (_M2M_CODEGEN,), ), "names": GeneratorEntry( name="names", @@ -301,6 +329,19 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: layer="capability", factory=lambda _ctx: trace_helper_generator(), source=trace_helper_generator, + runtime=(_EXTRACT_SCHEMA, _FR010_FIELD_MAPPING), + ), + "api-docs": GeneratorEntry( + name="api-docs", + description=( + "Per-entity/template SDK API reference (generated-code API; human + agent forms). " + "Run by `metaobjects docs`, not `gen`." + ), + tier="native", + layer="docs", + factory=lambda _ctx: api_docs_generator(), + source=api_docs_generator, + runtime=_API_DOCS_RUNTIME + _SHARED_PREDICATES + (_M2M_CODEGEN,), ), } diff --git a/server/python/src/metaobjects/codegen/generators/api_docs_generator.py b/server/python/src/metaobjects/codegen/generators/api_docs_generator.py new file mode 100644 index 000000000..a3c86fae1 --- /dev/null +++ b/server/python/src/metaobjects/codegen/generators/api_docs_generator.py @@ -0,0 +1,68 @@ +"""``api-docs`` — the Python SDK api reference (the ``api/python`` surface of the cross-port +SDK-docs contract): one page per entity, value object, projection, served report and +template, an index, and the agent-facing ``AGENT-API.md`` page. + +Run by ``metaobjects docs``, not by ``metaobjects gen``: it carries ``docs_surface``, which +``gen`` skips. Its files land under the docs output directory, prefixed with the api +subdirectory the run names (``ctx.docs``). + +The page IR, the builder that decides which units and symbols get a page, the renderer and +the page-path math live beside this module in ``codegen/runtime/`` and are handed over with +it: ``metaobjects eject api-docs`` copies all of them, so the copy owns every line that +writes a page. What each page says a generator emits for comes from +``..runtime.shared_predicates`` — the module the router and filter-allowlist generators read +their gates from. An owned ``api-docs`` and an owned ``routes`` therefore read ONE owned copy +of those predicates, and the reference describes the code the project's own generators emit. +The naming seam (:mod:`metaobjects.apidocs.naming`) is the engine: the same names the +generators emit under. +""" +from __future__ import annotations + +from metaobjects.codegen.generator import ( + DOCS_SURFACE_API, + DocsRunFacts, + EmittedFile, + GenContext, + Generator, +) + +from ..runtime.api_docs_builder import PythonApiModelBuilder +from ..runtime.api_docs_paths import Layout, doc_page_output_path, model_cross_href +from ..runtime.api_docs_renderer import PythonApiDocsRenderer + + +class ApiDocsGenerator: + """Writes the api reference pages for the run's loaded model.""" + + name = "api-docs" + #: Which docs surface this generator takes over (see ``metaobjects docs``). + docs_surface = DOCS_SURFACE_API + + def generate(self, ctx: GenContext) -> list[EmittedFile]: + if ctx.loaded_root is None: + return [] + # A docs run supplies the facts. A caller that runs this generator directly, as a + # registry-wide probe does, gets the defaults and a neutral project label. + facts = ctx.docs or DocsRunFacts() + model = PythonApiModelBuilder().build(ctx.loaded_root, facts.project) + renderer = PythonApiDocsRenderer() + layout = Layout.PACKAGE + + def page(path: str, content: str) -> EmittedFile: + return EmittedFile(path=f"{facts.api_subdir}/{path}", content=content) + + files: list[EmittedFile] = [] + for unit in model.units: + page_path = doc_page_output_path(layout, unit.package, unit.node) + href = model_cross_href( + f"{facts.api_subdir}/{page_path}", page_path, facts.model_base_url + ) + files.append(page(page_path, renderer.render_unit_page(unit, href))) + files.append(page("README.md", renderer.render_index(model, layout))) + files.append(page("AGENT-API.md", renderer.render_agent_api(model))) + return files + + +def api_docs_generator() -> Generator: + """Generator factory: the api reference ``metaobjects docs`` writes.""" + return ApiDocsGenerator() diff --git a/server/python/src/metaobjects/codegen/generators/entity_model.py b/server/python/src/metaobjects/codegen/generators/entity_model.py index 476c0fc7c..367139cef 100644 --- a/server/python/src/metaobjects/codegen/generators/entity_model.py +++ b/server/python/src/metaobjects/codegen/generators/entity_model.py @@ -43,11 +43,6 @@ from metaobjects.codegen.type_map import field_is_array, py_type_for from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator, per_entity -from metaobjects.codegen.generators import fr019_shared_enum as fr019 -from metaobjects.codegen.generators.m2m_codegen import ( - build_object_index, - resolve_m2m_descriptors, -) from metaobjects.codegen.generators.tph_plan import tph_subtype_binding from metaobjects.codegen.value_objects import ( model_class_name, @@ -55,6 +50,8 @@ object_ref_class_name, ) +from ..runtime import fr019_shared_enum as fr019 +from ..runtime.m2m_codegen import build_object_index, resolve_m2m_descriptors from ..runtime.shared_predicates import has_literal_default, is_sql_expr_default diff --git a/server/python/src/metaobjects/codegen/generators/extractor_generator.py b/server/python/src/metaobjects/codegen/generators/extractor_generator.py index 2659ed6bc..792e57105 100644 --- a/server/python/src/metaobjects/codegen/generators/extractor_generator.py +++ b/server/python/src/metaobjects/codegen/generators/extractor_generator.py @@ -38,8 +38,6 @@ from collections.abc import Callable from metaobjects.apidocs.naming import snake_case as _snake_case -from metaobjects.codegen import fr010_field_mapping as fm -from metaobjects.codegen import extract_delegate_emitter as rde from metaobjects.codegen.constants import generated_header from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator @@ -52,6 +50,8 @@ from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.meta.meta_data import MetaData +from ..runtime import extract_delegate_emitter as rde +from ..runtime import fr010_field_mapping as fm from ..runtime.shared_predicates import has_literal_default _GENERATOR_NAME = "extractor-generator" diff --git a/server/python/src/metaobjects/codegen/generators/filter_allowlist_generator.py b/server/python/src/metaobjects/codegen/generators/filter_allowlist_generator.py index 6b67406ab..ca20de679 100644 --- a/server/python/src/metaobjects/codegen/generators/filter_allowlist_generator.py +++ b/server/python/src/metaobjects/codegen/generators/filter_allowlist_generator.py @@ -36,13 +36,13 @@ from metaobjects.codegen.constants import generated_header from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator, per_entity -from metaobjects.codegen.generators.m2m_codegen import build_object_index from metaobjects.codegen.generators.tph_plan import tph_plan_for from metaobjects.codegen.instance_artifacts import is_sourceless_entity from metaobjects.meta.core.field import field_constants as fc from metaobjects.meta.core.field.meta_field import MetaField from metaobjects.meta.core.object.meta_object import MetaObject +from ..runtime.m2m_codegen import build_object_index from ..runtime.shared_predicates import emits_router diff --git a/server/python/src/metaobjects/codegen/generators/fr019_shared_enum.py b/server/python/src/metaobjects/codegen/generators/fr019_shared_enum.py index 8dd24b7b9..b6280c9bf 100644 --- a/server/python/src/metaobjects/codegen/generators/fr019_shared_enum.py +++ b/server/python/src/metaobjects/codegen/generators/fr019_shared_enum.py @@ -1,154 +1,4 @@ -"""FR-019 — shared + externally-provided enum resolution (Python port of the TS reference -``enum-shared.ts`` and the C#/Java/Kotlin ``Fr019SharedEnum``). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.fr019_shared_enum`.""" +from metaobjects.codegen._moved import alias -A reusable enum is a ROOT/package-level abstract ``field.enum`` (a sibling of ``object.entity``) -that concrete entity fields ``extends``. Per ADR-0026 such a declaration is materialized ONCE per -port as a module-level ``class (str, Enum)`` and referenced, instead of redeclaring the -members inline (a per-entity ``Literal[...]``) in every consumer. - -Two cases on the abstract declaration: - -* non-``@provided`` → metaobjects MATERIALIZES the type once (a shared ``enums.py`` module), and - consuming fields reference it (``from .enums import ``). -* ``@provided: true`` → metaobjects emits NOTHING for the type; consuming fields import an existing - type from a per-port-configured module (``GenConfig.provided_enum_packages`` keyed by the enum's - declaring metadata package, then the ``provided_enum_namespace`` fallback). The module never lives - in metadata (ADR-0001); a missing config for a referenced provided enum is a codegen-time error - naming the enum + its package. - -The common inline case (a concrete ``field.enum`` with own ``@values``, no root-extends) is -UNCHANGED — it stays a per-entity ``Literal[...]`` (byte-identical default). -""" -from __future__ import annotations - -import warnings -from dataclasses import dataclass - -from metaobjects.meta.core.field.meta_field import ( - MetaField, - resolve_shared_enum_declaration, -) -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.meta_data import MetaData -from metaobjects.shared.separators import PACKAGE_SEP -from metaobjects.codegen import type_map -from metaobjects.codegen.config import GenConfig - - -@dataclass(frozen=True) -class SharedEnum: - """A shared (package-level abstract) enum declaration codegen reasons about.""" - - name: str #: materialized type name — the decl's PascalCase short name (cross-port id) - values: list[str] #: member symbols (the @values SSOT), verbatim - provided: bool #: True when the decl carries @provided: true (reference, don't materialize) - meta_package: str #: declaring metadata package in "::" form (the provided-config key) - - -def _pascal(name: str) -> str: - """``priority`` → ``Priority`` (leading char only; no snake-splitting — cross-port rule).""" - return name[:1].upper() + name[1:] if name else name - - -def resolve_shared_enum_decl(field: MetaField) -> MetaData | None: - """Deprecated name of - :func:`metaobjects.meta.core.field.meta_field.resolve_shared_enum_declaration`, - kept so a generator ejected before the resolver moved to the metadata layer - still imports. Same answer, including through a multi-hop ``extends``.""" - warnings.warn( - "resolve_shared_enum_decl is deprecated; call " - "metaobjects.meta.core.field.meta_field.resolve_shared_enum_declaration", - DeprecationWarning, - stacklevel=2, - ) - return resolve_shared_enum_declaration(field) - - -def is_provided(decl: MetaData) -> bool: - """``@provided`` truth of an enum DECLARATION. - - ADR-0039 sanctioned own (Python naming inversion: ``attr()`` is the OWN read, - ``get_meta_attr()`` resolves). ``@provided`` is a declaration-layer provenance - marker — "THIS type is supplied by hand-written/third-party code", like - ``is_abstract`` — and does not flow down an ``extends`` chain. - - This is only ever called on the resolved declaration (see - ``shared_enum_for_field``), never on the consuming field, so own and resolving - agree for a plain ``field extends @provided decl``. They diverge on a CHAINED - declaration (root abstract ``B extends`` root abstract ``@provided A``): a - resolving read reports B provided and emits a reference to a hand-written ``B`` - the adopter never declared, instead of materializing B. Matches the JVM ports. - """ - return decl.attr(fc.FIELD_ATTR_PROVIDED) is True - - -def _meta_package_of(decl: MetaData) -> str: - """The decl's metadata package in ``::`` form (its resolution key minus the trailing name).""" - key = decl.resolution_key() - idx = key.rfind(PACKAGE_SEP) - return key[:idx] if idx >= 0 else "" - - -def shared_enum_for_field(field: MetaField) -> SharedEnum | None: - """The shared-enum descriptor *field* resolves to, or ``None`` for inline enums. - - The member set is read off the DECLARATION *decl* (a resolving ``attrs()`` - read, never off *field*) so materialization cannot depend on which consuming - field a generator happens to visit first — every field resolving to the same - shared declaration must describe that declaration identically.""" - decl = resolve_shared_enum_declaration(field) - if decl is None: - return None - values = type_map.effective_enum_values(decl) - if not values: - return None - return SharedEnum( - name=_pascal(decl.name), - values=values, - provided=is_provided(decl), - meta_package=_meta_package_of(decl), - ) - - -def collect_shared_enums(entities: list) -> list[SharedEnum]: - """All shared (package-level abstract) enum decls CONSUMED by at least one entity field, - keyed by materialized type name, sorted by name (deterministic). A decl nobody extends is - not materialized (no dangling type).""" - out: dict[str, SharedEnum] = {} - for entity in entities: - # ADR-0039 — RESOLVING (fields()): mirrors the TS collectSharedEnums - # (``entity.fields()``); an inherited field consuming a shared enum counts. - for f in entity.fields(): - shared = shared_enum_for_field(f) - if shared is None: - continue - out.setdefault(shared.name, shared) - return sorted(out.values(), key=lambda e: e.name) - - -def materialized_shared_enums(entities: list) -> list[SharedEnum]: - """The shared enums metaobjects MATERIALIZES (non-``@provided``, consumed).""" - return [e for e in collect_shared_enums(entities) if not e.provided] - - -def enum_type_and_import(shared: SharedEnum, config: GenConfig) -> tuple[str, str]: - """The ``(type_name, import_line)`` a consuming field emits for a shared/provided enum: - - * materialized → ``("", "from .enums import ")`` (the shared module). - * ``@provided`` → ``("", "from import ")``, *module* resolved from - ``config.provided_enum_packages`` (keyed by the enum's metadata package) then the single - ``config.provided_enum_namespace`` fallback. A missing config is a codegen-time ``ValueError`` - naming the enum + its package (ADR-0026 / ADR-0001).""" - name = shared.name - if not shared.provided: - return name, f"from .enums import {name}" - module = config.provided_enum_packages.get(shared.meta_package) or config.provided_enum_namespace - if not module: - raise ValueError( - f'provided enum "{name}" (declared in package "{shared.meta_package}") is marked ' - f"@provided but no Python import module is configured to reference it from. Map its " - f'package in GenConfig.provided_enum_packages["{shared.meta_package}"] = ' - f'"your_app.enums", or set the single provided_enum_namespace fallback, so the ' - f'generated code can import "{name}".' - ) - return name, f"from {module} import {name}" +alias(__name__, "metaobjects.codegen.runtime.fr019_shared_enum") diff --git a/server/python/src/metaobjects/codegen/generators/m2m_codegen.py b/server/python/src/metaobjects/codegen/generators/m2m_codegen.py index 813794dbb..c1cc0bcb5 100644 --- a/server/python/src/metaobjects/codegen/generators/m2m_codegen.py +++ b/server/python/src/metaobjects/codegen/generators/m2m_codegen.py @@ -1,240 +1,4 @@ -"""Shared M:N (many-to-many) codegen descriptor — the build-time resolution of a -source entity's ``relationship.* @cardinality:"many" + @through`` children into -the concrete coordinates each Python generator needs (Pydantic nested collection, -FastAPI traversal route, and the physical junction/target column names the -repository seam joins on). +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.m2m_codegen`.""" +from metaobjects.codegen._moved import alias -This mirrors the TS reference ``relation-resolver.ts`` (Unit 10): codegen derives -the descriptor at build time from the junction entity's two ``identity.reference`` -children (the cross-port SSOT for FK direction) via the shared -``derive_m2m_fields`` helper — the relationship never restates its FK columns. - -The descriptor carries: - * ``relation_name`` — the navigation member name (route segment + Pydantic field). - * ``target_entity`` — the related entity's bare name (Pydantic element type). - * ``source_plural`` / ``target_plural`` — pluralized entity names for the - REST contract: the source URL segment is the ENTITY name pluralized - (``Person`` → ``persons``), NOT the physical ``@table`` (cross-port grammar). - * ``source_column`` / ``target_column`` — the PHYSICAL junction FK columns - (e.g. ``follower_id`` / ``followee_id``); the repository seam joins on these. - * ``junction_table`` / ``target_table`` / ``target_pk_column`` — physical names - for the join. - * ``symmetric`` — selects the union-on-read traversal. -""" -from __future__ import annotations - -from dataclasses import dataclass - -from metaobjects.apidocs.naming import route_path -from metaobjects.source_resolution import resolve_table_name -from metaobjects.naming import ( - DEFAULT_COLUMN_NAMING, - apply_column_naming_strategy, - resolve_column_name, -) -from metaobjects.meta.core.field.meta_field import MetaField -from metaobjects.meta.meta_data import MetaData -from metaobjects.meta.core.identity.identity_constants import ( - IDENTITY_ATTR_FIELDS, - IDENTITY_SUBTYPE_PRIMARY, -) -from metaobjects.meta.core.object.meta_object import MetaObject -from metaobjects.meta.core.object.object_constants import OBJECT_ATTR_DISCRIMINATOR_VALUE -from metaobjects.codegen.generators.tph_plan import is_tph_subtype -from metaobjects.meta.core.relationship.derive_m2m_fields import ( - M2MDerivationError, - derive_m2m_fields, -) -from metaobjects.meta.core.relationship.meta_relationship import MetaRelationship -from metaobjects.meta.core.relationship.relationship_constants import CARDINALITY_MANY -from metaobjects.shared.base_types import TYPE_IDENTITY -from metaobjects.shared.separators import PACKAGE_SEP - - -@dataclass(frozen=True) -class M2mDescriptor: - """Build-time coordinates of one M:N navigation on a source entity.""" - - relation_name: str - target_entity: str - source_plural: str - target_plural: str - junction_table: str - target_table: str - source_column: str - target_column: str - target_pk_column: str - symmetric: bool - #: The target's OWN ``@discriminatorValue`` when ``@objectRef`` resolves to a - #: concrete TPH subtype (FW-8 follow-up; mirrors the TS reference's - #: ``targetDiscriminator``) — ``None`` for the overwhelming common case (a - #: vanilla, non-TPH target). The target's rows physically live in its - #: discriminator base's shared table, so a route generator threads this value - #: into the ``find_related_`` seam as an extra ``target_subtype`` - #: parameter, structurally handing the consumer's join what it needs to - #: filter to just that subtype rather than every row of the shared table. - #: Defaulted so existing positional construction (if any) stays valid. - target_discriminator: str | None = None - - -def _strip_package(name: str) -> str: - idx = name.rfind(PACKAGE_SEP) - return name[idx + len(PACKAGE_SEP):] if idx >= 0 else name - - -def _physical_table(entity: MetaObject, *, relation_context: str) -> str: - """The entity's physical SQL table name, via the shared - :func:`~metaobjects.source_resolution.resolve_table_name` resolver. - - Raises :class:`M2MDerivationError` — naming *entity* and *relation_context* - — when it declares (and inherits) no primary source at all. #248: DB - participation derives from a declared source, never from the object - subtype, so ``resolve_table_name`` returning ``None`` here means the - junction/target has no physical table to join through at all; a M:N - relationship's whole point is a real SQL join, so that is a metadata - authoring error, not a legitimate "doesn't participate in persistence" - shape. Coercing to the bare entity name — the pre-existing behaviour this - replaces — fabricated a table that was never created, precisely the defect - class the C# port's EF binding had to fix this same session (a - ``[Table(...)]`` attribute naming a class that was never generated).""" - table = resolve_table_name(entity) - if table is None: - raise M2MDerivationError( - f"{relation_context}: entity \"{entity.name}\" declares no primary " - f"source (source.rdb) — a M:N join needs a physical table on both " - f"the junction and the target." - ) - return table - - -def _column_of( - field: MetaField | None, fallback: str, strategy: str = DEFAULT_COLUMN_NAMING -) -> str: - """Physical column name for a junction FK. Delegates to the shared - :func:`metaobjects.naming.resolve_column_name` — the SAME resolver the runtime uses, - so generated junction SQL and runtime SQL cannot disagree about a column name. - - *fallback* covers the case where the field could not be resolved at all; it is a - bare metadata name, so it goes through the strategy too.""" - if field is None: - return apply_column_naming_strategy(fallback, strategy) - return resolve_column_name(field, strategy) - - -def _field_named(entity: MetaObject, field_name: str) -> MetaField | None: - for f in entity.fields(): - if f.name == field_name: - return f - return None - - -def pk_field_name(entity: MetaObject) -> str: - """The single PK field name (``identity.primary @fields``), default ``id``.""" - for c in entity.children(): - if c.type == TYPE_IDENTITY and c.sub_type == IDENTITY_SUBTYPE_PRIMARY: - fields = c.get_meta_attr(IDENTITY_ATTR_FIELDS) # ADR-0039: resolving (identity attr) - if isinstance(fields, (list, tuple)) and fields: - first = fields[0] - if isinstance(first, str): - return first - if isinstance(fields, str) and fields: - return fields.split(",")[0].strip() or "id" - return "id" - - -def m2m_relationships(entity: MetaObject) -> list[MetaRelationship]: - """The entity's ``relationship.* @cardinality:"many" + @through`` children - (declaration order). 1:N relationships (no ``@through``) are excluded. - - ADR-0039 — RESOLVING (children()): mirrors the TS ``relationships()`` (built on - the effective ``children()``); a relationship inherited via ``extends`` counts. - """ - out: list[MetaRelationship] = [] - for c in entity.children(): - if not isinstance(c, MetaRelationship): - continue - if c.cardinality() != CARDINALITY_MANY: - continue - if c.through() is None: - continue - out.append(c) - return out - - -def resolve_m2m_descriptors( - entity: MetaObject, - object_index: dict[str, MetaObject], - column_naming: str = DEFAULT_COLUMN_NAMING, -) -> list[M2mDescriptor]: - """Resolve every M:N navigation on *entity* into a build-time descriptor. - - *object_index* is a bare-name → entity map of the loaded model. A relationship - that fails derivation (missing/malformed junction, ambiguous self-join) raises - :class:`M2MDerivationError` — the loader validates this; reaching it here is a - metadata error, surfaced loudly (never silently skipped). - """ - descriptors: list[M2mDescriptor] = [] - # The source is *entity* for every relation, so its route segment is - # loop-invariant; only the target varies per relation. - source_plural = route_path(entity.name) - for rel in m2m_relationships(entity): - target_name = _strip_package(rel.object_ref() or "") - junction_name = _strip_package(rel.through() or "") - target = object_index.get(target_name) - junction = object_index.get(junction_name) - if target is None or junction is None: - raise M2MDerivationError( - f'M:N relationship "{entity.name}.{rel.name}": @objectRef ' - f'"{target_name}" / @through "{junction_name}" must resolve to ' - f"loaded entities" - ) - - # Widen to dict[str, MetaData] for the shared derivation helper (dict is - # invariant; MetaObject is a MetaData subtype). - meta_index: dict[str, MetaData] = dict(object_index) - fields = derive_m2m_fields(rel, entity, meta_index) - source_fk = _field_named(junction, fields.source_field) - target_fk = _field_named(junction, fields.target_field) - # FW-8 follow-up: the target's rows live in its discriminator base's shared - # table when @objectRef resolves to a concrete TPH subtype — @discriminatorValue - # is the subtype's OWN value (ADR-0039 sanctioned own, same idiom as tph_plan.py). - target_discriminator = ( - target.attr(OBJECT_ATTR_DISCRIMINATOR_VALUE) if is_tph_subtype(target) else None - ) - - descriptors.append( - M2mDescriptor( - relation_name=rel.name, - target_entity=target.name, - source_plural=source_plural, - target_plural=route_path(target.name), - junction_table=_physical_table( - junction, - relation_context=( - f'M:N relationship "{entity.name}.{rel.name}" @through junction' - ), - ), - target_table=_physical_table( - target, - relation_context=( - f'M:N relationship "{entity.name}.{rel.name}" @objectRef target' - ), - ), - source_column=_column_of(source_fk, fields.source_field, column_naming), - target_column=_column_of(target_fk, fields.target_field, column_naming), - target_pk_column=_column_of( - _field_named(target, pk_field_name(target)), - pk_field_name(target), - column_naming, - ), - symmetric=rel.symmetric(), - target_discriminator=target_discriminator, - ) - ) - return descriptors - - -def build_object_index(entities: list[MetaObject]) -> dict[str, MetaObject]: - """Bare-name → entity map (the codegen resolution surface, mirroring the - runtime resolver's ``object_index``).""" - return {e.name: e for e in entities} +alias(__name__, "metaobjects.codegen.runtime.m2m_codegen") diff --git a/server/python/src/metaobjects/codegen/generators/output_parser_generator.py b/server/python/src/metaobjects/codegen/generators/output_parser_generator.py index 00ee12a55..4aee3894e 100644 --- a/server/python/src/metaobjects/codegen/generators/output_parser_generator.py +++ b/server/python/src/metaobjects/codegen/generators/output_parser_generator.py @@ -31,7 +31,6 @@ from collections.abc import Callable from metaobjects.apidocs.naming import snake_case as _snake_case -from metaobjects.codegen import extract_delegate_emitter as rde from metaobjects.codegen.constants import generated_header from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator @@ -44,6 +43,8 @@ from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.meta.meta_data import MetaData +from ..runtime import extract_delegate_emitter as rde + _GENERATOR_NAME = "output-parser-generator" diff --git a/server/python/src/metaobjects/codegen/generators/output_prompt_generator.py b/server/python/src/metaobjects/codegen/generators/output_prompt_generator.py index a7f615d68..002ef037d 100644 --- a/server/python/src/metaobjects/codegen/generators/output_prompt_generator.py +++ b/server/python/src/metaobjects/codegen/generators/output_prompt_generator.py @@ -24,7 +24,6 @@ from collections.abc import Callable from metaobjects.apidocs.naming import snake_case as _snake_case -from metaobjects.codegen import output_format_spec_emitter as ofs from metaobjects.codegen.constants import generated_header from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator @@ -36,6 +35,8 @@ from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.meta.meta_data import MetaData +from ..runtime import output_format_spec_emitter as ofs + _GENERATOR_NAME = "output-prompt-generator" diff --git a/server/python/src/metaobjects/codegen/generators/router_generator.py b/server/python/src/metaobjects/codegen/generators/router_generator.py index e25c7a348..d2ada70d8 100644 --- a/server/python/src/metaobjects/codegen/generators/router_generator.py +++ b/server/python/src/metaobjects/codegen/generators/router_generator.py @@ -46,14 +46,7 @@ from metaobjects.apidocs.naming import snake_case as _snake_case from metaobjects.codegen.constants import GENERATED_MARKER, generated_header from metaobjects.codegen.format import ruff_format -from metaobjects.codegen.fr010_field_mapping import is_required from metaobjects.codegen.generator import EmittedFile, GenContext, Generator, per_entity -from metaobjects.codegen.generators.m2m_codegen import ( - M2mDescriptor, - build_object_index, - pk_field_name, - resolve_m2m_descriptors, -) from metaobjects.codegen.generators.tph_plan import TphPlan, is_tph_subtype, tph_plan_for from metaobjects.codegen.instance_artifacts import has_item_route from metaobjects.source_resolution import primary_rdb_source @@ -81,7 +74,14 @@ from metaobjects.shared.base_types import TYPE_IDENTITY from metaobjects.shared.separators import PACKAGE_SEP -from ..runtime.shared_predicates import READ_ONLY_ROUTED_KINDS, emits_router +from ..runtime.fr010_field_mapping import is_required +from ..runtime.m2m_codegen import ( + M2mDescriptor, + build_object_index, + pk_field_name, + resolve_m2m_descriptors, +) +from ..runtime.shared_predicates import emits_router, is_read_only_routed #: Where the emitted router imports its helper runtime (``filter_parser``, #: ``constraint_errors``) from. ``False`` — the packaged generator — imports the copy @@ -412,19 +412,6 @@ def reverse_fks_for(entity: MetaObject) -> list[ReverseFk]: return out -def is_read_only_routed(entity: MetaObject) -> bool: - """Whether :meth:`RouterGenerator.render_router` chooses the READ-ONLY router for - this object (a view / materialized-view source that is not write-through): GET list, - GET by id when it has an item route, and the 405 refusals, with no write surface. - - THE predicate behind that choice, shared with the API-docs builder so a page can never - present write verbs for an object whose router refuses them.""" - if not emits_router(entity) or entity.is_write_through(): - return False - src = primary_rdb_source(entity) - return src is not None and src.effective_kind() in READ_ONLY_ROUTED_KINDS - - def _sort_type_and_tables_lines(sort_field_nodes: list[MetaField]) -> list[str]: """The `_SortClause` model + the two sort lookup tables. diff --git a/server/python/src/metaobjects/codegen/generators/trace_helper_generator.py b/server/python/src/metaobjects/codegen/generators/trace_helper_generator.py index 453530943..500e47a2f 100644 --- a/server/python/src/metaobjects/codegen/generators/trace_helper_generator.py +++ b/server/python/src/metaobjects/codegen/generators/trace_helper_generator.py @@ -42,7 +42,6 @@ from collections.abc import Callable -from metaobjects.codegen import extract_schema_emitter as rse from metaobjects.codegen.constants import generated_header from metaobjects.codegen.format import ruff_format from metaobjects.codegen.generator import EmittedFile, GenContext, Generator @@ -56,6 +55,8 @@ from metaobjects.shared.base_types import TYPE_TEMPLATE from metaobjects.library.library_sources import LIBRARY_MANIFESTS +from ..runtime import extract_schema_emitter as rse + _GENERATOR_NAME = "trace-helper" def _anchor_fqns() -> frozenset[str]: diff --git a/server/python/src/metaobjects/codegen/metaobjects-config.schema.json b/server/python/src/metaobjects/codegen/metaobjects-config.schema.json index dfd8aa68a..a2db91104 100644 --- a/server/python/src/metaobjects/codegen/metaobjects-config.schema.json +++ b/server/python/src/metaobjects/codegen/metaobjects-config.schema.json @@ -55,6 +55,18 @@ } } }, + "docs": { + "type": "object", + "additionalProperties": false, + "description": "Options of `metaobjects docs`.", + "properties": { + "generators": { + "type": "array", + "items": { "type": "string", "pattern": "^[^:\\s]+:[^:\\s]+$" }, + "description": "Owned docs generators as 'module:symbol' (`metaobjects eject api-docs` prints the entry), resolved relative to this config file's directory. `metaobjects docs` runs one in place of the packaged generator for the surface it takes over; `gen` never runs it." + } + } + }, "targets": { "type": "object", "minProperties": 1, diff --git a/server/python/src/metaobjects/codegen/output_format_spec_emitter.py b/server/python/src/metaobjects/codegen/output_format_spec_emitter.py index cbd018d65..6c389b56c 100644 --- a/server/python/src/metaobjects/codegen/output_format_spec_emitter.py +++ b/server/python/src/metaobjects/codegen/output_format_spec_emitter.py @@ -1,83 +1,4 @@ -"""Turns a response value-object + its ``template.prompt`` node into a Python source -literal for an :class:`~metaobjects.render.OutputFormatSpec` — the artifact-1 prompt -descriptor used by the FR-010 response-format codegen. +"""Deprecated: moved to :mod:`metaobjects.codegen.runtime.output_format_spec_emitter`.""" +from metaobjects.codegen._moved import alias -Emits ``OutputFormatSpec(Format.X, "rootName", PromptStyle.X, [PromptField(...), …])``. -Mirrors the C# / Java ``OutputFormatSpecEmitter`` adapted to Python (keyword args for -the long ``PromptField`` ctor to keep the emitted source readable + order-robust). -Bounded scope: scalar / enum. Nested object → ``FieldKind.OBJECT`` placeholder. -""" -from __future__ import annotations - -from metaobjects.codegen import fr010_field_mapping as fm -from metaobjects.codegen.generators.find_inbound import is_xml, response_format_of -from metaobjects.meta.core.field import field_constants as fc -from metaobjects.meta.meta_data import MetaData -from metaobjects.meta.template import template_constants as tc - - -def _format_enum(template: MetaData) -> str: - """ADR-0053 — the fragment describes the REPLY, so its syntax is ``@responseFormat``, - NOT ``@format``, which is the syntax of the rendered prompt BODY. Reading ``@format`` - here typed the instruction "produce your answer like this" off the format of the - QUESTION.""" - return "Format.XML" if is_xml(response_format_of(template)) else "Format.JSON" - - -def _prompt_style_enum(template: MetaData) -> str: - style = template.get_meta_attr(tc.TEMPLATE_ATTR_PROMPT_STYLE) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) - if style == tc.PROMPT_STYLE_INLINE: - return "PromptStyle.INLINE" - if style == tc.PROMPT_STYLE_EXAMPLE_ONLY: - return "PromptStyle.EXAMPLE_ONLY" - return "PromptStyle.GUIDE" - - -def _opt_string_attr(field: MetaData, attr_name: str) -> str: - v = field.attrs().get(attr_name) - return fm.py_string_literal(v) if isinstance(v, str) else "None" - - -def _prompt_field_literal(field: MetaData) -> str: - name = field.name - req = "True" if fm.is_required(field) else "False" - array = "True" if fm.is_array(field) else "False" - - if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: - # FR-010: nested prompt deferred — emit a FieldKind.OBJECT placeholder (nested - # defaults to None on PromptField). The note lives HERE, not in the returned - # literal: spec_literal joins every field onto ONE line, and a Python '#' inline - # comment would swallow the rest of that line — including the closing `])` — so a - # nested-object payload emitted invalid Python (unterminated list) and crashed - # `gen` + `verify --codegen`. The other ports use `/* */` block comments, which - # are inline-safe; Python has none, so the literal must carry no comment. - return f'PromptField("{name}", FieldKind.OBJECT, {req}, array={array})' - - example = _opt_string_attr(field, fc.FIELD_ATTR_EXAMPLE) - instruction = _opt_string_attr(field, fc.FIELD_ATTR_INSTRUCTION) - - if field.sub_type == fc.FIELD_SUBTYPE_ENUM: - values_lit = fm.string_list_literal(fm.enum_values(field)) - enum_doc_lit = fm.properties_map_literal(field.attrs().get(fc.FIELD_ATTR_ENUM_DOC)) - return ( - f'PromptField("{name}", FieldKind.ENUM, {req}, array={array}, ' - f"enum_values={values_lit}, enum_doc={enum_doc_lit}, " - f"example={example}, instruction={instruction})" - ) - - kind = fm.scalar_kind(field.sub_type) or "STRING" - return ( - f'PromptField("{name}", FieldKind.{kind}, {req}, array={array}, ' - f"example={example}, instruction={instruction})" - ) - - -def spec_literal(vo: MetaData, template: MetaData, root_name: str) -> str: - """Emit ``OutputFormatSpec(Format.X, "rootName", PromptStyle.X, [PromptField(...), …])``.""" - format_enum = _format_enum(template) - style_enum = _prompt_style_enum(template) - field_lits = [_prompt_field_literal(f) for f in fm.fields(vo)] - body = ", ".join(field_lits) - return ( - f'OutputFormatSpec({format_enum}, "{root_name}", {style_enum}, [{body}])' - ) +alias(__name__, "metaobjects.codegen.runtime.output_format_spec_emitter") diff --git a/server/python/src/metaobjects/codegen/project_config.py b/server/python/src/metaobjects/codegen/project_config.py index f3ddf7ffa..15f931310 100644 --- a/server/python/src/metaobjects/codegen/project_config.py +++ b/server/python/src/metaobjects/codegen/project_config.py @@ -22,7 +22,7 @@ from __future__ import annotations import re -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path import yaml # type: ignore[import-untyped] # PyYAML ships no type stubs @@ -38,12 +38,17 @@ #: shipped schema has always said an unknown key is invalid; naming the set here is what #: lets the loader — the thing that actually runs — say the same. Kept in step with the #: schema by ``test_schema_and_loader_accept_EXACTLY_the_same_keys``. -TOP_LEVEL_KEYS: tuple[str, ...] = ("metadata", "providers", "libraries", "targets", "requirementTests") +TOP_LEVEL_KEYS: tuple[str, ...] = ( + "metadata", "providers", "libraries", "targets", "requirementTests", "docs", +) #: Every key a ``targets.`` mapping may carry. Same contract as above. TARGET_KEYS: tuple[str, ...] = ("outDir", "generators", "entities") +#: Every key the ``docs`` block may carry. Same contract as above. +DOCS_KEYS: tuple[str, ...] = ("generators",) + #: Every key the ``requirementTests`` block may carry (the ``requirement-tests`` generator's #: options). Same contract as above. REQUIREMENT_TESTS_KEYS: tuple[str, ...] = ( @@ -137,6 +142,9 @@ class ProjectConfig: targets: list[TargetConfig] #: Options of the ``requirement-tests`` generator; ``None`` => its defaults. requirement_tests: RequirementTestsConfig | None = None + #: The ``docs.generators`` block: owned docs generators (``module:symbol``) that + #: ``metaobjects docs`` runs in place of the packaged ones. ``[]`` => the packaged ones. + docs_generators: list[str] = field(default_factory=list) def metadata_dir(self) -> str: """The metadata dir resolved against ``config_dir`` (absolute path string).""" @@ -227,6 +235,20 @@ def symbol(key: str) -> str | None: ) +def _parse_docs(raw: object, ctx: str) -> list[str]: + if not isinstance(raw, dict): + raise ConfigError(f"{ctx} must be a mapping.") + _reject_unknown_keys(raw, DOCS_KEYS, ctx) + generators = _require_str_list(raw.get("generators", []), f"{ctx} 'generators'") + bad = [g for g in generators if not re.fullmatch(MODULE_SYMBOL_PATTERN, g)] + if bad: + raise ConfigError( + f"{ctx}: 'generators' takes owned generators in 'module:symbol' form " + f"(`metaobjects eject api-docs` prints the entry), got {bad}." + ) + return generators + + def load_project_config(path: Path) -> ProjectConfig: """Parse + validate ``metaobjects.config.yaml`` at ``path``. @@ -306,6 +328,8 @@ def load_project_config(path: Path) -> ProjectConfig: else None ) + docs_generators = _parse_docs(raw["docs"], f"{path}: 'docs'") if "docs" in raw else [] + return ProjectConfig( config_dir=path.parent.resolve(), metadata=metadata, @@ -313,4 +337,5 @@ def load_project_config(path: Path) -> ProjectConfig: libraries=libraries, targets=targets, requirement_tests=requirement_tests, + docs_generators=docs_generators, ) diff --git a/server/python/src/metaobjects/codegen/runner.py b/server/python/src/metaobjects/codegen/runner.py index 395402871..288b366eb 100644 --- a/server/python/src/metaobjects/codegen/runner.py +++ b/server/python/src/metaobjects/codegen/runner.py @@ -133,6 +133,17 @@ def run_gen( emitted: dict[str, tuple[str, str]] = {} # full_path -> (content, generated_by) for gen in generators: + docs_surface = getattr(gen, "docs_surface", None) + if docs_surface is not None: + # A docs generator writes pages under the docs output directory, and + # `metaobjects docs` runs it from `docs.generators`: here it would write them + # into the code tree, so it is skipped and the miswiring is named. + result.warnings.append( + f"{gen.name!r} writes the {docs_surface} reference, so gen skipped it. " + "List it under `docs.generators` in metaobjects.config.yaml: " + "`metaobjects docs` runs it from there." + ) + continue ctx = GenContext( entities=safe, loaded_root=metadata, diff --git a/server/python/src/metaobjects/codegen/runtime/__init__.py b/server/python/src/metaobjects/codegen/runtime/__init__.py index beabbcd3d..995a56f52 100644 --- a/server/python/src/metaobjects/codegen/runtime/__init__.py +++ b/server/python/src/metaobjects/codegen/runtime/__init__.py @@ -9,6 +9,20 @@ stay self-contained — standard library only, no ``metaobjects`` import and no FastAPI / SQLAlchemy / pg8000 dependency — or the copy would not run on its own. -``shared_predicates`` is what several GENERATORS import, at ``gen`` time: the emit rules -they have to agree on. No generated code imports it, so it may import ``metaobjects``. +Everything else is what GENERATORS import, at ``gen`` time. No generated code imports it, so +it may import ``metaobjects``. Each generator lists the ones it uses on its registry entry's +``runtime``, and an owned copy reads the copies eject placed beside it: + +* ``shared_predicates`` — the emit rules several generators have to agree on. +* ``fr019_shared_enum``, ``m2m_codegen`` — the fragment writers behind the entity, routes and + filter-allowlist generators (the shared-enum module, M:N navigation and the object index). +* ``fr010_field_mapping``, ``extract_delegate_emitter``, ``extract_schema_emitter``, + ``output_format_spec_emitter`` — the prompt-tier emitters behind the output-parser, + extractor, output-prompt and trace-helper generators. +* ``api_docs_model``, ``api_docs_builder``, ``api_docs_renderer``, ``api_docs_paths`` — the + page IR, builder, renderer and path math behind ``api-docs``. + +What stays a package import is the engine: the generator base and file writer, the naming +and type-mapping tables (``metaobjects.apidocs.naming``, ``type_map``, ``value_objects``), +the model walks (``find_inbound``, ``tph_plan``, ``instance_artifacts``) and the loader. """ diff --git a/server/python/src/metaobjects/codegen/runtime/api_docs_builder.py b/server/python/src/metaobjects/codegen/runtime/api_docs_builder.py new file mode 100644 index 000000000..0f3754c18 --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/api_docs_builder.py @@ -0,0 +1,488 @@ +"""Builds the Python SDK api-surface IR (:class:`ApiModel`) from a loaded MetaRoot. + +This is the ACCURATE-BY-CONSTRUCTION half of the api-docs pipeline: every +documented symbol name comes from the :mod:`metaobjects.apidocs.naming` seam (never +re-concatenated here — the SAME seam the real generators delegate to), and every +inclusion decision comes from an applies-predicate that REUSES the corresponding +generator's own gate helpers (``emits_instance_artifacts`` / ``primary_rdb_source`` ++ source kind / ``resolve_payload_vo`` / format check / ``is_tph_subtype``) rather +than re-implementing the gate. The result is that what this builder documents == +what the generators emit, by SHARING the single source of truth. + +Per-category enumeration (mirrors the Java ``JavaApiModelBuilder`` / C# +``CSharpApiModelBuilder``, Python/Pydantic-flavored): + +* Objects (``root.own_children()`` filtered to ``object.*``): each concrete object + → a MODEL symbol. A writable-table entity adds DATA_ACCESS (the repository + ``Protocol``) / REST (one per FastAPI route) / VALIDATION (Pydantic field + constraints on the create/update shape) / FILTER (the allowlist). A value object + → MODEL only. An abstract object / a TPH subtype yields no instance artifacts. +* Templates (EVERY subtype — ADR-0052): a resolvable ``@payloadRef`` gives the + PAYLOAD record it renders outbound, plus RENDER for a ``template.output`` (the + only subtype the render-helper generator emits for). A ``template.prompt`` + declaring ``@responseRef`` adds the INBOUND tier — the response PAYLOAD record, + PROMPT (the response-format fragment) and EXTRACTOR, plus OUTPUT_PARSER unless + the reply is XML, which gets no strict tier (ADR-0053). + +An object that yields no symbols (e.g. an abstract object) produces NO unit at all +(never an empty unit). +""" +from __future__ import annotations + +from metaobjects.apidocs import naming +from .api_docs_model import ( + ApiModel, + ApiSymbol, + ApiSymbolKind, + ApiUnit, + FieldShape, +) +from metaobjects.codegen import type_map +from .m2m_codegen import ( + build_object_index, + resolve_m2m_descriptors, +) +from metaobjects.codegen.generators.find_inbound import is_xml, response_shape +from metaobjects.codegen.value_objects import is_field_required, pkg_of, resolve_payload_vo +from metaobjects.codegen.instance_artifacts import ( + has_item_route, + is_abstract, + is_served_report, +) +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.core.field.meta_field import MetaField +from metaobjects.meta.core.object.meta_object import MetaObject +from metaobjects.meta.core.object.object_constants import ( + OBJECT_SUBTYPE_ENTITY, + OBJECT_SUBTYPE_REPORT, +) +from metaobjects.meta.core.reporting.report_read_model import report_read_model +from metaobjects.meta.meta_data import MetaData +from metaobjects.meta.template import template_constants as tc +from metaobjects.shared.base_types import TYPE_OBJECT, TYPE_TEMPLATE + +from .shared_predicates import emits_writable_router, is_read_only_routed + + +# --------------------------------------------------------------------------- +# Applies-predicates — each REUSES the matching generator's own gate helpers, so +# inclusion can never drift from emission. (The Python generators gate inline; +# these mirror that gate exactly via the shared helpers.) +# --------------------------------------------------------------------------- + + +def _payload_resolves(tmpl: MetaData, root: MetaData) -> MetaObject | None: + """The payload VO a template resolves to (``@payloadRef`` → ``object.value``), + or ``None`` — the shared gate for payload / render / prompt / parser / extractor.""" + payload_ref = tmpl.get_meta_attr(tc.TEMPLATE_ATTR_PAYLOAD_REF) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) + if not isinstance(payload_ref, str) or not payload_ref: + return None + # ADR-0042 (#228): the referrer is THIS template — a bare @payloadRef resolves + # in ITS OWN package first. + return resolve_payload_vo(root, payload_ref, pkg_of(tmpl)) + + +def _is_email_kind(tmpl: MetaData) -> bool: + kind = tmpl.get_meta_attr(tc.TEMPLATE_ATTR_KIND) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) + return isinstance(kind, str) and kind.lower() == tc.TEMPLATE_KIND_EMAIL + + +# --------------------------------------------------------------------------- +# Builder. +# --------------------------------------------------------------------------- + + +class PythonApiModelBuilder: + """Builds the :class:`ApiModel` SDK-surface IR from a loaded ``MetaRoot``.""" + + def build(self, root: MetaData, project: str) -> ApiModel: + objects = [ + # ADR-0039 sanctioned own: top-level scan on the loader ROOT (never extended, own == effective) + c + for c in root.own_children() + if c.type == TYPE_OBJECT and isinstance(c, MetaObject) + ] + object_index = build_object_index(objects) + + units: list[ApiUnit] = [] + for obj in objects: + # FR-044: a served report (Table A) has a generated read API, documented from + # its read model (its derived fields); every other report has no output. + if obj.sub_type == OBJECT_SUBTYPE_REPORT: + unit = ( + self._build_report_unit(obj, root) if is_served_report(obj) else None + ) + if unit is not None: + units.append(unit) + continue + unit = self._build_object_unit(obj, root, object_index) + if unit is not None: + units.append(unit) + + # Templates: ADR-0052 — EVERY template subtype gets a unit, not just + # template.output. A template.prompt carries a @payloadRef record and, when it + # declares @responseRef, the whole inbound tier — all of it generated, so all + # of it documentable. + # ADR-0039 sanctioned own: top-level scan on the loader ROOT (never extended, own == effective) + for tmpl in root.own_children(): + if tmpl.type == TYPE_TEMPLATE: + units.append(self._build_template_unit(tmpl, root)) + + return ApiModel(project, units) + + # ----- objects ----------------------------------------------------------- + + def _build_object_unit( + self, obj: MetaObject, root: MetaData, object_index: dict[str, MetaObject] + ) -> ApiUnit | None: + entity = obj.sub_type == OBJECT_SUBTYPE_ENTITY + module = naming.snake_case(obj.name) + unit_kind = "entity" if entity else "value" + symbols: list[ApiSymbol] = [] + + # MODEL — only for concrete objects (an abstract object cannot be + # instantiated → documented ⊆ generated). A value object → MODEL only. + if not is_abstract(obj): + model = naming.model_class_name(obj) + symbols.append( + ApiSymbol( + name=model, + kind=ApiSymbolKind.MODEL, + module=naming.model_import(obj), + signature=f"class {model}(BaseModel)", + usage=( + "the Pydantic v2 entity model" + if entity + else "the Pydantic v2 value-object model" + ), + ) + ) + + if entity and emits_writable_router(obj): + # VALIDATION — the Pydantic field constraints carried on the model + # (required / max-length / range / pattern). Names the model class. + model = naming.model_class_name(obj) + symbols.append( + ApiSymbol( + name=model, + kind=ApiSymbolKind.VALIDATION, + module=naming.model_import(obj), + signature=f"class {model}(BaseModel)", + usage="Pydantic field validation on the create/update shape", + fields=self._model_fields(obj), + ) + ) + + # DATA_ACCESS — the consumer-implemented repository ``Protocol`` seam. + repo = naming.repository_class_name(obj.name) + symbols.append( + ApiSymbol( + name=repo, + kind=ApiSymbolKind.DATA_ACCESS, + module=f"from .{naming.router_module_name(obj.name)} import {repo}", + signature=f"class {repo}(Protocol)", + usage="data access — the repository Protocol the consumer implements", + returns="list / Optional / bool", + ) + ) + + # REST — one symbol per FastAPI route the router registers. + self._add_rest_symbols(symbols, obj, module, root, object_index) + + # FILTER — the per-entity sort/filter allowlist. + fields_const = naming.filter_fields_const(obj.name) + symbols.append( + ApiSymbol( + name=fields_const, + kind=ApiSymbolKind.FILTER, + module=( + f"from .{naming.filter_allowlist_module_name(obj.name)} " + f"import {fields_const}" + ), + signature=f"{fields_const}: frozenset[str]", + usage="the filterable-field + filter-operator allowlist", + ) + ) + + elif is_read_only_routed(obj): + # A view-backed projection: the read-only router and its allowlist, no writes. + symbols.extend(self._read_only_symbols(obj)) + + if not symbols: + return None + return ApiUnit(obj.name, _package_of(obj), unit_kind, symbols) + + def _build_report_unit(self, obj: MetaObject, root: MetaData) -> ApiUnit: + """The unit for a SERVED report (FR-044 Table G): the row model plus the read-only + surface its router mounts (:meth:`_read_only_symbols`). Nothing else is generated + for it: no item route, no write verb, no validation model. Built over the + report's read model so the names are the generators'.""" + model_obj = report_read_model(obj, root) # type: ignore[arg-type] + model = naming.model_class_name(model_obj) + symbols = [ + ApiSymbol( + name=model, + kind=ApiSymbolKind.MODEL, + module=naming.model_import(model_obj), + signature=f"class {model}(BaseModel)", + usage="the Pydantic v2 report row model", + ), + *self._read_only_symbols(model_obj), + ] + return ApiUnit(obj.name, _package_of(obj), "report", symbols) + + def _read_only_symbols(self, obj: MetaObject) -> list[ApiSymbol]: + """What the READ-ONLY router (``is_read_only_routed``) actually mounts and emits, + and nothing more: the repository seam, ``GET`` list, ``GET`` by id ONLY when + :func:`has_item_route`, and the filter allowlist. The collection ``POST`` and the + item-verb refusals are 405s, not endpoints, so no write verb is documented; + a keyless object has no item route and its seam no ``find_by_id``.""" + router_module = naming.router_module_name(obj.name) + base_path = "/api/" + naming.route_path(obj.name) + repo = naming.repository_class_name(obj.name) + fields_const = naming.filter_fields_const(obj.name) + item = has_item_route(obj) + + def rest(verb_path: str, usage: str) -> ApiSymbol: + return ApiSymbol( + name=verb_path, + kind=ApiSymbolKind.REST, + module=f"# {router_module}.py — FastAPI APIRouter", + signature=verb_path, + usage=usage, + ) + + symbols = [ + ApiSymbol( + name=repo, + kind=ApiSymbolKind.DATA_ACCESS, + module=f"from .{router_module} import {repo}", + signature=f"class {repo}(Protocol)", + usage="data access — the read-only repository Protocol the consumer implements", + returns="list / count / Optional" if item else "list / count", + ), + rest("GET " + base_path, "list with pagination / sort / filters"), + ] + if item: + symbols.append( + rest("GET " + base_path + "/{" + naming.pk_param(obj.name) + "}", "fetch one by id") + ) + symbols.append( + ApiSymbol( + name=fields_const, + kind=ApiSymbolKind.FILTER, + module=( + f"from .{naming.filter_allowlist_module_name(obj.name)} " + f"import {fields_const}" + ), + signature=f"{fields_const}: frozenset[str]", + usage="the filterable-field + filter-operator allowlist", + ) + ) + return symbols + + def _add_rest_symbols( + self, + symbols: list[ApiSymbol], + entity: MetaObject, + module: str, + root: MetaData, + object_index: dict[str, MetaObject], + ) -> None: + router_module = naming.router_module_name(entity.name) + base_path = "/api/" + naming.route_path(entity.name) + # The router registers the item routes with a per-entity path param + # (`/{_id}`, e.g. `/{customer_id}`) — document that exact name, not + # a generic `{id}`, so the docs match the generated FastAPI routes. + pk = naming.pk_param(entity.name) + item = base_path + "/{" + pk + "}" + + def add(verb_path: str, usage: str) -> None: + symbols.append( + ApiSymbol( + name=verb_path, + kind=ApiSymbolKind.REST, + module=f"# {router_module}.py — FastAPI APIRouter", + signature=verb_path, + usage=usage, + ) + ) + + # The router registers GET list, GET item, POST, PATCH item, PUT item, + # DELETE item (PATCH + PUT share a handler but are two registered routes). + add("GET " + base_path, "list with pagination / sort / filters") + add("GET " + item, "fetch one by id") + add("POST " + base_path, "create") + add("PATCH " + item, "update") + add("PUT " + item, "update (PUT alias)") + add("DELETE " + item, "delete") + + # M:N traversal — GET //{id}/. `docs` has no + # `GenConfig`/`--column-naming` of its own (see `cli.py`'s docs handler) and + # this loop reads only `.target_entity` / `.relation_name` off the + # descriptor — never a physical column/table name — so resolving it under + # the port default here changes nothing observable; threading a naming + # strategy through would mean inventing a config seam this command doesn't + # have, for a value nothing here would read. + for d in resolve_m2m_descriptors(entity, object_index): + target = object_index.get(d.target_entity) + target_name = target.name if target is not None else d.target_entity + add( + "GET " + base_path + "/{" + pk + "}/" + d.relation_name, + f"M:N traversal — the related {target_name} rows", + ) + + # ----- templates --------------------------------------------------------- + + def _build_template_unit(self, tmpl: MetaData, root: MetaData) -> ApiUnit: + name = tmpl.name + module = naming.snake_case(name) + symbols: list[ApiSymbol] = [] + payload_vo = _payload_resolves(tmpl, root) + + if payload_vo is not None: + # PAYLOAD — the shape this template RENDERS. ADR-0056: it IS the @payloadRef + # value object's own model, emitted by the entity generator. + payload_class = naming.model_class_name(payload_vo) + symbols.append( + ApiSymbol( + name=payload_class, + kind=ApiSymbolKind.PAYLOAD, + module=naming.model_import(payload_vo), + signature=f"class {payload_class}(BaseModel)", + usage="the typed payload projection bound to the template", + fields=self._payload_fields(payload_vo), + ) + ) + + # RENDER — the typed render helper wrapping the render engine. OUTBOUND + # ONLY: render_helper_generator emits for a template.output alone, so + # documenting it for a prompt would name a function never generated. + if tmpl.sub_type == tc.TEMPLATE_SUBTYPE_OUTPUT: + render_fn = naming.render_helper_fn(name) + symbols.append( + ApiSymbol( + name=render_fn, + kind=ApiSymbolKind.RENDER, + module=f"from .{module}_render_helper import {render_fn}", + signature=f"def {render_fn}(payload: {payload_class}, provider)", + usage="renders the output template against a typed payload", + returns="EmailDocument" if _is_email_kind(tmpl) else "str", + ) + ) + + # ADR-0052 — the INBOUND symbols belong to a RESPONDING template.prompt, never + # to an output, and the gate is @responseRef PRESENCE rather than a format + # value. Shares `response_shape` with the generators, so the docs can never + # claim a symbol codegen suppressed. + inbound = response_shape(root, tmpl, pkg_of(tmpl)) + if inbound is not None: + # The RESPONSE shape the parser actually returns — the @responseRef value + # object's own model (ADR-0056). Documented once: when the prompt renders and + # parses the SAME value object, the model above already is it. + response_class = naming.model_class_name(inbound.vo) + if payload_vo is None or inbound.vo.resolution_key() != payload_vo.resolution_key(): + symbols.append( + ApiSymbol( + name=response_class, + kind=ApiSymbolKind.PAYLOAD, + module=naming.model_import(inbound.vo), + signature=f"class {response_class}(BaseModel)", + usage="the typed response shape a model reply is parsed into", + fields=self._payload_fields(inbound.vo), + ) + ) + + # OUTPUT_PARSER — the strict ``parse_*``. ADR-0053: JSON-only, so an XML + # reply gets no strict tier and documenting one would name a missing fn. + if not is_xml(inbound.format): + parse_fn = naming.output_parser_fn(name) + symbols.append( + ApiSymbol( + name=parse_fn, + kind=ApiSymbolKind.OUTPUT_PARSER, + module=f"from .{module}_response_parser import {parse_fn}", + signature=f"def {parse_fn}(text)", + usage="parses a model reply back into the typed response shape", + returns=response_class, + ) + ) + + prompt_fn = naming.output_prompt_fn(name) + symbols.append( + ApiSymbol( + name=prompt_fn, + kind=ApiSymbolKind.PROMPT, + module=f"from .{module}_response_format import {prompt_fn}", + signature=f"def {prompt_fn}(overrides=None)", + usage="builds the response-format prompt fragment", + returns="str", + ) + ) + extract_fn = naming.extractor_fn(name) + symbols.append( + ApiSymbol( + name=extract_fn, + kind=ApiSymbolKind.EXTRACTOR, + module=f"from .{module}_extractor import {extract_fn}", + signature=f"def {extract_fn}(root, text, opts=None)", + usage="extracts the strict typed response from dirty model output", + returns=response_class, + ) + ) + + return ApiUnit(name, _package_of(tmpl), "template", symbols) + + # ----- field shapes ------------------------------------------------------- + + def _model_fields(self, entity: MetaObject) -> list[FieldShape]: + """The entity's documented scalar/enum field shapes — one row per field the + Pydantic model maps (object-typed fields are owned-type navs, not a scalar + documented shape). Types via ``type_map.py_type_for``; required = own + ``@required`` is the boolean ``True`` (the same boundary the model uses).""" + rows: list[FieldShape] = [] + for f in entity.fields(): + if not isinstance(f, MetaField): + continue + if f.sub_type == fc.FIELD_SUBTYPE_OBJECT: + continue + type_expr = type_map.py_type_for(f).expr + note = None + if f.sub_type == fc.FIELD_SUBTYPE_ENUM: + values = type_map.effective_enum_values(f) + if values: + note = "allowed: " + " | ".join(values) + required = f.get_meta_attr(fc.FIELD_ATTR_REQUIRED) is True # ADR-0039: resolving (@required may be inherited) + rows.append(FieldShape(f.name, type_expr, optional=not required, note=note)) + return rows + + def _payload_fields(self, vo: MetaObject) -> list[FieldShape]: + """The payload model's documented field shapes — one row per field of the + value object, with optionality from ``is_field_required`` (the boundary the + value object's model draws between ``T`` and ``T | None``).""" + rows: list[FieldShape] = [] + for f in vo.fields(): + if not isinstance(f, MetaField): + continue + type_expr = type_map.py_type_for(f).expr + rows.append( + FieldShape(f.name, type_expr, optional=not is_field_required(f)) + ) + return rows + + +def _package_of(node: MetaData) -> str: + """The unit's metadata package (``"acme::shop"``) — drives the doc-page layout + path. Resolved from the nearest ancestor that carries one (objects inherit the + file/root package).""" + pkg = node.package + parent = node.parent + while pkg is None and parent is not None: + pkg = parent.package + parent = parent.parent + return pkg or "" + + +def build_api_model(root: MetaData, project: str) -> ApiModel: + """Module-level convenience: build the Python SDK api-surface IR.""" + return PythonApiModelBuilder().build(root, project) diff --git a/server/python/src/metaobjects/codegen/runtime/api_docs_model.py b/server/python/src/metaobjects/codegen/runtime/api_docs_model.py new file mode 100644 index 000000000..12766ddef --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/api_docs_model.py @@ -0,0 +1,111 @@ +"""The Python native SDK api-surface IR (intermediate representation). + +Mirrors the Java ``JavaApiModel`` IR (server/java/.../apidocs), the C# +``CSharpApiModel``, and the TS ``api-model.ts``, idiomatic to the Python +(Pydantic / FastAPI / ObjectManager) generated surface: Pydantic model / +``ObjectManager`` data-access / FastAPI routes / validation / extractor / +render helper / payload model / output-format prompt / output parser / filter +allowlist. + +NAMES on every symbol come from the :mod:`metaobjects.apidocs.naming` seam (never +re-concatenated in the builder), so what this model documents == what the real +generators emit. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + + +class ApiSymbolKind(str, Enum): + """The generated-category axis a documented symbol belongs to (Python / + Pydantic + FastAPI flavored).""" + + #: The Pydantic v2 model class (entity) or value-object model. + MODEL = "model" + #: Data access — the consumer-implemented repository ``Protocol`` seam. + DATA_ACCESS = "data_access" + #: A FastAPI route (named ``"VERB path"``). + REST = "rest" + #: Pydantic field validation on the create/update shape. + VALIDATION = "validation" + #: The tolerant + strict extractor for a template payload. + EXTRACTOR = "extractor" + #: The typed render helper wrapping the render engine. + RENDER = "render" + #: The typed payload model bound to a template. + PAYLOAD = "payload" + #: The output-format prompt fragment. + PROMPT = "prompt" + #: The output parser (``parse_*``) back into the typed payload. + OUTPUT_PARSER = "output_parser" + #: The per-entity sort/filter allowlist. + FILTER = "filter" + + +@dataclass(frozen=True) +class FieldShape: + """A documented field: name + Python type + optionality + an optional note + (e.g. enum values).""" + + name: str + type: str + optional: bool + note: str | None = None + + +@dataclass(frozen=True) +class UnitExample: + """A worked example for a unit (reserved for later phases; carried for + cross-port IR parity).""" + + title: str + code: str + + +@dataclass(frozen=True) +class ApiSymbol: + """One documented symbol of the generated Python SDK surface. + + :param name: the emitted Python identifier (or ``"VERB path"`` for a REST route). + :param kind: which generated category this symbol belongs to. + :param module: the import line / module path the symbol lives in. + :param signature: a human-readable Python signature line. + :param usage: a one-line "what you use this for". + :param returns: the symbol's return surface, or ``None``. + :param fields: per-field shapes for model/validation/payload symbols (may be empty). + """ + + name: str + kind: ApiSymbolKind + module: str + signature: str + usage: str + returns: str | None = None + fields: list[FieldShape] = field(default_factory=list) + + +@dataclass(frozen=True) +class ApiUnit: + """One documented unit (an entity / value object, or a template) + its symbols. + + :param node: the unit's short name (the doc-page basename). + :param package: the unit's metadata package (e.g. ``acme::shop``). + :param kind: ``"entity"`` | ``"value"`` | ``"report"`` | ``"template"``. + :param symbols: the documented symbols, in canonical IR order. + :param example: an optional unit-level worked example (reserved). + """ + + node: str + package: str + kind: str + symbols: list[ApiSymbol] + example: UnitExample | None = None + + +@dataclass(frozen=True) +class ApiModel: + """The full per-project Python SDK api surface IR.""" + + project: str + units: list[ApiUnit] diff --git a/server/python/src/metaobjects/codegen/runtime/api_docs_paths.py b/server/python/src/metaobjects/codegen/runtime/api_docs_paths.py new file mode 100644 index 000000000..6360d8d6a --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/api_docs_paths.py @@ -0,0 +1,75 @@ +"""Doc-page path math for the Python native SDK-docs surface (``api/python``). + +Byte-parity with the Java ``DocsPaths`` (server/java/.../apidocs/DocsPaths.java), +the C# ``DocsPaths`` (server/csharp/.../ApiDocs/DocsPaths.cs), and the TS +``docs-paths.ts`` contract: the same layout + relative-href math, so the shared +``fixtures/conformance/api-docs-cross-port/expected-paths.json`` resolves +identically across all ports. Nothing here is Python-specific — it is pure path +math. The api-docs renderer + the ``docs`` command call these helpers; they never +re-derive a path. +""" +from __future__ import annotations + +from enum import Enum + + +class Layout(str, Enum): + """Doc-page layout: one file per unit (:attr:`FLAT`) or foldered by package + (:attr:`PACKAGE`).""" + + #: One file per unit at the surface root (``.md``). + FLAT = "flat" + #: Foldered by package (``/.md``). + PACKAGE = "package" + + +def package_to_path(pkg: str | None) -> str: + """``"acme::shop"`` or ``"acme.shop"`` → ``"acme/shop"``; ``None``/"" → "".`""" + if not pkg: + return "" + return pkg.replace("::", "/").replace(".", "/") + + +def doc_page_output_path(layout: Layout, pkg: str | None, name: str) -> str: + """Flat → ``".md"``; Package → ``"/.md"``.""" + file = f"{name}.md" + if layout == Layout.FLAT: + return file + folder = package_to_path(pkg) + return file if not folder else f"{folder}/{file}" + + +def surface_cross_href(from_output_path: str, to_output_path: str) -> str: + """Relative posix href from *from_output_path*'s directory to *to_output_path* + (mirrors the TS ``surfaceCrossHref``).""" + slash = from_output_path.rfind("/") + from_dir = from_output_path[:slash] if slash >= 0 else "" + rel = _posix_relative(from_dir, to_output_path) + return rel if rel.startswith(".") else f"./{rel}" + + +def model_cross_href( + api_page_path: str, model_page_path: str, model_base_url: str | None = None +) -> str: + """From an api page to its model page: relative by default, absolute when + *model_base_url* is set (federated docs).""" + if model_base_url: + return f"{model_base_url.rstrip('/')}/{model_page_path}" + return surface_cross_href(api_page_path, model_page_path) + + +def _posix_relative(from_dir: str, to_path: str) -> str: + """``posixpath.relative(from_dir, to_path)``: drop the common prefix, ``..`` + per remaining *from_dir* segment, then the remaining *to_path* segments. + "" from_dir → to_path; identical → ".".""" + if not from_dir: + return to_path + frm = from_dir.split("/") + to = to_path.split("/") + common = 0 + cap = min(len(frm), len(to)) + while common < cap and frm[common] == to[common]: + common += 1 + parts: list[str] = [".."] * (len(frm) - common) + parts.extend(to[common:]) + return "/".join(parts) if parts else "." diff --git a/server/python/src/metaobjects/codegen/runtime/api_docs_renderer.py b/server/python/src/metaobjects/codegen/runtime/api_docs_renderer.py new file mode 100644 index 000000000..ce4c709af --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/api_docs_renderer.py @@ -0,0 +1,185 @@ +"""Renders the :class:`ApiModel` IR into Python-idiomatic api-doc markdown. + +One IR, three forms — all derived from the SAME model, never re-derived: + +* :func:`render_unit_page` — a per-unit HUMAN page (entity OR template), symbols + grouped into ordered sections + the ``**Model / metadata:**`` back-link. +* :func:`render_index` — the consolidated index (README.md), one bullet per unit. +* :func:`render_agent_api` — the token-frugal AGENT form (AGENT-API.md), symbols + grouped under their import line. + +PRESENTATION ONLY: every symbol name comes from the IR (which keys off the naming +seam) and every path comes from :mod:`metaobjects.codegen.runtime.api_docs_paths` — the renderer never +re-derives a name or a path. The section order + headings + the back-link literal +mirror the Java / C# renderers so the polyglot doc tree coheres. +""" +from __future__ import annotations + +from .api_docs_model import ApiModel, ApiSymbolKind, ApiUnit +from .api_docs_paths import Layout, doc_page_output_path, surface_cross_href + +# Canonical section order per kind (a unit renders only the kinds it carries). +_KIND_ORDER: tuple[ApiSymbolKind, ...] = ( + ApiSymbolKind.MODEL, + ApiSymbolKind.DATA_ACCESS, + ApiSymbolKind.REST, + ApiSymbolKind.VALIDATION, + ApiSymbolKind.EXTRACTOR, + ApiSymbolKind.RENDER, + ApiSymbolKind.PAYLOAD, + ApiSymbolKind.PROMPT, + ApiSymbolKind.OUTPUT_PARSER, + ApiSymbolKind.FILTER, +) + +_HEADINGS: dict[ApiSymbolKind, str] = { + ApiSymbolKind.MODEL: "Model", + ApiSymbolKind.DATA_ACCESS: "Data access", + ApiSymbolKind.REST: "REST", + ApiSymbolKind.VALIDATION: "Validation", + ApiSymbolKind.EXTRACTOR: "Extractor", + ApiSymbolKind.RENDER: "Render", + ApiSymbolKind.PAYLOAD: "Payload", + ApiSymbolKind.PROMPT: "Prompt", + ApiSymbolKind.OUTPUT_PARSER: "Output parser", + ApiSymbolKind.FILTER: "Filter", +} + +_SUMMARY_LABELS: dict[ApiSymbolKind, str] = { + ApiSymbolKind.MODEL: "model", + ApiSymbolKind.DATA_ACCESS: "data access", + ApiSymbolKind.REST: "REST", + ApiSymbolKind.VALIDATION: "validation", + ApiSymbolKind.EXTRACTOR: "extractor", + ApiSymbolKind.RENDER: "render", + ApiSymbolKind.PAYLOAD: "payload", + ApiSymbolKind.PROMPT: "prompt", + ApiSymbolKind.OUTPUT_PARSER: "output parser", + ApiSymbolKind.FILTER: "filter", +} + + +def _md_cell(text: str) -> str: + return text.replace("|", "\\|") + + +def render_unit_page(unit: ApiUnit, model_href: str | None) -> str: + """Render one per-unit human reference page. *model_href* (when non-empty) is a + pre-computed relative href back to this unit's model/metadata page — the caller + derives it via :func:`metaobjects.codegen.runtime.api_docs_paths.model_cross_href`; the renderer + only places it (as the contract ``**Model / metadata:**`` back-link).""" + out: list[str] = [f"# {unit.node} API\n"] + if model_href: + out.append(f"\n**Model / metadata:** [{unit.node}]({model_href})\n") + out.append( + "\n> Import paths are relative to the generated package; add it to your " + "Python path.\n" + ) + + for kind in _KIND_ORDER: + syms = [s for s in unit.symbols if s.kind == kind] + if not syms: + continue + out.append(f"\n## {_HEADINGS[kind]}\n") + for sym in syms: + out.append(f"\n### `{sym.signature}`\n") + out.append(f"\n{sym.usage}\n") + out.append(f"\n```python\n{sym.module}\n```\n") + if sym.returns: + out.append(f"\nReturns: {sym.returns}\n") + if sym.fields: + out.append("\n| Field | Type | Required | Notes |\n|---|---|---|---|\n") + for f in sym.fields: + required = "no" if f.optional else "yes" + out.append( + f"| `{f.name}` | `{_md_cell(f.type)}` | {required} | " + f"{_md_cell(f.note or '')} |\n" + ) + return "".join(out) + + +def render_index(model: ApiModel, layout: Layout) -> str: + """Render the consolidated api index (README.md): one bullet per unit, entities/ + values vs templates. Each unit's href is the relative link from the index to the + unit's page in the given *layout*.""" + entities = sorted( + (u for u in model.units if u.kind != "template"), key=lambda u: u.node + ) + templates = sorted( + (u for u in model.units if u.kind == "template"), key=lambda u: u.node + ) + + out: list[str] = [ + "# API Reference\n\nGenerated public API surface, one page per entity and " + "output template.\n" + ] + if entities: + out.append("\n## Entities\n\n") + for u in entities: + out.append(_index_row(u, layout)) + if templates: + out.append("\n## Templates\n\n") + for u in templates: + out.append(_index_row(u, layout)) + return "".join(out) + + +def _index_row(unit: ApiUnit, layout: Layout) -> str: + href = surface_cross_href( + "README.md", doc_page_output_path(layout, unit.package, unit.node) + ) + return ( + f"- [{unit.node}]({href}) — {_summary(unit)} " + f"({len(unit.symbols)} symbols)\n" + ) + + +def _summary(unit: ApiUnit) -> str: + parts: list[str] = [] + for kind in _KIND_ORDER: + n = sum(1 for s in unit.symbols if s.kind == kind) + if n == 0: + continue + label = _SUMMARY_LABELS[kind] + parts.append(label if n == 1 else f"{n} {label}") + return ", ".join(parts) if parts else "no public symbols" + + +def render_agent_api(model: ApiModel) -> str: + """Render the condensed agent/LLM form (AGENT-API.md): per unit, symbols grouped + under a single import-line header then one compact ``` `signature` — usage ``` line + each. NO prose / field-tables (token budget). Units + symbols keep their IR order.""" + out: list[str] = [ + f"# Agent API Reference\n\nGenerated Python API reference for " + f"{model.project}; call these exactly as written. Import paths are relative " + "to the generated package.\n" + ] + for unit in model.units: + if not unit.symbols: + continue + out.append(f"\n## {unit.node}\n") + groups: dict[str, list[str]] = {} + order: list[str] = [] + for s in unit.symbols: + if s.module not in groups: + groups[s.module] = [] + order.append(s.module) + groups[s.module].append(f"- `{s.signature}` — {s.usage}\n") + for module in order: + out.append(f"\n`{module}`\n") + out.extend(groups[module]) + return "".join(out) + + +class PythonApiDocsRenderer: + """Object wrapper around the module-level render functions (parity with the + Java / C# renderer classes; the functions are the canonical entry points).""" + + def render_unit_page(self, unit: ApiUnit, model_href: str | None) -> str: + return render_unit_page(unit, model_href) + + def render_index(self, model: ApiModel, layout: Layout) -> str: + return render_index(model, layout) + + def render_agent_api(self, model: ApiModel) -> str: + return render_agent_api(model) diff --git a/server/python/src/metaobjects/codegen/runtime/extract_delegate_emitter.py b/server/python/src/metaobjects/codegen/runtime/extract_delegate_emitter.py new file mode 100644 index 000000000..258011557 --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/extract_delegate_emitter.py @@ -0,0 +1,383 @@ +"""FR-010 nested-codegen-gap — the runtime-DELEGATING extract emitter (Python). + +The self-contained ``extract_(text)`` path (``extract_schema_emitter`` + the +baked ``ExtractSchema``) covers scalars / enums / scalar-arrays but leaves +nested-object and array-of-object components ``None`` — the historical FR-010 codegen +gap. This module emits the additive *delegating* entry that CLOSES that gap by +wrapping the runtime extract: + + extract__with_loader(root, text, opts=None) -> ExtractionResult[Extracted] + +It resolves this payload's ``MetaObject`` from the supplied loaded ``MetaRoot`` by its +baked simple name (``PAYLOAD_NAME``), delegates to ``extract_object`` in +:mod:`metaobjects.meta.core.object.object_extract` (which assembles the FULL nested +object graph reflection-free via the Phase A object model — ``MetaObject.new_instance()`` ++ the ``MetaField`` set-by-name SPI), then maps the assembled ``ValueObject`` graph into +the typed nullable mirror graph via generated ``_from__extracted(o)`` mapper +functions (payload + every reachable nested VO, deduped). + +This is the codegen-wrapping-runtime pattern (a generated DAO calling the +dynamic-metadata runtime), mirroring the Java / Kotlin / TS pilots. The generated +mappers read the assembled graph through a tiny ``_read_prop`` helper that mirrors the +``MetaField`` get SPI (``ValueObject.get(name)`` else plain-attribute access), so the +emitted code stays self-sufficient and reflection-free. + +Bounded by the cross-port ``MAX_NEST_DEPTH`` via the runtime — codegen here only mirrors +the runtime's resolved object graph, so depth/cycle guarding lives in ``object_extract``. +The emitter dedupes mirrors/mappers by ``resolution_key()`` (the package-qualified FQN, +cycle-safe) — NOT the bare VO ``name``, which would silently collapse two same-short-name +value-objects from different packages into one (ADR-0044, #228). + +ADR-0056 rule 3: a mirror is keyed by the VALUE OBJECT, never the template. Every mirror +and mapper — the root's included — is named after the value object's emitted model name +(:func:`~metaobjects.codegen.value_objects.model_class_name`: ``Extracted`` / +``_from__extracted``), the same name the entity generator declares the model under, so +a cross-package short-name collision qualifies the mirror exactly as it qualifies the +model. Mirrors stay module-local (declared in each parser module that needs them), like +TypeScript's: a Python module scopes them, so two parsers over one response each carry +their own copy without clashing. +""" +from __future__ import annotations + +from . import fr010_field_mapping as fm +from metaobjects.codegen.value_objects import model_class_name, pkg_of +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.meta_data import MetaData +from metaobjects.naming_refs import resolve_object_ref + + +def ref_vo(field: MetaData, root: MetaData) -> MetaData | None: + """The ``@objectRef`` target VO for a nested-object field, or ``None`` when + unresolvable. + + ADR-0042 (#228) — resolves via the canonical `resolve_object_ref` package-local + contract: an FQN ref resolves EXACTLY; a bare ref resolves in the DECLARING + field's own package first, else a root-level object. NO bare-tail short-name + fallback — that pattern (matching an FQN ref by its trailing simple-name segment + against ANY same-named object root-wide) is the #219/ADR-0042-banned "wrong + node" bug: under a cross-package short-name collision it silently binds + whichever same-named object happens to load first, regardless of which package + the ref actually pointed at. *referrer_pkg* is the field's OWN declaring + package (which differs from the VO's when the field is inherited via `extends` + from an abstract VO in another package).""" + ref = field.attrs().get(fc.FIELD_ATTR_OBJECT_REF) + if not isinstance(ref, str) or not ref: + return None + referrer_pkg = pkg_of(field.parent) if field.parent is not None else "" + return resolve_object_ref(root, ref, referrer_pkg) + + +def _is_object_field(field: MetaData) -> bool: + """True iff the field is a nested object reference (``field.object`` — distinct + from the string-backed ``field.enum``, treated as a scalar).""" + return field.sub_type == fc.FIELD_SUBTYPE_OBJECT + + +def mirror_name(vo: MetaData) -> str: + """The extracted-mirror dataclass name for a value-object: ``Extracted``, where + ```` is the value object's emitted model name (collision-qualified when another + object shares its short name).""" + return f"{model_class_name(vo)}Extracted" + + +def mapper_name(vo: MetaData) -> str: + """The mapper function name for a value-object (``_from__extracted``).""" + return f"_from_{_snake(model_class_name(vo))}_extracted" + + +def _snake(name: str) -> str: + """``NpcResponseOutput`` → ``npc_response_output`` (matches the cross-generator + ``_snake_case`` convention; no acronym handling).""" + out: list[str] = [] + for i, ch in enumerate(name): + if ch.isupper() and i > 0: + out.append("_") + out.append(ch.lower()) + return "".join(out) + + +# ============================================================================= +# Nested-aware mirror type (recurses into nested mirror names) +# ============================================================================= + + +def _nested_mirror_type(field: MetaData, root: MetaData) -> str: + """The nullable mirror annotation for one field — nested-aware (nested objects + become ``Extracted``; array-of-objects become ``list[...]``).""" + if _is_object_field(field): + target = ref_vo(field, root) + base = f'"{mirror_name(target)}"' if target is not None else "object" + elem = f"{base} | None" + return f"list[{elem}] | None" if fm.is_array(field) else elem + if fm.is_array(field): + return "list[str | None] | None" + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + return "str | None" + kind = fm.scalar_kind(field.sub_type) + if kind in ("INT", "LONG"): + return "int | None" + if kind == "DOUBLE": + return "float | None" + if kind == "BOOLEAN": + return "bool | None" + return "str | None" + + +# ============================================================================= +# Reachability +# ============================================================================= + + +def reachable_vos(vo: MetaData, root: MetaData) -> list[MetaData]: + """``vo`` + every value-object reachable through nested ``@objectRef`` fields, in + stable BFS order, deduped by ``resolution_key()`` (cycle-safe). + + ADR-0044 (#228) — deduping by the bare ``name`` (pre-fix) silently DROPPED a + second cross-package value-object sharing the first one's bare short name (e.g. + ``acme::alpha::Note`` + ``acme::beta::Note``): once the first ``Note`` was seen, + the second's bare name matched ``seen`` and it was never queued/emitted — a + silent shape loss, not merely a naming cosmetic. ``resolution_key()`` is the + package-qualified FQN, so two same-short-name VOs from different packages are + two distinct keys and both survive the walk.""" + out: list[MetaData] = [] + seen: set[str] = set() + queue: list[MetaData] = [vo] + while queue: + cur = queue.pop(0) + key = cur.resolution_key() + if key in seen: + continue + seen.add(key) + out.append(cur) + for f in fm.fields(cur): + if _is_object_field(f): + target = ref_vo(f, root) + if target is not None and target.resolution_key() not in seen: + queue.append(target) + return out + + +def has_nested(vo: MetaData, root: MetaData) -> bool: + """True iff ``vo`` (or any reachable nested VO) has a nested-object field.""" + for cur in reachable_vos(vo, root): + if any(_is_object_field(f) for f in fm.fields(cur)): + return True + return False + + +# ============================================================================= +# Nested-aware mirror dataclasses +# ============================================================================= + + +def nested_mirror_dataclasses(vo: MetaData, root: MetaData) -> list[str]: + """Emit the nested-aware mirror dataclass for ``vo`` and every reachable nested VO + (deduped), each named :func:`mirror_name`. Returns source lines (blank-line + separated).""" + lines: list[str] = [] + for i, cur in enumerate(reachable_vos(vo, root)): + if i > 0: + lines.append("") + lines.append("") + lines.extend(_one_mirror(cur, root)) + return lines + + +def _one_mirror(vo: MetaData, root: MetaData) -> list[str]: + lines: list[str] = [ + "@dataclass(frozen=True, slots=True)", + f"class {mirror_name(vo)}:", + f' """Best-effort extracted twin of ``{model_class_name(vo)}`` — every field nullable', + ' (``None`` where the value was lost or malformed)."""', + ] + field_lines = [ + f" {f.name}: {_nested_mirror_type(f, root)} = None" + for f in fm.fields(vo) + ] + lines.extend(field_lines or [" pass"]) + return lines + + +# ============================================================================= +# Mapper functions (assembled ValueObject graph -> typed nullable mirror graph) +# ============================================================================= + + +def nested_mappers(vo: MetaData, root: MetaData) -> list[str]: + """Emit one ``_from__extracted(o)`` mapper per reachable VO (payload + + nested, deduped), each named :func:`mapper_name`. Returns source lines + (blank-line separated).""" + lines: list[str] = [] + for i, cur in enumerate(reachable_vos(vo, root)): + if i > 0: + lines.append("") + lines.append("") + lines.extend(_one_mapper(cur, root)) + return lines + + +def _one_mapper(vo: MetaData, root: MetaData) -> list[str]: + fn = mapper_name(vo) + mirror = mirror_name(vo) + lines: list[str] = [ + f'def {fn}(o: object | None) -> "{mirror} | None":', + f" \"\"\"Map an assembled ValueObject graph into a typed ``{mirror}`` mirror;" + " null-tolerant.\"\"\"", + " if o is None:", + " return None", + f" return {mirror}(", + ] + for f in fm.fields(vo): + lines.append(f" {f.name}={_mapper_arg(f, root)},") + lines.append(" )") + return lines + + +def _mapper_arg(field: MetaData, root: MetaData) -> str: + """The mirror-field initializer that reads ``field`` from the assembled object ``o``.""" + key = f'"{field.name}"' + if _is_object_field(field): + target = ref_vo(field, root) + if target is None: + return "None # unresolved @objectRef" + fn = mapper_name(target) + if fm.is_array(field): + return f"_map_object_list(_read_prop(o, {key}), {fn})" + return f"{fn}(_read_prop(o, {key}))" + + # Enum / scalar / scalar-array: the runtime already coerced; read + light-coerce to + # the mirror's nullable shape via the locally-defined _dlg_* readers. + # ARRAY is checked BEFORE the scalar-enum branch: an enum ARRAY must route through + # the string-LIST reader, not the scalar enum reader — otherwise the list collapses + # to a single stringified scalar (the cross-port "enum-before-isArray" ordering bug). + if fm.is_array(field): + return f"_dlg_str_list(_read_prop(o, {key}))" + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + return f"_dlg_str(_read_prop(o, {key}))" + kind = fm.scalar_kind(field.sub_type) + if kind in ("INT", "LONG"): + return f"_dlg_int(_read_prop(o, {key}))" + if kind == "DOUBLE": + return f"_dlg_float(_read_prop(o, {key}))" + if kind == "BOOLEAN": + return f"_dlg_bool(_read_prop(o, {key}))" + return f"_dlg_str(_read_prop(o, {key}))" + + +# ============================================================================= +# Used-helper scoping + the shared helper block +# ============================================================================= + + +def used_helpers(vo: MetaData, root: MetaData) -> set[str]: + """The generated-helper names the mappers for ``vo`` (+ reachable nested VOs) + actually reference. ``_read_prop`` is always present once any mapper is emitted.""" + used: set[str] = {"_read_prop"} + for cur in reachable_vos(vo, root): + for f in fm.fields(cur): + if _is_object_field(f): + if fm.is_array(f): + used.add("_map_object_list") + continue + if fm.is_array(f): + # ARRAY before scalar-enum — an enum array uses the string-LIST reader + # (mirrors the _mapper_arg ordering; avoids the collapse-to-scalar bug). + used.add("_dlg_str_list") + elif f.sub_type == fc.FIELD_SUBTYPE_ENUM: + used.add("_dlg_str") + else: + kind = fm.scalar_kind(f.sub_type) + if kind in ("INT", "LONG"): + used.add("_dlg_int") + elif kind == "DOUBLE": + used.add("_dlg_float") + elif kind == "BOOLEAN": + used.add("_dlg_bool") + else: + used.add("_dlg_str") + return used + + +# Each helper: name -> source block (lines). +_HELPER_BLOCKS: dict[str, list[str]] = { + "_read_prop": [ + "def _read_prop(o: object | None, name: str) -> object | None:", + ' """Read a property from an assembled backing object, mirroring the', + " MetaField get SPI (``ValueObject.get(name)`` else plain-attribute access).", + ' Keeps the mappers reflection-free + backing-agnostic."""', + " if o is None:", + " return None", + ' getter = getattr(o, "get", None)', + " if callable(getter):", + " return getter(name)", + " return getattr(o, name, None)", + ], + "_map_object_list": [ + "def _map_object_list(v: object | None, fn) -> list | None:", + ' """Map each element of an assembled array via ``fn``; non-list -> None."""', + " if not isinstance(v, list):", + " return None", + " return [fn(e) for e in v]", + ], + "_dlg_str": [ + "def _dlg_str(v: object | None) -> str | None:", + " return None if v is None else str(v)", + ], + "_dlg_int": [ + "def _dlg_int(v: object | None) -> int | None:", + " if v is None:", + " return None", + " try:", + " return int(v) # type: ignore[arg-type, call-overload]", + " except (TypeError, ValueError):", + " return None", + ], + "_dlg_float": [ + "def _dlg_float(v: object | None) -> float | None:", + " if v is None:", + " return None", + " try:", + " return float(v) # type: ignore[arg-type]", + " except (TypeError, ValueError):", + " return None", + ], + "_dlg_bool": [ + "def _dlg_bool(v: object | None) -> bool | None:", + " if v is None:", + " return None", + " if isinstance(v, bool):", + " return v", + ' return str(v).lower() == "true"', + ], + "_dlg_str_list": [ + "def _dlg_str_list(v: object | None) -> list | None:", + " if not isinstance(v, list):", + " return None", + " return [None if e is None else str(e) for e in v]", + ], +} + +# Stable emission order (deterministic output). +_HELPER_ORDER: tuple[str, ...] = ( + "_read_prop", + "_map_object_list", + "_dlg_str", + "_dlg_int", + "_dlg_float", + "_dlg_bool", + "_dlg_str_list", +) + + +def delegate_helpers(used: set[str]) -> list[str]: + """The shared helper block the generated mappers rely on, scoped to ``used`` and + emitted in stable order. Returns source lines (blank-line separated).""" + lines: list[str] = [ + "# ---- runtime-delegating extract helpers (generated) ----" + ] + for name in _HELPER_ORDER: + if name in used: + lines.append("") + lines.append("") + lines.extend(_HELPER_BLOCKS[name]) + return lines diff --git a/server/python/src/metaobjects/codegen/runtime/extract_schema_emitter.py b/server/python/src/metaobjects/codegen/runtime/extract_schema_emitter.py new file mode 100644 index 000000000..d13306129 --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/extract_schema_emitter.py @@ -0,0 +1,139 @@ +"""Turns a payload value-object into Python source fragments for the FR-010 extract +codegen: + +* :func:`schema_literal` — a ``ExtractSchema(Format.X, "root", [FieldSpec(...), …])`` + baked descriptor for the emitted parser module. +* :func:`mirror_dataclass` — an all-nullable mirror dataclass ``Extracted`` + (every field ``X | None = None``; the extract entry + returns this nullable twin rather than the strict + Pydantic payload — same reasoning as the Kotlin / C# + nullable mirror). +* :func:`mirror_initializer` — ``Extracted(field=as_string(d, "field"), …)``. + +Mirrors the C# / Java ``ExtractSchemaEmitter`` adapted to Python syntax + the +nullable-mirror shape. Bounded scope: scalar / enum / scalar-array. Nested object + +array-of-enum deferred. +""" +from __future__ import annotations + +from . import fr010_field_mapping as fm +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.meta_data import MetaData + +# The extract_map accessors a generated module may import (sorted, deduped subset). +ALL_EXTRACT_MAP_HELPERS: tuple[str, ...] = ( + "as_bool", + "as_double", + "as_int", + "as_long", + "as_string", + "as_string_list", +) + + +def _format_enum(fmt: str) -> str: + return "Format.XML" if fmt.lower() == "xml" else "Format.JSON" + + +def _field_spec_literal(field: MetaData, owner: MetaData) -> str: + name = field.name + required = fm.is_required(field) + req = "True" if required else "False" + + # An enum ARRAY is checked BEFORE the scalar-enum branch below: it is baked as a + # string-list scalar slot (read via as_string_list), NOT a scalar enum_field — + # otherwise the runtime collapses the list to one stringified value (the cross-port + # "enum-before-isArray" ordering bug). The mirror type / extract_map helper already + # route arrays first; this keeps the baked schema spec consistent with them. Scoped + # to the enum case so NON-enum scalar arrays keep their real element FieldKind below + # (the self-contained baked path drops arrays as MALFORMED before kind is read, so + # the kind is inert either way — but emitting the true kind is correct + clearer). + if fm.is_array(field) and field.sub_type == fc.FIELD_SUBTYPE_ENUM: + return f'FieldSpec.scalar("{name}", FieldKind.STRING, {req})' + + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + values_lit = fm.string_list_literal(fm.enum_values(field)) + alias_lit = fm.properties_map_literal(field.attrs().get(fc.FIELD_ATTR_ENUM_ALIAS)) + # FR-011: resolve the three new enum args (field → object.value → "strip" for + # normalize). Keep the back-compat 4-arg form when nothing is set; otherwise + # emit the 7-arg form (..., coerce_default, normalize, default_value). + coerce_default = fm.coerce_default(field) + default_value = fm.default_value(field) + normalize = fm.resolve_normalize(field, owner) + if ( + coerce_default is None + and default_value is None + and normalize == fc.NORMALIZE_DEFAULT + ): + return f'FieldSpec.enum_field("{name}", {req}, {values_lit}, {alias_lit})' + cd_lit = "None" if coerce_default is None else fm.py_string_literal(coerce_default) + norm_lit = fm.py_string_literal(normalize) + dv_lit = "None" if default_value is None else fm.py_string_literal(default_value) + return ( + f'FieldSpec.enum_field("{name}", {req}, {values_lit}, {alias_lit}, ' + f"{cd_lit}, {norm_lit}, {dv_lit})" + ) + + if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: + # Self-contained path models a nested object as a plain (opaque) STRING scalar + # slot — it does not recurse. The runtime-delegating ``*_with_loader`` entry + # populates the real nested graph. (No trailing inline comment: this literal is + # joined into a ``[...]`` argument list, where a ``#`` comment would be illegal.) + return f'FieldSpec.scalar("{name}", FieldKind.STRING, {req})' + + kind = fm.scalar_kind(field.sub_type) or "STRING" + # @xmlText: a scalar field marked to receive its element's XML text content + # (JAXB @XmlValue / Jackson @JacksonXmlText / .NET [XmlText]). Mirrors the TS + # extract-schema-emitter textContentField branch. No effect for JSON. + if fm.xml_text(field): + return f'FieldSpec.text_content_field("{name}", FieldKind.{kind}, {req})' + return f'FieldSpec.scalar("{name}", FieldKind.{kind}, {req})' + + +def schema_literal(vo: MetaData, fmt: str, root_name: str) -> str: + """Emit ``ExtractSchema(Format.X, "rootName", [FieldSpec(...), …])``.""" + format_enum = _format_enum(fmt) + specs = [_field_spec_literal(f, vo) for f in fm.fields(vo)] + if not specs: + return f'ExtractSchema({format_enum}, "{root_name}", [])' + body = ", ".join(specs) + return f'ExtractSchema({format_enum}, "{root_name}", [{body}])' + + +def mirror_dataclass(vo: MetaData, record_name: str) -> list[str]: + """Emit the all-nullable mirror dataclass declaration (source lines).""" + base = ( + record_name[: -len("Extracted")] + if record_name.endswith("Extracted") + else record_name + ) + lines: list[str] = [ + "@dataclass(frozen=True, slots=True)", + f"class {record_name}:", + f' """Best-effort extracted twin of ``{base}`` — every field nullable', + ' (``None`` where the value was lost or malformed)."""', + ] + field_lines = [ + f" {f.name}: {fm.mirror_type(f)} = None" for f in fm.fields(vo) + ] + if field_lines: + lines.extend(field_lines) + else: + lines.append(" pass") + return lines + + +def mirror_initializer(vo: MetaData, record_name: str) -> str: + """Emit ``(field=as_string(d, "field"), …)``.""" + assigns = [f"{f.name}={fm.extract_map_call(f)}" for f in fm.fields(vo)] + return f"{record_name}({', '.join(assigns)})" + + +def extract_map_imports(vo: MetaData) -> list[str]: + """The sorted, deduped ``extract_map`` accessor names the mirror initializer needs.""" + used: set[str] = set() + for f in fm.fields(vo): + h = fm.extract_map_helper(f) + if h is not None: + used.add(h) + return [h for h in ALL_EXTRACT_MAP_HELPERS if h in used] diff --git a/server/python/src/metaobjects/codegen/runtime/fr010_field_mapping.py b/server/python/src/metaobjects/codegen/runtime/fr010_field_mapping.py new file mode 100644 index 000000000..ae2d7510b --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/fr010_field_mapping.py @@ -0,0 +1,228 @@ +"""Shared field-kind mapping for the FR-010 codegen emitters (``extract_schema_emitter`` ++ ``output_format_spec_emitter``). + +Maps a metadata field subtype onto the render engine's ``FieldKind`` member, the +idiomatic ``X | None`` Python type used by the extract mirror dataclass, and the +``extract_map.as_*`` accessor that reads it from the forgiving outcome map. + +Mirrors the C# ``Fr010FieldMapping`` and the Java ``ExtractSchemaEmitter`` ordering. +Bounded scope (parity with Java / Kotlin / C#): scalar / enum / scalar-array. Nested +object + array-of-enum are deferred (see ``render/extract/KNOWN_GAPS.md``). + +Python has a single ``int`` type, so the ``LONG`` and ``CURRENCY`` subtypes map to the +``LONG`` ``FieldKind`` (and ``extract_map.as_long`` — identical to ``as_int``), keeping +the cross-port ``FieldKind`` vocabulary intact even though the emitted accessor is the +same. The mirror annotation collapses ``INT``/``LONG`` to ``int`` (one numeric int type). +""" +from __future__ import annotations + +from collections.abc import Iterable + +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.meta_data import MetaData +from metaobjects.meta.template.template_constants import TEMPLATE_ATTR_XML_TEXT +from metaobjects.shared.base_types import TYPE_FIELD + + +def fields(vo: MetaData) -> list[MetaData]: + """The field children of a payload value-object, in declaration order + (effective children — own + inherited via ``extends``).""" + return [c for c in vo.children() if c.type == TYPE_FIELD] + + +def is_array(field: MetaData) -> bool: + """Effective array-ness (ADR-0039 resolving): the native ``is_array`` flag OR the + ``@isArray`` attr, resolved through the ``extends`` super chain so a concrete field + inheriting array-ness from an abstract parent is honored.""" + return field.resolved_is_array() + + +def is_required(field: MetaData) -> bool: + """``@required`` — accepts a bool ``True`` or the string ``"true"``.""" + v = field.attrs().get(fc.FIELD_ATTR_REQUIRED) + if v is True: + return True + return isinstance(v, str) and v.lower() == "true" + + +def xml_text(field: MetaData) -> bool: + """``@xmlText`` — the XML text-content extract marker (accepts a bool ``True`` or the + string ``"true"``). When set, codegen bakes a ``FieldSpec.text_content_field(...)``. + Mirrors the TS ``xmlText(field)`` helper.""" + v = field.attrs().get(TEMPLATE_ATTR_XML_TEXT) + if v is True: + return True + return isinstance(v, str) and v.lower() == "true" + + +def enum_values(field: MetaData) -> list[str]: + """The string members of an enum field's ``@values`` attr (empty when absent).""" + v = field.attrs().get(fc.FIELD_ATTR_VALUES) + if isinstance(v, (list, tuple)): + return [str(x) for x in v] + return [] + + +def _attr_string(node: MetaData, name: str) -> str | None: + """The RESOLVING string value of attr *name* (own + inherited via ``extends``), + or ``None``. + + ADR-0039 — effective read: a concrete enum field extending an abstract enum may + inherit ``@coerceDefault``/``@default``/``@normalize`` from the abstract parent, + so these resolve through the super chain (``get_meta_attr``), never own-only.""" + v = node.get_meta_attr(name) + return v if isinstance(v, str) else None + + +def coerce_default(field: MetaData) -> str | None: + """FR-011: the enum field's effective ``@coerceDefault`` member, or ``None``.""" + return _attr_string(field, fc.FIELD_ATTR_COERCE_DEFAULT) + + +def default_value(field: MetaData) -> str | None: + """FR-011: the enum field's effective ``@default`` absent-fill member, or ``None``.""" + return _attr_string(field, fc.FIELD_ATTR_DEFAULT) + + +def resolve_normalize(field: MetaData, owner: MetaData | None) -> str: + """FR-011: resolve the enum normalization mode — field-level ``@normalize``, else the + owning ``object.value``'s ``@normalize`` (the per-object default), else the global + default (``"strip"``). Mirrors the Java/Kotlin/C#/TS ``resolveNormalize``.""" + field_mode = _attr_string(field, fc.FIELD_ATTR_NORMALIZE) + if field_mode is not None: + return field_mode + if owner is not None: + owner_mode = _attr_string(owner, fc.FIELD_ATTR_NORMALIZE) + if owner_mode is not None: + return owner_mode + return fc.NORMALIZE_DEFAULT + + +def scalar_kind(sub_type: str) -> str | None: + """The render-engine ``FieldKind`` member name for a scalar field subtype, or + ``None`` if the subtype is non-scalar (enum / object).""" + if sub_type in ( + fc.FIELD_SUBTYPE_STRING, + fc.FIELD_SUBTYPE_DATE, + fc.FIELD_SUBTYPE_TIME, + fc.FIELD_SUBTYPE_TIMESTAMP, + # ADR-0036/0037 Wave 3: uri/inet are string scalars on the wire. + fc.FIELD_SUBTYPE_URI, + fc.FIELD_SUBTYPE_INET, + ): + return "STRING" + if sub_type == fc.FIELD_SUBTYPE_INT: + return "INT" + if sub_type in (fc.FIELD_SUBTYPE_LONG, fc.FIELD_SUBTYPE_CURRENCY): + return "LONG" + if sub_type in ( + fc.FIELD_SUBTYPE_DOUBLE, + fc.FIELD_SUBTYPE_FLOAT, + fc.FIELD_SUBTYPE_DECIMAL, + ): + return "DOUBLE" + if sub_type == fc.FIELD_SUBTYPE_BOOLEAN: + return "BOOLEAN" + return None + + +def mirror_type(field: MetaData) -> str: + """The ``X | None`` Python annotation for a field in the extract mirror dataclass. + + A extracted array can contain null elements where individual items were lost, so + the array type is ``list[str | None] | None`` (matches ``extract_map.as_string_list``). + """ + # Nested object (single OR array): the self-contained path defers to a typed null; + # the runtime-delegating path overrides this with the nested mirror type. Checked + # BEFORE the generic is_array branch so an array-of-objects is NOT mistyped as a + # string array. Mirrors the Java/TS fix. + if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: + return "object | None" # nested deferred (self-contained path) + if is_array(field): + return "list[str | None] | None" + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + return "str | None" # enum is string-backed + kind = scalar_kind(field.sub_type) + if kind in ("INT", "LONG"): + return "int | None" + if kind == "DOUBLE": + return "float | None" + if kind == "BOOLEAN": + return "bool | None" + return "str | None" + + +def extract_map_helper(field: MetaData) -> str | None: + """The ``extract_map`` accessor a field reads through, or ``None`` + (object/deferred fields read no helper). + + Single source of the field → accessor decision; ``extract_map_call`` builds + the ``as_*(d, "name")`` call on top of this.""" + # Nested object (single OR array) → no helper in the self-contained path. Checked + # BEFORE is_array so an array-of-objects is NOT read via as_string_list. Mirrors the + # Java/TS fix. + if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: + return None + if is_array(field): + return "as_string_list" + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + return "as_string" + return { + "INT": "as_int", + "LONG": "as_long", + "DOUBLE": "as_double", + "BOOLEAN": "as_bool", + }.get(scalar_kind(field.sub_type) or "", "as_string") + + +def extract_map_call(field: MetaData) -> str: + """The ``as_*(d, "name")`` call that reads this field from the forgiving map ``d``. + + The helpers are imported by name into the generated module (see + ``extract_schema_emitter.extract_map_imports``).""" + helper = extract_map_helper(field) + if helper is None: + # Nested object: the self-contained initializer leaves it None (the + # runtime-delegating path populates it). Bare ``None`` — no trailing inline + # comment: this expr is joined into a ``Name(field=..., …)`` call-args list. + return "None" + return f'{helper}(d, "{field.name}")' + + +def py_string_literal(value: str) -> str: + """A safe Python double-quoted string literal for embedding free text + (``@example`` / ``@instruction`` / ``@enumDoc`` values) into emitted source. + + Uses ``json.dumps`` — a JSON string is a valid Python string literal and escapes + backslashes / quotes / control chars correctly (NOT naive quoting).""" + import json + + return json.dumps(value, ensure_ascii=False) + + +def string_list_literal(values: Iterable[str]) -> str: + """A Python ``["a", "b"]`` list literal for enum members.""" + return "[" + ", ".join(py_string_literal(v) for v in values) + "]" + + +def properties_map_literal(attr: object) -> str: + """A Python ``{"k": "v", …}`` dict literal for a properties-shaped attr + (``@enumAlias`` / ``@enumDoc``), or ``"None"`` when absent/empty. + + Null values are dropped; keys are sorted to match the canonical-serializer + properties sort (deterministic output).""" + if not isinstance(attr, dict): + return "None" + entries = [ + (k, v) + for k, v in attr.items() + if v is not None + ] + if not entries: + return "None" + entries.sort(key=lambda kv: str(kv[0])) + parts = [ + f"{py_string_literal(str(k))}: {py_string_literal(str(v))}" + for k, v in entries + ] + return "{" + ", ".join(parts) + "}" diff --git a/server/python/src/metaobjects/codegen/runtime/fr019_shared_enum.py b/server/python/src/metaobjects/codegen/runtime/fr019_shared_enum.py new file mode 100644 index 000000000..8dd24b7b9 --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/fr019_shared_enum.py @@ -0,0 +1,154 @@ +"""FR-019 — shared + externally-provided enum resolution (Python port of the TS reference +``enum-shared.ts`` and the C#/Java/Kotlin ``Fr019SharedEnum``). + +A reusable enum is a ROOT/package-level abstract ``field.enum`` (a sibling of ``object.entity``) +that concrete entity fields ``extends``. Per ADR-0026 such a declaration is materialized ONCE per +port as a module-level ``class (str, Enum)`` and referenced, instead of redeclaring the +members inline (a per-entity ``Literal[...]``) in every consumer. + +Two cases on the abstract declaration: + +* non-``@provided`` → metaobjects MATERIALIZES the type once (a shared ``enums.py`` module), and + consuming fields reference it (``from .enums import ``). +* ``@provided: true`` → metaobjects emits NOTHING for the type; consuming fields import an existing + type from a per-port-configured module (``GenConfig.provided_enum_packages`` keyed by the enum's + declaring metadata package, then the ``provided_enum_namespace`` fallback). The module never lives + in metadata (ADR-0001); a missing config for a referenced provided enum is a codegen-time error + naming the enum + its package. + +The common inline case (a concrete ``field.enum`` with own ``@values``, no root-extends) is +UNCHANGED — it stays a per-entity ``Literal[...]`` (byte-identical default). +""" +from __future__ import annotations + +import warnings +from dataclasses import dataclass + +from metaobjects.meta.core.field.meta_field import ( + MetaField, + resolve_shared_enum_declaration, +) +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.meta_data import MetaData +from metaobjects.shared.separators import PACKAGE_SEP +from metaobjects.codegen import type_map +from metaobjects.codegen.config import GenConfig + + +@dataclass(frozen=True) +class SharedEnum: + """A shared (package-level abstract) enum declaration codegen reasons about.""" + + name: str #: materialized type name — the decl's PascalCase short name (cross-port id) + values: list[str] #: member symbols (the @values SSOT), verbatim + provided: bool #: True when the decl carries @provided: true (reference, don't materialize) + meta_package: str #: declaring metadata package in "::" form (the provided-config key) + + +def _pascal(name: str) -> str: + """``priority`` → ``Priority`` (leading char only; no snake-splitting — cross-port rule).""" + return name[:1].upper() + name[1:] if name else name + + +def resolve_shared_enum_decl(field: MetaField) -> MetaData | None: + """Deprecated name of + :func:`metaobjects.meta.core.field.meta_field.resolve_shared_enum_declaration`, + kept so a generator ejected before the resolver moved to the metadata layer + still imports. Same answer, including through a multi-hop ``extends``.""" + warnings.warn( + "resolve_shared_enum_decl is deprecated; call " + "metaobjects.meta.core.field.meta_field.resolve_shared_enum_declaration", + DeprecationWarning, + stacklevel=2, + ) + return resolve_shared_enum_declaration(field) + + +def is_provided(decl: MetaData) -> bool: + """``@provided`` truth of an enum DECLARATION. + + ADR-0039 sanctioned own (Python naming inversion: ``attr()`` is the OWN read, + ``get_meta_attr()`` resolves). ``@provided`` is a declaration-layer provenance + marker — "THIS type is supplied by hand-written/third-party code", like + ``is_abstract`` — and does not flow down an ``extends`` chain. + + This is only ever called on the resolved declaration (see + ``shared_enum_for_field``), never on the consuming field, so own and resolving + agree for a plain ``field extends @provided decl``. They diverge on a CHAINED + declaration (root abstract ``B extends`` root abstract ``@provided A``): a + resolving read reports B provided and emits a reference to a hand-written ``B`` + the adopter never declared, instead of materializing B. Matches the JVM ports. + """ + return decl.attr(fc.FIELD_ATTR_PROVIDED) is True + + +def _meta_package_of(decl: MetaData) -> str: + """The decl's metadata package in ``::`` form (its resolution key minus the trailing name).""" + key = decl.resolution_key() + idx = key.rfind(PACKAGE_SEP) + return key[:idx] if idx >= 0 else "" + + +def shared_enum_for_field(field: MetaField) -> SharedEnum | None: + """The shared-enum descriptor *field* resolves to, or ``None`` for inline enums. + + The member set is read off the DECLARATION *decl* (a resolving ``attrs()`` + read, never off *field*) so materialization cannot depend on which consuming + field a generator happens to visit first — every field resolving to the same + shared declaration must describe that declaration identically.""" + decl = resolve_shared_enum_declaration(field) + if decl is None: + return None + values = type_map.effective_enum_values(decl) + if not values: + return None + return SharedEnum( + name=_pascal(decl.name), + values=values, + provided=is_provided(decl), + meta_package=_meta_package_of(decl), + ) + + +def collect_shared_enums(entities: list) -> list[SharedEnum]: + """All shared (package-level abstract) enum decls CONSUMED by at least one entity field, + keyed by materialized type name, sorted by name (deterministic). A decl nobody extends is + not materialized (no dangling type).""" + out: dict[str, SharedEnum] = {} + for entity in entities: + # ADR-0039 — RESOLVING (fields()): mirrors the TS collectSharedEnums + # (``entity.fields()``); an inherited field consuming a shared enum counts. + for f in entity.fields(): + shared = shared_enum_for_field(f) + if shared is None: + continue + out.setdefault(shared.name, shared) + return sorted(out.values(), key=lambda e: e.name) + + +def materialized_shared_enums(entities: list) -> list[SharedEnum]: + """The shared enums metaobjects MATERIALIZES (non-``@provided``, consumed).""" + return [e for e in collect_shared_enums(entities) if not e.provided] + + +def enum_type_and_import(shared: SharedEnum, config: GenConfig) -> tuple[str, str]: + """The ``(type_name, import_line)`` a consuming field emits for a shared/provided enum: + + * materialized → ``("", "from .enums import ")`` (the shared module). + * ``@provided`` → ``("", "from import ")``, *module* resolved from + ``config.provided_enum_packages`` (keyed by the enum's metadata package) then the single + ``config.provided_enum_namespace`` fallback. A missing config is a codegen-time ``ValueError`` + naming the enum + its package (ADR-0026 / ADR-0001).""" + name = shared.name + if not shared.provided: + return name, f"from .enums import {name}" + module = config.provided_enum_packages.get(shared.meta_package) or config.provided_enum_namespace + if not module: + raise ValueError( + f'provided enum "{name}" (declared in package "{shared.meta_package}") is marked ' + f"@provided but no Python import module is configured to reference it from. Map its " + f'package in GenConfig.provided_enum_packages["{shared.meta_package}"] = ' + f'"your_app.enums", or set the single provided_enum_namespace fallback, so the ' + f'generated code can import "{name}".' + ) + return name, f"from {module} import {name}" diff --git a/server/python/src/metaobjects/codegen/runtime/m2m_codegen.py b/server/python/src/metaobjects/codegen/runtime/m2m_codegen.py new file mode 100644 index 000000000..813794dbb --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/m2m_codegen.py @@ -0,0 +1,240 @@ +"""Shared M:N (many-to-many) codegen descriptor — the build-time resolution of a +source entity's ``relationship.* @cardinality:"many" + @through`` children into +the concrete coordinates each Python generator needs (Pydantic nested collection, +FastAPI traversal route, and the physical junction/target column names the +repository seam joins on). + +This mirrors the TS reference ``relation-resolver.ts`` (Unit 10): codegen derives +the descriptor at build time from the junction entity's two ``identity.reference`` +children (the cross-port SSOT for FK direction) via the shared +``derive_m2m_fields`` helper — the relationship never restates its FK columns. + +The descriptor carries: + * ``relation_name`` — the navigation member name (route segment + Pydantic field). + * ``target_entity`` — the related entity's bare name (Pydantic element type). + * ``source_plural`` / ``target_plural`` — pluralized entity names for the + REST contract: the source URL segment is the ENTITY name pluralized + (``Person`` → ``persons``), NOT the physical ``@table`` (cross-port grammar). + * ``source_column`` / ``target_column`` — the PHYSICAL junction FK columns + (e.g. ``follower_id`` / ``followee_id``); the repository seam joins on these. + * ``junction_table`` / ``target_table`` / ``target_pk_column`` — physical names + for the join. + * ``symmetric`` — selects the union-on-read traversal. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from metaobjects.apidocs.naming import route_path +from metaobjects.source_resolution import resolve_table_name +from metaobjects.naming import ( + DEFAULT_COLUMN_NAMING, + apply_column_naming_strategy, + resolve_column_name, +) +from metaobjects.meta.core.field.meta_field import MetaField +from metaobjects.meta.meta_data import MetaData +from metaobjects.meta.core.identity.identity_constants import ( + IDENTITY_ATTR_FIELDS, + IDENTITY_SUBTYPE_PRIMARY, +) +from metaobjects.meta.core.object.meta_object import MetaObject +from metaobjects.meta.core.object.object_constants import OBJECT_ATTR_DISCRIMINATOR_VALUE +from metaobjects.codegen.generators.tph_plan import is_tph_subtype +from metaobjects.meta.core.relationship.derive_m2m_fields import ( + M2MDerivationError, + derive_m2m_fields, +) +from metaobjects.meta.core.relationship.meta_relationship import MetaRelationship +from metaobjects.meta.core.relationship.relationship_constants import CARDINALITY_MANY +from metaobjects.shared.base_types import TYPE_IDENTITY +from metaobjects.shared.separators import PACKAGE_SEP + + +@dataclass(frozen=True) +class M2mDescriptor: + """Build-time coordinates of one M:N navigation on a source entity.""" + + relation_name: str + target_entity: str + source_plural: str + target_plural: str + junction_table: str + target_table: str + source_column: str + target_column: str + target_pk_column: str + symmetric: bool + #: The target's OWN ``@discriminatorValue`` when ``@objectRef`` resolves to a + #: concrete TPH subtype (FW-8 follow-up; mirrors the TS reference's + #: ``targetDiscriminator``) — ``None`` for the overwhelming common case (a + #: vanilla, non-TPH target). The target's rows physically live in its + #: discriminator base's shared table, so a route generator threads this value + #: into the ``find_related_`` seam as an extra ``target_subtype`` + #: parameter, structurally handing the consumer's join what it needs to + #: filter to just that subtype rather than every row of the shared table. + #: Defaulted so existing positional construction (if any) stays valid. + target_discriminator: str | None = None + + +def _strip_package(name: str) -> str: + idx = name.rfind(PACKAGE_SEP) + return name[idx + len(PACKAGE_SEP):] if idx >= 0 else name + + +def _physical_table(entity: MetaObject, *, relation_context: str) -> str: + """The entity's physical SQL table name, via the shared + :func:`~metaobjects.source_resolution.resolve_table_name` resolver. + + Raises :class:`M2MDerivationError` — naming *entity* and *relation_context* + — when it declares (and inherits) no primary source at all. #248: DB + participation derives from a declared source, never from the object + subtype, so ``resolve_table_name`` returning ``None`` here means the + junction/target has no physical table to join through at all; a M:N + relationship's whole point is a real SQL join, so that is a metadata + authoring error, not a legitimate "doesn't participate in persistence" + shape. Coercing to the bare entity name — the pre-existing behaviour this + replaces — fabricated a table that was never created, precisely the defect + class the C# port's EF binding had to fix this same session (a + ``[Table(...)]`` attribute naming a class that was never generated).""" + table = resolve_table_name(entity) + if table is None: + raise M2MDerivationError( + f"{relation_context}: entity \"{entity.name}\" declares no primary " + f"source (source.rdb) — a M:N join needs a physical table on both " + f"the junction and the target." + ) + return table + + +def _column_of( + field: MetaField | None, fallback: str, strategy: str = DEFAULT_COLUMN_NAMING +) -> str: + """Physical column name for a junction FK. Delegates to the shared + :func:`metaobjects.naming.resolve_column_name` — the SAME resolver the runtime uses, + so generated junction SQL and runtime SQL cannot disagree about a column name. + + *fallback* covers the case where the field could not be resolved at all; it is a + bare metadata name, so it goes through the strategy too.""" + if field is None: + return apply_column_naming_strategy(fallback, strategy) + return resolve_column_name(field, strategy) + + +def _field_named(entity: MetaObject, field_name: str) -> MetaField | None: + for f in entity.fields(): + if f.name == field_name: + return f + return None + + +def pk_field_name(entity: MetaObject) -> str: + """The single PK field name (``identity.primary @fields``), default ``id``.""" + for c in entity.children(): + if c.type == TYPE_IDENTITY and c.sub_type == IDENTITY_SUBTYPE_PRIMARY: + fields = c.get_meta_attr(IDENTITY_ATTR_FIELDS) # ADR-0039: resolving (identity attr) + if isinstance(fields, (list, tuple)) and fields: + first = fields[0] + if isinstance(first, str): + return first + if isinstance(fields, str) and fields: + return fields.split(",")[0].strip() or "id" + return "id" + + +def m2m_relationships(entity: MetaObject) -> list[MetaRelationship]: + """The entity's ``relationship.* @cardinality:"many" + @through`` children + (declaration order). 1:N relationships (no ``@through``) are excluded. + + ADR-0039 — RESOLVING (children()): mirrors the TS ``relationships()`` (built on + the effective ``children()``); a relationship inherited via ``extends`` counts. + """ + out: list[MetaRelationship] = [] + for c in entity.children(): + if not isinstance(c, MetaRelationship): + continue + if c.cardinality() != CARDINALITY_MANY: + continue + if c.through() is None: + continue + out.append(c) + return out + + +def resolve_m2m_descriptors( + entity: MetaObject, + object_index: dict[str, MetaObject], + column_naming: str = DEFAULT_COLUMN_NAMING, +) -> list[M2mDescriptor]: + """Resolve every M:N navigation on *entity* into a build-time descriptor. + + *object_index* is a bare-name → entity map of the loaded model. A relationship + that fails derivation (missing/malformed junction, ambiguous self-join) raises + :class:`M2MDerivationError` — the loader validates this; reaching it here is a + metadata error, surfaced loudly (never silently skipped). + """ + descriptors: list[M2mDescriptor] = [] + # The source is *entity* for every relation, so its route segment is + # loop-invariant; only the target varies per relation. + source_plural = route_path(entity.name) + for rel in m2m_relationships(entity): + target_name = _strip_package(rel.object_ref() or "") + junction_name = _strip_package(rel.through() or "") + target = object_index.get(target_name) + junction = object_index.get(junction_name) + if target is None or junction is None: + raise M2MDerivationError( + f'M:N relationship "{entity.name}.{rel.name}": @objectRef ' + f'"{target_name}" / @through "{junction_name}" must resolve to ' + f"loaded entities" + ) + + # Widen to dict[str, MetaData] for the shared derivation helper (dict is + # invariant; MetaObject is a MetaData subtype). + meta_index: dict[str, MetaData] = dict(object_index) + fields = derive_m2m_fields(rel, entity, meta_index) + source_fk = _field_named(junction, fields.source_field) + target_fk = _field_named(junction, fields.target_field) + # FW-8 follow-up: the target's rows live in its discriminator base's shared + # table when @objectRef resolves to a concrete TPH subtype — @discriminatorValue + # is the subtype's OWN value (ADR-0039 sanctioned own, same idiom as tph_plan.py). + target_discriminator = ( + target.attr(OBJECT_ATTR_DISCRIMINATOR_VALUE) if is_tph_subtype(target) else None + ) + + descriptors.append( + M2mDescriptor( + relation_name=rel.name, + target_entity=target.name, + source_plural=source_plural, + target_plural=route_path(target.name), + junction_table=_physical_table( + junction, + relation_context=( + f'M:N relationship "{entity.name}.{rel.name}" @through junction' + ), + ), + target_table=_physical_table( + target, + relation_context=( + f'M:N relationship "{entity.name}.{rel.name}" @objectRef target' + ), + ), + source_column=_column_of(source_fk, fields.source_field, column_naming), + target_column=_column_of(target_fk, fields.target_field, column_naming), + target_pk_column=_column_of( + _field_named(target, pk_field_name(target)), + pk_field_name(target), + column_naming, + ), + symmetric=rel.symmetric(), + target_discriminator=target_discriminator, + ) + ) + return descriptors + + +def build_object_index(entities: list[MetaObject]) -> dict[str, MetaObject]: + """Bare-name → entity map (the codegen resolution surface, mirroring the + runtime resolver's ``object_index``).""" + return {e.name: e for e in entities} diff --git a/server/python/src/metaobjects/codegen/runtime/output_format_spec_emitter.py b/server/python/src/metaobjects/codegen/runtime/output_format_spec_emitter.py new file mode 100644 index 000000000..be0193de5 --- /dev/null +++ b/server/python/src/metaobjects/codegen/runtime/output_format_spec_emitter.py @@ -0,0 +1,83 @@ +"""Turns a response value-object + its ``template.prompt`` node into a Python source +literal for an :class:`~metaobjects.render.OutputFormatSpec` — the artifact-1 prompt +descriptor used by the FR-010 response-format codegen. + +Emits ``OutputFormatSpec(Format.X, "rootName", PromptStyle.X, [PromptField(...), …])``. +Mirrors the C# / Java ``OutputFormatSpecEmitter`` adapted to Python (keyword args for +the long ``PromptField`` ctor to keep the emitted source readable + order-robust). +Bounded scope: scalar / enum. Nested object → ``FieldKind.OBJECT`` placeholder. +""" +from __future__ import annotations + +from . import fr010_field_mapping as fm +from metaobjects.codegen.generators.find_inbound import is_xml, response_format_of +from metaobjects.meta.core.field import field_constants as fc +from metaobjects.meta.meta_data import MetaData +from metaobjects.meta.template import template_constants as tc + + +def _format_enum(template: MetaData) -> str: + """ADR-0053 — the fragment describes the REPLY, so its syntax is ``@responseFormat``, + NOT ``@format``, which is the syntax of the rendered prompt BODY. Reading ``@format`` + here typed the instruction "produce your answer like this" off the format of the + QUESTION.""" + return "Format.XML" if is_xml(response_format_of(template)) else "Format.JSON" + + +def _prompt_style_enum(template: MetaData) -> str: + style = template.get_meta_attr(tc.TEMPLATE_ATTR_PROMPT_STYLE) # ADR-0039: template attr resolves via extends (not origin; templates CAN extend) + if style == tc.PROMPT_STYLE_INLINE: + return "PromptStyle.INLINE" + if style == tc.PROMPT_STYLE_EXAMPLE_ONLY: + return "PromptStyle.EXAMPLE_ONLY" + return "PromptStyle.GUIDE" + + +def _opt_string_attr(field: MetaData, attr_name: str) -> str: + v = field.attrs().get(attr_name) + return fm.py_string_literal(v) if isinstance(v, str) else "None" + + +def _prompt_field_literal(field: MetaData) -> str: + name = field.name + req = "True" if fm.is_required(field) else "False" + array = "True" if fm.is_array(field) else "False" + + if field.sub_type == fc.FIELD_SUBTYPE_OBJECT: + # FR-010: nested prompt deferred — emit a FieldKind.OBJECT placeholder (nested + # defaults to None on PromptField). The note lives HERE, not in the returned + # literal: spec_literal joins every field onto ONE line, and a Python '#' inline + # comment would swallow the rest of that line — including the closing `])` — so a + # nested-object payload emitted invalid Python (unterminated list) and crashed + # `gen` + `verify --codegen`. The other ports use `/* */` block comments, which + # are inline-safe; Python has none, so the literal must carry no comment. + return f'PromptField("{name}", FieldKind.OBJECT, {req}, array={array})' + + example = _opt_string_attr(field, fc.FIELD_ATTR_EXAMPLE) + instruction = _opt_string_attr(field, fc.FIELD_ATTR_INSTRUCTION) + + if field.sub_type == fc.FIELD_SUBTYPE_ENUM: + values_lit = fm.string_list_literal(fm.enum_values(field)) + enum_doc_lit = fm.properties_map_literal(field.attrs().get(fc.FIELD_ATTR_ENUM_DOC)) + return ( + f'PromptField("{name}", FieldKind.ENUM, {req}, array={array}, ' + f"enum_values={values_lit}, enum_doc={enum_doc_lit}, " + f"example={example}, instruction={instruction})" + ) + + kind = fm.scalar_kind(field.sub_type) or "STRING" + return ( + f'PromptField("{name}", FieldKind.{kind}, {req}, array={array}, ' + f"example={example}, instruction={instruction})" + ) + + +def spec_literal(vo: MetaData, template: MetaData, root_name: str) -> str: + """Emit ``OutputFormatSpec(Format.X, "rootName", PromptStyle.X, [PromptField(...), …])``.""" + format_enum = _format_enum(template) + style_enum = _prompt_style_enum(template) + field_lits = [_prompt_field_literal(f) for f in fm.fields(vo)] + body = ", ".join(field_lits) + return ( + f'OutputFormatSpec({format_enum}, "{root_name}", {style_enum}, [{body}])' + ) diff --git a/server/python/src/metaobjects/codegen/runtime/shared_predicates.py b/server/python/src/metaobjects/codegen/runtime/shared_predicates.py index 1ed052e6c..0a0cdded2 100644 --- a/server/python/src/metaobjects/codegen/runtime/shared_predicates.py +++ b/server/python/src/metaobjects/codegen/runtime/shared_predicates.py @@ -20,6 +20,7 @@ import re +from metaobjects.codegen.generators.tph_plan import is_tph_subtype from metaobjects.codegen.instance_artifacts import emits_instance_artifacts from metaobjects.meta.core.field import field_constants as fc from metaobjects.meta.core.field.meta_field import MetaField @@ -93,3 +94,30 @@ def emits_router(entity: MetaObject) -> bool: or kind == SOURCE_KIND_TABLE or kind in READ_ONLY_ROUTED_KINDS ) + + +def is_read_only_routed(entity: MetaObject) -> bool: + """Whether the router generator chooses the READ-ONLY router for this object (a view / + materialized-view source that is not write-through): GET list, GET by id when it has an + item route, and the 405 refusals, with no write surface. + + THE predicate behind that choice, shared with the API-docs generator so a page can never + present write verbs for an object whose router refuses them.""" + if not emits_router(entity) or entity.is_write_through(): + return False + src = primary_rdb_source(entity) + return src is not None and src.effective_kind() in READ_ONLY_ROUTED_KINDS + + +def emits_writable_router(entity: MetaObject) -> bool: + """Whether the router generator emits the full CRUD router for this object: it gets a + router, that router is not the read-only one, and it is not a table-per-hierarchy subtype + (a subtype is folded into its base's single table, so its CRUD lives under the base). + + THE predicate behind that choice, shared with the API-docs generator so a page can never + document write routes for an object the router does not give them.""" + return ( + emits_router(entity) + and not is_read_only_routed(entity) + and not is_tph_subtype(entity) + ) diff --git a/server/python/tests/codegen/test_api_docs_paths.py b/server/python/tests/codegen/test_api_docs_paths.py index 0c4e6683b..96f824227 100644 --- a/server/python/tests/codegen/test_api_docs_paths.py +++ b/server/python/tests/codegen/test_api_docs_paths.py @@ -11,7 +11,7 @@ import json from pathlib import Path -from metaobjects.apidocs.paths import ( +from metaobjects.codegen.runtime.api_docs_paths import ( Layout, doc_page_output_path, model_cross_href, diff --git a/server/python/tests/codegen/test_cli_registry.py b/server/python/tests/codegen/test_cli_registry.py index 55377eb3a..0ead45d7e 100644 --- a/server/python/tests/codegen/test_cli_registry.py +++ b/server/python/tests/codegen/test_cli_registry.py @@ -39,8 +39,8 @@ def test_gen_list_prints_all_and_exits_zero(capsys, tmp_path: Path) -> None: # Every registered stable name appears in the listing. for name in GENERATOR_REGISTRY: assert name in out, f"--list omitted {name!r}" - # Exactly the 11 python-slice names are registered (ADR-0056 removed `payload`). - assert len(GENERATOR_REGISTRY) == 11 + # Exactly the 12 python-slice names are registered (ADR-0056 removed `payload`). + assert len(GENERATOR_REGISTRY) == 12 assert "payload" not in GENERATOR_REGISTRY # A template-tier generator says it needs the entity generator's models. assert "output-parser — " in out diff --git a/server/python/tests/codegen/test_docs_generators.py b/server/python/tests/codegen/test_docs_generators.py new file mode 100644 index 000000000..6bdcd8109 --- /dev/null +++ b/server/python/tests/codegen/test_docs_generators.py @@ -0,0 +1,205 @@ +"""`api-docs` is a generator `metaobjects docs` runs, and a project can own it. + +An owned copy is listed under ``docs.generators`` in ``metaobjects.config.yaml`` and ``docs`` +runs it in place of the packaged one. ``gen`` never runs a docs generator: it skips one found +under a target and says so. What the reference documents comes from the same +``shared_predicates`` module the owned ``routes`` and ``filter-allowlist`` read, so an edit +to the owned predicate moves the router, the allowlist and the reference together. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from metaobjects.cli import main +from metaobjects.codegen.project_config import ConfigError, load_project_config + +REPO = Path(__file__).parents[4] +FITNESS = REPO / "fixtures" / "persistence-conformance" / "canonical" / "meta.fitness.json" +OWNED_DOCS = "codegen.generators.api_docs:api_docs_generator" + + +def _project(root: Path, extra: str = "", generators: str = "entity") -> Path: + (root / "metaobjects").mkdir(parents=True, exist_ok=True) + (root / "metaobjects" / FITNESS.name).write_text(FITNESS.read_text(encoding="utf-8")) + cfg = root / "metaobjects.config.yaml" + cfg.write_text( + f"targets:\n main:\n outDir: gen\n generators: [{generators}]\n{extra}", + encoding="utf-8", + ) + return cfg + + +@pytest.fixture(autouse=True) +def _fresh_owned_modules(): + """A real run is a fresh process; here one process runs several projects, and an edit to + an owned module must be seen by the next run, so each run starts with no `codegen` loaded.""" + _forget_owned() + yield + _forget_owned() + + +def _forget_owned() -> None: + for name in [k for k in sys.modules if k == "codegen" or k.startswith("codegen.")]: + del sys.modules[name] + + +def _docs(root: Path, monkeypatch, *args: str) -> int: + _forget_owned() + monkeypatch.chdir(root) + return main(["docs", "--out", "docs", "--project", "model", *args]) + + +def _pages(root: Path) -> dict[str, str]: + base = root / "docs" + return {p.relative_to(base).as_posix(): p.read_text(encoding="utf-8") + for p in sorted(base.rglob("*.md"))} + + +# --- config ----------------------------------------------------------------------------- + + +def test_the_docs_block_is_read(tmp_path: Path) -> None: + cfg = _project(tmp_path, f"docs:\n generators: [{OWNED_DOCS}]\n") + assert load_project_config(cfg).docs_generators == [OWNED_DOCS] + assert load_project_config(_project(tmp_path / "plain")).docs_generators == [] + + +@pytest.mark.parametrize("block, message", [ + ("docs:\n generators: [api-docs]\n", "module:symbol"), + ("docs:\n generators: nope\n", "list of strings"), + ("docs:\n generator: [a:b]\n", "unknown key"), + ("docs: []\n", "must be a mapping"), +]) +def test_a_bad_docs_block_is_refused_by_name(tmp_path: Path, block: str, message: str) -> None: + with pytest.raises(ConfigError, match=message): + load_project_config(_project(tmp_path, block)) + + +# --- docs runs the owned copy ----------------------------------------------------------- + + +def test_docs_runs_an_owned_api_docs_in_place_of_the_packaged_one(tmp_path: Path, monkeypatch, capsys) -> None: + _project(tmp_path, f"docs:\n generators: [{OWNED_DOCS}]\n") + monkeypatch.chdir(tmp_path) + assert main(["eject", "api-docs"]) == 0 + out = capsys.readouterr().out + assert "docs.generators" in out and OWNED_DOCS in out and "`gen` never does" in out + assert _docs(tmp_path, monkeypatch) == 0 + owned_pages = _pages(tmp_path) + + packaged = tmp_path / "packaged" + _project(packaged) + assert _docs(packaged, monkeypatch) == 0 + # Unchanged, the copy writes what the packaged generator writes. + assert owned_pages == _pages(packaged) and owned_pages + + # An edit to the owned renderer reaches the page: the copy, not the package, ran. + renderer = tmp_path / "codegen" / "runtime" / "api_docs_renderer.py" + source = renderer.read_text(encoding="utf-8") + assert source.count("# Agent API Reference") == 1 + renderer.write_text(source.replace("# Agent API Reference", "# Owned Agent API Reference"), + encoding="utf-8") + assert _docs(tmp_path, monkeypatch) == 0 + assert _pages(tmp_path)["api/python/AGENT-API.md"].startswith("# Owned Agent API Reference") + + +def test_docs_without_a_docs_block_runs_the_packaged_generator_even_when_one_is_owned( + tmp_path: Path, monkeypatch, +) -> None: + _project(tmp_path) + monkeypatch.chdir(tmp_path) + assert main(["eject", "api-docs"]) == 0 + renderer = tmp_path / "codegen" / "runtime" / "api_docs_renderer.py" + renderer.write_text(renderer.read_text(encoding="utf-8").replace( + "# Agent API Reference", "# Owned Agent API Reference"), encoding="utf-8") + assert _docs(tmp_path, monkeypatch) == 0 + assert _pages(tmp_path)["api/python/AGENT-API.md"].startswith("# Agent API Reference") + + +def test_the_positional_docs_form_reads_no_config(tmp_path: Path, monkeypatch) -> None: + _project(tmp_path, f"docs:\n generators: [{OWNED_DOCS}]\n") + monkeypatch.chdir(tmp_path) + assert main(["eject", "api-docs"]) == 0 + renderer = tmp_path / "codegen" / "runtime" / "api_docs_renderer.py" + renderer.write_text(renderer.read_text(encoding="utf-8").replace( + "# Agent API Reference", "# Owned Agent API Reference"), encoding="utf-8") + assert _docs(tmp_path, monkeypatch, "metaobjects") == 0 + assert _pages(tmp_path)["api/python/AGENT-API.md"].startswith("# Agent API Reference") + + +def test_a_docs_entry_that_is_not_a_docs_generator_is_refused_by_name( + tmp_path: Path, monkeypatch, capsys, +) -> None: + _project(tmp_path, "docs:\n generators: [codegen.generators.entity:entity_model]\n") + monkeypatch.chdir(tmp_path) + assert main(["eject", "entity"]) == 0 + assert _docs(tmp_path, monkeypatch) == 1 + err = capsys.readouterr().err + assert "codegen.generators.entity:entity_model" in err and "not a docs generator" in err + assert not (tmp_path / "docs").exists() + + +def test_two_owned_generators_for_the_api_surface_are_refused(tmp_path: Path, monkeypatch, capsys) -> None: + _project(tmp_path, f"docs:\n generators: [{OWNED_DOCS}, {OWNED_DOCS}]\n") + monkeypatch.chdir(tmp_path) + assert main(["eject", "api-docs"]) == 0 + assert _docs(tmp_path, monkeypatch) == 1 + assert "two generators for the 'api' surface" in capsys.readouterr().err + + +# --- gen leaves a docs generator out ---------------------------------------------------- + + +def test_gen_skips_a_docs_generator_under_a_target_and_says_so(tmp_path: Path, monkeypatch, capsys) -> None: + _project(tmp_path, generators="entity, api-docs") + monkeypatch.chdir(tmp_path) + assert main(["gen"]) == 0 + out = capsys.readouterr().out + assert "warning: 'api-docs' writes the api reference, so gen skipped it" in out + assert "docs.generators" in out + written = {p.relative_to(tmp_path / "gen").as_posix() for p in (tmp_path / "gen").rglob("*") + if p.is_file()} + assert written and not any(w.startswith("api/") or w.endswith("AGENT-API.md") for w in written) + assert not (tmp_path / "gen" / "api").exists() + + +# --- the reference describes the code the owned generators emit -------------------------- + + +def test_one_owned_predicate_moves_the_router_the_allowlist_and_the_reference( + tmp_path: Path, monkeypatch, +) -> None: + owned = tmp_path / "owned" + _project(owned, f"docs:\n generators: [{OWNED_DOCS}]\n", + generators="entity, codegen.generators.filter_allowlist:filter_allowlist_generator, " + "codegen.generators.routes:router_generator") + monkeypatch.chdir(owned) + assert main(["eject", "api-docs", "filter-allowlist", "routes"]) == 0 + assert main(["gen"]) == 0 + assert _docs(owned, monkeypatch) == 0 + before = _pages(owned) + assert any(f.name == "post_router.py" for f in (owned / "gen").glob("*_router.py")) + assert any("GET /api/posts" in page for page in before.values()) + + rule = owned / "codegen" / "runtime" / "shared_predicates.py" + rule.write_text( + rule.read_text(encoding="utf-8") + + "\n\n_reference_emits_router = emits_router\n\n\n" + "def emits_router(entity):\n" + " return entity.name != 'Post' and _reference_emits_router(entity)\n", + encoding="utf-8", + ) + for stale in (owned / "gen").glob("*"): + if stale.is_file(): + stale.unlink() + _forget_owned() + assert main(["gen"]) == 0 + assert _docs(owned, monkeypatch) == 0 + after = _pages(owned) + assert not any(f.name == "post_router.py" for f in (owned / "gen").glob("*_router.py")) + assert not any(f.name == "post_filter_allowlist.py" for f in (owned / "gen").glob("*.py")) + assert not any("GET /api/posts" in page for page in after.values()) + assert after != before diff --git a/server/python/tests/codegen/test_eject_alone.py b/server/python/tests/codegen/test_eject_alone.py new file mode 100644 index 000000000..ab473c435 --- /dev/null +++ b/server/python/tests/codegen/test_eject_alone.py @@ -0,0 +1,323 @@ +"""Each ejectable generator, ejected ALONE, owns everything that writes its output. + +ADR-0034 Amendment 3: the adopter owns the generator and the code that code calls, when +that code is a helper. So for every generator the registry makes ejectable, derived from the +registry and not from a list kept beside it, an empty project that ejects only that generator +must get a copy that + +* imports no packaged emit module: the copy and the helper source eject placed beside it + import only the engine surface written below (the generator base, the file writer and + formatter, the naming and type-mapping tables, the model walks and the loader); +* loads and runs with every packaged emit module made unimportable; and +* writes output byte-identical to the packaged generator over the same model. + +The one documented difference is the router: an owned ``routes`` emits its runtime into the +generated package and imports it from there, so its router modules differ from the packaged +ones in those import lines and in the ``_runtime/`` files (``test_eject_runtime.py``). + +The census at the bottom keeps the guard honest: a helper module no registry entry hands over, +or a generator module no registry entry names, fails here instead of staying packaged. +""" +from __future__ import annotations + +import ast +import json +import os +import shutil +import subprocess +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +import pytest + +import metaobjects +from metaobjects.codegen import eject as owned +from metaobjects.codegen.generator_registry import GENERATOR_REGISTRY + +REPO = Path(__file__).parents[4] +FIXTURES = REPO / "fixtures" +CODEGEN = Path(metaobjects.__file__).parent / "codegen" + +#: The packaged ``metaobjects.codegen`` modules an owned copy may import: the engine. Everything +#: that returns emitted text is handed over instead, and everything outside ``metaobjects.codegen`` +#: (the loader, the metamodel, naming, render, extract, the runtime) is core by ruling. +ENGINE_CODEGEN = frozenset({ + "metaobjects.codegen.config", # GenConfig + "metaobjects.codegen.constants", # the @generated marker and header + "metaobjects.codegen.format", # the formatter + "metaobjects.codegen.generator", # the generator base, EmittedFile, GenContext + "metaobjects.codegen.instance_artifacts", # model questions: what an object emits + "metaobjects.codegen.model_walk", + "metaobjects.codegen.project_config", # the requirementTests options + "metaobjects.codegen.requirement_hooks", + "metaobjects.codegen.requirement_walk", # shared with `verify` + "metaobjects.codegen.type_map", # the type-mapping table + "metaobjects.codegen.value_objects", # naming and value-object resolution + "metaobjects.codegen.generators.find_inbound", # model walk: the inbound tier + "metaobjects.codegen.generators.tph_plan", # model walk: table-per-hierarchy +}) +#: Of ``metaobjects.apidocs``, only the naming seam: the names the generators emit under. +ENGINE_APIDOCS = frozenset({"metaobjects.apidocs.naming"}) + +EJECTABLE = sorted(e.name for e in owned.ejectable()) + + +def _runtime_imports(node: ast.AST) -> list[ast.stmt]: + """The import statements under *node* that run: an ``if TYPE_CHECKING:`` block is only an + annotation for a type checker, so it is not walked.""" + found: list[ast.stmt] = [] + for child in ast.iter_child_nodes(node): + if isinstance(child, ast.If) and isinstance(child.test, ast.Name) \ + and child.test.id == "TYPE_CHECKING": + found.extend(_runtime_imports_of(child.orelse)) + elif isinstance(child, (ast.Import, ast.ImportFrom)): + found.append(child) + else: + found.extend(_runtime_imports(child)) + return found + + +def _runtime_imports_of(body: list[ast.stmt]) -> list[ast.stmt]: + found: list[ast.stmt] = [] + for stmt in body: + if isinstance(stmt, (ast.Import, ast.ImportFrom)): + found.append(stmt) + else: + found.extend(_runtime_imports(stmt)) + return found + + +def _imports(source: str) -> list[tuple[int, str | None, list[str]]]: + """``(level, module, imported names)`` for each import statement in *source* that runs.""" + out: list[tuple[int, str | None, list[str]]] = [] + for node in _runtime_imports(ast.parse(source)): + if isinstance(node, ast.ImportFrom): + out.append((node.level, node.module, [a.name for a in node.names])) + elif isinstance(node, ast.Import): + out.extend((0, a.name, []) for a in node.names) + return out + + +def _outside_the_engine(source: str, handed_over: set[str]) -> list[str]: + """What *source* imports that is neither the engine surface nor a helper handed over.""" + offenders: list[str] = [] + for level, module, names in _imports(source): + if level: + # A relative import is the handed-over helper: `..runtime.` from a generator, + # `.` or `from . import ` from a helper. + if module is None: + targets = names + elif module.startswith("runtime."): + targets = [module.split(".", 1)[1]] + elif module == "runtime": + targets = names + else: + targets = [module] + offenders.extend( + f"relative import of {t!r}, which eject does not hand over" + for t in targets if t not in handed_over + ) + continue + if module is None or not module.startswith("metaobjects."): + continue + # `from metaobjects.codegen import type_map` names a module through the package. + packages = ("metaobjects.codegen", "metaobjects.codegen.generators", + "metaobjects.codegen.runtime", "metaobjects.apidocs") + candidates = [f"{module}.{n}" for n in names] if module in packages else [module] + for c in candidates: + if c.startswith("metaobjects.codegen"): + if c not in ENGINE_CODEGEN: + offenders.append(c) + elif c.startswith("metaobjects.apidocs"): + if c not in ENGINE_APIDOCS: + offenders.append(c) + return offenders + + +@pytest.mark.parametrize("name", EJECTABLE) +def test_an_ejected_generator_imports_only_the_engine(name: str, tmp_path: Path) -> None: + entry = GENERATOR_REGISTRY[name] + assert owned.eject([name], tmp_path).code == 0 + handed_over = set(entry.runtime) + copied = [owned.owned_path(tmp_path, name)] + [ + owned.runtime_path(tmp_path, m) for m in entry.runtime + ] + offenders: list[str] = [] + for path in copied: + assert path.is_file(), f"eject did not write {path.relative_to(tmp_path)}" + offenders += [ + f"{path.relative_to(tmp_path)}: {o}" + for o in _outside_the_engine(path.read_text(encoding="utf-8"), handed_over) + ] + assert not offenders, "\n".join(offenders) + # Nothing else was written: the copy is the generator and its helpers. + written = {p.relative_to(tmp_path).as_posix() for p in (tmp_path / "codegen").rglob("*.py") + if p.name != "__init__.py"} + assert written == {p.relative_to(tmp_path).as_posix() for p in copied} + + +# --- the same project, run twice ------------------------------------------------------- + +_PROMPTS = { + "metadata.root": { + "package": "acme::prompts", + "children": [ + {"object.value": {"name": "ClassifyRequest", "children": [ + {"field.string": {"name": "text", "@required": True}}]}}, + {"object.value": {"name": "ClassifyResponse", "children": [ + {"field.string": {"name": "label", "@required": True}}, + {"field.int": {"name": "score"}}, + {"field.string": {"name": "tags", "isArray": True}}]}}, + {"template.prompt": { + "name": "ClassifyPrompt", "@payloadRef": "ClassifyRequest", + "@responseRef": "ClassifyResponse", "@textRef": "p/classify", "@format": "json"}}, + {"template.prompt": { + "name": "ClassifyXmlPrompt", "@payloadRef": "ClassifyRequest", + "@responseRef": "ClassifyResponse", "@textRef": "p/classify", "@format": "xml"}}, + {"object.entity": { + "name": "ClassifyCall", "extends": "metaobjects::ai::LlmCallBase", + "children": [ + {"source.rdb": {"@table": "classify_call", "@role": "primary"}}, + {"identity.primary": {"name": "id", "@fields": ["traceId"]}}, + {"template.prompt": { + "name": "ClassifyTraced", "@payloadRef": "ClassifyRequest", + "@responseRef": "ClassifyResponse", "@textRef": "p/classify", + "@format": "json"}}]}}, + ], + } +} + + +def _model_project(root: Path, generators: str, docs: str = "") -> Path: + """A project over every metadata shape the ejectable generators read, with one target + that runs *generators* (and, when given, a ``docs.generators`` block).""" + meta = root / "metaobjects" + meta.mkdir(parents=True) + canonical = FIXTURES / "persistence-conformance" / "canonical" / "meta.fitness.json" + (meta / canonical.name).write_text(canonical.read_text(encoding="utf-8"), encoding="utf-8") + render = FIXTURES / "template-output-render-conformance" + (meta / "meta.render.json").write_text((render / "meta.json").read_text(encoding="utf-8")) + shutil.copytree(render / "templates", root / "templates") + shutil.copytree(canonical.parent / "prompts", root / "templates", dirs_exist_ok=True) + (root / "templates" / "p").mkdir() + (root / "templates" / "p" / "classify.mustache").write_text("Classify {{text}}\n") + (meta / "meta.prompts.json").write_text(json.dumps(_PROMPTS), encoding="utf-8") + shop = FIXTURES / "requirement-test-identity-conformance" / "worked-example" / "input" + for doc in shop.iterdir(): + (meta / doc.name).write_text(doc.read_text(encoding="utf-8"), encoding="utf-8") + config = f"libraries: [ai]\ntargets:\n main:\n outDir: gen\n generators: [{generators}]\n" + if docs: + config += f"docs:\n generators: [{docs}]\n" + (root / "metaobjects.config.yaml").write_text(config, encoding="utf-8") + return root + + +#: Run in a fresh interpreter: load the CLI, then (for an owned run) make every packaged emit +#: module unimportable, run the command, and print the tree it wrote. +_SCRIPT = """ +import json, os, sys +from pathlib import Path +from metaobjects.cli import main +block = json.loads(sys.argv[3]) +for name in list(sys.modules): + if any(name == b or name.startswith(b + ".") for b in block): + sys.modules[name] = None +sys.path.insert(0, os.getcwd()) +rc = main(json.loads(sys.argv[2])) +out = Path(sys.argv[1]) +tree = {p.relative_to(out).as_posix(): p.read_text(encoding="utf-8") + for p in sorted(out.rglob("*")) if p.is_file() and "__pycache__" not in p.parts} +print(json.dumps({"rc": rc, "tree": tree})) +""" + + +def _run(root: Path, argv: list[str], out: str, block: list[str]) -> dict[str, str]: + src = str(Path(metaobjects.__file__).resolve().parents[1]) + env = {**os.environ, "PYTHONPATH": os.pathsep.join( + p for p in (src, os.environ.get("PYTHONPATH", "")) if p)} + proc = subprocess.run( + [sys.executable, "-c", _SCRIPT, str(root / out), json.dumps(argv), json.dumps(block)], + cwd=root, env=env, capture_output=True, text=True, + ) + assert proc.returncode == 0, proc.stderr + result: dict[str, Any] = json.loads(proc.stdout.splitlines()[-1]) + assert result["rc"] == 0, proc.stderr + return result["tree"] # type: ignore[no-any-return] + + +def _packaged_modules() -> list[str]: + """Every packaged module an owned copy must not reach: each ejectable generator's own and + the helper runtime beside it.""" + return sorted({"metaobjects.codegen.runtime"} + | {e.source_module for e in owned.ejectable() if e.source_module}) + + +def _lines(text: str) -> Counter[str]: + return Counter(ln for ln in text.splitlines() if ln.strip()) + + +@pytest.mark.parametrize("name", EJECTABLE) +def test_an_ejected_generator_writes_what_the_packaged_one_writes(name: str, tmp_path: Path) -> None: + entry = GENERATOR_REGISTRY[name] + docs = entry.docs_surface is not None + # The docs label defaults to the directory's name, which differs between the two projects. + argv = ["docs", "--out", "docs", "--project", "model"] if docs else ["gen"] + out = "docs" if docs else "gen" + + packaged_root = _model_project(tmp_path / "packaged", "" if docs else name) + packaged = _run(packaged_root, argv, out, block=[]) + assert packaged, f"{name} wrote nothing over the test model, so it proves nothing" + + owned_root = _model_project( + tmp_path / "owned", + "" if docs else owned.owned_token(entry), + owned.owned_token(entry) if docs else "", + ) + assert owned.eject([name], owned_root).code == 0 + mine = _run(owned_root, argv, out, block=_packaged_modules()) + + if entry.runtime and {"constraint_errors", "filter_parser"} <= set(entry.runtime): + # An owned router emits its runtime into the generated package and imports it there. + runtime_files = {k for k in mine if k.startswith("_runtime/")} + assert runtime_files and runtime_files == { + f"_runtime/{m}.py" for m in ("__init__", "constraint_errors", "filter_parser") + } + mine = {k: v for k, v in mine.items() if k not in runtime_files} + packaged = {k: v.replace("metaobjects.codegen.runtime.", "._runtime.") + for k, v in packaged.items()} + assert set(mine) == set(packaged) + # Import sorting moves the now-relative imports, so compare lines, not positions. + assert {k: _lines(v) for k, v in mine.items()} == {k: _lines(v) for k, v in packaged.items()} + else: + assert mine == packaged + + +# --- the census ------------------------------------------------------------------------- + + +def test_every_helper_module_is_handed_over_by_some_generator() -> None: + """A module in ``codegen/runtime/`` no registry entry lists is one eject would leave + packaged behind an owned generator.""" + on_disk = {p.stem for p in (CODEGEN / "runtime").glob("*.py") if p.stem != "__init__"} + listed = {m for e in GENERATOR_REGISTRY.values() for m in e.runtime} + assert on_disk == listed + + +def test_every_generator_module_is_a_registry_entry_or_a_model_walk() -> None: + """The census for Python: every module in ``codegen/generators/`` is a registered + generator or one of the two model walks the engine surface names.""" + registered = {Path(e.source_module.replace(".", "/")).name + for e in GENERATOR_REGISTRY.values() if e.source_module} + registered |= {"template_generator"} # the primitive: exempt, it has no emit logic of its own + walks = {Path(m.replace(".", "/")).name for m in ENGINE_CODEGEN + if m.startswith("metaobjects.codegen.generators.")} + # A module that moved into `runtime/` leaves a deprecated alias at its old path. + on_disk = {p.stem for p in (CODEGEN / "generators").glob("*.py") + if p.stem != "__init__" and "codegen._moved" not in p.read_text(encoding="utf-8")} + assert on_disk == registered | walks + + +def test_template_is_the_only_registered_generator_without_an_eject_source() -> None: + assert {e.name for e in GENERATOR_REGISTRY.values() if e.source is None} == {"template"} diff --git a/server/python/tests/codegen/test_eject_independence.py b/server/python/tests/codegen/test_eject_independence.py index 420d4f79b..de13c7121 100644 --- a/server/python/tests/codegen/test_eject_independence.py +++ b/server/python/tests/codegen/test_eject_independence.py @@ -71,9 +71,14 @@ def test_loading_one_ejectable_generator_loads_no_other_generators_module(tmp_pa assert not offenders, "\n".join(offenders) +#: The docs generator reads the router gates (`is_read_only_routed`, `emits_writable_router`) from +#: the same module, so the reference describes the routers an owned `routes` emits. +DOCS_READERS = {"api-docs"} + + def test_the_generators_that_share_a_predicate_hand_it_over() -> None: users = {n for n, e in GENERATOR_REGISTRY.items() if SHARED in e.runtime} - assert users == set(SHARED_USERS) + assert users == set(SHARED_USERS) | DOCS_READERS def test_each_owned_generator_ejected_alone_reads_the_owned_predicates(tmp_path, monkeypatch) -> None: diff --git a/server/python/tests/codegen/test_eject_runtime.py b/server/python/tests/codegen/test_eject_runtime.py index 5e98e47bd..9872687cf 100644 --- a/server/python/tests/codegen/test_eject_runtime.py +++ b/server/python/tests/codegen/test_eject_runtime.py @@ -85,10 +85,11 @@ def _block_installed_runtime(monkeypatch) -> None: def test_routes_entry_lists_the_runtime_its_output_imports_and_nothing_else_does() -> None: - """``shared_predicates`` is handed over through the same field but no generated code - imports it (``test_eject_independence.py`` covers it), so it is set aside here.""" + """The emitters and predicates a generator reads at ``gen`` time are handed over through + the same field, but no generated code imports them (``test_eject_independence.py`` and + ``test_eject_alone.py`` cover them), so only the two modules a router imports count.""" output_runtime = { - n: tuple(m for m in e.runtime if m != "shared_predicates") + n: tuple(m for m in e.runtime if m in RUNTIME_MODULES) for n, e in GENERATOR_REGISTRY.items() } assert output_runtime.pop("routes") == RUNTIME_MODULES @@ -151,7 +152,7 @@ def test_eject_never_overwrites_an_owned_runtime_copy(tmp_path, monkeypatch, cap def test_eject_of_a_generator_without_runtime_writes_no_runtime(tmp_path, monkeypatch) -> None: monkeypatch.chdir(tmp_path) - assert main(["eject", "names", "output-prompt", "trace-helper"]) == 0 + assert main(["eject", "names", "render-helper", "requirement-tests"]) == 0 assert not (tmp_path / "codegen" / "runtime").exists() diff --git a/server/python/tests/codegen/test_fr010_nested_object_prompt.py b/server/python/tests/codegen/test_fr010_nested_object_prompt.py index 9e446d5ee..52336423c 100644 --- a/server/python/tests/codegen/test_fr010_nested_object_prompt.py +++ b/server/python/tests/codegen/test_fr010_nested_object_prompt.py @@ -14,7 +14,7 @@ import ast import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes -from metaobjects.codegen import output_format_spec_emitter as ofs +from metaobjects.codegen.runtime import output_format_spec_emitter as ofs from metaobjects.meta.core.field import field_constants as fc from metaobjects.meta.core.field.meta_field import MetaField from metaobjects.meta.core.object.meta_object import MetaObject diff --git a/server/python/tests/codegen/test_fr010_output_codegen.py b/server/python/tests/codegen/test_fr010_output_codegen.py index eee02ea5a..448569de0 100644 --- a/server/python/tests/codegen/test_fr010_output_codegen.py +++ b/server/python/tests/codegen/test_fr010_output_codegen.py @@ -15,7 +15,7 @@ import sys import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes -from metaobjects.codegen import ( +from metaobjects.codegen.runtime import ( output_format_spec_emitter as ofs, extract_schema_emitter as rse, ) diff --git a/server/python/tests/codegen/test_fr019_shared_provided_conformance.py b/server/python/tests/codegen/test_fr019_shared_provided_conformance.py index f1f037d83..05ba7d5a8 100644 --- a/server/python/tests/codegen/test_fr019_shared_provided_conformance.py +++ b/server/python/tests/codegen/test_fr019_shared_provided_conformance.py @@ -27,7 +27,7 @@ from metaobjects.codegen.config import GenConfig from metaobjects.codegen.generator import GenContext from metaobjects.codegen.generators.entity_model import EntityModelGenerator, entity_model -from metaobjects.codegen.generators.fr019_shared_enum import ( +from metaobjects.codegen.runtime.fr019_shared_enum import ( is_provided, materialized_shared_enums, resolve_shared_enum_decl, diff --git a/server/python/tests/codegen/test_m2m_codegen.py b/server/python/tests/codegen/test_m2m_codegen.py index 408edb3f7..ace6ca006 100644 --- a/server/python/tests/codegen/test_m2m_codegen.py +++ b/server/python/tests/codegen/test_m2m_codegen.py @@ -28,7 +28,7 @@ EntityModelGenerator, render_entity_model, ) -from metaobjects.codegen.generators.m2m_codegen import ( +from metaobjects.codegen.runtime.m2m_codegen import ( M2mDescriptor, build_object_index, resolve_m2m_descriptors, diff --git a/server/python/tests/codegen/test_moved_modules.py b/server/python/tests/codegen/test_moved_modules.py new file mode 100644 index 000000000..10fe687c8 --- /dev/null +++ b/server/python/tests/codegen/test_moved_modules.py @@ -0,0 +1,55 @@ +"""The emitters that moved into ``metaobjects.codegen.runtime`` keep answering at their old paths. + +``metaobjects eject`` copies the emitters a generator writes through from +``metaobjects/codegen/runtime/``, so they moved there. A generator ejected before they moved +imports them from the old paths; each old path is a deprecated alias that IS the new module +(same object, so private names and monkeypatching behave as before) and warns once. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +import metaobjects + +MOVED = { + "metaobjects.codegen.extract_delegate_emitter": "metaobjects.codegen.runtime.extract_delegate_emitter", + "metaobjects.codegen.fr010_field_mapping": "metaobjects.codegen.runtime.fr010_field_mapping", + "metaobjects.codegen.output_format_spec_emitter": "metaobjects.codegen.runtime.output_format_spec_emitter", + "metaobjects.codegen.extract_schema_emitter": "metaobjects.codegen.runtime.extract_schema_emitter", + "metaobjects.codegen.generators.fr019_shared_enum": "metaobjects.codegen.runtime.fr019_shared_enum", + "metaobjects.codegen.generators.m2m_codegen": "metaobjects.codegen.runtime.m2m_codegen", + "metaobjects.apidocs.api_model": "metaobjects.codegen.runtime.api_docs_model", + "metaobjects.apidocs.builder": "metaobjects.codegen.runtime.api_docs_builder", + "metaobjects.apidocs.renderer": "metaobjects.codegen.runtime.api_docs_renderer", + "metaobjects.apidocs.paths": "metaobjects.codegen.runtime.api_docs_paths", +} + + +@pytest.mark.parametrize("old, new", sorted(MOVED.items())) +def test_an_old_path_is_the_new_module_and_warns(old: str, new: str, tmp_path: Path) -> None: + # A fresh interpreter: in this process the old path may already be imported, which warns once. + src = str(Path(metaobjects.__file__).resolve().parents[1]) + code = ( + "import importlib, json, warnings\n" + "with warnings.catch_warnings(record=True) as w:\n" + " warnings.simplefilter('always')\n" + f" old = importlib.import_module({old!r})\n" + f"new = importlib.import_module({new!r})\n" + "print(json.dumps({'same': old is new, " + "'warned': [str(x.message) for x in w if issubclass(x.category, DeprecationWarning)]}))\n" + ) + proc = subprocess.run( + [sys.executable, "-c", code], cwd=tmp_path, capture_output=True, text=True, + env={**os.environ, "PYTHONPATH": os.pathsep.join( + p for p in (src, os.environ.get("PYTHONPATH", "")) if p)}, + ) + assert proc.returncode == 0, proc.stderr + result = json.loads(proc.stdout) + assert result["same"] is True + assert len(result["warned"]) == 1 and old in result["warned"][0] and new in result["warned"][0] diff --git a/server/python/tests/codegen/test_reverse_finders.py b/server/python/tests/codegen/test_reverse_finders.py index e7b7ce39a..a6fb2e25d 100644 --- a/server/python/tests/codegen/test_reverse_finders.py +++ b/server/python/tests/codegen/test_reverse_finders.py @@ -32,7 +32,7 @@ reverse_finder_fn, reverse_finder_in_fn, ) -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import ( render_router, reverse_fks_for, diff --git a/server/python/tests/codegen/test_router_autoset.py b/server/python/tests/codegen/test_router_autoset.py index 69557480d..0e9c852ca 100644 --- a/server/python/tests/codegen/test_router_autoset.py +++ b/server/python/tests/codegen/test_router_autoset.py @@ -11,7 +11,7 @@ from pathlib import Path from metaobjects import MetaDataLoader -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.shared.base_types import TYPE_OBJECT diff --git a/server/python/tests/codegen/test_router_generator.py b/server/python/tests/codegen/test_router_generator.py index b50c93d99..53b6acfa1 100644 --- a/server/python/tests/codegen/test_router_generator.py +++ b/server/python/tests/codegen/test_router_generator.py @@ -12,7 +12,7 @@ import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes from metaobjects import MetaDataLoader -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.codegen.runtime.filter_parser import parse_filter from metaobjects.meta.core.field.meta_field import MetaField diff --git a/server/python/tests/codegen/test_router_generator_m2m_target_tph.py b/server/python/tests/codegen/test_router_generator_m2m_target_tph.py index 5416acc96..3fa29813f 100644 --- a/server/python/tests/codegen/test_router_generator_m2m_target_tph.py +++ b/server/python/tests/codegen/test_router_generator_m2m_target_tph.py @@ -30,7 +30,7 @@ import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes from metaobjects import MetaDataLoader -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.shared.base_types import TYPE_OBJECT diff --git a/server/python/tests/codegen/test_router_generator_tph_m2m.py b/server/python/tests/codegen/test_router_generator_tph_m2m.py index 2ffe574dc..f13af41f9 100644 --- a/server/python/tests/codegen/test_router_generator_tph_m2m.py +++ b/server/python/tests/codegen/test_router_generator_tph_m2m.py @@ -41,7 +41,7 @@ import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes from metaobjects import MetaDataLoader -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.shared.base_types import TYPE_OBJECT diff --git a/server/python/tests/codegen/test_router_generator_tph_m2m_shadowed.py b/server/python/tests/codegen/test_router_generator_tph_m2m_shadowed.py index 7196098cf..9bad9710b 100644 --- a/server/python/tests/codegen/test_router_generator_tph_m2m_shadowed.py +++ b/server/python/tests/codegen/test_router_generator_tph_m2m_shadowed.py @@ -30,7 +30,7 @@ import metaobjects.core_types # noqa: F401 — side-effect: registers attr classes from metaobjects import MetaDataLoader -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.shared.base_types import TYPE_OBJECT diff --git a/server/python/tests/codegen/test_tph_codegen.py b/server/python/tests/codegen/test_tph_codegen.py index fa44a9892..9b405a9ac 100644 --- a/server/python/tests/codegen/test_tph_codegen.py +++ b/server/python/tests/codegen/test_tph_codegen.py @@ -19,7 +19,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.codegen.generators.tph_plan import ( is_tph_base, diff --git a/server/python/tests/conformance/test_eject_contract.py b/server/python/tests/conformance/test_eject_contract.py index c603a41c4..7dc4426bb 100644 --- a/server/python/tests/conformance/test_eject_contract.py +++ b/server/python/tests/conformance/test_eject_contract.py @@ -269,6 +269,8 @@ def test_depth_every_engine_surface_entry_names_a_real_module() -> None: from metaobjects import MetaDataLoader from metaobjects.codegen import eject as owned from metaobjects.codegen.config import GenConfig +from metaobjects.codegen.docs_run import run_docs +from metaobjects.codegen.generator import DocsRunFacts from metaobjects.codegen.generator_registry import GENERATOR_REGISTRY, GeneratorBuildContext from metaobjects.codegen.runner import run_gen from metaobjects.loader.sources import FileSource @@ -282,6 +284,12 @@ def test_depth_every_engine_surface_entry_names_a_real_module() -> None: mine, err = owned.build_owned(owned.owned_token(entry), root, ctx) assert err is None, err for side, generator in (("owned", mine), ("packaged", entry.factory(ctx))): + if getattr(generator, "docs_surface", None) is not None: + # A docs generator is run by `metaobjects docs`, not by `gen`, which skips it. + outcome = run_docs(loaded.root, "docs", DocsRunFacts(project="model"), [generator]) + assert outcome.error is None, outcome.error + out[side][corpus] = dict(outcome.files) + continue with tempfile.TemporaryDirectory() as tmp: run_gen(GenConfig(out_dir=tmp, emit_package_init=False), loaded.root, generators=[generator]) out[side][corpus] = { diff --git a/server/python/tests/integration/generated_m2m_app.py b/server/python/tests/integration/generated_m2m_app.py index c88baa038..d9359afbd 100644 --- a/server/python/tests/integration/generated_m2m_app.py +++ b/server/python/tests/integration/generated_m2m_app.py @@ -53,7 +53,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import ( +from metaobjects.codegen.runtime.m2m_codegen import ( M2mDescriptor, build_object_index, pk_field_name, diff --git a/server/python/tests/integration/generated_tph_app.py b/server/python/tests/integration/generated_tph_app.py index c0070a1a5..e87ddddcb 100644 --- a/server/python/tests/integration/generated_tph_app.py +++ b/server/python/tests/integration/generated_tph_app.py @@ -31,7 +31,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.codegen.generators.tph_plan import tph_plan_for from metaobjects.meta.core.object.meta_object import MetaObject diff --git a/server/python/tests/integration/test_m2m_target_tph_generated.py b/server/python/tests/integration/test_m2m_target_tph_generated.py index eb5bffaba..c4427c067 100644 --- a/server/python/tests/integration/test_m2m_target_tph_generated.py +++ b/server/python/tests/integration/test_m2m_target_tph_generated.py @@ -36,7 +36,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.meta.core.object.meta_object import MetaObject from metaobjects.shared.base_types import TYPE_OBJECT diff --git a/server/python/tests/integration/test_tph_m2m_generated.py b/server/python/tests/integration/test_tph_m2m_generated.py index 852d57b9f..3b28ffb84 100644 --- a/server/python/tests/integration/test_tph_m2m_generated.py +++ b/server/python/tests/integration/test_tph_m2m_generated.py @@ -35,7 +35,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.codegen.generators.tph_plan import tph_plan_for from metaobjects.meta.core.object.meta_object import MetaObject diff --git a/server/python/tests/integration/test_tph_m2m_shadowed_generated.py b/server/python/tests/integration/test_tph_m2m_shadowed_generated.py index d12d6d224..7427d4d9a 100644 --- a/server/python/tests/integration/test_tph_m2m_shadowed_generated.py +++ b/server/python/tests/integration/test_tph_m2m_shadowed_generated.py @@ -30,7 +30,7 @@ from metaobjects import MetaDataLoader from metaobjects.codegen.generators.entity_model import render_entity_model from metaobjects.codegen.generators.filter_allowlist_generator import render_filter_allowlist -from metaobjects.codegen.generators.m2m_codegen import build_object_index +from metaobjects.codegen.runtime.m2m_codegen import build_object_index from metaobjects.codegen.generators.router_generator import render_router from metaobjects.codegen.generators.tph_plan import tph_plan_for from metaobjects.meta.core.object.meta_object import MetaObject diff --git a/server/python/tests/test_reporting_inert.py b/server/python/tests/test_reporting_inert.py index ab334df5c..f40eaea2b 100644 --- a/server/python/tests/test_reporting_inert.py +++ b/server/python/tests/test_reporting_inert.py @@ -26,9 +26,9 @@ import pytest from metaobjects import load_uris -from metaobjects.apidocs.builder import PythonApiModelBuilder -from metaobjects.apidocs.paths import Layout, doc_page_output_path -from metaobjects.apidocs.renderer import render_agent_api, render_index, render_unit_page +from metaobjects.codegen.runtime.api_docs_builder import PythonApiModelBuilder +from metaobjects.codegen.runtime.api_docs_paths import Layout, doc_page_output_path +from metaobjects.codegen.runtime.api_docs_renderer import render_agent_api, render_index, render_unit_page from metaobjects.codegen.config import GenConfig from metaobjects.codegen.generator import Generator from metaobjects.codegen.generator_registry import ( From 57891195755336a16eabbb2faa0fe44fe5b2e09c Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 12 Oct 2026 01:31:49 -0400 Subject: [PATCH 3/8] no-mistakes(review): test(python): defer engine surface to shared eject manifest --- CHANGELOG.md | 14 +- docs/features/cli.md | 3 + docs/features/own-your-codegen.md | 10 +- .../python/src/metaobjects/codegen/eject.py | 6 +- .../metaobjects/codegen/generator_registry.py | 2 +- .../python/tests/codegen/test_eject_alone.py | 148 +++--------------- .../tests/codegen/test_project_config.py | 5 + 7 files changed, 53 insertions(+), 135 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9a76eea5..353971098 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -259,12 +259,14 @@ generator catalog and the Atlas evaluation are 1.2 work. To own them, eject the same name into a scratch directory and copy the generator's `runtime/` files over (see [own-your-codegen](docs/features/own-your-codegen.md#the-emitters-a-generator-writes-through-come-with-it)). - - **A gate for the rule.** For every ejectable generator the registry lists, a test ejects it - alone into an empty project, fails on any import outside a written engine surface, runs it - with every packaged emit module made unimportable, and compares its output byte for byte - with the packaged generator's over a model that exercises each one (the router's own - runtime imports are the one documented difference). Another fails on a helper module no - registry entry hands over, or a generator module no registry entry names. + - **A gate for the rule.** The engine surface is written once, in the shared manifest + (`eject.engineSurface.python`), and `tests/conformance/test_eject_contract.py` ejects every + ejectable generator alone into an empty project and fails on any import outside it. A + second test (`tests/codegen/test_eject_alone.py`) runs each owned copy with every packaged + emit module made unimportable and compares its output byte for byte with the packaged + generator's over a model that exercises each one (the router's own runtime imports are the + one documented difference). Another fails on a helper module no registry entry hands over, + or a generator module no registry entry names. The nine Python entries of the shared eject contract's `eject.pending` list are closed, and the contract test runs `api-docs` through `metaobjects docs`, the way it is run. diff --git a/docs/features/cli.md b/docs/features/cli.md index 4c940c412..806641525 100644 --- a/docs/features/cli.md +++ b/docs/features/cli.md @@ -376,6 +376,9 @@ requirementTests: # optional; options of the `requirement-tests` filter: codegen.requirement_hooks:include # "module:symbol", resolved config-relative renderer: codegen.requirement_hooks:render # "module:symbol", resolved config-relative warnUncovered: true # the default +docs: # optional; options of `metaobjects docs` + generators: # owned docs generators as "module:symbol", run in place of the packaged one + - codegen.generators.api_docs:api_docs_generator targets: api: outDir: src/generated/api diff --git a/docs/features/own-your-codegen.md b/docs/features/own-your-codegen.md index 3659a6f06..56d0001e5 100644 --- a/docs/features/own-your-codegen.md +++ b/docs/features/own-your-codegen.md @@ -705,10 +705,12 @@ package: the generator base and file writer (`metaobjects.codegen.generator`), t formatter, the naming and type-mapping tables (`metaobjects.apidocs.naming`, `metaobjects.codegen.type_map`, `metaobjects.codegen.value_objects`), the model walks (`find_inbound`, `tph_plan`, `instance_artifacts`) and the loader. Those module paths are -the surface an owned generator builds on. `tests/codegen/test_eject_alone.py` ejects each -generator alone into an empty project, rejects any import outside that surface, runs it with -the packaged emitters made unimportable, and compares its output byte for byte with the -packaged generator's. +the surface an owned generator builds on. It is written once, in the shared manifest +(`fixtures/generator-registry-conformance/registry.json`, `eject.engineSurface.python`), and +`tests/conformance/test_eject_contract.py` ejects each generator alone into an empty project +and rejects any import outside it. `tests/codegen/test_eject_alone.py` then runs each owned +copy with the packaged emitters made unimportable and compares its output byte for byte with +the packaged generator's. `requirement-tests` ejects the same way, to `codegen/generators/requirement_tests.py`, wired as `codegen.generators.requirement_tests:requirement_tests_generator`. The copy holds diff --git a/server/python/src/metaobjects/codegen/eject.py b/server/python/src/metaobjects/codegen/eject.py index 1ea0ab2c7..891c6d0ca 100644 --- a/server/python/src/metaobjects/codegen/eject.py +++ b/server/python/src/metaobjects/codegen/eject.py @@ -26,8 +26,10 @@ ``runtime`` sibling, so the verbatim copy in ``codegen/generators/`` reads ``codegen/runtime/.py`` with no edit, and two owned generators read the one copy. Only the engine stays a package import: the generator base and file writer, the naming and -type-mapping tables, the model walks and the loader. ``tests/codegen/test_eject_alone.py`` -ejects each generator alone and holds it to exactly that. +type-mapping tables, the model walks and the loader. That surface is written in the shared +manifest (``eject.engineSurface.python`` in ``fixtures/generator-registry-conformance/registry.json``) +and ``tests/conformance/test_eject_contract.py`` ejects each generator alone and holds it to +exactly that. A docs generator (``api-docs``) is not wired into ``generators``: ``metaobjects docs`` runs it from the config's ``docs.generators``, and ``gen`` skips one it finds under a target. diff --git a/server/python/src/metaobjects/codegen/generator_registry.py b/server/python/src/metaobjects/codegen/generator_registry.py index a8b1fb003..bc77fedc2 100644 --- a/server/python/src/metaobjects/codegen/generator_registry.py +++ b/server/python/src/metaobjects/codegen/generator_registry.py @@ -206,7 +206,7 @@ def _render_helper_default(ctx: GeneratorBuildContext) -> Generator: #: that return emitted text, which an owned copy therefore owns instead of calling the #: packaged copy. Eject copies them next to the generator (``codegen/runtime/``) and the #: generator imports them relatively. Naming and type-mapping tables, the model walks and the -#: generator base stay in the package as the engine. See ``tests/codegen/test_eject_alone.py``. +#: generator base stay in the package as the engine. See ``tests/conformance/test_eject_contract.py``. _FR010_FIELD_MAPPING = "fr010_field_mapping" _M2M_CODEGEN = "m2m_codegen" _FR019_SHARED_ENUM = "fr019_shared_enum" diff --git a/server/python/tests/codegen/test_eject_alone.py b/server/python/tests/codegen/test_eject_alone.py index ab473c435..9a0b1c7fb 100644 --- a/server/python/tests/codegen/test_eject_alone.py +++ b/server/python/tests/codegen/test_eject_alone.py @@ -5,12 +5,14 @@ registry and not from a list kept beside it, an empty project that ejects only that generator must get a copy that -* imports no packaged emit module: the copy and the helper source eject placed beside it - import only the engine surface written below (the generator base, the file writer and - formatter, the naming and type-mapping tables, the model walks and the loader); +* is the generator and the helper modules its registry entry lists, and nothing else; * loads and runs with every packaged emit module made unimportable; and * writes output byte-identical to the packaged generator over the same model. +What an owned copy may still import from the package (the engine surface) is defined once, in +``fixtures/generator-registry-conformance/registry.json`` (``eject.engineSurface.python``), and +asserted by ``tests/conformance/test_eject_contract.py``. + The one documented difference is the router: an owned ``routes`` emits its runtime into the generated package and imports it from there, so its router modules differ from the packaged ones in those import lines and in the ``_runtime/`` files (``test_eject_runtime.py``). @@ -20,12 +22,13 @@ """ from __future__ import annotations -import ast +import importlib import json import os import shutil import subprocess import sys +import warnings from collections import Counter from pathlib import Path from typing import Any @@ -35,124 +38,22 @@ import metaobjects from metaobjects.codegen import eject as owned from metaobjects.codegen.generator_registry import GENERATOR_REGISTRY +from tests.conformance import eject_contract as contract REPO = Path(__file__).parents[4] FIXTURES = REPO / "fixtures" CODEGEN = Path(metaobjects.__file__).parent / "codegen" -#: The packaged ``metaobjects.codegen`` modules an owned copy may import: the engine. Everything -#: that returns emitted text is handed over instead, and everything outside ``metaobjects.codegen`` -#: (the loader, the metamodel, naming, render, extract, the runtime) is core by ruling. -ENGINE_CODEGEN = frozenset({ - "metaobjects.codegen.config", # GenConfig - "metaobjects.codegen.constants", # the @generated marker and header - "metaobjects.codegen.format", # the formatter - "metaobjects.codegen.generator", # the generator base, EmittedFile, GenContext - "metaobjects.codegen.instance_artifacts", # model questions: what an object emits - "metaobjects.codegen.model_walk", - "metaobjects.codegen.project_config", # the requirementTests options - "metaobjects.codegen.requirement_hooks", - "metaobjects.codegen.requirement_walk", # shared with `verify` - "metaobjects.codegen.type_map", # the type-mapping table - "metaobjects.codegen.value_objects", # naming and value-object resolution - "metaobjects.codegen.generators.find_inbound", # model walk: the inbound tier - "metaobjects.codegen.generators.tph_plan", # model walk: table-per-hierarchy -}) -#: Of ``metaobjects.apidocs``, only the naming seam: the names the generators emit under. -ENGINE_APIDOCS = frozenset({"metaobjects.apidocs.naming"}) - EJECTABLE = sorted(e.name for e in owned.ejectable()) -def _runtime_imports(node: ast.AST) -> list[ast.stmt]: - """The import statements under *node* that run: an ``if TYPE_CHECKING:`` block is only an - annotation for a type checker, so it is not walked.""" - found: list[ast.stmt] = [] - for child in ast.iter_child_nodes(node): - if isinstance(child, ast.If) and isinstance(child.test, ast.Name) \ - and child.test.id == "TYPE_CHECKING": - found.extend(_runtime_imports_of(child.orelse)) - elif isinstance(child, (ast.Import, ast.ImportFrom)): - found.append(child) - else: - found.extend(_runtime_imports(child)) - return found - - -def _runtime_imports_of(body: list[ast.stmt]) -> list[ast.stmt]: - found: list[ast.stmt] = [] - for stmt in body: - if isinstance(stmt, (ast.Import, ast.ImportFrom)): - found.append(stmt) - else: - found.extend(_runtime_imports(stmt)) - return found - - -def _imports(source: str) -> list[tuple[int, str | None, list[str]]]: - """``(level, module, imported names)`` for each import statement in *source* that runs.""" - out: list[tuple[int, str | None, list[str]]] = [] - for node in _runtime_imports(ast.parse(source)): - if isinstance(node, ast.ImportFrom): - out.append((node.level, node.module, [a.name for a in node.names])) - elif isinstance(node, ast.Import): - out.extend((0, a.name, []) for a in node.names) - return out - - -def _outside_the_engine(source: str, handed_over: set[str]) -> list[str]: - """What *source* imports that is neither the engine surface nor a helper handed over.""" - offenders: list[str] = [] - for level, module, names in _imports(source): - if level: - # A relative import is the handed-over helper: `..runtime.` from a generator, - # `.` or `from . import ` from a helper. - if module is None: - targets = names - elif module.startswith("runtime."): - targets = [module.split(".", 1)[1]] - elif module == "runtime": - targets = names - else: - targets = [module] - offenders.extend( - f"relative import of {t!r}, which eject does not hand over" - for t in targets if t not in handed_over - ) - continue - if module is None or not module.startswith("metaobjects."): - continue - # `from metaobjects.codegen import type_map` names a module through the package. - packages = ("metaobjects.codegen", "metaobjects.codegen.generators", - "metaobjects.codegen.runtime", "metaobjects.apidocs") - candidates = [f"{module}.{n}" for n in names] if module in packages else [module] - for c in candidates: - if c.startswith("metaobjects.codegen"): - if c not in ENGINE_CODEGEN: - offenders.append(c) - elif c.startswith("metaobjects.apidocs"): - if c not in ENGINE_APIDOCS: - offenders.append(c) - return offenders - - @pytest.mark.parametrize("name", EJECTABLE) -def test_an_ejected_generator_imports_only_the_engine(name: str, tmp_path: Path) -> None: +def test_eject_writes_the_generator_and_the_helpers_its_entry_lists(name: str, tmp_path: Path) -> None: entry = GENERATOR_REGISTRY[name] assert owned.eject([name], tmp_path).code == 0 - handed_over = set(entry.runtime) copied = [owned.owned_path(tmp_path, name)] + [ owned.runtime_path(tmp_path, m) for m in entry.runtime ] - offenders: list[str] = [] - for path in copied: - assert path.is_file(), f"eject did not write {path.relative_to(tmp_path)}" - offenders += [ - f"{path.relative_to(tmp_path)}: {o}" - for o in _outside_the_engine(path.read_text(encoding="utf-8"), handed_over) - ] - assert not offenders, "\n".join(offenders) - # Nothing else was written: the copy is the generator and its helpers. written = {p.relative_to(tmp_path).as_posix() for p in (tmp_path / "codegen").rglob("*.py") if p.name != "__init__.py"} assert written == {p.relative_to(tmp_path).as_posix() for p in copied} @@ -305,19 +206,22 @@ def test_every_helper_module_is_handed_over_by_some_generator() -> None: assert on_disk == listed +def _is_deprecated_alias(module: str) -> bool: + """A module that moved into ``runtime/`` leaves an alias at its old path: importing the + old name returns the module at the new one.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return importlib.import_module(module).__name__ != module + + def test_every_generator_module_is_a_registry_entry_or_a_model_walk() -> None: """The census for Python: every module in ``codegen/generators/`` is a registered - generator or one of the two model walks the engine surface names.""" - registered = {Path(e.source_module.replace(".", "/")).name - for e in GENERATOR_REGISTRY.values() if e.source_module} - registered |= {"template_generator"} # the primitive: exempt, it has no emit logic of its own - walks = {Path(m.replace(".", "/")).name for m in ENGINE_CODEGEN - if m.startswith("metaobjects.codegen.generators.")} - # A module that moved into `runtime/` leaves a deprecated alias at its old path. - on_disk = {p.stem for p in (CODEGEN / "generators").glob("*.py") - if p.stem != "__init__" and "codegen._moved" not in p.read_text(encoding="utf-8")} - assert on_disk == registered | walks - - -def test_template_is_the_only_registered_generator_without_an_eject_source() -> None: - assert {e.name for e in GENERATOR_REGISTRY.values() if e.source is None} == {"template"} + generator or one of the model walks the shared engine surface names.""" + package = "metaobjects.codegen.generators" + registered = {e.source_module for e in GENERATOR_REGISTRY.values() if e.source_module} + # The primitive: exempt, it has no emit logic of its own. + registered |= {f"{package}.template_generator"} + walks = {m for m in contract.EngineSurface().engine if m.startswith(f"{package}.")} + on_disk = {f"{package}.{p.stem}" for p in (CODEGEN / "generators").glob("*.py") + if p.stem != "__init__"} + assert {m for m in on_disk if not _is_deprecated_alias(m)} == registered | walks diff --git a/server/python/tests/codegen/test_project_config.py b/server/python/tests/codegen/test_project_config.py index b8b3fc544..6a0bd7714 100644 --- a/server/python/tests/codegen/test_project_config.py +++ b/server/python/tests/codegen/test_project_config.py @@ -10,6 +10,7 @@ CONFIG_FILENAME, DEFAULT_METADATA_DIR, DEFAULT_REQUIREMENT_WITNESS_MODULE, + DOCS_KEYS, DOTTED_MODULE_PATTERN, MODULE_SYMBOL_PATTERN, REQUIREMENT_TESTS_KEYS, @@ -215,6 +216,10 @@ def test_schema_and_loader_accept_EXACTLY_the_same_keys(tmp_path: Path) -> None: assert set(block["properties"]) == set(REQUIREMENT_TESTS_KEYS) assert block["additionalProperties"] is False assert block["properties"]["grain"]["enum"] == list(REQUIREMENT_TEST_GRAIN_VALUES) + # So is the `docs` block. + docs = schema["properties"]["docs"] + assert set(docs["properties"]) == set(DOCS_KEYS) + assert docs["additionalProperties"] is False # Both levels must keep saying unknown keys are invalid — that claim is what the # loader now enforces. assert schema["additionalProperties"] is False From 3d93a3f1d2a305580d7653fd4d682e349b9ecb10 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 12 Oct 2026 02:02:34 -0400 Subject: [PATCH 4/8] no-mistakes(review): fix(python): eject reports stale kept runtime copies --- CHANGELOG.md | 8 +++ docs/features/own-your-codegen.md | 18 +++++- .../python/src/metaobjects/codegen/eject.py | 33 ++++++++--- .../tests/codegen/test_eject_independence.py | 58 ++++++++++++++++++- .../tests/codegen/test_eject_runtime.py | 4 +- 5 files changed, 109 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 353971098..ecd01ac80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -259,6 +259,14 @@ generator catalog and the Atlas evaluation are 1.2 work. To own them, eject the same name into a scratch directory and copy the generator's `runtime/` files over (see [own-your-codegen](docs/features/own-your-codegen.md#the-emitters-a-generator-writes-through-come-with-it)). + - **A `codegen/runtime/shared_predicates.py` you already own needs two new functions.** + `routes` now reads `is_read_only_routed` from it and `api-docs` reads that and + `emits_writable_router`, and eject never overwrites a runtime copy. When eject keeps a + runtime copy it now says how the copy compares with the installed module + (`kept your copy — identical`, or `kept your copy — DIFFERS: N behind, M of your own`), + and for a copy that is behind it prints the command that refreshes it: `cp` of the + installed file over yours when you have not edited it, otherwise a `diff -u` against it + whose hunks you apply. - **A gate for the rule.** The engine surface is written once, in the shared manifest (`eject.engineSurface.python`), and `tests/conformance/test_eject_contract.py` ejects every ejectable generator alone into an empty project and fails on any import outside it. A diff --git a/docs/features/own-your-codegen.md b/docs/features/own-your-codegen.md index 56d0001e5..e9bc66e9e 100644 --- a/docs/features/own-your-codegen.md +++ b/docs/features/own-your-codegen.md @@ -755,9 +755,11 @@ like the rest of the package. --codegen` reports the stale `_runtime/` file until you run `gen`. That is the right answer, because the committed package is out of date. - **Eject never overwrites your runtime.** An existing `codegen/runtime/.py` is - kept, even with `--force`, and eject says so. `metaobjects gen --list` marks each - runtime copy `identical` or `DIFFERS: N behind, M of your own` against the installed - one. + kept, even with `--force`, and eject says so, with how the copy compares with the + installed one (`kept your copy — identical` or `kept your copy — DIFFERS: N behind, M of + your own`). When the copy is behind, the next line names the generator that may need the + newer one and the command that brings yours up. `metaobjects gen --list` marks each + runtime copy the same way. - **A project that has not ejected `routes` is unchanged.** The packaged generator still imports `metaobjects.codegen.runtime.*` and emits no `_runtime/`. @@ -840,6 +842,16 @@ into a scratch directory (`metaobjects eject --root `), copy the files over into `codegen/runtime/`, and change your imports to the relative form the new copy uses (`from ..runtime import fr010_field_mapping as fm`). +**An existing `shared_predicates.py` needs two new functions.** `routes` now reads +`is_read_only_routed` from it, and `api-docs` reads that and `emits_writable_router`. A +`codegen/runtime/shared_predicates.py` you already own does not have them, and eject keeps +it, so `metaobjects eject api-docs` or `metaobjects eject routes --force` leaves a generator +that fails to import (`cannot import name 'emits_writable_router'`). Eject marks the kept +copy `DIFFERS: N behind` and prints the command that refreshes it: `cp` of the installed +file over yours when you have not edited it, otherwise a `diff -u` against the installed +file whose hunks you apply (the recipe under +[The runtime your generated code imports comes with it](#the-runtime-your-generated-code-imports-comes-with-it)). + ### `api-docs`: the api reference as an owned generator `metaobjects docs` writes the Python api reference (`api/python`). `metaobjects eject diff --git a/server/python/src/metaobjects/codegen/eject.py b/server/python/src/metaobjects/codegen/eject.py index 891c6d0ca..3bb4d5b26 100644 --- a/server/python/src/metaobjects/codegen/eject.py +++ b/server/python/src/metaobjects/codegen/eject.py @@ -16,7 +16,8 @@ ``codegen/runtime/`` and the copy's ``OWNED_RUNTIME`` switch is flipped, so the owned generator emits that source into the generated package and its output imports the local copy. Core modules (loader, registry, render, extract, ``ObjectManager``) stay package -imports. An existing runtime copy is never overwritten, even with ``--force``. +imports. An existing runtime copy is never overwritten, even with ``--force``: eject +reports how it compares with the packaged module, and how to refresh one that is behind. Everything a generator writes its output THROUGH is handed over the same way: the emit rules several generators share (``shared_predicates``), the fragment emitters behind a full @@ -159,12 +160,23 @@ def eject(names: list[str], root: Path, *, force: bool = False) -> EjectResult: for module in entry.runtime: rt = runtime_path(root, module) rel = rt.relative_to(root) + packaged = packaged_runtime_path(module) + reference = packaged.read_text(encoding="utf-8") if rt.exists(): - out.append(f" runtime {rel}: kept your copy (eject never overwrites runtime)") + behind, own = _line_diff(rt.read_text(encoding="utf-8"), reference) + out.append(f" runtime {rel}: kept your copy — {_status(behind, own)} " + "(eject never overwrites runtime)") + if behind: + refresh = ( + "it carries edits of your own, so apply the hunks you need from: " + f'diff -u {rel} "{packaged}"' + if own else f'refresh it: cp "{packaged}" {rel.parent}/' + ) + out.append(f" {n} may need the newer copy: yours lacks lines the " + f"packaged one has; {refresh}") continue rt.parent.mkdir(parents=True, exist_ok=True) - rt.write_text(packaged_runtime_path(module).read_text(encoding="utf-8"), - encoding="utf-8") + rt.write_text(reference, encoding="utf-8") out.append(f" runtime {rel}: copied — your owned generator reads this copy") out.append("") out.append( @@ -191,16 +203,23 @@ def owned_status(root: Path, entry: GeneratorEntry) -> str | None: return _compare(path.read_text(encoding="utf-8"), ejected_source(entry)) -def _compare(mine_text: str, ref_text: str) -> str: +def _line_diff(mine_text: str, ref_text: str) -> tuple[int, int]: + """``(behind, of your own)``: reference lines the copy lacks, copy lines the reference lacks.""" mine = _normalised(mine_text) ref = _normalised(ref_text) - behind = sum((ref - mine).values()) - own = sum((mine - ref).values()) + return sum((ref - mine).values()), sum((mine - ref).values()) + + +def _status(behind: int, own: int) -> str: if behind == 0 and own == 0: return "identical" return f"DIFFERS: {behind} behind, {own} of your own" +def _compare(mine_text: str, ref_text: str) -> str: + return _status(*_line_diff(mine_text, ref_text)) + + def runtime_status(root: Path) -> list[tuple[Path, str]]: """``(relative path, status)`` for each owned runtime copy, against the installed one. diff --git a/server/python/tests/codegen/test_eject_independence.py b/server/python/tests/codegen/test_eject_independence.py index de13c7121..dff1d5e0c 100644 --- a/server/python/tests/codegen/test_eject_independence.py +++ b/server/python/tests/codegen/test_eject_independence.py @@ -14,15 +14,20 @@ """ from __future__ import annotations +import ast import json import os +import re import subprocess import sys import uuid from pathlib import Path from typing import Any +import pytest + import metaobjects +import metaobjects.codegen.runtime from metaobjects.cli import main from metaobjects.codegen import eject as owned from metaobjects.codegen.generator_registry import GENERATOR_REGISTRY @@ -133,7 +138,9 @@ def test_an_edit_to_the_owned_predicate_moves_owned_routes_and_owned_allowlist_t assert main(["eject", "routes"]) == 0 capsys.readouterr() assert main(["eject", "filter-allowlist"]) == 0 - assert f"codegen/runtime/{SHARED}.py: kept your copy" in capsys.readouterr().out + kept = capsys.readouterr().out + assert f"codegen/runtime/{SHARED}.py: kept your copy — identical" in kept + assert "may need the newer copy" not in kept # One owned rule, edited once: the Post object no longer gets a router. rule = root / "codegen" / "runtime" / f"{SHARED}.py" @@ -154,3 +161,52 @@ def test_an_edit_to_the_owned_predicate_moves_owned_routes_and_owned_allowlist_t assert main(["gen", "--list"]) == 0 assert (f"runtime codegen/runtime/{SHARED}.py [owned — DIFFERS: 0 behind, 3 of your own]" in capsys.readouterr().out) + + +#: The predicates `routes` and `api-docs` read from the shared module that an earlier +#: hand-over of it did not carry. +ADDED_PREDICATES = {"is_read_only_routed", "emits_writable_router"} + + +def _shared_predicates_before_the_added_ones() -> str: + source = owned.packaged_runtime_path(SHARED).read_text(encoding="utf-8") + lines = source.splitlines(keepends=True) + removed = set() + for node in reversed(ast.parse(source).body): + if isinstance(node, ast.FunctionDef) and node.name in ADDED_PREDICATES: + del lines[node.lineno - 1:node.end_lineno] + removed.add(node.name) + assert removed == ADDED_PREDICATES + return "".join(lines) + + +@pytest.mark.parametrize("own_edit", ["", "\n# my fix\n"], ids=["unedited", "edited"]) +def test_eject_onto_an_older_shared_copy_keeps_it_and_says_how_to_refresh_it( + own_edit: str, tmp_path, monkeypatch, capsys, +) -> None: + """A project that ejected a generator before `api-docs` existed owns a + `shared_predicates.py` without the predicates the api reference imports. Eject keeps + that copy, so it has to say the copy is behind and name the way to bring it up.""" + older = _shared_predicates_before_the_added_ones() + own_edit + mine = tmp_path / "codegen" / "runtime" / f"{SHARED}.py" + mine.parent.mkdir(parents=True) + mine.write_text(older, encoding="utf-8") + monkeypatch.chdir(tmp_path) + + assert main(["eject", "api-docs"]) == 0 + assert mine.read_text(encoding="utf-8") == older + lines = capsys.readouterr().out.splitlines() + at = next(i for i, ln in enumerate(lines) if f"{SHARED}.py: kept your copy" in ln) + behind, own = re.search(r"kept your copy — DIFFERS: (\d+) behind, (\d+) of your own", + lines[at]).groups() + assert int(behind) > 0 and int(own) == (1 if own_edit else 0) + + packaged = Path(metaobjects.codegen.runtime.__path__[0]) / f"{SHARED}.py" + hint = lines[at + 1] + assert "api-docs may need the newer copy" in hint + if own_edit: + assert f'diff -u codegen/runtime/{SHARED}.py "{packaged}"' in hint + assert "cp " not in hint + else: + assert f'cp "{packaged}" codegen/runtime/' in hint + assert [ln for ln in lines if "may need the newer copy" in ln] == [hint] diff --git a/server/python/tests/codegen/test_eject_runtime.py b/server/python/tests/codegen/test_eject_runtime.py index 9872687cf..3774d7c40 100644 --- a/server/python/tests/codegen/test_eject_runtime.py +++ b/server/python/tests/codegen/test_eject_runtime.py @@ -144,7 +144,9 @@ def test_eject_never_overwrites_an_owned_runtime_copy(tmp_path, monkeypatch, cap capsys.readouterr() assert main(["eject", "routes", "--force"]) == 0 assert mine.read_text(encoding="utf-8").endswith("# my fix\n") - assert "filter_parser.py: kept your copy" in capsys.readouterr().out + kept = capsys.readouterr().out + assert "filter_parser.py: kept your copy — DIFFERS: 0 behind, 1 of your own" in kept + assert "may need the newer copy" not in kept assert main(["gen", "--list"]) == 0 assert ("runtime codegen/runtime/filter_parser.py [owned — DIFFERS: 0 behind, 1 of your own]" in capsys.readouterr().out) From dbcbdc38d3f5c25d3bcb6278915b252e32207280 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 12 Oct 2026 02:30:22 -0400 Subject: [PATCH 5/8] no-mistakes(review): fix(python): eject refresh hint prints diff first, cwd-valid paths --- CHANGELOG.md | 7 +-- docs/features/own-your-codegen.md | 11 ++--- .../python/src/metaobjects/codegen/eject.py | 9 +--- .../tests/codegen/test_eject_independence.py | 43 +++++++++++++------ 4 files changed, 41 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecd01ac80..126740adf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -264,9 +264,10 @@ generator catalog and the Atlas evaluation are 1.2 work. `emits_writable_router`, and eject never overwrites a runtime copy. When eject keeps a runtime copy it now says how the copy compares with the installed module (`kept your copy — identical`, or `kept your copy — DIFFERS: N behind, M of your own`), - and for a copy that is behind it prints the command that refreshes it: `cp` of the - installed file over yours when you have not edited it, otherwise a `diff -u` against it - whose hunks you apply. + and for a copy that is behind it prints two commands that run from the working + directory, `--root` or not: the `diff -u` of your copy against the installed file, whose + hunks you apply, and the `cp` of the installed file over yours, for a copy you have not + changed. - **A gate for the rule.** The engine surface is written once, in the shared manifest (`eject.engineSurface.python`), and `tests/conformance/test_eject_contract.py` ejects every ejectable generator alone into an empty project and fails on any import outside it. A diff --git a/docs/features/own-your-codegen.md b/docs/features/own-your-codegen.md index e9bc66e9e..4e93b7614 100644 --- a/docs/features/own-your-codegen.md +++ b/docs/features/own-your-codegen.md @@ -758,8 +758,9 @@ like the rest of the package. kept, even with `--force`, and eject says so, with how the copy compares with the installed one (`kept your copy — identical` or `kept your copy — DIFFERS: N behind, M of your own`). When the copy is behind, the next line names the generator that may need the - newer one and the command that brings yours up. `metaobjects gen --list` marks each - runtime copy the same way. + newer one and two commands, both written to run from the directory you ran eject in: the + `diff -u` of your copy against the installed file, and the `cp` to use only if you have + not changed yours. `metaobjects gen --list` marks each runtime copy the same way. - **A project that has not ejected `routes` is unchanged.** The packaged generator still imports `metaobjects.codegen.runtime.*` and emits no `_runtime/`. @@ -847,9 +848,9 @@ uses (`from ..runtime import fr010_field_mapping as fm`). `codegen/runtime/shared_predicates.py` you already own does not have them, and eject keeps it, so `metaobjects eject api-docs` or `metaobjects eject routes --force` leaves a generator that fails to import (`cannot import name 'emits_writable_router'`). Eject marks the kept -copy `DIFFERS: N behind` and prints the command that refreshes it: `cp` of the installed -file over yours when you have not edited it, otherwise a `diff -u` against the installed -file whose hunks you apply (the recipe under +copy `DIFFERS: N behind` and prints how to bring it up: first the `diff -u` of your copy +against the installed file, whose hunks you apply, then the `cp` of the installed file over +yours, for a copy you have not changed. Eject does not decide which applies (the recipe under [The runtime your generated code imports comes with it](#the-runtime-your-generated-code-imports-comes-with-it)). ### `api-docs`: the api reference as an owned generator diff --git a/server/python/src/metaobjects/codegen/eject.py b/server/python/src/metaobjects/codegen/eject.py index 3bb4d5b26..36944f968 100644 --- a/server/python/src/metaobjects/codegen/eject.py +++ b/server/python/src/metaobjects/codegen/eject.py @@ -167,13 +167,8 @@ def eject(names: list[str], root: Path, *, force: bool = False) -> EjectResult: out.append(f" runtime {rel}: kept your copy — {_status(behind, own)} " "(eject never overwrites runtime)") if behind: - refresh = ( - "it carries edits of your own, so apply the hunks you need from: " - f'diff -u {rel} "{packaged}"' - if own else f'refresh it: cp "{packaged}" {rel.parent}/' - ) - out.append(f" {n} may need the newer copy: yours lacks lines the " - f"packaged one has; {refresh}") + out.append(f' {n} may need the newer copy: diff -u "{rt}" "{packaged}"; ' + f'if you have not changed it: cp "{packaged}" "{rt}"') continue rt.parent.mkdir(parents=True, exist_ok=True) rt.write_text(reference, encoding="utf-8") diff --git a/server/python/tests/codegen/test_eject_independence.py b/server/python/tests/codegen/test_eject_independence.py index dff1d5e0c..3ab7961bf 100644 --- a/server/python/tests/codegen/test_eject_independence.py +++ b/server/python/tests/codegen/test_eject_independence.py @@ -18,6 +18,8 @@ import json import os import re +import shlex +import shutil import subprocess import sys import uuid @@ -180,33 +182,46 @@ def _shared_predicates_before_the_added_ones() -> str: return "".join(lines) -@pytest.mark.parametrize("own_edit", ["", "\n# my fix\n"], ids=["unedited", "edited"]) +@pytest.mark.parametrize("root_arg", [None, "services/api"], ids=["cwd", "--root"]) def test_eject_onto_an_older_shared_copy_keeps_it_and_says_how_to_refresh_it( - own_edit: str, tmp_path, monkeypatch, capsys, + root_arg: str | None, tmp_path, monkeypatch, capsys, ) -> None: """A project that ejected a generator before `api-docs` existed owns a `shared_predicates.py` without the predicates the api reference imports. Eject keeps - that copy, so it has to say the copy is behind and name the way to bring it up.""" - older = _shared_predicates_before_the_added_ones() + own_edit - mine = tmp_path / "codegen" / "runtime" / f"{SHARED}.py" + that copy, so it has to say the copy is behind and name the way to bring it up, in + commands that work from the directory eject was run in.""" + root = tmp_path / root_arg if root_arg else tmp_path + older = _shared_predicates_before_the_added_ones() + mine = root / "codegen" / "runtime" / f"{SHARED}.py" mine.parent.mkdir(parents=True) mine.write_text(older, encoding="utf-8") + # With --root, the working directory is another project with a copy of its own. + elsewhere = tmp_path / "codegen" / "runtime" / f"{SHARED}.py" + if root_arg: + elsewhere.parent.mkdir(parents=True) + elsewhere.write_text("# another project's copy\n", encoding="utf-8") monkeypatch.chdir(tmp_path) - assert main(["eject", "api-docs"]) == 0 + assert main(["eject", "api-docs", *(["--root", root_arg] if root_arg else [])]) == 0 assert mine.read_text(encoding="utf-8") == older lines = capsys.readouterr().out.splitlines() at = next(i for i, ln in enumerate(lines) if f"{SHARED}.py: kept your copy" in ln) behind, own = re.search(r"kept your copy — DIFFERS: (\d+) behind, (\d+) of your own", lines[at]).groups() - assert int(behind) > 0 and int(own) == (1 if own_edit else 0) + assert int(behind) > 0 and int(own) == 0 - packaged = Path(metaobjects.codegen.runtime.__path__[0]) / f"{SHARED}.py" hint = lines[at + 1] - assert "api-docs may need the newer copy" in hint - if own_edit: - assert f'diff -u codegen/runtime/{SHARED}.py "{packaged}"' in hint - assert "cp " not in hint - else: - assert f'cp "{packaged}" codegen/runtime/' in hint assert [ln for ln in lines if "may need the newer copy" in ln] == [hint] + assert "api-docs may need the newer copy" in hint + packaged = Path(metaobjects.codegen.runtime.__path__[0]) / f"{SHARED}.py" + diff, copy = hint.split(": ", 1)[1].split("; if you have not changed it: ") + assert shlex.split(diff)[:2] == ["diff", "-u"] + assert [Path(p).resolve() for p in shlex.split(diff)[2:]] == [mine.resolve(), packaged.resolve()] + + # The copy command, run from the working directory, refreshes this project's copy. + command, source, target = shlex.split(copy) + assert command == "cp" + shutil.copy(source, target) + assert mine.read_text(encoding="utf-8") == packaged.read_text(encoding="utf-8") + if root_arg: + assert elsewhere.read_text(encoding="utf-8") == "# another project's copy\n" From 1e3b829d2c0f5605ab0df38be734bd12317af40a Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 12 Oct 2026 02:58:53 -0400 Subject: [PATCH 6/8] no-mistakes(review): fix(python): gen skip warning distinguishes packaged and owned docs generators --- CHANGELOG.md | 3 ++- docs/features/own-your-codegen.md | 3 ++- .../python/src/metaobjects/codegen/runner.py | 15 ++++++++----- .../tests/codegen/test_docs_generators.py | 21 ++++++++++++++----- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 126740adf..7d06b489a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -239,7 +239,8 @@ generator catalog and the Atlas evaluation are 1.2 work. generator (`docs_surface = "api"`) that `metaobjects docs` runs: an owned copy is listed under the new `docs.generators` block of `metaobjects.config.yaml` and runs in place of the packaged one. `gen` and `verify --codegen` skip a docs generator they find under a target - and say so. The reference asks `shared_predicates` which objects get a router (the new + and say so: the packaged `api-docs` name is to be removed (`metaobjects docs` already runs + it), an owned copy is to be listed under `docs.generators`. The reference asks `shared_predicates` which objects get a router (the new `emits_writable_router` and the moved `is_read_only_routed`, beside `emits_router`), the module your owned `routes` and `filter-allowlist` read, so one edit to the owned predicate moves the router, the allowlist and the pages together. The cross-port manifest lists diff --git a/docs/features/own-your-codegen.md b/docs/features/own-your-codegen.md index 4e93b7614..362332f60 100644 --- a/docs/features/own-your-codegen.md +++ b/docs/features/own-your-codegen.md @@ -874,7 +874,8 @@ runs the packaged one, and the positional `metaobjects docs ` form config, so it always does. An entry that is not a docs generator, or a second one for the `api` surface, is refused by name. `gen` and `verify --codegen` skip a docs generator they find under a target and say so (`'api-docs' writes the api reference, so gen skipped -it.`). +it.`), with what to do about the entry: the packaged `api-docs` name is removed, because +`metaobjects docs` already runs it, and an owned copy moves to `docs.generators`. **It describes the code your generators emit.** The page builder asks `shared_predicates` which objects get a router, the same module your owned `routes` and `filter-allowlist` diff --git a/server/python/src/metaobjects/codegen/runner.py b/server/python/src/metaobjects/codegen/runner.py index 288b366eb..c4482fc3d 100644 --- a/server/python/src/metaobjects/codegen/runner.py +++ b/server/python/src/metaobjects/codegen/runner.py @@ -136,12 +136,17 @@ def run_gen( docs_surface = getattr(gen, "docs_surface", None) if docs_surface is not None: # A docs generator writes pages under the docs output directory, and - # `metaobjects docs` runs it from `docs.generators`: here it would write them - # into the code tree, so it is skipped and the miswiring is named. + # `metaobjects docs` runs it: the packaged one always, an owned copy from + # `docs.generators`. Here it would write them into the code tree, so it is + # skipped and the miswiring is named. + remedy = ( + "`metaobjects docs` already runs it, so remove the entry." + if type(gen).__module__.startswith("metaobjects.") + else "List it under `docs.generators` in metaobjects.config.yaml: " + "`metaobjects docs` runs it from there." + ) result.warnings.append( - f"{gen.name!r} writes the {docs_surface} reference, so gen skipped it. " - "List it under `docs.generators` in metaobjects.config.yaml: " - "`metaobjects docs` runs it from there." + f"{gen.name!r} writes the {docs_surface} reference, so gen skipped it. {remedy}" ) continue ctx = GenContext( diff --git a/server/python/tests/codegen/test_docs_generators.py b/server/python/tests/codegen/test_docs_generators.py index 6bdcd8109..764b70618 100644 --- a/server/python/tests/codegen/test_docs_generators.py +++ b/server/python/tests/codegen/test_docs_generators.py @@ -153,13 +153,24 @@ def test_two_owned_generators_for_the_api_surface_are_refused(tmp_path: Path, mo # --- gen leaves a docs generator out ---------------------------------------------------- -def test_gen_skips_a_docs_generator_under_a_target_and_says_so(tmp_path: Path, monkeypatch, capsys) -> None: - _project(tmp_path, generators="entity, api-docs") +@pytest.mark.parametrize("entry, remedy, not_the_remedy", [ + ("api-docs", "`metaobjects docs` already runs it, so remove the entry.", "docs.generators"), + (OWNED_DOCS, "List it under `docs.generators` in metaobjects.config.yaml", "already runs it"), +], ids=["packaged", "owned"]) +def test_gen_skips_a_docs_generator_under_a_target_and_says_so( + entry: str, remedy: str, not_the_remedy: str, tmp_path: Path, monkeypatch, capsys, +) -> None: + """The packaged name cannot go under ``docs.generators`` (that block takes owned copies + only), so the warning sends each kind of entry where it can actually go.""" + _project(tmp_path, generators=f"entity, {entry}") monkeypatch.chdir(tmp_path) + assert main(["eject", "api-docs"]) == 0 + capsys.readouterr() assert main(["gen"]) == 0 - out = capsys.readouterr().out - assert "warning: 'api-docs' writes the api reference, so gen skipped it" in out - assert "docs.generators" in out + skipped = [ln for ln in capsys.readouterr().out.splitlines() if "so gen skipped it" in ln] + assert len(skipped) == 1 + assert "warning: 'api-docs' writes the api reference, so gen skipped it" in skipped[0] + assert remedy in skipped[0] and not_the_remedy not in skipped[0] written = {p.relative_to(tmp_path / "gen").as_posix() for p in (tmp_path / "gen").rglob("*") if p.is_file()} assert written and not any(w.startswith("api/") or w.endswith("AGENT-API.md") for w in written) From 0ddcd53d0df09fb23dc956dd04c5a7048682b350 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 12 Oct 2026 03:22:33 -0400 Subject: [PATCH 7/8] no-mistakes(review): docs(adr-0034): record Python eject gaps 4-6 as closed --- spec/decisions/ADR-0034-codegen-scaffold-and-own.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/decisions/ADR-0034-codegen-scaffold-and-own.md b/spec/decisions/ADR-0034-codegen-scaffold-and-own.md index fe8b74f20..f9a95dff6 100644 --- a/spec/decisions/ADR-0034-codegen-scaffold-and-own.md +++ b/spec/decisions/ADR-0034-codegen-scaffold-and-own.md @@ -828,9 +828,9 @@ refusal is then `scripts/finish-release.mjs`, which withholds the tag. | 1 | Java `routes`, `dto` and `value-object` did not compile when ejected alone. | Closed before this amendment | | 2 | TypeScript `requirementsFile` and `agentDocsFile` were in no registry. | Registered here as `requirements` and `agent-docs`; reference templates pending | | 3 | TypeScript reference templates that call packaged composers. | Pending | -| 4 | Prompt-tier emitters that stay packaged in C#, Python, Java and Kotlin. | Closed in C#. Pending in Python, Java and Kotlin | -| 5 | Fragment emitters behind otherwise whole generators (shared enums, M:N navigation, doc comments, record builders). | Closed in C#. Pending in Python, Java and Kotlin | -| 6 | The docs command's emitter in C#, Python, Java and Kotlin is not a registered `api-docs`. | Closed in C#. Pending in Python, Java and Kotlin | +| 4 | Prompt-tier emitters that stay packaged in C#, Python, Java and Kotlin. | Closed in C# and Python. Pending in Java and Kotlin | +| 5 | Fragment emitters behind otherwise whole generators (shared enums, M:N navigation, doc comments, record builders). | Closed in C# and Python. Pending in Java and Kotlin | +| 6 | The docs command's emitter in C#, Python, Java and Kotlin is not a registered `api-docs`. | Closed in C# and Python. Pending in Java and Kotlin | | 7 | Java generators outside the registry. | Registered here as `plantuml`, `html-docs`, `ai-docs`, `json-schema` and `mustache`; eject source pending for the first four | | 8 | The docs-site builders. | Exempt (ruling 3) | | 9 | Python generators that called another generator's packaged module. | Closed before this amendment | @@ -911,8 +911,8 @@ is. - **The older registry-conformance tests in C#, Java, Kotlin and Python subtract the names with a pending census entry.** The manifest lists `api-docs` for those ports (gap 6) and - the Java, Kotlin and Python registries do not hold it yet. The pending entry is the one - record of that difference. Deleting the entry removes the subtraction, as it has in C#. + the Java and Kotlin registries do not hold it yet. The pending entry is the one record of + that difference. Deleting the entry removes the subtraction, as it has in C# and Python. - **Two registry-driven Java suites pass an `outputFilename` argument** to generators that write one file, so the five newly registered rows run in them. It is a test argument; no generator changed. From 7ee5523b96655ef09c90966ef0628d880df24bc6 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 12 Oct 2026 04:13:03 -0400 Subject: [PATCH 8/8] no-mistakes(document): Drop Python from ADR-0034 Amendment 8 pending wording --- spec/decisions/ADR-0034-codegen-scaffold-and-own.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/decisions/ADR-0034-codegen-scaffold-and-own.md b/spec/decisions/ADR-0034-codegen-scaffold-and-own.md index f9a95dff6..d0f4f1008 100644 --- a/spec/decisions/ADR-0034-codegen-scaffold-and-own.md +++ b/spec/decisions/ADR-0034-codegen-scaffold-and-own.md @@ -909,7 +909,7 @@ is. *What changed in existing tests so the new rows could run.* -- **The older registry-conformance tests in C#, Java, Kotlin and Python subtract the names +- **The older registry-conformance tests in C#, Java and Kotlin subtract the names with a pending census entry.** The manifest lists `api-docs` for those ports (gap 6) and the Java and Kotlin registries do not hold it yet. The pending entry is the one record of that difference. Deleting the entry removes the subtraction, as it has in C# and Python.