diff --git a/.dagger/modules/engine-e2e/dagger-module.toml b/.dagger/modules/engine-e2e/dagger-module.toml index bc5fffb..d370e49 100644 --- a/.dagger/modules/engine-e2e/dagger-module.toml +++ b/.dagger/modules/engine-e2e/dagger-module.toml @@ -6,4 +6,4 @@ engineVersion = "v1.0.0-0" [[dependencies]] name = "engine-dev" - source = "github.com/dagger/dagger/.dagger/modules/engine-dev@0d031c08ef3e379c6f4eb7f8f5cad4638a168863" + source = "github.com/dagger/dagger/.dagger/modules/engine-dev@6bf59d50654ce9244ebeee1cc090b7dce3fe3083" diff --git a/.dagger/modules/engine-e2e/main.dang b/.dagger/modules/engine-e2e/main.dang index 94938cb..88a0b82 100644 --- a/.dagger/modules/engine-e2e/main.dang +++ b/.dagger/modules/engine-e2e/main.dang @@ -1,9 +1,9 @@ """ -Checks the Python SDK against a pinned main engine. +Checks the Python SDK against the v1.0.0-beta.13 engine, pinned by commit. Keep engineCommit and the engine-dev dependency in dagger-module.toml aligned. """ type EngineE2e { - let engineCommit: String! = "0d031c08ef3e379c6f4eb7f8f5cad4638a168863" + let engineCommit: String! = "6bf59d50654ce9244ebeee1cc090b7dce3fe3083" let modulePath: String! = ".dagger/modules/sdk-smoke" let assert(condition: Boolean!, message: String!): Void { diff --git a/README.md b/README.md index d62d3c0..da102c5 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ It uses the engine's native `Workspace` and `ModuleSource` APIs. It uses | --- | --- | | `python-sdk.dang`, `mod.dang`, `templates/` | authoring: `findClientRoot`, `generateScope`, `mod` (generate, config), templates | | `sdk/` | the `dagger-io` client library and code generator | -| `runtime/` | the module runtime the engine calls to run a module | +| `runtime/` | the module runtime the engine calls to run a module, and the container build the static entrypoint shares | Code generation happens at `dagger generate`, which calls `generateScope` for every recorded scope. It runs the code generator in `sdk/` and vendors the @@ -73,6 +73,51 @@ runtime runs the module — so switching back is just editing the line again. Within this repository, a path relative to the module works too, which is how the end-to-end fixture exercises the runtime before the ref exists. +## Static entrypoint + +Not available yet. A module's types are discovered by running it: the engine +builds the module's container and starts Python once per session to register +the types, then again for every call. The SDK can instead compute the types +once, at `dagger generate`, and write them into a generated entrypoint the +engine loads without running Python. The setting that turns this on is not +exposed until the engine loads manifest version 2 (dagger/dagger#14038), so +`dagger module init python` has no `--static-entrypoint` flag yet and every +new module is generated on the default path. + +Once enabled, generating a module writes a manifest version 2 instead of a +runtime manifest, and `sdk/entrypoint/` next to the vendored library: + +| File | What it is | +| --- | --- | +| `types.dang` | the module's types, as a literal list of `TypeDef` values | +| `main.dang` | the `ModuleEntrypoint`: returns the types and runs calls in the module's container | +| `build.dang` | the container build, copied from `runtime/build.dang` | + +The types come from importing the module in its own container and reading +what its decorators registered, so they are the ones the runtime would +register. `main.dang` bakes a content digest of every source file that can +change them; a call after an edit is refused with a message to run +`dagger generate`. A lock file added afterwards is refused the same way, and +only file contents count, not permissions. + +What the static path cannot do yet, and refuses at `dagger generate`: +module clients (manifest version 2 has no dependencies), any `cache=` value +on a function (the entrypoint's exec is content-cached and receives no +per-call signal), the `legacy` template, and a manifest with `include`, +`disableDefaultFunctionCaching`, a runtime other than `python`, a `source` +other than `.`, or `codegen`, `clients` or `dependencies` tables. Such +modules keep the default path. There is no `debug` terminal on the static +path, and a function error reaches the caller as the exec failure with the +process's stderr. + +A module that already has a manifest version 2 keeps its static entrypoint +when generated directly, with `dagger call python-sdk mod --path +generate`. `dagger generate` moves it back to the default path: it removes +`sdk/entrypoint/` and rewrites a runtime manifest with the generating engine's +version. See +[`future/done/static-module-entrypoint.md`](./future/done/static-module-entrypoint.md) +for the design and the plan to make it the default. + ## Install From your workspace root: @@ -174,6 +219,6 @@ one to a Python scope is refused and the workspace is left unchanged. dagger check ``` -`engine-e-2-e:dev-sdk-check` builds the pinned dagger/dagger#13992 engine. It +`engine-e-2-e:dev-sdk-check` builds the dagger/dagger `v1.0.0-beta.13` engine. It runs the SDK interface checks, initializes Python modules with default and explicit settings, and calls a generated module. diff --git a/dagger-module.toml b/dagger-module.toml index d073573..f6bc375 100644 --- a/dagger-module.toml +++ b/dagger-module.toml @@ -1,6 +1,6 @@ name = "python-sdk" engineVersion = "v1.0.0-beta.11" -include = ["!runtime", "!.dagger", "!future", "!docs"] +include = ["!.dagger", "!future", "!docs"] [runtime] source = "dang" @@ -8,3 +8,7 @@ source = "dang" [[dependencies]] name = "sdk-helpers" source = "dagger.io/sdk/helpers@v1" + +[[dependencies]] +name = "python-sdk-runtime" +source = "runtime" diff --git a/dagger.json b/dagger.json index 92bc692..8da7783 100644 --- a/dagger.json +++ b/dagger.json @@ -5,7 +5,6 @@ "source": "dang" }, "include": [ - "!runtime", "!.dagger", "!future", "!docs" @@ -14,6 +13,10 @@ { "name": "sdk-helpers", "source": "dagger.io/sdk/helpers@v1" + }, + { + "name": "python-sdk-runtime", + "source": "runtime" } ] } diff --git a/dagger.lock b/dagger.lock index 6108ad1..aa5fbc0 100644 --- a/dagger.lock +++ b/dagger.lock @@ -1,17 +1,12 @@ +# Generated by Dagger. Pins the images and git refs this workspace uses. +# Any Dagger command may update this file; always commit it with your changes. +# Run `dagger workspace update` to refresh the pinned versions. [["version","2"]] -["","git-latest",["github.com/dagger/sdk-helpers"],"refs/tags/v1.0.2",[["version","v1"]]] -["","git-latest",["https://github.com/dagger/sdk-sdk"],"refs/heads/main"] -["","git-sha",["github.com/dagger/sdk-helpers","refs/tags/v1.0.2"],"e5e38a5f0778c5a804931d95e5d0d9953668049a"] -["","git-sha",["https://github.com/containernetworking/plugins","refs/tags/v1.9.0"],"9b3772e1a7abf93cbb7c6526a28bc0d27b830e02"] -["","git-sha",["https://github.com/dagger/sdk-helpers","main"],"64645f1967d3dba6fce951dd61ae4acd8d9b0861"] -["","git-sha",["https://github.com/dagger/sdk-sdk","refs/heads/main"],"334448911a8292fba0d677e5f31926c79ad80ad3"] -["","git-sha",["https://github.com/libfuse/sshfs.git","refs/tags/sshfs-3.7.6"],"7a2d988775446ebe7af9b01c99b3b8e86bddb05a"] -["","git-sha",["https://github.com/opencontainers/runc","refs/tags/v1.4.2"],"c241c0bb5e60a8e8c1b2e53d4eca8d0068d8d57e"] -["","oci-latest",["docker.io/library/busybox"],"1.38.0"] -["","oci-sha",["docker.io/library/alpine:3.22"],"sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce"] -["","oci-sha",["docker.io/library/busybox:1.38.0"],"sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616"] -["","oci-sha",["docker.io/library/golang:1.25-alpine"],"sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59"] +["","git-latest",["github.com/dagger/sdk-helpers",["version","v1"]],"refs/tags/v1.0.4"] +["","git-sha",["github.com/containernetworking/plugins","refs/tags/v1.9.0"],"9b3772e1a7abf93cbb7c6526a28bc0d27b830e02"] +["","git-sha",["github.com/dagger/sdk-helpers","refs/tags/v1.0.4"],"6c883f657933ecb06ae86d56d9cd1f8785346bb1"] +["","git-sha",["github.com/libfuse/sshfs","refs/tags/sshfs-3.7.6"],"7a2d988775446ebe7af9b01c99b3b8e86bddb05a"] +["","git-sha",["github.com/opencontainers/runc","refs/tags/v1.4.2"],"c241c0bb5e60a8e8c1b2e53d4eca8d0068d8d57e"] ["","oci-sha",["docker.io/library/golang:1.26-alpine"],"sha256:ce864e7223ac17b1775e6fd0b4c0db580c2eb50e7953a427916379e4b92a1628"] ["","oci-sha",["docker.io/tonistiigi/xx:1.2.1"],"sha256:8879a398dedf0aadaacfbd332b29ff2f84bc39ae6d4e9c0a1109db27ac5ba012"] -["","oci-sha",["ghcr.io/astral-sh/uv:python3.14-alpine"],"sha256:706cd9faf6d3fa7476fc470463bfca625d4b3ea5e46d6dc166ec1fb8ad3976e6"] -["","vanity-url",["https://dagger.io/sdk/helpers"],"https://github.com/dagger/sdk-helpers"] +["","vanity-url",["https://dagger.io/sdk/helpers"],"https://github.com/dagger/sdk-helpers"] \ No newline at end of file diff --git a/future/done/static-module-entrypoint.md b/future/done/static-module-entrypoint.md new file mode 100644 index 0000000..e0b0088 --- /dev/null +++ b/future/done/static-module-entrypoint.md @@ -0,0 +1,1047 @@ +# Static module entrypoints for Python modules + +author: yves +created: 2026-09-11 +status: done (phase 1 landed as draft PR dagger/python-sdk#28; phase 2 flips the +default once released engines load manifest version 2) +related: `dagger/dagger#14038` (manifest v2 entrypoints, draft, head +`75c777223ccc4baaf5819a04d70d060034a94dbb`); `dagger/dagger#13992` (SDK +interface, merged 2026-09-09 as `908d48ebda4b3dd1c33c8bebba49b90abb4f2953`, +released in `v1.0.0-beta.12`); `dagger/dagger#11803`, `#13095`, `#13251` +and `#13235` (the Python AST analyzer shipped in v0.20.7, its fixes, its +removal in v0.21.1, and the follow-up design that this document adopts); +`dagger/java-sdk#19` (head `12a2682d2976c4eb59f8d6c6a501c446ce54a499`) and +`dagger/go-sdk#36` (head `4dfd447d58344835a0d4692ec0c8e5683c18bd6f`), two +prototypes of the same adaptation; the first attempt at this design, +`future/manifest-v2-entrypoint.md` at commit `691de2b` on branch +`python-sdk-manifest-v2-sdk-ux-lead-9a51fff5` of the fork +`eunomie/python-sdk` (never merged); `future/done/self-contained-python-sdk.md`; +`hack/designs/2026-08-25-python-module-performance-ideas.md`; +`dagger/sdk-helpers` (`dagger.io/sdk/helpers@v1`, locked at v1.0.2). + +## Summary + +A Python module is loaded and called through code that runs at call time. +The engine starts the module's Python interpreter to learn which types the +module exposes, and starts it again to run a function. The first start +exists only to discover types. This document moves that discovery to +`dagger generate`: the SDK imports the module once, in the module's own +container, reads what its decorators registered, and writes the result into +a generated module entrypoint. The engine then reads the types from the +entrypoint and never boots Python to discover them. + +The extraction imports the module instead of parsing its source. A static +`ast` analyzer was measured (5 ms against about 1 s) and rejected on the +evidence of `dagger/dagger#11803`: an analyzer of that kind shipped in +v0.20.7, needed five fix releases, and was removed in v0.21.1 because it +could not see what Python computes at import time. Importing is exact by +construction and is about 150 lines. + +The change is gated by one SDK setting, `staticEntrypoint`, in two phases. +In phase A the setting defaults to `false` and nothing changes for a module +that does not set it. In phase B the default becomes `true`, and a module +that needs the dynamic path sets it to `false`. The dynamic path is today's +path. It is not removed, and this document plans no date for its removal. + +The static path depends on an engine change that is still a draft, +`dagger/dagger#14038`. Everything that does not depend on it, which is the +extraction, the rendering, the manifest and their tests, lands and runs in +this repository's CI. Everything that does is verified against a +development engine and recorded here. + +## Terms + +| Term | Meaning | +|---|---| +| dynamic path | What every Python module does today: a version 1 `dagger-module.toml` with `[runtime]`, a runtime module that builds the module's container, and types discovered by running the module once per session. Selected by `staticEntrypoint = false`. | +| static path | A version 2 `dagger-module.toml` with `[entrypoint]`, a generated Dang entrypoint under `sdk/entrypoint/`, and types written into it at `dagger generate`. Selected by `staticEntrypoint = true`. | +| runtime registry | The `dagger.mod.Module` instance that the module's decorators populate when the module is imported. It holds the object, interface and enum definitions and dispatches calls. | +| description | The plain-data form of what the registry holds: `ModuleDescription` in `dagger.mod._describe`. Both paths derive their type definitions from it. | +| renderer | The code that turns a description into Dang source. | +| the first attempt | `future/manifest-v2-entrypoint.md` at `691de2b` on `eunomie/python-sdk`. | +| the v0.20.7 analyzer | The `ast`-based analyzer of `dagger/dagger#11803`, removed by `#13251`. | + +"Legacy" appears below only in two proper names that other code owns: the +`legacy` starter template of this repository, and the `withLegacy*` functions +of `dagger/sdk-helpers`. + +## Requirements + +The repository owner set these requirements on 2026-09-11 and confirmed the +approach on 2026-09-12. + +1. The types and function signatures a module exposes must be computed once + at `dagger generate` and written down, so that the entrypoint can be + generated from them. Discovering them by loading the module at call time + is no longer acceptable as the default. +2. The rollout has two phases behind one SDK flag. First the static path is + opt-in. Then the default inverts and the dynamic path is opt-out. The + dynamic path stays supported for as long as users want it. There is no + hard cutover. +3. The scope is the extraction of types and function signatures. The + module's functions are never executed to extract them. Execution and + dispatch are in scope only as far as the generated entrypoint needs to + call a function. +4. Static analysis of the source was the owner's first idea, not a mandate. + Alternatives were evaluated on correctness for the type-hint surface the + SDK accepts, on implementation complexity, and on analysis speed. The + owner chose importing at generate time on that evaluation. +5. Implementation started after the owner approved this design. + +## Problem + +### What the dynamic path costs + +On the dynamic path the engine calls the module's runtime module, gets a +`Container`, execs it once with an empty function call to learn the module's +types, then once per constructor or function call. Each exec is a fresh +Python interpreter. `hack/designs/2026-08-25-python-module-performance-ideas.md` +measured a warm `dagger call` on the default template at about 4.4 s, of +which about 3.1 s is three interpreter boots; the type-discovery boot is one +of the three, and the SDK's import chain inside a boot costs 665 to 684 ms. +Discovery itself, once the module is imported, costs about 4 ms. + +### What the engine changes + +`dagger/dagger#14038` defines manifest version 2 and the module entrypoint. +A version 2 `dagger-module.toml` has exactly three keys, and the engine +rejects any other (`core/modules/config_format.go` at `75c77722`, +`validateModuleManifestV2TOML`): + +```toml +manifestVersion = 2 +name = "hello" + +[entrypoint] +kind = "dang" +source = "./sdk/entrypoint" +``` + +The engine loads every `.dang` file in `entrypoint.source` as one program. +That program must define exactly one type that implements this interface, +constructible with no arguments (`core/sdk/dang/v2/entrypoint.go` at +`75c77722`, `findModuleEntrypoint`): + +```graphql +interface ModuleEntrypoint { + types(workspace: Workspace!): [TypeDef!]! + call( + workspace: Workspace! + receiverType: String! + receiverValue: JSON + fnName: String! + fnArgs: JSON! + ): JSON! +} +``` + +The engine calls `types` to install the module and `call` for every +constructor or function invocation. No SDK module is called at load or call +time. `types` can be a literal list of `TypeDef` expressions, which is what +the Go and Java prototypes and the engine's own reference module +(`.dagger/modules/tiny/entrypoint/main.dang` at `75c77722`) return. + +### The v0.20.7 analyzer + +`dagger/dagger#11803` (v0.20.7, 2026-04-22) replaced runtime registration +with a pure-`ast` analyzer, `dagger.mod._analyzer`, run at load time for +every module with no opt-out. It grew through `#13090`, `#13091`, `#13093`, +`#13095`, `#13162` and `#13171` to about 5,000 lines of analyzer and 6,700 +lines of tests (a differential suite against `typing.get_type_hints`, +Hypothesis property tests, and a daggerverse corpus), and was removed by +`#13251` (v0.21.1, 2026-05-28). The owner's reasons on `#13251`: every +release found new edge cases; modules that worked stopped working; and "the +current AST based analysis can't find all the things people are doing in +python. Some can be solved at the price of breaking changes, some can't be +fixed currently, by design." The by-design cases named in `#13230` and +`#13234`: defaults that are runtime values (`logging.INFO` recorded as +`"INFO"` instead of `20`), members added by decorators at runtime, enums +built dynamically, `enum.auto()`. + +The owner's follow-up design, `#13235` (`hack/designs/python-sdk-no-codegen-at-runtime.md` +on that branch), concluded: "Python can import it... use its richest model +(execution) at generate time, get real values, and still emit a fully +static runtime artifact", because a self-call such as `dag.hello().greet()` +lives in a function body that does not run at import. This document is that +conclusion applied to manifest version 2. + +### What the first attempt concluded, and what changed + +The first attempt designed a Python entrypoint whose `types()` execs the +module's container at load time to describe the types, cached by the +content of the module directory. It rejected computing the types at +generate time because "every signature edit would then need `dagger +generate`". Requirement 1 decides that the other way, as it is for the Go, +Java and TypeScript SDKs, and the staleness guard below turns a stale +entrypoint into an actionable error. Since then `dagger/dagger#13992` +merged (2026-09-09) and this repository adopted its SDK interface in +`dagger/python-sdk#25`, `#26` and `#27`; `#14038` did not move (head +`75c77722`, draft, no review, not on `main` at `4932411f` on 2026-09-11). + +## Goals + +1. `dagger generate` on a Python module with `staticEntrypoint = true` + writes a version 2 `dagger-module.toml` and a generated Dang entrypoint + under `sdk/entrypoint/`. The entrypoint's `types()` is a literal list of + `TypeDef` expressions. `call()` runs the module's function. +2. The static path exposes exactly the types the dynamic path exposes for + the same module, because both derive them from one description built by + importing the module and reading the runtime registry. +3. The extraction is one interpreter boot in the module's container at + `dagger generate`, and nothing at load time. +4. The dynamic path keeps working, unchanged, for every module that does not + opt in during phase A and for every module that opts out during phase B. + A module moves between the two paths by changing one setting and running + `dagger generate`, in both directions, leaving no file of the other path + behind. A module whose version 1 manifest holds fields version 2 cannot + carry is refused on the static path, naming the fields. +5. The parts that do not depend on `dagger/dagger#14038` (extraction, + rendering, manifest, the switch in both directions, and their checks) are + verified in this repository's CI. The parts that do are verified on a + development engine and recorded here. + +### Accepted differences + +These follow from the `ModuleEntrypoint` interface at `75c77722` and from +requirement 1. Each is either refused at `dagger generate` or documented in +the README. + +- A change to the module's source is visible to the engine only after + `dagger generate`. Until then `call()` refuses to run the module with a + message that says to run `dagger generate` (the staleness guard). +- No cache policy other than the default can be honoured: the entrypoint's + container exec is cached by the content of its inputs and receives no + per-call nonce, so after the engine's own cache entry expires the + identical exec still returns its cached result. The static path refuses + every explicit `cache=` value and a version 1 manifest with + `disableDefaultFunctionCaching = true`. +- A function error reaches the user as an exec failure carrying the + process's stderr. The structured values the dynamic path attaches to a + `dagger.Error` are lost: `Query.currentFunctionCall` is reachable inside + the Dang program but not inside the exec it starts (first attempt, probe + 1; unchanged at this head). +- The module docstring (`Module.withDescription` on the dynamic path) is not + exposed. `types()` has no place for it. +- The `debug` setting of the dynamic path has no equivalent: a version 2 + manifest carries no SDK configuration. +- A module with `staticEntrypoint = true` cannot depend on other modules. + Manifest version 2 has no dependency list + (`future/module-manifest-v2/compat-bridge.md` at `75c77722`, + *Dependencies*). `generateScope` refuses a non-empty `clients` list on the + static path. +- Only the module directory is mounted into the module's container, and + version 2 has no `include` list. A Python dependency outside the module + directory fails the container build, at generate time and at call time + alike, with the installer's error. +- A version 1 manifest field that version 2 cannot carry is refused rather + than dropped: `include`, a `source` other than `.`, a runtime other than + `python`, `disableDefaultFunctionCaching`, `codegen` and `clients` tables, + and dependencies. Such a module stays on the dynamic path or removes the + field first. On the way back to the dynamic path the manifest is rebuilt + with the generating engine's `engineVersion`. +- The static path runs the module in the container this repository's + `runtime/` builds. A module on the dynamic path with `[runtime] source = + "python"` runs in the engine's built-in Python runtime instead. The two + builds differ in base image pin, `uv` version and install flow. +- Two behaviours inside the nested exec are verified first thing after + approval (*Testing*, dev-engine step 1): `DefaultPath` and + `DefaultAddress` arguments, which the engine resolves before it calls + `call` (`future/module-manifest-v2/spec.md`, *Call rules*), and + `dag.current_module()`, which is an open question because + `currentFunctionCall` does not reach the exec. If one does not hold, it + becomes an entry in this list. + +## Non-goals (YAGNI) + +- Generating a static Python dispatcher (a switch over receiver and function + names, as the Go, Java and TypeScript prototypes do). Importing the module + is unavoidable to run a function, the import populates the registry, and + the registry's lookup costs about 1 ms after it. The description holds + every name a dispatcher would need, so one can be generated later if the + lean-boot work changes the balance. +- Making the call-time boot faster. That is idea 3 (lean boot) and idea 2 + (warm worker) of the performance-ideas document. +- Removing the dynamic path, `runtime/`, or any function of the authoring + module. Requirement 2. +- Loading a static module from a git ref or from another workspace. The + entrypoint receives the caller's workspace and finds its module by a path + baked at generate time (*Verified constraints*). +- Dependencies for version 2 modules, and a dynamic version 2 entrypoint. + Both wait on the engine (*Rollout*). +- Supporting the `legacy` template on the static path. It scaffolds a + `dagger.json`-era module whose `.gitignore` ignores `/sdk`, which a static + module must commit. +- Self-calls (bindings that include the module's own types). The description + is what they need; wiring them is `#13235`'s steps 4 and 5 and a separate + change. +- Writing the version 2 manifest through `dagger/sdk-helpers`. Its + `tomlContents` writes no `manifestVersion` at v1.0.2, v1.0.4 or `main` + (*Verified constraints*), so this change writes the five lines itself. + Moving to the helper once it supports version 2 is a one-function change. +- Source maps. The registry emits none. + +## Measurements + +All numbers are from one shared, noisy x86-64 Linux host: Python 3.14.7, +`uv` 0.11.16, the `dagger-io` library at `dagger/python-sdk` commit +`d551f327ae111b5213b4462186e04ed35817b9a2` installed in a virtual +environment. Each number is the middle of three runs. + +**Fixture.** A module of 131 lines in two files using the surface +`dagger.mod` accepts: two object types, one interface, two enums with +member docstrings, 16 functions and 2 constructors; `from __future__ import +annotations`; a `create` classmethod; `field()` with `name=`, `default=`, +`default=list`, an enum default and an `InitVar`; `Annotated` metadata of +every kind; `str | None`, `list[str] | None`, `list[list[str] | None]`, +`Self`, `list[Self]`; a `function()(Other)` attribute; core object, scalar +and enum types; a relative import. + +| Extraction | Time | +|---|---| +| Import the module in a process that already has the SDK loaded: decorators register 3 classes, 16 signatures resolved | 4.3 ms | +| `import dagger.mod` before that | 200 ms warm, 552 ms cold bytecode cache; 665 to 684 ms inside a fresh module container | +| A 400-line `ast` prototype: parse, resolve, serialize | 4.8 ms in process, 20 ms wall | +| `ty check` 0.0.80 | 100 to 110 ms | +| `mypy` 2.3.1 | 270 to 300 ms warm cache, 15.5 s first run | +| `pyright` 1.1.414 | 720 to 820 ms | +| `libcst` parse plus scope metadata | about 130 ms | + +**Dang builder chain.** A scratch Dang module with every `TypeDef` and +`Function` call the renderer emits type-checked and evaluated on engine +`v1.0.0-beta.12` (commit `4932411f`) through `dagger call`. Two spellings +matter: the `JSON` scalar must be written `Dagger.JSON` in a Dang module +(bare `JSON` is Dang's own JSON namespace), and the cache policy members are +`FunctionCachePolicy.Never` and `.Default`. + +**Digests.** On `v1.0.0-beta.12`, `Directory.digest` changes when only a +file's permissions change. `File.digest(excludeMetadata: true)` depends only +on the file's bytes (`core/file.go` at `4932411f`, `File.Digest`): it is +`"sha256:" + hex(sha256(sha256(bytes)))`, `sha256` over the content and +then `digest.FromBytes` over that 32-byte hash. The guard uses this field. + +## Extraction approaches evaluated + +| # | Approach | Speed | Correctness | Size | Operational cost | +|---|---|---|---|---|---| +| 1 | Import the module at generate time in its own container and read the registry | about 1 s per module (one boot), plus the dependency install the call path needs anyway | Exact: the producer is the registry the dynamic path uses | about 150 lines plus a container exec in Dang | The module's dependencies must install at generate time; module-level code runs at generate time | +| 2 | Standard library `ast` with an import and name resolver | 20 ms | A subset; the v0.20.7 analyzer shows the tail (aliases, constants, inheritance, runtime-valued defaults) is long and some of it unfixable | about 5,000 lines to reach v0.20.7's coverage, plus refusals | None | +| 3 | A type checker (`ty`, `mypy`, or `pyright`'s type server) | 100 ms to 800 ms | Resolves aliases; still needs the walker over decorated classes; cannot see runtime values | Large glue over unstable APIs | The module's dependencies must be importable anyway | +| 4 | `libcst` | about 130 ms | Same reach as 2 | as 2 plus a dependency | A dependency | +| 5 | A parser in Rust or Go | estimate: 5 ms | Same reach as 2 | as 2 plus a toolchain | A second language for Python semantics | + +**Decision: approach 1.** It is exact, it is the smallest, and it is what +the owner concluded after shipping approach 2. Its cost is one interpreter +boot per module at `dagger generate`, which is the boot the dynamic path +pays at every session load today, moved to a development-time command. +Approach 2 wins on generate-time speed alone, and would strand every module +outside its subset when the default flips. + +### Where the extraction runs + +In the module's own container, built by this repository's runtime module +(`runtime/`, `moduleRuntime`), which the authoring module gains as a +dependency. `dagger generate` first vendors the library and the bindings +into `sdk/`, then builds the container from the module with that `sdk/` and +execs: + +```text +python -m dagger.mod entrypoint --name hello --path .dagger/modules/hello --output /dagger/entrypoint +``` + +The subcommand imports the module through the runtime's own loader +(`dagger.mod.cli.load_module`), builds the description, renders `types.dang` +and `main.dang`, and writes them to the output directory. It opens no +engine connection. + +## Verified constraints + +Engine references are to `dagger/dagger` at `75c77722` unless marked +`main`, in which case they are to `4932411ff7a9b0771dd53779d9a085406cefb531` +(2026-09-11, tagged `v1.0.0-beta.12`). + +**How the engine drives an entrypoint** (`core/sdk/dang/v2/entrypoint.go`). + +- `runEntrypointDir` copies every `.dang` file in `entrypoint.source` into + one temporary directory, appends the interface as + `__module_entrypoint.dang` (that name is reserved), and runs the whole + program on every `types` and every `call`. +- `findModuleEntrypoint` requires exactly one public type implementing + `ModuleEntrypoint` with a zero-argument constructor. `let` fields with + defaults are not constructor arguments. +- `types` results are `TypeDef` values the program built, loaded by ID. + `validateEntrypointConstructors` rejects more than one object with a + constructor. +- `call` receives `receiverValue` and `fnArgs` as `JSON` scalars. When the + program encodes a record holding them with `JSON.encode`, each is written + as a JSON text string (first attempt, probe 1). The result must be a + `JSON` scalar or null. +- The `workspace` argument is `Query.currentWorkspace`; its `cwd` is `/` + from the root and from inside the module (probe 1). The entrypoint finds + its module by a path baked at generate time. +- Inside a container exec started by the program, `Query.currentFunctionCall` + fails; inside the Dang program it resolves. An exec whose inputs are + unchanged is reported `CACHED` by a later CLI invocation. +- `resolveEntrypointSourceDirectory` reads `entrypoint.source` relative to + the manifest's directory and rejects absolute paths and paths that escape + the module directory. +- `ModuleSource.introspectionSchemaJSON` loads the module's dependencies and + asks the schema builder for the module-facing schema; it does not run the + module. + +**Version 1 manifests on the `#14038` engine.** `parseCurrentModuleConfigTOML` +reads `manifestVersion` first; when absent, the version 1 parser runs, and +`SDKForModule` takes the entrypoint path only when `Entrypoint != nil`. The +specification text says the opposite. The code is what runs; *Rollout* +makes it a condition. + +**What the dynamic path runs on** (`main`). `core/sdk/loader.go` resolves +the runtime name `python` to the engine-baked Python SDK. A module can +instead name this repository's `runtime/` by ref. + +**Where CI runs.** This repository's `dagger.toml` registers one module, +`engine-e2e`. Its `devSdkCheck` builds an engine from `dagger/dagger` at +`0d031c08ef3e379c6f4eb7f8f5cad4638a168863` (after `#13992`, without +`#14038`), mounts this checkout as the `python` SDK in a scratch workspace +whose `dagger.toml` is `.dagger/modules/engine-e2e/workspace.toml`, and runs +`dagger check` inside it, which is where every `e2e:*` check runs. The +checks on `dagger/python-sdk#27` are `engine-e-2-e:dev-sdk-check` and +`load`. "CI" below means that development engine. + +**How SDK settings reach the SDK** (`main`). `PythonSdk`'s constructor +fields are the SDK's settings, exposed as kebab-case flags of `dagger module +init python` and persisted on the scope in `dagger.toml` +(`SDKScope.Settings`). `effectiveSDKModuleSettings` +(`core/schema/workspace_sdk_init.go`) applies, highest first, the scope's +own settings, then the SDK module's `[modules..settings]` as seen +through user and environment overlays. `withInitModule` on an existing scope +merges new explicit settings in, and an explicit `false` is stored. `dagger +module init` already owns an `--entrypoint` flag, so the setting is not +named `entrypoint`. + +**Manifest builder** (`dagger/sdk-helpers` v1.0.2). `withDangEntrypoint` +records an entrypoint, `withoutLegacyFields` drops the runtime fields, but +`tomlContents` never writes `manifestVersion` and preserves `$schema` and +`disableDefaultFunctionCaching`, which version 2 rejects. A manifest it +writes for an entrypoint parses as version 1 on the `#14038` engine and +reaches `errMissingSDKRef`. + +**Schema view for a version 2 manifest.** On the `#14038` engine a version +2 module gets the current schema view. On an engine without `#14038` the +version 2 keys are unknown, no `engineVersion` is present, and the engine +serves its oldest view (`v0.9.9`, whose `Workspace` has no `cwd`). Bindings +must never be generated from that view; the static path reads the schema +from a version 1 manifest. + +**The runtime registry** (`sdk/src/dagger/mod/` at `d551f32`). Decorators +populate `Module._objects` and `Module._enums` at import; +`Module._typedefs()` turns them into `TypeDef` selections through +`_converter.to_typedef`, `Function.parameters` and `Parameter`. The main +object is the class named `DAGGER_MAIN_OBJECT`. Two of its behaviours are +worth naming because the description preserves them: a field's description +is `get_doc(field.type)` on the raw annotation (a class's docstring when the +annotation is a class, and nothing under `from __future__ import +annotations`), and a nullable argument's `TypeDef` gets `withOptional(true)` +twice. + +## Proposed approach + +### 0. One setting, two phases + +`PythonSdk` gains one constructor field: + +```dang +""" +Generate a static entrypoint that carries the module's types, so the +engine loads them without running the module. +""" +pub staticEntrypoint: Boolean! = false +``` + +`dagger module init python --static-entrypoint` sets it; the engine persists +it on the scope and passes it back on every `dagger generate`. A workspace +sets it for every Python module through `[modules.python-sdk.settings]`, +which a scope's own setting overrides. + +| | Phase A (this change) | Phase B (a later change) | +|---|---|---| +| Default | `false` | `true` | +| Opt in / out | `--static-entrypoint` selects the static path | `--static-entrypoint=false` selects the dynamic path | +| Existing module, setting unset, next `dagger generate` | unchanged | migrates to the static path, or fails with an actionable error if refused | +| Engine floor of this SDK module | `v1.0.0-beta.11`, as today | the first release that loads manifest version 2 | + +Silent fallback to the dynamic path was rejected: requirement 2 makes the +dynamic path a stated choice. A boolean is what "invert the flag" means; a +string setting was considered and rejected as having no third value. + +### 1. What each path writes + +For a module scope named `hello` at `.dagger/modules/hello`: + +| | dynamic path | static path | +|---|---|---| +| `dagger-module.toml` | version 1: `name`, `engineVersion`, `[runtime] source = "python"`, dependencies (as today) | version 2: `manifestVersion = 2`, `name`, `[entrypoint] kind = "dang" source = "./sdk/entrypoint"` | +| `sdk/` | client library and `src/dagger/client/gen.py` (as today) | the same, plus `sdk/entrypoint/` | +| `sdk/entrypoint/main.dang` | absent, removed if present | `type Entrypoint implements ModuleEntrypoint`: `types()`, `call()`, the staleness guard | +| `sdk/entrypoint/types.dang` | absent, removed if present | `type ModuleTypes`: the literal `TypeDef` list | +| `sdk/entrypoint/build.dang` | absent, removed if present | `type PythonModuleBuild`: the module container build, copied from `runtime/build.dang` with its externals inlined | +| Module load | engine calls the runtime module, execs Python once for types | engine evaluates `types.dang`; no exec | +| Function call | engine execs the runtime container | `call()` checks the source digests, builds the container from `build.dang`, execs `python -m dagger.mod call` | + +**Who decides the path.** `generateScope` reads the setting and stages the +version 1 manifest the schema is read from. `Mod.generated` writes the +version 2 manifest, next to the entrypoint it generated, so generating a +single module directly writes the same files. `mod(...)` reads the setting +too, and also treats a module whose manifest has `[entrypoint]` as static, so +`dagger generate` through either entry regenerates a static module +consistently. + +`generateScope(ws, isModule, name, clients)`: + +1. As today: refuse standalone clients; return `ws` when `isModule` is + false; render the template when the scope has no config. +2. Static path: refuse a non-empty `clients` list and the `legacy` template; + refuse an existing version 1 manifest that carries a field version 2 + cannot (*Accepted differences*), naming the fields. +3. Write the version 1 manifest as today (for a version 2 manifest on the + way back to the dynamic path, build it fresh from the name and the + clients, because a version 2 manifest holds nothing else). This is the + staging manifest: `ws.moduleSource(path).introspectionSchemaJSON` needs a + manifest the engine understands, and a version 1 one yields the current + schema view on every engine. +4. `Mod.generated`: vendor `sdk/` from the library and the bindings as + today. Static path: build the module's container from the staging + workspace with the fresh `sdk/` through the runtime module, run the + `entrypoint` subcommand, add `build.dang`, place the result at + `sdk/entrypoint/`, and replace the manifest with the version 2 text. + `name` is written as a TOML basic string with `"` and `\` escaped; a name + with a control character is refused. Dynamic path: remove + `sdk/entrypoint/` when present (`Workspace.withoutDirectory`). +5. Return the workspace. `cwd` is untouched; `dagger.toml` is never written. + +### 2. The description + +`dagger.mod._describe` holds plain dataclasses (`ModuleDescription`, +`ObjectDescription`, `FunctionDescription`, `ArgumentDescription`, +`FieldDescription`, `EnumDescription`, `EnumMemberDescription`, `TypeRef`) +and two functions: + +- `describe_type(annotation, context)` is the decision procedure of + `_converter.to_typedef` as data: kind, name, description for scalars and + enums, optionality, element for lists. `to_typedef` becomes a + materialisation of it into a `TypeDef` selection in the same call order, + so the existing tests that compare `TypeDef` chains pass unchanged. +- `describe_module(module)` is the decision procedure of + `Module._typedefs()` as data: the main-object check, the module docstring, + each object with its fields, functions and constructor, each enum with its + members. `_typedefs()` becomes a materialisation of it into `dag.module()`. + +Both paths therefore build their type definitions from one description. +There is no static subset, no rejection table and no differential test: what +the dynamic path registers is, by construction, what the static path writes. + +The `entrypoint` subcommand refuses a description that the static path +cannot honour: any function with a `cache=` value. The refusal names the +function and the setting. + +### 3. The generated entrypoint + +`dagger.mod._entrypoint` renders a description to `types.dang` and +`main.dang`. For the module `hello` at `.dagger/modules/hello` with the +default template: + +`types.dang`: + +```dang +# Code generated by dagger. DO NOT EDIT. + +type ModuleTypes { + pub all: [TypeDef!]! { + [ + typeDef + .withObject("Hello") + .withFunction( + function("container", typeDef.withObject("Container")) + .withDescription("A container with the workspace source, ready to build.") + ) + .withConstructor( + function("", typeDef.withObject("Hello")) + .withArg("ws", typeDef.withObject("Workspace")) + .withArg("baseImageAddress", typeDef.withKind(TypeDefKind.STRING_KIND), defaultValue: ("\"alpine:3.24\"" :: Dagger.JSON!)) + ), + ] + } +} +``` + +`main.dang`: + +```dang +# Code generated by dagger. DO NOT EDIT. + +type Entrypoint implements ModuleEntrypoint { + let moduleName: String! = "hello" + let modulePath: String! = ".dagger/modules/hello" + let skippedDirs: [String!]! = [".venv", "__pycache__", "sdk"] + let sourceFiles: [SourceFile!]! = [ + SourceFile(path: ".python-version", digest: ""), + SourceFile(path: "pyproject.toml", digest: "sha256:..."), + SourceFile(path: "requirements.lock", digest: ""), + SourceFile(path: "uv.lock", digest: ""), + SourceFile(path: "src/hello/__init__.py", digest: "sha256:..."), + ] + + pub types(workspace: Workspace!): [TypeDef!]! { + ModuleTypes().all + } + + pub call( + workspace: Workspace!, + receiverType: String!, + receiverValue: JSON, + fnName: String!, + fnArgs: JSON!, + ): JSON! { + let request = JSON.encode({{ + receiverType: receiverType, + receiverValue: receiverValue, + fnName: fnName, + fnArgs: fnArgs, + }}) + let result = runtime(workspace) + .withExec(["python", "-m", "dagger.mod", "call", "--output", "/dagger/result.json"], stdin: request, experimentalPrivilegedNesting: true) + .file("/dagger/result.json") + .contents + (result :: JSON!) + } + + let runtime(workspace: Workspace!): Container! { + let module = workspace.directory("/" + modulePath) + if (module.exists("pyproject.toml") == false) { + raise "module \"" + moduleName + "\" was generated at \"" + modulePath + "\" and is not there; run `dagger generate` after moving it" + } else { + let changed = sourceFiles.filter { f => + if (f.digest == "") { + module.exists(f.path) + } else { + module.exists(f.path) == false or module.file(f.path).digest(excludeMetadata: true) != f.digest + } + }.map { f => f.path } + let added = module.glob("**/*.py").filter { p => + isSource(p) and sourceFiles.filter { f => f.path == p }.length == 0 + } + if ((changed + added).length > 0) { + raise "module \"" + moduleName + "\" changed since its entrypoint was generated (" + (changed + added).join(", ") + "); run `dagger generate`" + } else { + PythonModuleBuild( + contextDir: workspace.directory("/", include: [modulePath + "/**"], exclude: ["**/.venv", "**/__pycache__"]), + subPath: modulePath, + moduleName: moduleName, + ).installed + } + } + } + + let isSource(path: String!): Boolean! { + path.split("/").dropLast(1).filter { segment => + segment.hasPrefix(".") or skippedDirs.contains(segment) + }.length == 0 + } +} + +type SourceFile { + pub path: String! + pub digest: String! + + new(path: String!, digest: String!) { + self.path = path + self.digest = digest + self + } +} +``` + +Rules of the renderer: + +- **One builder call per description member, in `_typedefs()`'s order**, so + the engine sees the same `TypeDef` on both paths: `withOptional(true)` + before the kind; on a function `withDescription`, `withDeprecated`, + `withCheck`, `withGenerator`, `withUp`, `withAgent`, then `withArg` per + argument with a nullable argument's second `withOptional(true)`; on an + object `withField` per field, `withFunction` per function, then + `withConstructor`. `withCachePolicy` is never emitted. +- **The `JSON` scalar is written `Dagger.JSON`** (`defaultValue: ("..." + :: Dagger.JSON!)`), verified in a Dang module; the `call` result uses the + interface's own `JSON!`, as the engine's example does. Whether + `Dagger.JSON` resolves inside an entrypoint program is dev-engine step 1; + the contingency is one spelling in the renderer. +- **Every string goes through one quoting routine** that escapes `"`, `\` + and control characters. +- **Only the main object has `withConstructor`.** +- **`types.dang` is a plain Dang type**, so that CI can load and evaluate it + as a version 1 Dang module (*Testing*). `main.dang` is the only file that + names `ModuleEntrypoint`. +- **The staleness guard.** The `entrypoint` subcommand hashes every file + that can affect the description: `pyproject.toml`, `.python-version`, + `requirements.lock`, `uv.lock`, and every `.py` file under the module + directory outside `sdk/`, hidden directories and `__pycache__`. The digest + is the engine's `File.digest(excludeMetadata: true)` value + (*Measurements*). One of those manifests that is absent is recorded with an + empty digest, so adding a lock file later, which changes what gets + installed, is refused too. The skipped directory names are rendered into + the entrypoint, so the guard's scan for an added `.py` file skips exactly + what the hashing skipped. `call()` recomputes each digest and refuses to + run when anything differs, naming the paths. It ignores permissions and + timestamps. It costs one cached digest per file per call. + +Why `call()` looks as it does: the request is one JSON object on stdin +whose two `JSON` members arrive as text strings; Python decodes each once. +The result is read from a file because user code prints. A non-zero exit +fails the exec, which fails `call()` with the process's stderr; a failure is +never cached. Only the module directory is mounted. `types()` ignores its +`workspace` argument. + +### 4. The container build and the Python side + +**`runtime/build.dang`** (new) holds `PythonModuleBuild(contextDir: +Directory!, subPath: String!, moduleName: String!)`, taken out of +`runtime/main.dang`: `base`, `install`, `pyConfig` and its TOML helpers, +image selection, `packageNameFor`, `mainObjectName`, `checkGeneratedFiles`, +`PyConfig`, `CamelState`. Its `installed` field is the installed container +with the `DAGGER_*` variables set and no entrypoint; a field named +`container` would shadow the global `container` constructor inside the +type and evaluate forever. The two reads of +`currentModule.source` (the image pins) are fenced between `#` +and `#` comment markers. `runtime/main.dang` keeps `type +PythonSdkRuntime` as a thin adapter that adds `runtime.py` and the +entrypoint after the install. Behaviour-neutral; the existing runtime checks +prove it. + +The authoring module gains `runtime/` as a dependency, so `mod.dang` can ask +the runtime module for the module's container at generate time (the same +container `call()` builds later), and its `include` list admits `runtime/` +so `mod.dang` can read `runtime/build.dang` and the Dockerfiles to splice +`build.dang`. `mod.dang` refuses to emit a `build.dang` that still mentions +`currentModule`. + +**`python -m dagger.mod call --output `** (new `__main__.py`): read +one request object from stdin; decode `receiverValue` (null becomes `{}`) +and `fnArgs` once each; import the module through `cli.load_module()`; +dispatch through the existing `Module.get_result`; create the output +directory and write the JSON result. Telemetry is initialised and shut down +as `app()` does. No connection is opened up front. A `ModuleError` or API +error is logged as today and the process exits 2; an unexpected exception +exits 1; neither writes the result file. `Module.dispatch(request)` is the +unit-tested core. + +### 5. What a call costs + +| | dynamic path | static path | +|---|---|---| +| module load | runtime module evaluation and one Python boot per session | Dang evaluation of `types.dang`; no exec | +| constructor or function call | one Python boot each | Dang evaluation, one cached digest per source file, then one Python boot each | +| `dagger generate` | bindings exec | bindings exec plus the module's container build (cached by content) and one Python boot | +| source edit | visible on the next call | refused until `dagger generate` | + +## Alternatives considered + +**A static `ast` analyzer** (the owner's first idea, the doc's first +version, and v0.20.7). Rejected on the evidence of `#11803` to `#13251`; +see *Extraction approaches evaluated*. + +**Dynamic `types()` in the entrypoint** (the first attempt). Keeps the boot +on first load after an edit. Rejected by requirement 1. + +**A dynamic version 2 entrypoint as the opt-out**, instead of the dynamic +path. Needed only if the engine stops loading version 1 manifests. +*Rollout* makes that a condition; the first attempt is the fallback design. + +**Emit a JSON description and decode it in Dang** at load time. Two +representations of one decision and a decoder shipped into every module. +Literal rendering has one representation and no runtime decoder. + +**Render the entrypoint from Dang.** Dang has no model of the module's +types; the renderer needs escaping and recursion, which Python does in a few +hundred lines. + +**Generate a static Python dispatcher.** See *Non-goals*. + +**Refuse the static path on an engine that cannot load it**, by reading +`__schemaVersion`. Rejected: CI's engine cannot load version 2 either, so +every check would fail. The engine floor in phase B enforces it. + +**Guard staleness with `Directory.digest`** over a pattern. It hashes +permissions, so a fresh checkout with another umask is "stale". Per-file +content digests do not. + +**Pin CI's dev engine to an engine that carries `#14038`.** The commit +would have to be a merge pushed to a fork and fetched from the fork's URL, +and it moves every `e2e:*` check onto an unreviewed draft. Offered as an +optional patch, not assumed. + +## Rollout + +Phase B has these gates, all of which must hold: + +1. A `dagger/dagger` release loads manifest version 2 with the + `ModuleEntrypoint` interface at `75c77722` or a successor this SDK has + been adapted to. +2. That release still loads a version 1 `dagger-module.toml` with + `[runtime]`. If it does not, the dynamic path needs the dynamic version 2 + entrypoint first. +3. The engine defines how a version 2 module uses other modules, and this + SDK implements it. +4. The engine passes a cache-policy signal to `call`, or the owner accepts + that functions with any `cache=` value stay on the dynamic path. +5. This SDK module's `engineVersion` is raised to the release in gate 1, so + that an older engine refuses to load the SDK instead of generating + modules it cannot run. A workspace on an older engine then keeps the + older SDK for its dynamic-path modules too. + +## Affected components + +- `future/done/static-module-entrypoint.md` (this document) +- `sdk/src/dagger/mod/_describe.py` (new); `_converter.py` and `_module.py` + materialise the description; `_entrypoint.py` (new, the renderer and the + digests); `__main__.py` (new, `entrypoint` and `call`) +- `sdk/tests/mod/test_describe.py`, `test_entrypoint.py`, + `test_dispatch.py` (new) +- `runtime/build.dang` (new), `runtime/main.dang` (thin adapter) +- `python-sdk.dang`: the setting, the version 2 manifest, the refusals; + `mod.dang`: the static path in `generated` +- `dagger.json`, `dagger-module.toml`, `dagger.lock`: `runtime/` included + and depended on +- `.dagger/modules/e2e/main.dang` and fixtures: the static checks +- `README.md` + +## Testing + +### Unit tests (`sdk/tests/mod`) + +- `test_describe.py`: `describe_module` on modules built with `Module()` + covers objects, interfaces, enums with member docs, fields with the + description quirk, constructors (`dataclass`, `create`, `__init__`, + `InitVar`, `init=False`), `function()(Other)`, every `Annotated` + metadata, `Self`, nested optional lists, name normalisation, the raw + `cache` value; `to_typedef` still equals the expected `TypeDef` chains + (the existing `test_registration.py` assertions). +- `test_entrypoint.py`: the rendered `types.dang` and `main.dang` compared + to committed golden files for a module covering the surface; a description + with a double quote, a backslash, a newline and a tab; exactly one + `withConstructor` and no `withCachePolicy`; refusal of a `cache=` + function; the digest of a file equals `"sha256:" + + sha256(sha256(bytes))`; the file set excludes `sdk/`. +- `test_dispatch.py`: a request with `receiverValue: null` and `fnName: ""` + calls the constructor; the two JSON members arrive as text and are + decoded once; an omitted argument with a default is not passed; a null + value is passed as `None`; the `call` subcommand run as a subprocess with + `--output` under a temporary directory writes the file on success and + exits 2 without a file when the function raises. + +### e2e checks on CI's engine (`.dagger/modules/e2e`) + +- `staticScopeInitCheck`: `generateScope` with `staticEntrypoint: true` on + an empty scope: a manifest whose whole content is the five version 2 + lines; the template files; a `gen.py` that defines `cwd` and + `with_new_file` on `Workspace` (the current schema view); a `main.dang` + with `implements ModuleEntrypoint`, the name, the path, and source files + whose digests equal `File.digest(excludeMetadata: true)` of the same + files; a `types.dang` naming the template's class and its `container` + function with no `withField`; a `build.dang` with both image pins and no + `currentModule`. +- `staticScopeSwitchCheck`: on a committed dynamic-path fixture, generate + with the setting on, then off, then on again: the version 2 manifest and + `sdk/entrypoint/` appear, then a version 1 manifest equal to the fixture's + except `engineVersion` and no `sdk/entrypoint/`, then the first result + again. User files are untouched throughout. +- `staticScopeRefusalsCheck`: `clients` non-empty, the `legacy` template, a + manifest with `include`, one with `disableDefaultFunctionCaching = true`, + and a fixture with `cache="never"` each raise before any file is written. +- `staticTypesLoadCheck`: the rendered `types.dang` of a fixture, next to a + one-type wrapper `main.dang` and a version 1 manifest with `[runtime] + source = "dang"` and `engineVersion = "v1.0.0-0"`, is called through the + `sdk-sdk` harness and returns the fixture's type count. This proves the + rendered Dang type-checks against the engine's schema and evaluates. +- The existing checks guard the dynamic path, including the + `runtime/main.dang` split. + +### On the dev engine + +The dev engine is `dagger/dagger` at `75c77722` merged onto +`0d031c08ef3e379c6f4eb7f8f5cad4638a168863`. The merge's only conflict is in +`dagger.toml` (`75c77722` adds `[modules.tiny]`, `0d031c08` moves +`[modules.tla-check]` and adds a comment); keeping both, `tiny` first, +gives tree `d8e9df6331fb3cbafbd3d1cb80a58c0f74c2c074`. The engine image and +CLI are built with `dagger/dagger`'s `.dagger/modules/dev` and +`.dagger/modules/cli-dev`. + +1. Probe, inside a nested exec: `Dagger.JSON` in an entrypoint program, + `DefaultPath` and `DefaultAddress` resolution, `dag.current_module()` + and its `source()`. Record differences in *Accepted differences*. +2. Scratch workspace: install this repository as SDK `python`, `dagger + module init python --name=hello --static-entrypoint`, inspect the files. +3. `dagger call hello container`; then the *Measurements* fixture as a + module: `dagger functions` and the GraphQL `__type` of every object + compared with the dynamic path on the same engine, with `[runtime] + source = "python"` and with this repository's `runtime/`. +4. Edit a signature and call without `dagger generate`: refused, naming the + file. Change only a file's permissions: the call runs. `dagger generate`: + the new signature is served. +5. `dagger module init python --path --static-entrypoint=false`, + then `dagger generate`: back on the dynamic path, `sdk/entrypoint/` gone, + the module runs. Edit the setting in `dagger.toml` to `true` and + generate: back. Set `[modules.python-sdk.settings] staticEntrypoint = + true` with no scope setting: the same. +6. Wall clock and the engine's `loading type definitions` span for the + fixture on both paths, three runs each. +7. A module with `cache="never"` fails `dagger generate` on the static + path; the same module runs on the dynamic path. + +## Risks + +- **`dagger/dagger#14038` may change.** Insulated: the description, the + renderer's `types.dang`, the manifest text. Exposed: `main.dang`'s + template and the `call` subcommand's request decoding. +- **The engine may stop loading version 1 manifests.** Rollout gate 2; the + first attempt's dynamic `types()` is the named contingency. +- **Version 2 modules cannot have dependencies, and cache policies cannot be + honoured.** Both refused; both gate phase B; reported upstream. +- **Behaviours inside the nested exec are unverified** until dev-engine + step 1, which runs before any code beyond this document. +- **Generate needs the module's dependencies.** A module whose dependencies + cannot install at generate time cannot generate a static entrypoint. The + same install is needed to run the module. +- **Phase B rewrites manifests on `dagger generate`.** The error path is + explicit and the settings exist to pre-empt it. +- **The staleness guard adds a failure mode.** A source edit that does not + change the types still needs `dagger generate` before the module can be + called. `dagger generate` is the command users already run after editing. + +## How the earlier conclusions held up + +Held: every engine mechanic in the first attempt's *Verified constraints*, +its six upstream findings, the `call()` design (request on stdin, result in +a file, module directory baked in), the `runtime/build.dang` split, "do not +cut over". + +Did not hold: + +- "There is no static analyzer for Python modules." There was one, in + v0.20.7, and it failed on the long tail the first attempt did not know + about. The first attempt was right that types come from importing the + classes, and wrong that this had to happen at load time. +- "Every signature edit would then need `dagger generate`" as a reason to + keep discovery at load time. Decided the other way by requirement 1. +- "An engine with `#13992` but not `#14038` is transient." It is the + released `v1.0.0-beta.12` and the engine CI runs. +- The first version of this document recommended a static `ast` analyzer at + generate time. Rejected after the owner pointed at the v0.20.7 history. + +Where `#14038` disagrees with itself or with what runs: the specification +says version 1 manifests are rejected, the code accepts them; the +specification says `fnArgs` values are embedded JSON, the request the +entrypoint builds carries them as text; the specification says the +entrypoint can read files above the module directory, but `cwd` is `/`. + +## Findings to report + +To the author of `dagger/dagger#14038`, restating the first attempt's +findings 1 to 6 (workspace `cwd` is `/`; no `include` list; no cache-policy +signal; no structured error channel; no ID loading from Dang; no module +description) and adding: a version 2 module has no dependency model; an +engine without version 2 support silently serves the oldest schema view for +a version 2 manifest; the specification and the code disagree on version 1 +manifests. + +To the author of `dagger/sdk-helpers`: `tomlContents` never writes +`manifestVersion = 2`, and carries `$schema` and +`disableDefaultFunctionCaching`, which version 2 rejects. + +To the author of `dagger/java-sdk#19`: `defaultValue: JSON.decode("...")` +is likely a type error; `("..." :: Dagger.JSON!)` type-checks. + +## Implementation plan + +StGit patch series on top of `dagger/python-sdk` +`d551f327ae111b5213b4462186e04ed35817b9a2`. Each patch carries +`Signed-off-by: Yves Brissaud ` and no other trailer, and +leaves `dagger check` and `uv run --frozen pytest` green. + +1. `future: design static module entrypoints for Python modules` — this + document. +2. `sdk: describe module types as data` — `_describe.py`; `to_typedef` and + `_typedefs()` materialise it; `test_describe.py`. +3. `sdk: render a static Dang entrypoint from a module description` — + `_entrypoint.py`; the `entrypoint` subcommand; `test_entrypoint.py` and + golden files. +4. `sdk: dispatch a module call from a JSON request` — `Module.dispatch`; + the `call` subcommand; `test_dispatch.py`. +5. `runtime: move the container build into a shared type` — + `runtime/build.dang`; `runtime/main.dang` as the adapter. +6. `python-sdk: generate a static entrypoint behind the staticEntrypoint + setting` — the setting, the manifest, the refusals, `Mod.generated`, the + `runtime/` dependency and include, the four e2e checks. +7. `docs: describe the static path and the rollout` — `README.md`. +8. `future: record the dev-engine results` — *Progress* updated. +9. Optional: `engine-e2e` pinned to an engine that loads version 2, on the + owner's decision. + +## Progress + +- Phase 0 — orientation: done 2026-09-11. Repository `dagger/python-sdk`, + base `d551f327ae111b5213b4462186e04ed35817b9a2`; feature branch + `python-sdk-static-entrypoint-lead-pythonsdk-080b6d74` on + `eunomie/python-sdk`. Design home `future/`. StGit; sign-off + `Signed-off-by: Yves Brissaud `; no AI attribution. CI: + Dagger Cloud checks from `dagger.toml`. +- Phase 1/2 — feature doc and plan: `ff41fcd`, revised as `568b35b`, + `c778c8c`, `ef38625` (a static `ast` analyzer at generate time). +- Phase 3 — three adversarial review rounds (a skeptic and a design/spec + reviewer). Round 1: both "rework"; blockers fixed (sdk-helpers writes no + `manifestVersion`; CI is the dev engine at `0d031c08`; the result file's + directory). Rounds 2 and 3: "approve with changes"; all findings folded in + (per-file content digests, cache policies refused, version 1 fields + refused rather than dropped, the double-hash digest format). +- Plan gate — 2026-09-11: the owner asked whether the v0.20.7 analyzer had + been considered; it had not. Re-evaluation with `#11803`, `#13095`, + `#13251` and `#13235` in hand moved the recommendation to importing at + generate time. Approved by the owner on 2026-09-12: import at generate, + static `types()`, `call()` through the registry, no generated Python + dispatcher, a boolean setting with the dynamic path kept. +- Phase 4 — implementation: done 2026-09-12, patches 2 to 7 of the plan. + Found on the way: a `pub container` field in `PythonModuleBuild` shadowed + the global `container` constructor that `base` starts from, and the + evaluation never ended; the field is `installed`. Unit tests: 232 pass; + ruff clean. e2e on a local `v1.0.0-beta.12` engine through the + `engine-e2e` workspace: `runtime-call`, `runtime-requires-generated-files`, + `toml-generate`, `generate-scope-init`, `generate`, `static-scope-init`, + `static-scope-switch`, `static-scope-refusals` and `static-types-load` + pass. +- Dev-engine results (a `dagger/dagger` build with the version 2 loader, + `7ebd6da5`, 2026-09-03): its SDK interface predates `generateScope`, so + `hello` was generated on the beta.12 engine and loaded on the dev engine. + `dagger functions` and `dagger call hello --help` match the same module + on the dynamic path; `dagger call hello container with-exec ... stdout` + runs. An edited signature is refused naming `src/hello/__init__.py`; a + permission-only change runs. Wall clock, three runs each, warm: + + | | dynamic (this `runtime/`) | static | + |---|---|---| + | `dagger functions` | 2.7 s to 3.1 s (7.7 s cold) | 1.2 s | + | `call hello container with-exec echo stdout` | 5.5 s to 6.4 s | 5.8 s to 7.4 s | + + The load is the win; a call costs the same Python boot on both paths. +- Phase 5 — code review, 2026-09-12: two reviewers (correctness and + safety; design and simplicity), both "approve with changes". Fixed: the + guard records every manifest, absent ones with an empty digest, and + `.python-version`; the rendered `added` scan skips the same directories + the renderer skips; `Mod.generated` writes the version 2 manifest so a + module generated directly matches one generated through the scope; + `source = "."` is accepted; the install path is keyed on the module + directory's digest. Dropped: digesting `dagger-module.toml` (the staging + manifest at generate time is not the final one; the version 2 manifest + carries only the name). Re-verified on the dev engine: a `.venv/` or a + hidden directory with Python files does not trip the guard; an added + `uv.lock` or `src/hello/extra.py` is refused naming the file. +- Phase 6 and 7 — 2026-09-12: draft PR `dagger/python-sdk#28`, head + `a3b8d6f`; CI green (`engine-e-2-e:dev-sdk-check`, `load`). Phase 2 of + the rollout waits for a released engine that loads version 2. diff --git a/mod.dang b/mod.dang index 49565fd..3fc819c 100644 --- a/mod.dang +++ b/mod.dang @@ -12,6 +12,12 @@ type Mod { """ let ws: Workspace! + """ + Whether the module gets a static entrypoint, from the SDK setting or from + the manifest it already has. + """ + pub staticEntrypoint: Boolean! + """ Module root relative to the client's cwd. """ @@ -46,7 +52,9 @@ type Mod { } """ - The workspace with this module's generated files merged in. + The workspace with this module's generated files merged in. On the static + path the manifest version 2 is written here too, so a module generated + directly gets the same files as one generated through the SDK scope. """ pub generated: Workspace! { let files = if (isModern) { @@ -62,15 +70,65 @@ type Mod { } # Merge, don't replace: the generated context holds only generated files. - ws.withDirectory("/" + rootPath, files) + let merged = ws.withDirectory("/" + rootPath, files) + let entrypointPath = vendorDirName + "/" + entrypointDirName + if (staticEntrypoint) { + merged.withNewFile("/" + manifestPath, entrypointManifest(moduleName)) + } else if (ws.directory("/" + rootPath).exists(entrypointPath)) { + merged.withoutDirectory("/" + rootPath + "/" + entrypointPath) + } else { + merged + } + } + + """ + The manifest of a static module: the three keys manifest version 2 allows. + """ + let entrypointManifest(name: String!): String! { + if (name.containsMatch("[\\x00-\\x1f\"\\\\]")) { + raise "module name cannot be written in a manifest: " + name + } else { + "manifestVersion = 2\nname = \"" + name + "\"\n\n[entrypoint]\nkind = \"dang\"\nsource = \"./" + vendorDirName + "/" + entrypointDirName + "\"\n" + } } """ Whether this module uses the 1.0 dagger-module.toml config. """ let isModern: Boolean! { - let configPath = if (rootPath == ".") { "dagger-module.toml" } else { rootPath + "/dagger-module.toml" } - ws.directory("/", include: [configPath]).exists(configPath) + ws.directory("/", include: [manifestPath]).exists(manifestPath) + } + + let manifestPath: String! { + if (rootPath == ".") { "dagger-module.toml" } else { rootPath + "/dagger-module.toml" } + } + + let hasEntrypointManifest: Boolean! { + isModern and ws.file("/" + manifestPath).contents.containsMatch("(?m)^\\s*\\[entrypoint\\]") + } + + let moduleName: String! { + let m = ws.file("/" + manifestPath).contents.match("(?m)^\\s*name\\s*=\\s*\"([^\"]*)\"") + if (m == null) { + raise "no module name in " + manifestPath + } else { + m.captures[0] ?? "" + } + } + + """ + The workspace the module's schema is read from and its container is built + in. A manifest version 2 is served the oldest schema view by an engine that + does not know it, so the module is staged with a runtime manifest instead. + """ + let stagingWs: Workspace! { + if (hasEntrypointManifest) { + sdkHelpers.moduleManifest.withLegacyPythonRuntime.withName(name: moduleName) + .generate(ws.withWorkdir(rootPath), lock: false, legacyJson: false) + .withWorkdir(".") + } else { + ws + } } """ @@ -78,12 +136,53 @@ type Mod { holding only `sdk/` so it merges onto the module without touching anything else. """ let vendoredDir: Directory! { - let schemaJSON = ws.moduleSource("/" + rootPath).introspectionSchemaJSON + let schemaJSON = stagingWs.moduleSource("/" + rootPath).introspectionSchemaJSON + let vendored = library.withFile(generatedBindingsPath, bindings(schemaJSON)) + let files = if (staticEntrypoint) { + vendored.withDirectory(entrypointDirName, entrypointDir(vendored)) + } else { + vendored + } + + directory.withDirectory(vendorDirName, files) + } - directory.withDirectory( - vendorDirName, - library.withFile(generatedBindingsPath, bindings(schemaJSON)), - ) + """ + The entrypoint the engine loads instead of calling a runtime: the module's + own container renders its types, and the container build travels with it. + """ + let entrypointDir(vendored: Directory!): Directory! { + let staged = stagingWs.withDirectory("/" + rootPath + "/" + vendorDirName, vendored) + pythonSdkRuntime + .moduleRuntime(modSource: staged.moduleSource("/" + rootPath), introspectionJson: null) + .withExec(["python", "-m", "dagger.mod", "entrypoint", "--name", moduleName, "--path", rootPath, "--output", entrypointOutput]) + .directory(entrypointOutput) + .withNewFile("build.dang", buildDang) + } + + """ + runtime/build.dang with its reads of this module's source inlined, so the + copy in a generated entrypoint needs nothing but itself. + """ + let buildDang: String! { + let pins = "let defaultBaseImage: String! = \"" + imageFrom("runtime/images/base/Dockerfile") + "\"\n" + + " let defaultUvImage: String! = \"" + imageFrom("runtime/images/uv/Dockerfile") + "\"" + let inlined = currentModule.source.file("runtime/build.dang").contents + .replaceMatches("(?s)#.*?#", pins) + if (inlined.contains("currentModule")) { + raise "runtime/build.dang reads currentModule outside its externals block" + } else { + inlined + } + } + + let imageFrom(path: String!): String! { + let line = currentModule.source.file(path).contents.match("(?m)^FROM\\s+(\\S+)") + if (line == null) { + raise "no FROM line in " + path + } else { + line.captures[0] ?? "" + } } """ @@ -163,6 +262,8 @@ type Mod { let stripScriptPath: String! = "/strip-dev-sections.py" let vendorDirName: String! = "sdk" + let entrypointDirName: String! = "entrypoint" + let entrypointOutput: String! = "/dagger/entrypoint" let generatedBindingsPath: String! = "src/dagger/client/gen.py" let schemaPath: String! = "/schema.json" diff --git a/python-sdk.dang b/python-sdk.dang index 9aa650b..e018ea0 100644 --- a/python-sdk.dang +++ b/python-sdk.dang @@ -25,6 +25,13 @@ type PythonSdk { """ pub baseImage: String! = "" + """ + Generate a static entrypoint that carries the module's types, so the engine + loads them without running the module. Needs an engine that loads manifest + version 2. + """ + let staticEntrypoint: Boolean! = false + """ Find the Python client root containing the workspace cwd: the directory of the nearest pyproject.toml, relative to the workspace root. A module's @@ -70,17 +77,23 @@ type PythonSdk { } let scopeHasFile(ws: Workspace!, scope: String!, filename: String!): Boolean! { - let path = if (scope == ".") { filename } else { scope + "/" + filename } + let path = scopePath(scope, filename) ws.directory("/", include: [path]).exists(path) } + let scopePath(scope: String!, filename: String!): String! { + if (scope == ".") { filename } else { scope + "/" + filename } + } + """ Generate one SDK scope: the module at the workspace cwd, when the scope has one. A module without a config file is initialized from the configured template. Every module receives a dagger-module.toml from the manifest builder module and is then generated. A pre-1.0 dagger.json is migrated and removed. The scope's module clients become the module's dependencies, - so the generated bindings include their types. + so the generated bindings include their types. With staticEntrypoint, the + module gets a manifest version 2 and a generated entrypoint instead of a + runtime. Standalone clients, in a scope without a module, are not generated yet. """ pub generateScope(ws: Workspace!, isModule: Boolean!, name: String!, clients: [ModuleSource!]!): Workspace! { @@ -100,9 +113,13 @@ type PythonSdk { } else { rooted.withDirectory("/" + scope, moduleTemplate(name, template, pythonVersion, useUv, baseImage)) } + if (staticEntrypoint) { + checkStaticScope(initialized, scope, hadConfig, clients) + } + # The runtime manifest is what every engine serves a schema for; the + # static path swaps it for the manifest version 2 as it generates. let configured = generateScopeManifest(initialized, scope, name, clients) - let module = mod(configured, path: scope, findUp: false) - module.generated.withWorkdir(scope) + mod(configured, path: scope, findUp: false).generated.withWorkdir(scope) } } @@ -115,7 +132,10 @@ type PythonSdk { let root = "/" + scope let hasToml = scopeHasFile(ws, scope, "dagger-module.toml") let hasJson = scopeHasFile(ws, scope, "dagger.json") - let base = if (hasToml) { + let base = if (hasToml and isEntrypointManifest(ws.file(root + "/dagger-module.toml").contents)) { + # A manifest version 2 holds nothing the runtime manifest keeps. + sdkHelpers.moduleManifest.withLegacyPythonRuntime + } else if (hasToml) { sdkHelpers.moduleManifest(loadToml: ws.file(root + "/dagger-module.toml")) } else if (hasJson) { sdkHelpers.moduleManifest(loadJson: ws.file(root + "/dagger.json")) @@ -127,6 +147,54 @@ type PythonSdk { }.generate(ws.withWorkdir(scope), lock: lock, legacyJson: false).withWorkdir(".") } + """ + What a static entrypoint cannot carry, refused before anything is written. + """ + let checkStaticScope(ws: Workspace!, scope: String!, hadConfig: Boolean!, clients: [ModuleSource!]!): Void { + if (clients.length > 0) { + raise "a static entrypoint cannot use other modules yet: manifest version 2 has no dependencies; set staticEntrypoint = false" + } + if (hadConfig == false and template == "legacy") { + raise "the legacy template is not available with a static entrypoint" + } + if (scopeHasFile(ws, scope, "dagger-module.toml")) { + let objections = staticObjections(ws.file("/" + scopePath(scope, "dagger-module.toml")).contents) + if (objections.length > 0) { + raise "dagger-module.toml has settings a static entrypoint cannot carry (" + objections.join(", ") + "); remove them or set staticEntrypoint = false" + } + } + null + } + + """ + Manifest keys and tables that manifest version 2 has no place for. + """ + let staticObjections(toml: String!): [String!]! { + let head = toml.split("\n[").takeFirst(1).join("") + let keys = ["include", "disableDefaultFunctionCaching"].filter { key => + head.containsMatch(Regexp("(?m)^\\s*" + key + "\\s*=")) + } + # Manifest version 2 roots a module at its own directory, which is what + # `source = "."` says; any other source is what it cannot carry. + let sourceMatch = head.match("(?m)^\\s*source\\s*=\\s*\"([^\"]*)\"") + let sourceValue = if (sourceMatch == null) { "." } else { sourceMatch.captures[0] ?? "." } + let source = if (sourceValue == ".") { [] :: [String!]! } else { ["source"] } + let tables = ["codegen", "clients", "dependencies"].filter { name => + toml.containsMatch(Regexp("(?m)^\\s*\\[\\[?" + name + "\\]\\]?")) + } + let pythonRuntime = toml.containsMatch("(?m)^\\s*\\[runtime\\]\\s*\\n\\s*source\\s*=\\s*\"python\"") + let runtime = if (toml.containsMatch("(?m)^\\s*\\[runtime\\]") and pythonRuntime == false) { ["runtime"] } else { [] :: [String!]! } + keys + source + tables + runtime + } + + let isEntrypointManifest(toml: String!): Boolean! { + toml.containsMatch("(?m)^\\s*\\[entrypoint\\]") + } + + let hasEntrypointManifest(ws: Workspace!, scope: String!): Boolean! { + scopeHasFile(ws, scope, "dagger-module.toml") and isEntrypointManifest(ws.file("/" + scopePath(scope, "dagger-module.toml")).contents) + } + """ Workspace-root-relative roots of the modules registered to this SDK. """ @@ -198,6 +266,7 @@ type PythonSdk { Mod( rootPath: modPath, ws: ws, + staticEntrypoint: staticEntrypoint or hasEntrypointManifest(ws, modPath), ) } diff --git a/runtime/build.dang b/runtime/build.dang new file mode 100644 index 0000000..85aa318 --- /dev/null +++ b/runtime/build.dang @@ -0,0 +1,418 @@ +""" +The container a Python module runs in, built from its committed files. +The runtime module and the generated static entrypoint share this build; +the entrypoint carries a copy with the block below inlined. +""" +type PythonModuleBuild { + """ + Context the module lives in, mounted whole so relative paths resolve. + """ + pub contextDir: Directory! + + """ + Module directory, relative to the context. + """ + pub subPath: String! + + pub moduleName: String! + + new(contextDir: Directory! = directory, subPath: String! = ".", moduleName: String! = "") { + self.contextDir = contextDir + self.subPath = subPath + self.moduleName = moduleName + self + } + + let modSourceDirPath: String! = "/src" + let venvPath: String! = "/opt/venv" + let genDir: String! = "sdk" + let sdkGenPath: String! = "src/dagger/client/gen.py" + let userGenPath: String! = "src/dagger_gen.py" + let projectCfg: String! = "pyproject.toml" + let pipCompileLock: String! = "requirements.lock" + let uvLock: String! = "uv.lock" + let uvCacheVolume: String! = "modpython-uv" + let pipCacheVolume: String! = "modpython-pip" + + # + # Committed single-FROM Dockerfiles so Dependabot keeps the digests fresh. + let defaultBaseImage: String! { imageFrom("images/base/Dockerfile") } + let defaultUvImage: String! { imageFrom("images/uv/Dockerfile") } + + let imageFrom(path: String!): String! { + let line = currentModule.source.file(path).contents.match("(?m)^FROM\\s+(\\S+)") + if (line == null) { + raise "no FROM line in " + path + } else { + line.captures[0] ?? "" + } + } + # + + """ + The module directory. + """ + pub source: Directory! { + subdir(contextDir, subPath) + } + + """ + The installed module, without an entrypoint. Not named `container`: a field + of that name would shadow the global `container` that `base` starts from. + """ + pub installed: Container! { + if (source.exists(projectCfg) == false) { + raise "module \"" + moduleName + "\" has no " + projectCfg + "; run `dagger generate` and commit the generated files" + } else if (hasPythonFiles(source) == false) { + raise "no python files found in module source" + } else { + let cfg = pyConfig(source) + let vendorPath = vendorPathFor(source, cfg) + checkGeneratedFiles(source, moduleName, vendorPath) + + let baseImage = baseImageFor(source, cfg) + let uvImage = uvImageFor(cfg) + # Unique per module source: the uv cache keys some entries on source paths. + let contextDirPath = modSourceDirPath + "/" + source.digest + let packageName = packageNameFor(moduleName, cfg) + + let built = base(baseImage, uvImage, cfg) + .withWorkdir(join(contextDirPath, subPath)) + .withMountedDirectory(contextDirPath, withoutVenv) + # Last, so the layers above are shared between modules. + .withEnvVariable("DAGGER_MODULE", moduleName) + .withEnvVariable("DAGGER_DEFAULT_PYTHON_PACKAGE", packageName) + .withEnvVariable("DAGGER_MAIN_OBJECT", mainObjectName(moduleName)) + + install(built, source, cfg) + } + } + + """ + The class name of a module's main object, converted like the engine's builtin + Python runtime (strcase.ToCamel) so a module loads the same on both. + """ + pub mainObjectName(name: String!): String! { + name + .trimSpace + .split("") + .reduce(CamelState(out: "", capNext: true, prevIsCap: false)) { state, c => + let isCap = c.containsMatch("^[A-Z]$") + let isLow = c.containsMatch("^[a-z]$") + let isNum = c.containsMatch("^[0-9]$") + if (isCap or isLow) { + let v = if (state.capNext and isLow) { + c.toUpper + } else if (state.capNext == false and state.prevIsCap and isCap) { + c.toLower + } else { + c + } + CamelState(out: state.out + v, capNext: false, prevIsCap: isCap) + } else if (isNum) { + CamelState(out: state.out + c, capNext: true, prevIsCap: false) + } else { + CamelState( + out: state.out, + capNext: c == "_" or c == " " or c == "-" or c == ".", + prevIsCap: false, + ) + } + } + .out + } + + let base(baseImage: String!, uvImage: String!, cfg: PyConfig!): Container! { + let uvBins = container.from(uvImage).rootfs + + let ctr = container + .from(baseImage) + # Informational only, in case a module finds them useful. + .withEnvVariable("DAGGER_BASE_IMAGE", baseImage) + .withEnvVariable("PYTHONUNBUFFERED", "1") + .withEnvVariable("PIP_DISABLE_PIP_VERSION_CHECK", "1") + .withEnvVariable("PIP_ROOT_USER_ACTION", "ignore") + .withMountedFile("/usr/local/bin/uv", uvBins.file("uv")) + .withMountedFile("/usr/local/bin/uvx", uvBins.file("uvx")) + .withMountedCache("/root/.cache/uv", cacheVolume(uvCacheVolume)) + .withEnvVariable("DAGGER_UV_IMAGE", uvImage) + .withEnvVariable("DAGGER_UV_VERSION", imageTag(uvImage)) + .withEnvVariable("UV_SYSTEM_PYTHON", "1") + .withEnvVariable("UV_LINK_MODE", "copy") + .withEnvVariable("UV_NATIVE_TLS", "1") + .withEnvVariable("UV_PROJECT_ENVIRONMENT", venvPath) + + let withPip = if (cfg.useUv) { + ctr + } else { + ctr.withMountedCache("/root/.cache/pip", cacheVolume(pipCacheVolume)) + } + let withIndex = if (cfg.indexURL == "") { + withPip + } else { + withPip.withEnvVariable("UV_INDEX_URL", cfg.indexURL) + } + if (cfg.extraIndexURL == "") { + withIndex + } else { + withIndex.withEnvVariable("UV_EXTRA_INDEX_URL", cfg.extraIndexURL) + } + } + + """ + The vendored library is installed non-editable so uv compiles its bytecode + once, not on every call into a throwaway mount; the module's own package + stays editable. + """ + let install(ctr: Container!, source: Directory!, cfg: PyConfig!): Container! { + let compiled = ctr.withEnvVariable("UV_COMPILE_BYTECODE", "1") + + if (cfg.useUv and source.exists(uvLock)) { + # --locked: fail loudly on a stale lockfile instead of re-resolving. + compiled + .withExec(["uv", "sync", "--no-dev", "--locked", "--no-editable", "--no-install-project"]) + .withEnvVariable("VIRTUAL_ENV", "$UV_PROJECT_ENVIRONMENT", expand: true) + .withEnvVariable("PATH", "$VIRTUAL_ENV/bin:$PATH", expand: true) + # uv pip ignores VIRTUAL_ENV while UV_SYSTEM_PYTHON is set. + .withExec(["uv", "pip", "install", "--python", venvPath + "/bin/python", "--no-deps", "-e", "."]) + } else if (cfg.useUv) { + let deps = if (source.exists(pipCompileLock)) { + # The lockfile is complete, so skip resolving. + ["--no-deps", "-r", pipCompileLock] + } else { + ["-r", projectCfg] + } + compiled + .withExec(["uv", "pip", "install", "--no-editable", "./" + genDir] + deps) + .withExec(["uv", "pip", "install", "--no-deps", "-e", "."]) + } else { + compiled.withExec(["pip", "install", "./" + genDir, "-e", "."]) + } + } + + """ + Fail early when the committed generated files are missing. + """ + let checkGeneratedFiles(source: Directory!, modName: String!, vendorPath: String!): Void { + let required = if (vendorPath == "") { + [userGenPath] + } else { + [vendorPath + "/" + projectCfg, vendorPath + "/" + sdkGenPath] + } + required.each { rel => + if (source.exists(rel, expectedType: ExistsType.REGULAR_TYPE) == false) { + raise "module \"" + modName + "\": generated file \"" + rel + "\" is missing; run `dagger generate` and commit the generated files" + } + null + } + null + } + + """ + Empty when a uv.lock module depends on a published dagger-io instead. + """ + let vendorPathFor(source: Directory!, cfg: PyConfig!): String! { + if (cfg.useUv and source.exists(uvLock)) { cfg.vendorPath } else { genDir } + } + + let packageNameFor(modName: String!, cfg: PyConfig!): String! { + let project = if (cfg.projectName == "") { + modName.toLower.replaceMatches("[^a-z0-9]+", "-").trim("-") + } else { + cfg.projectName + } + project.replace("-", "_") + } + + """ + Prefers the digest-pinned default image when it matches the requested version. + """ + let baseImageFor(source: Directory!, cfg: PyConfig!): String! { + if (cfg.baseImage != "") { + cfg.baseImage + } else { + let version = pythonVersionFor(source, cfg) + let wanted = "python:" + version + "-slim" + if (version == "" or defaultBaseImage.hasPrefix(wanted)) { + defaultBaseImage + } else { + wanted + } + } + } + + let pythonVersionFor(source: Directory!, cfg: PyConfig!): String! { + if (source.exists(".python-version")) { + source.file(".python-version").contents.trimSpace + } else { + # requires-python is a floor, but with `>=` or `==` it names a supported version. + let minimum = cfg.requiresPython.trimSpace + if (minimum.hasPrefix("==") or minimum.hasPrefix(">=")) { + minimum.trimPrefix("==").trimPrefix(">=").trimSpace + } else { + "" + } + } + } + + let uvImageFor(cfg: PyConfig!): String! { + if (cfg.uvVersion == "" or cfg.uvVersion == imageTag(defaultUvImage)) { + defaultUvImage + } else { + # Uv's image tag matches the version exactly. + "ghcr.io/astral-sh/uv:" + cfg.uvVersion + } + } + + let hasPythonFiles(source: Directory!): Boolean! { + source.glob("src/**/*.py").length > 0 or source.glob("*.py").length > 0 + } + + """ + A virtualenv left in the module by the host must not reach the build. + """ + let withoutVenv: Directory! { + if (source.exists(".venv")) { + contextDir.withoutDirectory(join(subPath, ".venv")) + } else { + contextDir + } + } + + let subdir(dir: Directory!, path: String!): Directory! { + if (path == "" or path == ".") { dir } else { dir.directory(path) } + } + + let join(base: String!, rel: String!): String! { + if (rel == "" or rel == ".") { base } else { base + "/" + rel } + } + + let imageTag(ref: String!): String! { + let named = ref.split("@").takeFirst(1).join("") + let parts = named.split(":") + if (parts.length < 2) { "" } else { parts.takeLast(1).join("") } + } + + """ + Not a TOML parser: regular expressions scoped to a table, for the keys the + templates and `mod config set` write. Single-line booleans and quoted strings + only; arrays, multi-line strings and dotted keys are not seen. + """ + let pyConfig(source: Directory!): PyConfig! { + let toml = source.file(projectCfg).contents + let project = tomlTable(toml, "project") + let dagger = tomlTable(toml, "tool.dagger") + let sources = tomlTable(toml, "tool.uv.sources") + let indexes = tomlTables(toml, "tool.uv.index") + .filter { table => tomlString(table, "name") == "" } + + PyConfig( + projectName: tomlString(project, "name"), + requiresPython: tomlString(project, "requires-python"), + baseImage: tomlString(dagger, "base-image"), + useUv: tomlBool(dagger, "use-uv", true), + uvVersion: tomlString(dagger, "uv-version"), + vendorPath: inlineTableString(sources, "dagger-io", "path"), + indexURL: indexes + .filter { table => tomlBool(table, "default", false) } + .map { table => tomlString(table, "url") } + .takeFirst(1) + .join(""), + extraIndexURL: indexes + .filter { table => tomlBool(table, "default", false) == false } + .map { table => tomlString(table, "url") } + .takeFirst(1) + .join(""), + ) + } + + let tomlTable(toml: String!, name: String!): String! { + tomlTables(toml, name).takeFirst(1).join("") + } + + """ + Every `[name]` and `[[name]]` body; headers may be indented and commented. + """ + let tomlTables(toml: String!, name: String!): [String!]! { + toml + .splitMatches("(?m)^\\s*\\[") + .filter { section => + let header = section + .split("\n") + .takeFirst(1) + .join("") + .replaceMatches("#.*$", "") + .trimSpace + header == name + "]" or header == "[" + name + "]]" + } + } + + let tomlString(table: String!, key: String!): String! { + let m = table.match(Regexp("(?m)^\\s*" + Regexp.escape(key) + "\\s*=\\s*[\"']([^\"']*)[\"']")) + if (m == null) { "" } else { m.captures[0] ?? "" } + } + + let tomlBool(table: String!, key: String!, default: Boolean!): Boolean! { + let m = table.match(Regexp("(?m)^\\s*" + Regexp.escape(key) + "\\s*=\\s*(true|false)")) + if (m == null) { default } else { m.captures[0] == "true" } + } + + """ + e.g. `path` in `dagger-io = { path = "sdk", editable = true }`. + """ + let inlineTableString(table: String!, key: String!, field: String!): String! { + let inline = table.match(Regexp("(?m)^\\s*" + Regexp.escape(key) + "\\s*=\\s*\\{([^}]*)\\}")) + if (inline == null) { "" } else { tomlString(inline.captures[0] ?? "", field) } + } +} + +""" +What this runtime reads from a module's pyproject.toml. +""" +type PyConfig { + pub projectName: String! + pub requiresPython: String! + pub baseImage: String! + pub useUv: Boolean! + pub uvVersion: String! + pub vendorPath: String! + pub indexURL: String! + pub extraIndexURL: String! + + new( + projectName: String! = "", + requiresPython: String! = "", + baseImage: String! = "", + useUv: Boolean! = true, + uvVersion: String! = "", + vendorPath: String! = "", + indexURL: String! = "", + extraIndexURL: String! = "", + ) { + self.projectName = projectName + self.requiresPython = requiresPython + self.baseImage = baseImage + self.useUv = useUv + self.uvVersion = uvVersion + self.vendorPath = vendorPath + self.indexURL = indexURL + self.extraIndexURL = extraIndexURL + self + } +} + +""" +Accumulator for the CamelCase conversion of a module name. +""" +type CamelState { + pub out: String! + pub capNext: Boolean! + pub prevIsCap: Boolean! + + new(out: String! = "", capNext: Boolean! = true, prevIsCap: Boolean! = false) { + self.out = out + self.capNext = capNext + self.prevIsCap = prevIsCap + self + } +} diff --git a/runtime/main.dang b/runtime/main.dang index 0096b0f..a2159aa 100644 --- a/runtime/main.dang +++ b/runtime/main.dang @@ -4,21 +4,7 @@ Runtime for Python modules that commit their generated client library: Dang costs no exec of its own. """ type PythonSdkRuntime { - let modSourceDirPath: String! = "/src" let runtimeExecutablePath: String! = "/runtime" - let venvPath: String! = "/opt/venv" - let genDir: String! = "sdk" - let sdkGenPath: String! = "src/dagger/client/gen.py" - let userGenPath: String! = "src/dagger_gen.py" - let projectCfg: String! = "pyproject.toml" - let pipCompileLock: String! = "requirements.lock" - let uvLock: String! = "uv.lock" - let uvCacheVolume: String! = "modpython-uv" - let pipCacheVolume: String! = "modpython-pip" - - # Committed single-FROM Dockerfiles so Dependabot keeps the digests fresh. - let defaultBaseImage: String! { imageFrom("images/base/Dockerfile") } - let defaultUvImage: String! { imageFrom("images/uv/Dockerfile") } """ Container for executing the Python module runtime. introspectionJson is @@ -26,375 +12,28 @@ type PythonSdkRuntime { """ pub moduleRuntime(modSource: ModuleSource!, introspectionJson: File): Container! { let modName = modSource.moduleOriginalName - let subPath = modSource.sourceSubpath - let contextDir = modSource.contextDirectory - let source = subdir(contextDir, subPath) + let build = PythonModuleBuild( + contextDir: modSource.contextDirectory, + subPath: modSource.sourceSubpath, + moduleName: modName, + ) - if (modSource.configExists == false or source.exists(projectCfg) == false) { + if (modSource.configExists == false or build.source.exists("pyproject.toml") == false) { raise "module \"" + modName + "\" has no source to trust; run `dagger generate` and commit the generated files" - } else if (hasPythonFiles(source) == false) { - raise "no python files found in module source" } else { - let cfg = pyConfig(source) - let vendorPath = vendorPathFor(source, cfg) - checkGeneratedFiles(source, modName, vendorPath) - - let baseImage = baseImageFor(source, cfg) - let uvImage = uvImageFor(cfg) - # Unique per module source: the uv cache keys some entries on source paths. - let contextDirPath = modSourceDirPath + "/" + modSource.digest - let packageName = packageNameFor(modName, cfg) - - let built = base(baseImage, uvImage, cfg) + # After the install, so those layers are shared with the static entrypoint. + let installed = build.installed .withFile(runtimeExecutablePath, currentModule.source.file("runtime.py"), permissions: 493) .withEntrypoint([runtimeExecutablePath]) - .withWorkdir(join(contextDirPath, subPath)) - .withMountedDirectory(contextDirPath, withoutVenv(contextDir, source, subPath)) - # Last, so the layers above are shared between modules. - .withEnvVariable("DAGGER_MODULE", modName) - .withEnvVariable("DAGGER_DEFAULT_PYTHON_PACKAGE", packageName) - .withEnvVariable("DAGGER_MAIN_OBJECT", mainObjectName(modName)) - - let installed = install(built, source, cfg) if (modSource.sdk.debug == true) { installed.terminal } else { installed } } } """ - The class name of a module's main object, converted like the engine's builtin - Python runtime (strcase.ToCamel) so a module loads the same on both. + The class name of a module's main object. """ pub mainObjectName(name: String!): String! { - name - .trimSpace - .split("") - .reduce(CamelState(out: "", capNext: true, prevIsCap: false)) { state, c => - let isCap = c.containsMatch("^[A-Z]$") - let isLow = c.containsMatch("^[a-z]$") - let isNum = c.containsMatch("^[0-9]$") - if (isCap or isLow) { - let v = if (state.capNext and isLow) { - c.toUpper - } else if (state.capNext == false and state.prevIsCap and isCap) { - c.toLower - } else { - c - } - CamelState(out: state.out + v, capNext: false, prevIsCap: isCap) - } else if (isNum) { - CamelState(out: state.out + c, capNext: true, prevIsCap: false) - } else { - CamelState( - out: state.out, - capNext: c == "_" or c == " " or c == "-" or c == ".", - prevIsCap: false, - ) - } - } - .out - } - - let base(baseImage: String!, uvImage: String!, cfg: PyConfig!): Container! { - let uvBins = container.from(uvImage).rootfs - - let ctr = container - .from(baseImage) - # Informational only, in case a module finds them useful. - .withEnvVariable("DAGGER_BASE_IMAGE", baseImage) - .withEnvVariable("PYTHONUNBUFFERED", "1") - .withEnvVariable("PIP_DISABLE_PIP_VERSION_CHECK", "1") - .withEnvVariable("PIP_ROOT_USER_ACTION", "ignore") - .withMountedFile("/usr/local/bin/uv", uvBins.file("uv")) - .withMountedFile("/usr/local/bin/uvx", uvBins.file("uvx")) - .withMountedCache("/root/.cache/uv", cacheVolume(uvCacheVolume)) - .withEnvVariable("DAGGER_UV_IMAGE", uvImage) - .withEnvVariable("DAGGER_UV_VERSION", imageTag(uvImage)) - .withEnvVariable("UV_SYSTEM_PYTHON", "1") - .withEnvVariable("UV_LINK_MODE", "copy") - .withEnvVariable("UV_NATIVE_TLS", "1") - .withEnvVariable("UV_PROJECT_ENVIRONMENT", venvPath) - - let withPip = if (cfg.useUv) { - ctr - } else { - ctr.withMountedCache("/root/.cache/pip", cacheVolume(pipCacheVolume)) - } - let withIndex = if (cfg.indexURL == "") { - withPip - } else { - withPip.withEnvVariable("UV_INDEX_URL", cfg.indexURL) - } - if (cfg.extraIndexURL == "") { - withIndex - } else { - withIndex.withEnvVariable("UV_EXTRA_INDEX_URL", cfg.extraIndexURL) - } - } - - """ - The vendored library is installed non-editable so uv compiles its bytecode - once, not on every call into a throwaway mount; the module's own package - stays editable. - """ - let install(ctr: Container!, source: Directory!, cfg: PyConfig!): Container! { - let compiled = ctr.withEnvVariable("UV_COMPILE_BYTECODE", "1") - - if (cfg.useUv and source.exists(uvLock)) { - # --locked: fail loudly on a stale lockfile instead of re-resolving. - compiled - .withExec(["uv", "sync", "--no-dev", "--locked", "--no-editable", "--no-install-project"]) - .withEnvVariable("VIRTUAL_ENV", "$UV_PROJECT_ENVIRONMENT", expand: true) - .withEnvVariable("PATH", "$VIRTUAL_ENV/bin:$PATH", expand: true) - # uv pip ignores VIRTUAL_ENV while UV_SYSTEM_PYTHON is set. - .withExec(["uv", "pip", "install", "--python", venvPath + "/bin/python", "--no-deps", "-e", "."]) - } else if (cfg.useUv) { - let deps = if (source.exists(pipCompileLock)) { - # The lockfile is complete, so skip resolving. - ["--no-deps", "-r", pipCompileLock] - } else { - ["-r", projectCfg] - } - compiled - .withExec(["uv", "pip", "install", "--no-editable", "./" + genDir] + deps) - .withExec(["uv", "pip", "install", "--no-deps", "-e", "."]) - } else { - compiled.withExec(["pip", "install", "./" + genDir, "-e", "."]) - } - } - - """ - Fail early when the committed generated files are missing. - """ - let checkGeneratedFiles(source: Directory!, modName: String!, vendorPath: String!): Void { - let required = if (vendorPath == "") { - [userGenPath] - } else { - [vendorPath + "/" + projectCfg, vendorPath + "/" + sdkGenPath] - } - required.each { rel => - if (source.exists(rel, expectedType: ExistsType.REGULAR_TYPE) == false) { - raise "module \"" + modName + "\": generated file \"" + rel + "\" is missing; run `dagger generate` and commit the generated files" - } - null - } - null - } - - """ - Empty when a uv.lock module depends on a published dagger-io instead. - """ - let vendorPathFor(source: Directory!, cfg: PyConfig!): String! { - if (cfg.useUv and source.exists(uvLock)) { cfg.vendorPath } else { genDir } - } - - let packageNameFor(modName: String!, cfg: PyConfig!): String! { - let project = if (cfg.projectName == "") { - modName.toLower.replaceMatches("[^a-z0-9]+", "-").trim("-") - } else { - cfg.projectName - } - project.replace("-", "_") - } - - """ - Prefers the digest-pinned default image when it matches the requested version. - """ - let baseImageFor(source: Directory!, cfg: PyConfig!): String! { - if (cfg.baseImage != "") { - cfg.baseImage - } else { - let version = pythonVersionFor(source, cfg) - let wanted = "python:" + version + "-slim" - if (version == "" or defaultBaseImage.hasPrefix(wanted)) { - defaultBaseImage - } else { - wanted - } - } - } - - let pythonVersionFor(source: Directory!, cfg: PyConfig!): String! { - if (source.exists(".python-version")) { - source.file(".python-version").contents.trimSpace - } else { - # requires-python is a floor, but with `>=` or `==` it names a supported version. - let minimum = cfg.requiresPython.trimSpace - if (minimum.hasPrefix("==") or minimum.hasPrefix(">=")) { - minimum.trimPrefix("==").trimPrefix(">=").trimSpace - } else { - "" - } - } - } - - let uvImageFor(cfg: PyConfig!): String! { - if (cfg.uvVersion == "" or cfg.uvVersion == imageTag(defaultUvImage)) { - defaultUvImage - } else { - # Uv's image tag matches the version exactly. - "ghcr.io/astral-sh/uv:" + cfg.uvVersion - } - } - - let hasPythonFiles(source: Directory!): Boolean! { - source.glob("src/**/*.py").length > 0 or source.glob("*.py").length > 0 - } - - """ - A virtualenv left in the module by the host must not reach the build. - """ - let withoutVenv(contextDir: Directory!, source: Directory!, subPath: String!): Directory! { - if (source.exists(".venv")) { - contextDir.withoutDirectory(join(subPath, ".venv")) - } else { - contextDir - } - } - - let subdir(dir: Directory!, subPath: String!): Directory! { - if (subPath == "" or subPath == ".") { dir } else { dir.directory(subPath) } - } - - let join(base: String!, rel: String!): String! { - if (rel == "" or rel == ".") { base } else { base + "/" + rel } - } - - let imageFrom(path: String!): String! { - let line = currentModule.source.file(path).contents.match("(?m)^FROM\\s+(\\S+)") - if (line == null) { - raise "no FROM line in " + path - } else { - line.captures[0] ?? "" - } - } - - let imageTag(ref: String!): String! { - let named = ref.split("@").takeFirst(1).join("") - let parts = named.split(":") - if (parts.length < 2) { "" } else { parts.takeLast(1).join("") } - } - - """ - Not a TOML parser: regular expressions scoped to a table, for the keys the - templates and `mod config set` write. Single-line booleans and quoted strings - only; arrays, multi-line strings and dotted keys are not seen. - """ - let pyConfig(source: Directory!): PyConfig! { - let toml = source.file(projectCfg).contents - let project = tomlTable(toml, "project") - let dagger = tomlTable(toml, "tool.dagger") - let sources = tomlTable(toml, "tool.uv.sources") - let indexes = tomlTables(toml, "tool.uv.index") - .filter { table => tomlString(table, "name") == "" } - - PyConfig( - projectName: tomlString(project, "name"), - requiresPython: tomlString(project, "requires-python"), - baseImage: tomlString(dagger, "base-image"), - useUv: tomlBool(dagger, "use-uv", true), - uvVersion: tomlString(dagger, "uv-version"), - vendorPath: inlineTableString(sources, "dagger-io", "path"), - indexURL: indexes - .filter { table => tomlBool(table, "default", false) } - .map { table => tomlString(table, "url") } - .takeFirst(1) - .join(""), - extraIndexURL: indexes - .filter { table => tomlBool(table, "default", false) == false } - .map { table => tomlString(table, "url") } - .takeFirst(1) - .join(""), - ) - } - - let tomlTable(toml: String!, name: String!): String! { - tomlTables(toml, name).takeFirst(1).join("") - } - - """ - Every `[name]` and `[[name]]` body; headers may be indented and commented. - """ - let tomlTables(toml: String!, name: String!): [String!]! { - toml - .splitMatches("(?m)^\\s*\\[") - .filter { section => - let header = section - .split("\n") - .takeFirst(1) - .join("") - .replaceMatches("#.*$", "") - .trimSpace - header == name + "]" or header == "[" + name + "]]" - } - } - - let tomlString(table: String!, key: String!): String! { - let m = table.match(Regexp("(?m)^\\s*" + Regexp.escape(key) + "\\s*=\\s*[\"']([^\"']*)[\"']")) - if (m == null) { "" } else { m.captures[0] ?? "" } - } - - let tomlBool(table: String!, key: String!, default: Boolean!): Boolean! { - let m = table.match(Regexp("(?m)^\\s*" + Regexp.escape(key) + "\\s*=\\s*(true|false)")) - if (m == null) { default } else { m.captures[0] == "true" } - } - - """ - e.g. `path` in `dagger-io = { path = "sdk", editable = true }`. - """ - let inlineTableString(table: String!, key: String!, field: String!): String! { - let inline = table.match(Regexp("(?m)^\\s*" + Regexp.escape(key) + "\\s*=\\s*\\{([^}]*)\\}")) - if (inline == null) { "" } else { tomlString(inline.captures[0] ?? "", field) } - } -} - -""" -What this runtime reads from a module's pyproject.toml. -""" -type PyConfig { - pub projectName: String! - pub requiresPython: String! - pub baseImage: String! - pub useUv: Boolean! - pub uvVersion: String! - pub vendorPath: String! - pub indexURL: String! - pub extraIndexURL: String! - - new( - projectName: String! = "", - requiresPython: String! = "", - baseImage: String! = "", - useUv: Boolean! = true, - uvVersion: String! = "", - vendorPath: String! = "", - indexURL: String! = "", - extraIndexURL: String! = "", - ) { - self.projectName = projectName - self.requiresPython = requiresPython - self.baseImage = baseImage - self.useUv = useUv - self.uvVersion = uvVersion - self.vendorPath = vendorPath - self.indexURL = indexURL - self.extraIndexURL = extraIndexURL - self - } -} - -""" -Accumulator for the CamelCase conversion of a module name. -""" -type CamelState { - pub out: String! - pub capNext: Boolean! - pub prevIsCap: Boolean! - - new(out: String! = "", capNext: Boolean! = true, prevIsCap: Boolean! = false) { - self.out = out - self.capNext = capNext - self.prevIsCap = prevIsCap - self + PythonModuleBuild().mainObjectName(name) } } diff --git a/sdk/ruff.toml b/sdk/ruff.toml index 907fb75..b83e4ee 100644 --- a/sdk/ruff.toml +++ b/sdk/ruff.toml @@ -88,6 +88,8 @@ ignore = [ ] # Same as above, for dev module "src/dagger_gen.py" = ["A", "D", "E501", "SLF001", "PLR0913"] +# The Dang entrypoint template has lines as long as the Dang they produce. +"src/dagger/mod/_entrypoint.py" = ["E501"] # Ignore built-in shadowing in test mocks. "./tests/client/test_inputs.py" = ["A", "ERA001"] "./tests/*.py" = [ diff --git a/sdk/src/dagger/mod/__main__.py b/sdk/src/dagger/mod/__main__.py new file mode 100644 index 0000000..65328c1 --- /dev/null +++ b/sdk/src/dagger/mod/__main__.py @@ -0,0 +1,85 @@ +"""Commands the generated entrypoint runs in the module's container.""" + +import argparse +import json +import logging +import pathlib +import sys +from typing import Any + +import anyio + +import dagger +from dagger import telemetry +from dagger.mod._exceptions import ModuleError + +logger = logging.getLogger(__package__) + + +def main(argv: list[str] | None = None) -> int: + """Run one command and return the exit status.""" + parser = argparse.ArgumentParser(prog="python -m dagger.mod") + commands = parser.add_subparsers(required=True) + + entrypoint = commands.add_parser( + "entrypoint", + help="render the static entrypoint of the module in the current directory", + ) + entrypoint.add_argument("--name", required=True, help="module name") + entrypoint.add_argument( + "--path", required=True, help="module directory, relative to the workspace" + ) + entrypoint.add_argument("--output", required=True, type=pathlib.Path) + entrypoint.set_defaults(run=_entrypoint) + + call = commands.add_parser( + "call", + help="run the call read from standard input and write its JSON result", + ) + call.add_argument("--output", required=True, type=pathlib.Path) + call.set_defaults(run=_call) + + args = parser.parse_args(argv) + try: + args.run(args) + except (ModuleError, dagger.QueryError) as e: + logger.error(str(e)) # noqa: TRY400 - the message is the whole story + return 2 + except Exception: + logger.exception("Unhandled exception") + return 1 + return 0 + + +def _entrypoint(args: argparse.Namespace) -> None: + from dagger.mod._entrypoint import write_entrypoint + from dagger.mod.cli import load_module + + write_entrypoint( + load_module().describe(), + name=args.name, + path=args.path, + root=pathlib.Path.cwd(), + output=args.output, + ) + + +def _call(args: argparse.Namespace) -> None: + request = json.load(sys.stdin) + telemetry.initialize() + try: + result = anyio.run(_dispatch, request) + finally: + telemetry.shutdown() + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result)) + + +async def _dispatch(request: dict[str, Any]) -> Any: + from dagger.mod.cli import load_module + + return await load_module().dispatch(request) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sdk/src/dagger/mod/_converter.py b/sdk/src/dagger/mod/_converter.py index 059a525..24ad207 100644 --- a/sdk/src/dagger/mod/_converter.py +++ b/sdk/src/dagger/mod/_converter.py @@ -1,6 +1,4 @@ -import enum import functools -import inspect import logging import typing @@ -11,20 +9,14 @@ from dagger.client._core import Arg, configure_converter_enum from dagger.client._guards import is_id_type, is_id_type_subclass from dagger.client.base import Interface, Scalar, Type +from dagger.mod._describe import TypeRef, describe_type from dagger.mod._resolver import Function from dagger.mod._utils import ( - get_doc, get_module, get_object_type, - is_annotated, is_dagger_interface_type, is_dagger_object_type, - is_initvar, - is_nullable, - is_subclass, - is_union, list_of, - non_null, strip_annotations, syncify, to_camel_case, @@ -163,59 +155,29 @@ async def exec_method(self, *args, **kwargs): @functools.cache -def to_typedef(annotation: typing.Any, context: str = "type") -> "TypeDef": # noqa: C901, PLR0911 +def to_typedef(annotation: typing.Any, context: str = "type") -> "TypeDef": """Convert Python object to API type.""" - if is_initvar(annotation): - return to_typedef(annotation.type, context) + return typedef_from(describe_type(annotation, context)) - if is_annotated(annotation): - return to_typedef(strip_annotations(annotation), context) +def typedef_from(ref: TypeRef) -> "TypeDef": + """Build the API type from its description.""" td = dag.type_def() - typ = type(None) if annotation is None else annotation - error_msg = f"unsupported {context}: {typ!r}" - - if is_nullable(typ): + if ref.optional: td = td.with_optional(True) - typ = non_null(typ) - - # Can't represent unions in the API. - if is_union(typ): - raise TypeError(error_msg) - - builtins = { - str: dagger.TypeDefKind.STRING_KIND, - int: dagger.TypeDefKind.INTEGER_KIND, - float: dagger.TypeDefKind.FLOAT_KIND, - bool: dagger.TypeDefKind.BOOLEAN_KIND, - type(None): dagger.TypeDefKind.VOID_KIND, - } - - if typ in builtins: - return td.with_kind(builtins[typ]) - - if el := list_of(typ): - return td.with_list_of(to_typedef(el)) - - if inspect.isclass(cls := typ): - name = cls.__name__ - - if is_subclass(cls, enum.Enum): - return td.with_enum(name, description=get_doc(cls)) - - if is_subclass(cls, Scalar): - return td.with_scalar(name, description=get_doc(cls)) - - # object defined in this module - if obj_type := get_object_type(cls): - if obj_type.interface: - return td.with_interface(name) - return td.with_object(name) - - # object type from API (codegen) - if is_id_type_subclass(cls): - return td.with_object(name) - - raise TypeError(error_msg) + match ref.kind: + case dagger.TypeDefKind.LIST_KIND: + assert ref.elem is not None + return td.with_list_of(typedef_from(ref.elem)) + case dagger.TypeDefKind.ENUM_KIND: + return td.with_enum(ref.name, description=ref.description) + case dagger.TypeDefKind.SCALAR_KIND: + return td.with_scalar(ref.name, description=ref.description) + case dagger.TypeDefKind.INTERFACE_KIND: + return td.with_interface(ref.name) + case dagger.TypeDefKind.OBJECT_KIND: + return td.with_object(ref.name) + case _: + return td.with_kind(ref.kind) diff --git a/sdk/src/dagger/mod/_describe.py b/sdk/src/dagger/mod/_describe.py new file mode 100644 index 0000000..9d04d02 --- /dev/null +++ b/sdk/src/dagger/mod/_describe.py @@ -0,0 +1,170 @@ +"""Module type definitions as plain data. + +Both ways of loading a module derive their type definitions from a +:class:`ModuleDescription`: the runtime materialises it into API calls when +the engine asks for the types, and the static entrypoint renders it to Dang +at generate time. +""" + +from __future__ import annotations + +import dataclasses +import enum +import inspect +from typing import Any + +import dagger +from dagger.client._guards import is_id_type_subclass +from dagger.client.base import Scalar +from dagger.mod._utils import ( + get_doc, + get_object_type, + is_annotated, + is_initvar, + is_nullable, + is_subclass, + is_union, + list_of, + non_null, + strip_annotations, +) + +Kind = dagger.TypeDefKind + +_BUILTINS: dict[Any, dagger.TypeDefKind] = { + str: Kind.STRING_KIND, + int: Kind.INTEGER_KIND, + float: Kind.FLOAT_KIND, + bool: Kind.BOOLEAN_KIND, + type(None): Kind.VOID_KIND, +} + + +@dataclasses.dataclass(frozen=True, slots=True) +class TypeRef: + """A reference to an API type.""" + + kind: dagger.TypeDefKind + name: str = "" + description: str | None = None + optional: bool = False + elem: TypeRef | None = None + + +@dataclasses.dataclass(frozen=True, slots=True) +class ArgumentDescription: + name: str + type: TypeRef + # The runtime marks a nullable argument optional a second time, on top + # of the type reference; kept so both paths produce the same definition. + nullable: bool = False + description: str | None = None + default_value: str | None = None + default_path: str | None = None + default_address: str | None = None + ignore: tuple[str, ...] | None = None + deprecated: str | None = None + + +@dataclasses.dataclass(frozen=True, slots=True) +class FunctionDescription: + name: str + returns: TypeRef + description: str | None = None + cache: str | None = None + deprecated: str | None = None + check: bool = False + generator: bool = False + service: bool = False + agent: bool = False + args: tuple[ArgumentDescription, ...] = () + + +@dataclasses.dataclass(frozen=True, slots=True) +class FieldDescription: + name: str + type: TypeRef + description: str | None = None + deprecated: str | None = None + + +@dataclasses.dataclass(frozen=True, slots=True) +class EnumMemberDescription: + name: str + value: str + description: str | None = None + deprecated: str | None = None + + +@dataclasses.dataclass(frozen=True, slots=True) +class EnumDescription: + name: str + description: str | None = None + members: tuple[EnumMemberDescription, ...] = () + + +@dataclasses.dataclass(frozen=True, slots=True) +class ObjectDescription: + name: str + interface: bool = False + description: str | None = None + deprecated: str | None = None + fields: tuple[FieldDescription, ...] = () + functions: tuple[FunctionDescription, ...] = () + constructor: FunctionDescription | None = None + + +@dataclasses.dataclass(frozen=True, slots=True) +class ModuleDescription: + main_object: str + description: str | None = None + objects: tuple[ObjectDescription, ...] = () + enums: tuple[EnumDescription, ...] = () + + +def describe_type( # noqa: C901, PLR0911 + annotation: Any, + context: str = "type", +) -> TypeRef: + """Describe a Python annotation as an API type reference.""" + if is_initvar(annotation): + return describe_type(annotation.type, context) + + if is_annotated(annotation): + return describe_type(strip_annotations(annotation), context) + + typ = type(None) if annotation is None else annotation + error_msg = f"unsupported {context}: {typ!r}" + + optional = is_nullable(typ) + typ = non_null(typ) + + # Can't represent unions in the API. + if is_union(typ): + raise TypeError(error_msg) + + if typ in _BUILTINS: + return TypeRef(_BUILTINS[typ], optional=optional) + + if el := list_of(typ): + return TypeRef(Kind.LIST_KIND, optional=optional, elem=describe_type(el)) + + if inspect.isclass(cls := typ): + name = cls.__name__ + + if is_subclass(cls, enum.Enum): + return TypeRef(Kind.ENUM_KIND, name, get_doc(cls), optional) + + if is_subclass(cls, Scalar): + return TypeRef(Kind.SCALAR_KIND, name, get_doc(cls), optional) + + # object defined in this module + if obj_type := get_object_type(cls): + kind = Kind.INTERFACE_KIND if obj_type.interface else Kind.OBJECT_KIND + return TypeRef(kind, name, optional=optional) + + # object type from API (codegen) + if is_id_type_subclass(cls): + return TypeRef(Kind.OBJECT_KIND, name, optional=optional) + + raise TypeError(error_msg) diff --git a/sdk/src/dagger/mod/_entrypoint.py b/sdk/src/dagger/mod/_entrypoint.py new file mode 100644 index 0000000..594ce45 --- /dev/null +++ b/sdk/src/dagger/mod/_entrypoint.py @@ -0,0 +1,313 @@ +"""Render a module description as a static Dang entrypoint.""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import pathlib +from collections.abc import Iterator + +import dagger +from dagger.mod._describe import ( + ArgumentDescription, + EnumDescription, + FunctionDescription, + ModuleDescription, + ObjectDescription, + TypeRef, +) +from dagger.mod._exceptions import BadUsageError + +HEADER = "# Code generated by dagger. DO NOT EDIT.\n" + +MANIFESTS = (".python-version", "pyproject.toml", "requirements.lock", "uv.lock") + +SKIPPED_DIRS = frozenset({"sdk", ".venv", "__pycache__"}) +"""Directory names that hold no module source, at any depth of the module. + +Together with every dotted directory: the vendored library is generated, and a +virtual environment or a bytecode cache never reaches the module's container. +The rendered guard skips the same names, so both sides scan one set of files. +""" + +Kind = dagger.TypeDefKind + + +@dataclasses.dataclass(frozen=True, slots=True) +class SourceFile: + path: str + digest: str + + +def write_entrypoint( + desc: ModuleDescription, + *, + name: str, + path: str, + root: pathlib.Path, + output: pathlib.Path, +) -> None: + """Write types.dang and main.dang for the module at root.""" + output.mkdir(parents=True, exist_ok=True) + (output / "types.dang").write_text(render_types(desc)) + (output / "main.dang").write_text(render_main(name, path, source_files(root))) + + +def source_files(root: pathlib.Path) -> list[SourceFile]: + """The files that can change the description, with their engine digests. + + A manifest absent at generation gets an empty digest: the guard refuses the + module when such a file appears later and changes what gets installed. + """ + files = [root / name for name in MANIFESTS] + files += sorted(f for f in root.rglob("*.py") if _is_source(f.relative_to(root))) + return [ + SourceFile( + f.relative_to(root).as_posix(), + file_digest(f.read_bytes()) if f.is_file() else "", + ) + for f in files + ] + + +def _is_source(rel: pathlib.PurePath) -> bool: + return not any(d in SKIPPED_DIRS or d.startswith(".") for d in rel.parts[:-1]) + + +def file_digest(data: bytes) -> str: + """What File.digest(excludeMetadata: true) returns for these bytes.""" + inner = hashlib.sha256(data).digest() + return "sha256:" + hashlib.sha256(inner).hexdigest() + + +def render_types(desc: ModuleDescription) -> str: + lines = [HEADER, "type ModuleTypes {", " pub all: [TypeDef!]! {", " ["] + for obj in desc.objects: + lines += _indent(_object(obj), 6) + lines[-1] += "," + for enum in desc.enums: + lines += _indent(_enum(enum), 6) + lines[-1] += "," + lines += [" ]", " }", "}", ""] + return "\n".join(lines) + + +def render_main(name: str, path: str, files: list[SourceFile]) -> str: + _check_path(path) + file_lines = "".join( + f" SourceFile(path: {_quote(f.path)}, digest: {_quote(f.digest)}),\n" + for f in files + ) + return MAIN_TEMPLATE.format( + header=HEADER, + name=_quote(name), + path=_quote(path), + files=file_lines, + skipped=", ".join(_quote(d) for d in sorted(SKIPPED_DIRS)), + ) + + +def _check_path(path: str) -> None: + parts = pathlib.PurePosixPath(path).parts + if pathlib.PurePosixPath(path).is_absolute() or ".." in parts: + msg = f"module path must be relative to the workspace and not escape it: {path}" + raise BadUsageError(msg) + + +def _object(obj: ObjectDescription) -> list[str]: + description = _opt("description", obj.description) + if obj.interface: + head = f"typeDef.withInterface({_quote(obj.name)}{description})" + else: + deprecated = _opt("deprecated", obj.deprecated) + head = f"typeDef.withObject({_quote(obj.name)}{description}{deprecated})" + lines = [head] + lines.extend( + f".withField({_quote(field.name)}, {_type(field.type)}" + f"{_opt('description', field.description)}" + f"{_opt('deprecated', field.deprecated)})" + for field in obj.fields + ) + for func in obj.functions: + lines += _wrap("withFunction", _function(func)) + if obj.constructor is not None: + lines += _wrap("withConstructor", _function(obj.constructor)) + return lines + + +def _function(func: FunctionDescription) -> list[str]: + if func.cache is not None: + msg = ( + f"function '{func.name}' sets cache={func.cache!r}: a static entrypoint " + "cannot honour cache policies; set staticEntrypoint = false to keep " + "discovering types at runtime" + ) + raise BadUsageError(msg) + lines = [f"function({_quote(func.name)}, {_type(func.returns)})"] + if func.description: + lines.append(f".withDescription({_quote(func.description)})") + if func.deprecated: + lines.append(f".withDeprecated(reason: {_quote(func.deprecated)})") + for flag, call in ( + (func.check, "withCheck"), + (func.generator, "withGenerator"), + (func.service, "withUp"), + (func.agent, "withAgent"), + ): + if flag: + lines.append(f".{call}") + lines += [_argument(arg) for arg in func.args] + return lines + + +def _argument(arg: ArgumentDescription) -> str: + type_expr = _type(arg.type) + if arg.nullable: + type_expr += ".withOptional(true)" + default = "" + if arg.default_value is not None: + default = f", defaultValue: ({_quote(arg.default_value)} :: Dagger.JSON!)" + ignore = "" + if arg.ignore is not None: + ignore = f", ignore: [{', '.join(_quote(p) for p in arg.ignore)}]" + return ( + f".withArg({_quote(arg.name)}, {type_expr}" + f"{_opt('description', arg.description)}{default}" + f"{_opt('defaultPath', arg.default_path)}" + f"{_opt('defaultAddress', arg.default_address)}{ignore}" + f"{_opt('deprecated', arg.deprecated)})" + ) + + +def _enum(enum: EnumDescription) -> list[str]: + lines = [ + f"typeDef.withEnum({_quote(enum.name)}{_opt('description', enum.description)})" + ] + lines += [ + f".withEnumMember({_quote(m.name)}, value: {_quote(m.value)}" + f"{_opt('description', m.description)}{_opt('deprecated', m.deprecated)})" + for m in enum.members + ] + return lines + + +def _type(ref: TypeRef) -> str: + expr = "typeDef" + if ref.optional: + expr += ".withOptional(true)" + match ref.kind: + case Kind.LIST_KIND: + assert ref.elem is not None + return f"{expr}.withListOf({_type(ref.elem)})" + case Kind.ENUM_KIND: + described = f"{_quote(ref.name)}{_opt('description', ref.description)}" + return f"{expr}.withEnum({described})" + case Kind.SCALAR_KIND: + described = f"{_quote(ref.name)}{_opt('description', ref.description)}" + return f"{expr}.withScalar({described})" + case Kind.INTERFACE_KIND: + return f"{expr}.withInterface({_quote(ref.name)})" + case Kind.OBJECT_KIND: + return f"{expr}.withObject({_quote(ref.name)})" + case _: + return f"{expr}.withKind(TypeDefKind.{ref.kind.value})" + + +def _opt(arg: str, value: str | None) -> str: + """A named string argument, omitted when there is nothing to say.""" + return f", {arg}: {_quote(value)}" if value else "" + + +def _quote(value: str) -> str: + return json.dumps(value, ensure_ascii=False) + + +def _wrap(call: str, body: list[str]) -> Iterator[str]: + yield f".{call}(" + yield from _indent(body, 2) + yield ")" + + +def _indent(lines: list[str], width: int) -> list[str]: + pad = " " * width + return [pad + line if i == 0 else pad + " " + line for i, line in enumerate(lines)] + + +MAIN_TEMPLATE = """{header} +type Entrypoint implements ModuleEntrypoint {{ + let moduleName: String! = {name} + let modulePath: String! = {path} + let skippedDirs: [String!]! = [{skipped}] + let sourceFiles: [SourceFile!]! = [ +{files} ] + + pub types(workspace: Workspace!): [TypeDef!]! {{ + ModuleTypes().all + }} + + pub call( + workspace: Workspace!, + receiverType: String!, + receiverValue: JSON, + fnName: String!, + fnArgs: JSON!, + ): JSON! {{ + let request = JSON.encode({{{{ + receiverType: receiverType, + receiverValue: receiverValue, + fnName: fnName, + fnArgs: fnArgs, + }}}}) + let result = runtime(workspace) + .withExec(["python", "-m", "dagger.mod", "call", "--output", "/dagger/result.json"], stdin: request, experimentalPrivilegedNesting: true) + .file("/dagger/result.json") + .contents + (result :: JSON!) + }} + + let runtime(workspace: Workspace!): Container! {{ + let module = workspace.directory(if (modulePath == ".") {{ "/" }} else {{ "/" + modulePath }}) + if (module.exists("pyproject.toml") == false) {{ + raise "module \\"" + moduleName + "\\" was generated at \\"" + modulePath + "\\" and is not there; run `dagger generate` after moving it" + }} else {{ + let changed = sourceFiles.filter {{ f => + if (f.digest == "") {{ + module.exists(f.path) + }} else {{ + module.exists(f.path) == false or module.file(f.path).digest(excludeMetadata: true) != f.digest + }} + }}.map {{ f => f.path }} + let added = module.glob("**/*.py").filter {{ p => + isSource(p) and sourceFiles.filter {{ f => f.path == p }}.length == 0 + }} + if ((changed + added).length > 0) {{ + raise "module \\"" + moduleName + "\\" changed since its entrypoint was generated (" + (changed + added).join(", ") + "); run `dagger generate`" + }} else {{ + PythonModuleBuild( + contextDir: workspace.directory("/", include: [if (modulePath == ".") {{ "**" }} else {{ modulePath + "/**" }}], exclude: ["**/.venv", "**/__pycache__"]), + subPath: modulePath, + moduleName: moduleName, + ).installed + }} + }} + }} + + let isSource(path: String!): Boolean! {{ + path.split("/").dropLast(1).filter {{ segment => + segment.hasPrefix(".") or skippedDirs.contains(segment) + }}.length == 0 + }} +}} + +type SourceFile {{ + pub path: String! + pub digest: String! + + new(path: String!, digest: String!) {{ + self.path = path + self.digest = digest + self + }} +}} +""" diff --git a/sdk/src/dagger/mod/_module.py b/sdk/src/dagger/mod/_module.py index 16e730e..c0c515a 100644 --- a/sdk/src/dagger/mod/_module.py +++ b/sdk/src/dagger/mod/_module.py @@ -19,7 +19,17 @@ import dagger from dagger import dag from dagger.client._core import configure_converter_enum -from dagger.mod._converter import make_converter, to_typedef +from dagger.mod._converter import make_converter, typedef_from +from dagger.mod._describe import ( + ArgumentDescription, + EnumDescription, + EnumMemberDescription, + FieldDescription, + FunctionDescription, + ModuleDescription, + ObjectDescription, + describe_type, +) from dagger.mod._exceptions import ( BadUsageError, FunctionError, @@ -123,7 +133,11 @@ async def register(self): raise RegistrationError(str(e), e) from e await anyio.Path(TYPE_DEF_FILE).write_text(output) - async def _typedefs(self) -> str: # noqa: C901, PLR0912, PLR0915 + async def _typedefs(self) -> str: + return await _module_from(self.describe()).id() + + def describe(self) -> ModuleDescription: + """Describe the registered types as plain data.""" if not self._main_name: msg = "Main object name can't be empty" raise ValueError(msg) @@ -138,140 +152,22 @@ async def _typedefs(self) -> str: # noqa: C901, PLR0912, PLR0915 ) raise ObjectNotFoundError(msg, extra=e.extra) from None - mod = dag.module() - - # Object types + description = None + objects = [] for obj_name, obj_type in self._objects.items(): if self.is_main(obj_type): # Only the main object's constructor is needed. # It's the entrypoint to the module. obj_type.get_constructor(self._converter) - - # Module description from main object's parent module - if desc := get_parent_module_doc(obj_type.cls): - mod = mod.with_description(desc) - - # Object/interface type - type_def = dag.type_def() - if obj_type.interface: - type_def = type_def.with_interface( - obj_name, - description=get_doc(obj_type.cls), - ) - else: - type_def = type_def.with_object( - obj_name, - description=get_doc(obj_type.cls), - deprecated=obj_type.deprecated, - ) - - # Object fields - if obj_type.fields: - types = typing.get_type_hints(obj_type.cls) - - for field_name, field in obj_type.fields.items(): - ctx = f"type for field '{field.original_name}' in {obj_type}" - type_def = type_def.with_field( - field_name, - to_typedef(types[field.original_name], ctx), - description=get_doc(field.return_type), - deprecated=field.meta.deprecated, - ) - - # Object/interface functions - for func_name, func in obj_type.functions.items(): - what = f"function '{func_name}'" if func_name else "constructor" - - func_def = dag.function( - func_name, - to_typedef( - func.return_type, - f"return type for {what} in {obj_type}", - ), - ) - - if doc := func.doc: - func_def = func_def.with_description(doc) - - if func.cache_policy is not None: - if func.cache_policy == "never": - func_def = func_def.with_cache_policy( - dagger.FunctionCachePolicy.Never, - ) - elif func.cache_policy == "session": - func_def = func_def.with_cache_policy( - dagger.FunctionCachePolicy.PerSession, - ) - elif func.cache_policy != "": - func_def = func_def.with_cache_policy( - dagger.FunctionCachePolicy.Default, - time_to_live=func.cache_policy, - ) - if deprecated := func.deprecated: - func_def = func_def.with_deprecated(reason=deprecated) - if func.check: - func_def = func_def.with_check() - if func.generate: - func_def = func_def.with_generator() - if func.service: - func_def = func_def.with_up() - if func.agent: - func_def = func_def.with_agent() - - for param in func.parameters.values(): - arg_def = to_typedef( - param.resolved_type, - f"parameter type for '{param.name}' in {what} and {obj_type}", - ) - - if param.is_nullable: - arg_def = arg_def.with_optional(True) - - func_def = func_def.with_arg( - param.name, - arg_def, - description=param.doc, - default_value=param.default_value, - default_path=param.default_path, - default_address=param.default_address, - ignore=param.ignore, - deprecated=param.deprecated, - ) - - type_def = ( - type_def.with_constructor(func_def) - if func_name == "" - else type_def.with_function(func_def) - ) - - # Add object/interface to module - mod = ( - mod.with_interface(type_def) - if obj_type.interface - else mod.with_object(type_def) - ) - - # Enum types - for name, cls in self._enums.items(): - enum_def = dag.type_def().with_enum(name, description=get_doc(cls)) - member_docs = extract_enum_member_doc(cls) - - for member in cls: - description = getattr(member, "description", None) - meta = member_docs.get(member.name) - - if description is None and meta and meta.description is not None: - description = meta.description - - enum_def = enum_def.with_enum_member( - member.name, - value=str(member.value), - description=description, - deprecated=meta.deprecated if meta else None, - ) - mod = mod.with_enum(enum_def) - - return await mod.id() + description = get_parent_module_doc(obj_type.cls) + objects.append(_describe_object(obj_name, obj_type)) + + return ModuleDescription( + main_object=self._main_name, + description=description, + objects=tuple(objects), + enums=tuple(_describe_enum(name, cls) for name, cls in self._enums.items()), + ) async def invoke(self) -> str: """Invoke a function and return its result. @@ -348,6 +244,25 @@ async def invoke(self) -> str: return result + async def dispatch(self, request: Mapping[str, Any]) -> Any: + """Run the call a ModuleEntrypoint forwards. + + The receiver state and the arguments arrive as JSON text. + """ + receiver = request.get("receiverValue") or "" + try: + parent_state = json.loads(receiver) if receiver.strip() else None + inputs = json.loads(request["fnArgs"]) + except ValueError as e: + msg = "Unable to decode the call request" + raise InvalidInputError(msg, extra={"request": request}) from e + return await self.get_result( + request["receiverType"], + parent_state or {}, + request["fnName"], + inputs, + ) + async def get_result( self, parent_name: str, @@ -1040,3 +955,193 @@ def wrapper(cls: T) -> T: return cls return wrapper(cls) if cls else wrapper + + +def _describe_object(name: str, obj_type: ObjectType) -> ObjectDescription: + fields: tuple[FieldDescription, ...] = () + if obj_type.fields: + types = typing.get_type_hints(obj_type.cls) + fields = tuple( + FieldDescription( + name=field.name, + type=describe_type( + types[field.original_name], + f"type for field '{field.original_name}' in {obj_type}", + ), + description=get_doc(field.return_type), + deprecated=field.meta.deprecated, + ) + for field in obj_type.fields.values() + ) + + functions = [] + constructor = None + for func_name, func in obj_type.functions.items(): + described = _describe_function(func_name, func, obj_type) + if func_name == "": + constructor = described + else: + functions.append(described) + + return ObjectDescription( + name=name, + interface=obj_type.interface, + description=get_doc(obj_type.cls), + deprecated=obj_type.deprecated, + fields=fields, + functions=tuple(functions), + constructor=constructor, + ) + + +def _describe_function( + name: str, + func: Function, + obj_type: ObjectType, +) -> FunctionDescription: + what = f"function '{name}'" if name else "constructor" + returns = describe_type(func.return_type, f"return type for {what} in {obj_type}") + args = tuple( + ArgumentDescription( + name=param.name, + type=describe_type( + param.resolved_type, + f"parameter type for '{param.name}' in {what} and {obj_type}", + ), + nullable=param.is_nullable, + description=param.doc, + default_value=param.default_value, + default_path=param.default_path, + default_address=param.default_address, + ignore=tuple(param.ignore) if param.ignore is not None else None, + deprecated=param.deprecated, + ) + for param in func.parameters.values() + ) + return FunctionDescription( + name=name, + returns=returns, + description=func.doc, + cache=func.cache_policy, + deprecated=func.deprecated, + check=func.check, + generator=func.generate, + service=func.service, + agent=func.agent, + args=args, + ) + + +def _describe_enum(name: str, cls: type[enum.Enum]) -> EnumDescription: + member_docs = extract_enum_member_doc(cls) + members = [] + for member in cls: + description = getattr(member, "description", None) + meta = member_docs.get(member.name) + if description is None and meta and meta.description is not None: + description = meta.description + members.append( + EnumMemberDescription( + name=member.name, + value=str(member.value), + description=description, + deprecated=meta.deprecated if meta else None, + ) + ) + return EnumDescription(name, get_doc(cls), tuple(members)) + + +def _module_from(desc: ModuleDescription) -> dagger.Module: + mod = dag.module() + for obj in desc.objects: + if obj.name == desc.main_object and desc.description: + mod = mod.with_description(desc.description) + type_def = _object_from(obj) + mod = ( + mod.with_interface(type_def) if obj.interface else mod.with_object(type_def) + ) + for enum_desc in desc.enums: + mod = mod.with_enum(_enum_from(enum_desc)) + return mod + + +def _object_from(obj: ObjectDescription) -> dagger.TypeDef: + type_def = dag.type_def() + if obj.interface: + type_def = type_def.with_interface(obj.name, description=obj.description) + else: + type_def = type_def.with_object( + obj.name, + description=obj.description, + deprecated=obj.deprecated, + ) + for field in obj.fields: + type_def = type_def.with_field( + field.name, + typedef_from(field.type), + description=field.description, + deprecated=field.deprecated, + ) + for func in obj.functions: + type_def = type_def.with_function(_function_from(func)) + if obj.constructor is not None: + type_def = type_def.with_constructor(_function_from(obj.constructor)) + return type_def + + +def _function_from(func: FunctionDescription) -> dagger.Function: # noqa: C901 + func_def = dag.function(func.name, typedef_from(func.returns)) + if func.description: + func_def = func_def.with_description(func.description) + if func.cache == "never": + func_def = func_def.with_cache_policy(dagger.FunctionCachePolicy.Never) + elif func.cache == "session": + func_def = func_def.with_cache_policy(dagger.FunctionCachePolicy.PerSession) + elif func.cache: + func_def = func_def.with_cache_policy( + dagger.FunctionCachePolicy.Default, + time_to_live=func.cache, + ) + if func.deprecated: + func_def = func_def.with_deprecated(reason=func.deprecated) + if func.check: + func_def = func_def.with_check() + if func.generator: + func_def = func_def.with_generator() + if func.service: + func_def = func_def.with_up() + if func.agent: + func_def = func_def.with_agent() + for arg in func.args: + arg_def = typedef_from(arg.type) + if arg.nullable: + arg_def = arg_def.with_optional(True) + func_def = func_def.with_arg( + arg.name, + arg_def, + description=arg.description, + default_value=( + dagger.JSON(arg.default_value) + if arg.default_value is not None + else None + ), + default_path=arg.default_path, + default_address=arg.default_address, + ignore=list(arg.ignore) if arg.ignore is not None else None, + deprecated=arg.deprecated, + ) + return func_def + + +def _enum_from(enum_desc: EnumDescription) -> dagger.TypeDef: + enum_def = dag.type_def().with_enum( + enum_desc.name, description=enum_desc.description + ) + for member in enum_desc.members: + enum_def = enum_def.with_enum_member( + member.name, + value=member.value, + description=member.description, + deprecated=member.deprecated, + ) + return enum_def diff --git a/sdk/tests/mod/golden/main.dang b/sdk/tests/mod/golden/main.dang new file mode 100644 index 0000000..f2fa2f4 --- /dev/null +++ b/sdk/tests/mod/golden/main.dang @@ -0,0 +1,83 @@ +# Code generated by dagger. DO NOT EDIT. + +type Entrypoint implements ModuleEntrypoint { + let moduleName: String! = "main" + let modulePath: String! = ".dagger/modules/main" + let skippedDirs: [String!]! = [".venv", "__pycache__", "sdk"] + let sourceFiles: [SourceFile!]! = [ + SourceFile(path: ".python-version", digest: ""), + SourceFile(path: "pyproject.toml", digest: "sha256:aefe03a941c71de872fd1c1f4b9eda3c91e1c32b4a80d1e50c91cd9405950ef5"), + SourceFile(path: "requirements.lock", digest: ""), + SourceFile(path: "uv.lock", digest: ""), + SourceFile(path: "src/main/__init__.py", digest: "sha256:7a18d55cea9842deb80ff288c43596d282d4121879ebe6ec06055ea057b1af52"), + SourceFile(path: "src/main/extra.py", digest: "sha256:29a8358f62c5e515e9e5e903df20673f5db196b8ae119d521b0ffc1f88267f00"), + ] + + pub types(workspace: Workspace!): [TypeDef!]! { + ModuleTypes().all + } + + pub call( + workspace: Workspace!, + receiverType: String!, + receiverValue: JSON, + fnName: String!, + fnArgs: JSON!, + ): JSON! { + let request = JSON.encode({{ + receiverType: receiverType, + receiverValue: receiverValue, + fnName: fnName, + fnArgs: fnArgs, + }}) + let result = runtime(workspace) + .withExec(["python", "-m", "dagger.mod", "call", "--output", "/dagger/result.json"], stdin: request, experimentalPrivilegedNesting: true) + .file("/dagger/result.json") + .contents + (result :: JSON!) + } + + let runtime(workspace: Workspace!): Container! { + let module = workspace.directory(if (modulePath == ".") { "/" } else { "/" + modulePath }) + if (module.exists("pyproject.toml") == false) { + raise "module \"" + moduleName + "\" was generated at \"" + modulePath + "\" and is not there; run `dagger generate` after moving it" + } else { + let changed = sourceFiles.filter { f => + if (f.digest == "") { + module.exists(f.path) + } else { + module.exists(f.path) == false or module.file(f.path).digest(excludeMetadata: true) != f.digest + } + }.map { f => f.path } + let added = module.glob("**/*.py").filter { p => + isSource(p) and sourceFiles.filter { f => f.path == p }.length == 0 + } + if ((changed + added).length > 0) { + raise "module \"" + moduleName + "\" changed since its entrypoint was generated (" + (changed + added).join(", ") + "); run `dagger generate`" + } else { + PythonModuleBuild( + contextDir: workspace.directory("/", include: [if (modulePath == ".") { "**" } else { modulePath + "/**" }], exclude: ["**/.venv", "**/__pycache__"]), + subPath: modulePath, + moduleName: moduleName, + ).installed + } + } + } + + let isSource(path: String!): Boolean! { + path.split("/").dropLast(1).filter { segment => + segment.hasPrefix(".") or skippedDirs.contains(segment) + }.length == 0 + } +} + +type SourceFile { + pub path: String! + pub digest: String! + + new(path: String!, digest: String!) { + self.path = path + self.digest = digest + self + } +} diff --git a/sdk/tests/mod/golden/types.dang b/sdk/tests/mod/golden/types.dang new file mode 100644 index 0000000..3c37ddf --- /dev/null +++ b/sdk/tests/mod/golden/types.dang @@ -0,0 +1,54 @@ +# Code generated by dagger. DO NOT EDIT. + +type ModuleTypes { + pub all: [TypeDef!]! { + [ + typeDef.withInterface("Greeter") + .withFunction( + function("greet", typeDef.withKind(TypeDefKind.STRING_KIND)) + .withArg("name", typeDef.withKind(TypeDefKind.STRING_KIND)) + ), + typeDef.withObject("Helper", description: "A helper.") + .withField("level", typeDef.withEnum("Color", description: "A color."), description: "A color."), + typeDef.withObject("Main", description: "The main object.") + .withField("greeting", typeDef.withKind(TypeDefKind.STRING_KIND)) + .withField("howMany", typeDef.withKind(TypeDefKind.INTEGER_KIND), description: "How many") + .withFunction( + function("container", typeDef.withKind(TypeDefKind.STRING_KIND)) + .withDescription("A \"container\".\n\nBuilt from\\a base image.") + .withArg("from", typeDef.withKind(TypeDefKind.STRING_KIND), defaultValue: ("\"alpine\"" :: Dagger.JSON!)) + ) + .withFunction( + function("greeter", typeDef.withInterface("Greeter")) + .withArg("g", typeDef.withInterface("Greeter")) + ) + .withFunction( + function("helpers", typeDef.withListOf(typeDef.withObject("Helper"))) + .withArg("src", typeDef.withObject("Directory"), defaultPath: ".", ignore: [".venv"]) + ) + .withFunction( + function("lint", typeDef.withOptional(true).withKind(TypeDefKind.VOID_KIND)) + .withCheck + ) + .withFunction( + function("mob", typeDef.withListOf(typeDef.withObject("Main"))) + .withArg("who", typeDef.withOptional(true).withKind(TypeDefKind.STRING_KIND).withOptional(true), defaultValue: ("null" :: Dagger.JSON!)) + ) + .withFunction( + function("old", typeDef.withScalar("JSON", description: "An arbitrary JSON-encoded value.")) + .withDeprecated(reason: "use container") + .withArg("p", typeDef.withScalar("Platform", description: "The platform config OS and architecture in a Container. The format\nis [os]/[platform]/[version] (e.g., \"darwin/arm64/v7\",\n\"windows/amd64\", \"linux/arm64\")."), description: "The platform config OS and architecture in a Container. The format\nis [os]/[platform]/[version] (e.g., \"darwin/arm64/v7\",\n\"windows/amd64\", \"linux/arm64\").") + ) + .withConstructor( + function("", typeDef.withObject("Main")) + .withDescription("The main object.") + .withArg("source", typeDef.withObject("Directory"), description: "A directory.") + .withArg("greeting", typeDef.withKind(TypeDefKind.STRING_KIND), defaultValue: ("\"hello\"" :: Dagger.JSON!)) + .withArg("count", typeDef.withKind(TypeDefKind.INTEGER_KIND), description: "How many", defaultValue: ("1" :: Dagger.JSON!)) + ), + typeDef.withEnum("Color", description: "A color.") + .withEnumMember("RED", value: "red", description: "The red one.") + .withEnumMember("BLUE", value: "blue"), + ] + } +} diff --git a/sdk/tests/mod/test_describe.py b/sdk/tests/mod/test_describe.py new file mode 100644 index 0000000..6fa92fa --- /dev/null +++ b/sdk/tests/mod/test_describe.py @@ -0,0 +1,256 @@ +import enum +import typing +from dataclasses import InitVar +from typing import Annotated + +import pytest +from typing_extensions import Doc, Self + +import dagger +from dagger import DefaultPath, Ignore, Name, dag +from dagger.mod import Module +from dagger.mod._converter import to_typedef, typedef_from +from dagger.mod._describe import TypeRef, describe_type +from dagger.mod._module import _module_from + +Kind = dagger.TypeDefKind + + +class Color(enum.Enum): + """A color.""" + + RED = "red" + """The red one.""" + + BLUE = "blue" + + +@pytest.fixture +def mod() -> Module: + m = Module("Main") + m.enum_type(Color) + + @m.interface + class Greeter(typing.Protocol): + @m.function + def greet(self, name: str) -> str: ... + + @m.object_type + class Helper: + """A helper.""" + + level: Color = m.field() + + @m.object_type + class Main: + """The main object.""" + + source: dagger.Directory + greeting: str = m.field(default="hello") + count: Annotated[int, Doc("How many")] = m.field(default=1, name="howMany") + extra: InitVar[str] = "" + + @m.function + def container(self, base: Annotated[str, Name("from")] = "alpine") -> str: + """A container.""" + return base + + @m.function(name="shout", doc="Shout it", cache="never") + async def loud(self, who: str | None = None, times: int = 1) -> str: + return (who or "nobody") * times + + @m.function + @m.check + def lint(self) -> None: ... + + @m.function + def helpers( + self, src: Annotated[dagger.Directory, DefaultPath("."), Ignore([".venv"])] + ) -> list[Helper]: ... + + @m.function + def mob(self) -> list[Self]: ... + + @m.function + def maybe(self) -> list[str] | None: ... + + @m.function + def greeter(self, g: Greeter) -> Greeter: ... + + @m.function(deprecated="use shout") + def old(self, p: dagger.Platform) -> dagger.JSON: ... + + make_helper = m.function()(Helper) + + return m + + +def test_objects_and_enums(mod: Module): + desc = mod.describe() + + assert desc.main_object == "Main" + assert [o.name for o in desc.objects] == ["Greeter", "Helper", "Main"] + assert [e.name for e in desc.enums] == ["Color"] + + greeter, helper, main = desc.objects + assert greeter.interface is True + assert greeter.constructor is None + assert [f.name for f in greeter.functions] == ["greet"] + + assert helper.description == "A helper." + assert helper.fields[0].type == TypeRef(Kind.ENUM_KIND, "Color", "A color.") + assert helper.constructor is None + + assert main.description == "The main object." + assert [f.name for f in main.functions] == [ + "make_helper", + "container", + "greeter", + "helpers", + "lint", + "shout", + "maybe", + "mob", + "old", + ] + + +def test_fields(mod: Module): + main = mod.describe().objects[2] + + assert [(f.name, f.type.kind) for f in main.fields] == [ + ("greeting", Kind.STRING_KIND), + ("howMany", Kind.INTEGER_KIND), + ] + assert main.fields[1].description == "How many" + + +def test_constructor(mod: Module): + ctor = mod.describe().objects[2].constructor + + assert ctor is not None + assert ctor.name == "" + assert ctor.returns == TypeRef(Kind.OBJECT_KIND, "Main") + assert [a.name for a in ctor.args] == ["source", "greeting", "count", "extra"] + assert ctor.args[0].type == TypeRef(Kind.OBJECT_KIND, "Directory") + assert ctor.args[1].default_value == '"hello"' + assert ctor.args[2].description == "How many" + assert ctor.args[3].default_value == '""' + + +def test_function_metadata(mod: Module): + functions = {f.name: f for f in mod.describe().objects[2].functions} + + shout = functions["shout"] + assert shout.description == "Shout it" + assert shout.cache == "never" + assert shout.args[0].nullable is True + assert shout.args[0].type == TypeRef(Kind.STRING_KIND, optional=True) + assert shout.args[0].default_value == "null" + assert shout.args[1].default_value == "1" + + assert functions["lint"].check is True + assert functions["lint"].returns == TypeRef(Kind.VOID_KIND, optional=True) + + src = functions["helpers"].args[0] + assert src.default_path == "." + assert src.ignore == (".venv",) + assert functions["helpers"].returns == TypeRef( + Kind.LIST_KIND, elem=TypeRef(Kind.OBJECT_KIND, "Helper") + ) + + assert functions["mob"].returns == TypeRef( + Kind.LIST_KIND, elem=TypeRef(Kind.OBJECT_KIND, "Main") + ) + assert functions["maybe"].returns == TypeRef( + Kind.LIST_KIND, optional=True, elem=TypeRef(Kind.STRING_KIND) + ) + assert functions["greeter"].returns == TypeRef(Kind.INTERFACE_KIND, "Greeter") + assert functions["container"].args[0].name == "from" + + old = functions["old"] + assert old.deprecated == "use shout" + assert old.args[0].type.kind == Kind.SCALAR_KIND + assert old.args[0].type.name == "Platform" + assert old.returns.kind == Kind.SCALAR_KIND + + make_helper = functions["make_helper"] + assert make_helper.description == "A helper." + assert [a.name for a in make_helper.args] == ["level"] + + +def test_enum_members(mod: Module): + color = mod.describe().enums[0] + + assert color.description == "A color." + assert [(m.name, m.value, m.description) for m in color.members] == [ + ("RED", "red", "The red one."), + ("BLUE", "blue", None), + ] + + +def test_typedef_from_matches_to_typedef(): + for annotation in (str, list[str] | None, list[list[str] | None], None): + assert typedef_from(describe_type(annotation)) == to_typedef(annotation) + + +def test_unsupported_type(): + with pytest.raises(TypeError, match=r"unsupported type: int \| str"): + describe_type(int | str) + + +def test_module_materialisation(): + mod = Module("Foo") + + @mod.object_type + class Foo: + """Foo doc.""" + + name: str = mod.field(default="foo") + + @mod.function + def hello(self, who: str | None = None) -> str: + """Say hello.""" + return who or self.name + + string = dag.type_def().with_kind(Kind.STRING_KIND) + expected = dag.module().with_object( + dag.type_def() + .with_object("Foo", description="Foo doc.", deprecated=None) + .with_field("name", string, description=None, deprecated=None) + .with_function( + dag.function("hello", string) + .with_description("Say hello.") + .with_arg( + "who", + dag.type_def() + .with_optional(True) + .with_kind(Kind.STRING_KIND) + .with_optional(True), + description=None, + default_value=dagger.JSON("null"), + default_path=None, + default_address=None, + ignore=None, + deprecated=None, + ) + ) + .with_constructor( + dag.function("", dag.type_def().with_object("Foo")) + .with_description("Foo doc.") + .with_arg( + "name", + string, + description=None, + default_value=dagger.JSON('"foo"'), + default_path=None, + default_address=None, + ignore=None, + deprecated=None, + ) + ) + ) + + desc = mod.describe() + assert desc.description is None + assert _module_from(desc) == expected diff --git a/sdk/tests/mod/test_dispatch.py b/sdk/tests/mod/test_dispatch.py new file mode 100644 index 0000000..8ea5f0a --- /dev/null +++ b/sdk/tests/mod/test_dispatch.py @@ -0,0 +1,137 @@ +import json +import pathlib +import subprocess +import sys + +import pytest + +from dagger.mod import Module +from dagger.mod._exceptions import InvalidInputError + +pytestmark = pytest.mark.anyio + + +@pytest.fixture +def mod() -> Module: + m = Module("Foo") + + @m.object_type + class Foo: + name: str = m.field(default="foo") + + @m.function + def hello(self, who: str | None = None, times: int = 1) -> str: + return (who if who is not None else self.name) * times + + return m + + +async def test_constructor(mod: Module): + request = { + "receiverType": "Foo", + "receiverValue": None, + "fnName": "", + "fnArgs": "{}", + } + assert await mod.dispatch(request) == {"name": "foo"} + + +async def test_function_with_receiver_state(mod: Module): + request = { + "receiverType": "Foo", + "receiverValue": '{"name": "bar"}', + "fnName": "hello", + "fnArgs": '{"times": 2}', + } + assert await mod.dispatch(request) == "barbar" + + +async def test_arguments_are_decoded_once(mod: Module): + request = { + "receiverType": "Foo", + "receiverValue": "{}", + "fnName": "hello", + "fnArgs": '{"who": "null"}', + } + assert await mod.dispatch(request) == "null" + + +async def test_null_argument(mod: Module): + request = { + "receiverType": "Foo", + "receiverValue": "{}", + "fnName": "hello", + "fnArgs": '{"who": null}', + } + assert await mod.dispatch(request) == "foo" + + +async def test_malformed_request(mod: Module): + request = { + "receiverType": "Foo", + "receiverValue": "{", + "fnName": "hello", + "fnArgs": "{}", + } + with pytest.raises(InvalidInputError, match="decode the call request"): + await mod.dispatch(request) + + +@pytest.fixture +def module_dir(tmp_path: pathlib.Path) -> pathlib.Path: + (tmp_path / "pyproject.toml").write_text('[project]\nname = "hello"\n') + pkg = tmp_path / "src" / "hello" + pkg.mkdir(parents=True) + (pkg / "__init__.py").write_text( + "from dagger import function, object_type\n\n" + "@object_type\nclass Hello:\n" + " @function\n" + " def hi(self, who: str) -> str:\n" + " return 'hi ' + who\n" + " @function\n" + " def boom(self) -> str:\n" + " raise RuntimeError('boom')\n" + ) + return tmp_path + + +def _call(module_dir: pathlib.Path, request: dict) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-m", "dagger.mod", "call", "--output", "out/result.json"], + cwd=module_dir, + env={ + "PATH": "", + "PYTHONPATH": str(module_dir / "src"), + "DAGGER_DEFAULT_PYTHON_PACKAGE": "hello", + "DAGGER_MAIN_OBJECT": "Hello", + }, + input=json.dumps(request), + capture_output=True, + text=True, + check=False, + ) + + +def test_command_writes_result(module_dir: pathlib.Path): + request = { + "receiverType": "Hello", + "receiverValue": "{}", + "fnName": "hi", + "fnArgs": '{"who": "you"}', + } + proc = _call(module_dir, request) + assert proc.returncode == 0, proc.stderr + assert json.loads((module_dir / "out" / "result.json").read_text()) == "hi you" + + +def test_command_failure_writes_nothing(module_dir: pathlib.Path): + request = { + "receiverType": "Hello", + "receiverValue": "{}", + "fnName": "boom", + "fnArgs": "{}", + } + proc = _call(module_dir, request) + assert proc.returncode == 2 + assert "boom" in proc.stderr + assert not (module_dir / "out").exists() diff --git a/sdk/tests/mod/test_entrypoint.py b/sdk/tests/mod/test_entrypoint.py new file mode 100644 index 0000000..10d3a58 --- /dev/null +++ b/sdk/tests/mod/test_entrypoint.py @@ -0,0 +1,222 @@ +import enum +import hashlib +import os +import pathlib +import subprocess +import sys +import typing +from typing import Annotated + +import pytest +from typing_extensions import Doc, Self + +import dagger +from dagger import DefaultPath, Ignore, Name +from dagger.mod import Module +from dagger.mod._entrypoint import ( + _quote, + file_digest, + render_main, + render_types, + source_files, + write_entrypoint, +) +from dagger.mod._exceptions import BadUsageError + +GOLDEN = pathlib.Path(__file__).parent / "golden" + + +class Color(enum.Enum): + """A color.""" + + RED = "red" + """The red one.""" + + BLUE = "blue" + + +@pytest.fixture +def mod() -> Module: + m = Module("Main") + m.enum_type(Color) + + @m.interface + class Greeter(typing.Protocol): + @m.function + def greet(self, name: str) -> str: ... + + @m.object_type + class Helper: + """A helper.""" + + level: Color = m.field() + + @m.object_type + class Main: + """The main object.""" + + source: dagger.Directory + greeting: str = m.field(default="hello") + count: Annotated[int, Doc("How many")] = m.field(default=1, name="howMany") + + @m.function + def container(self, base: Annotated[str, Name("from")] = "alpine") -> str: + r"""A "container". + + Built from\a base image. + """ + return base + + @m.function + @m.check + def lint(self) -> None: ... + + @m.function + def helpers( + self, src: Annotated[dagger.Directory, DefaultPath("."), Ignore([".venv"])] + ) -> list[Helper]: ... + + @m.function + def mob(self, who: str | None = None) -> list[Self]: ... + + @m.function + def greeter(self, g: Greeter) -> Greeter: ... + + @m.function(deprecated="use container") + def old(self, p: dagger.Platform) -> dagger.JSON: ... + + return m + + +@pytest.fixture +def root(tmp_path: pathlib.Path) -> pathlib.Path: + (tmp_path / "dagger-module.toml").write_text('name = "main"\n') + (tmp_path / "pyproject.toml").write_text('[project]\nname = "main"\n') + (tmp_path / "src" / "main").mkdir(parents=True) + (tmp_path / "src" / "main" / "__init__.py").write_text("x = 1\n") + (tmp_path / "src" / "main" / "extra.py").write_text("y = 2\n") + for skipped in ("sdk/src/dagger", ".venv/lib", "src/main/__pycache__", ".hidden"): + (tmp_path / skipped).mkdir(parents=True) + (tmp_path / skipped / "ignored.py").write_text("z = 3\n") + return tmp_path + + +def _assert_golden(name: str, rendered: str): + path = GOLDEN / name + if os.environ.get("UPDATE_GOLDEN"): + path.write_text(rendered) + assert rendered == path.read_text() + + +def test_types_golden(mod: Module): + _assert_golden("types.dang", render_types(mod.describe())) + + +def test_main_golden(root: pathlib.Path): + rendered = render_main("main", ".dagger/modules/main", source_files(root)) + _assert_golden("main.dang", rendered) + + +def test_source_files(root: pathlib.Path): + assert [f.path for f in source_files(root)] == [ + ".python-version", + "pyproject.toml", + "requirements.lock", + "uv.lock", + "src/main/__init__.py", + "src/main/extra.py", + ] + + +def test_absent_manifest_is_recorded_without_a_digest(root: pathlib.Path): + rendered = render_main("main", ".", source_files(root)) + assert 'SourceFile(path: "uv.lock", digest: ""),' in rendered + + +def test_file_digest(): + data = b"x = 1\n" + inner = hashlib.sha256(data).digest() + assert file_digest(data) == "sha256:" + hashlib.sha256(inner).hexdigest() + + +def test_quoting(): + mod = Module("Foo") + + @mod.object_type + class Foo: + r"""Say "hi"\now. + + On a new line. + """ + + rendered = render_types(mod.describe()) + assert r'description: "Say \"hi\"\\now.\n\nOn a new line."' in rendered + assert _quote("a\tb") == r'"a\tb"' + + +def test_one_constructor_no_cache_policy(mod: Module): + rendered = render_types(mod.describe()) + assert rendered.count("withConstructor(") == 1 + assert "withCachePolicy" not in rendered + + +def test_refuses_cache_policy(): + mod = Module("Foo") + + @mod.object_type + class Foo: + @mod.function(cache="never") + def fresh(self) -> str: ... + + with pytest.raises(BadUsageError, match="cache='never'"): + render_types(mod.describe()) + + +@pytest.mark.parametrize("path", ["/abs", "../up", "a/../../b"]) +def test_path_must_stay_inside(path: str): + with pytest.raises(BadUsageError, match="relative"): + render_main("main", path, []) + + +def test_write_entrypoint(mod: Module, root: pathlib.Path): + out = root / "out" + write_entrypoint(mod.describe(), name="main", path=".", root=root, output=out) + assert (out / "types.dang").read_text().startswith("# Code generated") + assert 'SourceFile(path: "src/main/extra.py"' in (out / "main.dang").read_text() + + +def test_command(tmp_path: pathlib.Path): + (tmp_path / "pyproject.toml").write_text('[project]\nname = "hello"\n') + pkg = tmp_path / "src" / "hello" + pkg.mkdir(parents=True) + (pkg / "__init__.py").write_text( + "from dagger import function, object_type\n\n" + "@object_type\nclass Hello:\n" + " @function\n def hi(self) -> str:\n return 'hi'\n" + ) + subprocess.run( + [ + sys.executable, + "-m", + "dagger.mod", + "entrypoint", + "--name", + "hello", + "--path", + ".dagger/modules/hello", + "--output", + "out", + ], + cwd=tmp_path, + env={ + "PATH": "", + "PYTHONPATH": str(pkg.parent), + "DAGGER_DEFAULT_PYTHON_PACKAGE": "hello", + "DAGGER_MAIN_OBJECT": "Hello", + }, + check=True, + ) + types = (tmp_path / "out" / "types.dang").read_text() + assert 'typeDef.withObject("Hello")' in types + assert 'function("hi", typeDef.withKind(TypeDefKind.STRING_KIND))' in types + assert "SourceFile" in (tmp_path / "out" / "main.dang").read_text()