diff --git a/.claude/lint-rules/example_microflow.star b/.claude/lint-rules/example_microflow.star index 04bb742bee..91285bd55e 100644 --- a/.claude/lint-rules/example_microflow.star +++ b/.claude/lint-rules/example_microflow.star @@ -7,8 +7,8 @@ # VAL_ - Validation microflows # SCH_ - Scheduled event microflows # -# Customize the prefixes in lint-config.yaml rather than here (`mxcli init` -# rewrites this file): +# Customize the prefixes in lint-config.yaml rather than here (an edited copy +# of this file is kept, and no longer receives mxcli's updates to the rule): # # rules: # CUSTOM002: diff --git a/.claude/lint-rules/orphaned_elements.star b/.claude/lint-rules/orphaned_elements.star index a13b400043..b9b22f4dbf 100644 --- a/.claude/lint-rules/orphaned_elements.star +++ b/.claude/lint-rules/orphaned_elements.star @@ -18,7 +18,8 @@ # as a YAML list or comma-separated text. Replaces # the built-in list below; unset keeps it. # -# Edit the option, not this file: `mxcli init` rewrites the shipped rules. +# Edit the option, not this file: an edited copy is kept, and no longer +# receives mxcli's updates to the rule. RULE_ID = "QUAL004" RULE_NAME = "Orphaned Elements" diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-21-init-sync-overwrites-edited-bundled-lint-rule.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-21-init-sync-overwrites-edited-bundled-lint-rule.json new file mode 100644 index 0000000000..aefece7b4d --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-21-init-sync-overwrites-edited-bundled-lint-rule.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-10-21", "symptom": "`mxcli init .` and `mxcli init --sync-skills .` (run at every session start by the generated SessionStart hook) overwrite an edited bundled lint rule — `Refreshed 1 bundled lint rule(s) in .claude/lint-rules/ to match this mxcli: example_microflow.star` — and, when the edit is moved to my_prefixes.star, write the stock example_microflow.star back beside it so CUSTOM002 fires from the stock list again", "cause": "Bundled rules were recognised by file name only: `syncBundledLintRules` rewrote any bundled-name file whose bytes differed from the binary's, and `init`'s two WalkDir loops wrote every bundled file unconditionally; neither could tell an old stock copy from a project's edit, and a missing bundled name was always recreated", "file": "`cmd/mxcli/lint_rules_bundled.go` (`writeBundledLintRules`, the one write path for init, init --sync-skills and add-tool), `cmd/mxcli/lint_rules_shipped.txt` (sha256 of every version ever shipped, seeded from git history); tests `cmd/mxcli/lint_rules_bundled_test.go`", "insight": "'Is this file still ours?' needs the set of everything we ever shipped, not the current version: with only the current one, an unedited copy from any older mxcli is indistinguishable from an edit, and the choice is between overwriting edits and freezing every older project. The set is recoverable after the fact from git (every blob of the directory at every commit reachable from main, `git fetch --unshallow --filter=blob:none` first in a shallow clone); the v0.27.0 stock md5 in the report (cec8b599) matched one of them, which is the check that fork history covers what was released. Keep the set current with a test that the embedded version's hash is listed — it fails on the commit that changes a rule and prints the line to add. For the renamed copy, the reliable signal is the rule ID, not a manifest of names written: another file defining the bundled rule's RULE_ID is the edit moved, and it needs no state on disk, so it works for projects initialised before the fix. Normalise CRLF before hashing, or a Windows checkout of a stock rule reads as an edit. Two existing comments in shipped rules said '`mxcli init` rewrites this file' and became false with the fix — changing them is itself a new shipped version, which the currency test caught", "refs": ["mendixlabs/mxcli#1615", "ako/mxcli#1543", "ako/mxcli#952", "mendixlabs/mxcli#1509"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-22-ensure-db-password-protected-superuser-windows.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-22-ensure-db-password-protected-superuser-windows.json new file mode 100644 index 0000000000..41b39109b0 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-22-ensure-db-password-protected-superuser-windows.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-22","symptom":"`mxcli run --local --ensure-db` on a standard Windows PostgreSQL install: \"no local PostgreSQL superuser available to create the role/database (tried a direct 'psql -U postgres' connection over the cluster socket and TCP, and non-interactive 'sudo -u postgres' ...)\" — the installer always gives postgres a password, there is no sudo, and the error offered no way out","cause":"`resolveSuperuser` only knew passwordless paths (trust socket, peer auth via sudo) and had no way to accept an admin login; its error listed what was tried, not what to do","file":"`cmd/mxcli/docker/ensuredb.go` (`dbAdmin`, `resolveSuperuser`, `noSuperuserHint`, `superuser.psql`); flags in `cmd/mxcli/cmd_run.go`; refusal without --ensure-db in `runlocal.go` `applyDatabaseDefaults`","insight":"`--db-admin-user`/`--db-admin-password` are tried ALONE when given — falling back to sudo after they fail would hide that the user's own login was rejected, so their psql error is returned verbatim. The password goes through PGPASSWORD on the child env, never argv. **PGPASSWORD for postgres already worked before this fix** (the direct TCP candidate inherits the environment, and -w only forbids prompting, not reading PGPASSWORD/.pgpass) — measured with the pre-fix binary against a real PG16 cluster initdb'd with `-U postgres --auth=scram-sha-256 --pwfile`: no env → the reported error verbatim, PGPASSWORD=… → role and database created. So the defect was as much an undiscoverable escape hatch as a missing feature; the error now names it, the flags, and the two CREATE statements (with names filled in, the app password NOT echoed — this text lands in logs). A Linux scram-only cluster run by a non-root user is a faithful stand-in for the Windows shape: no trust socket mxcli knows of, no sudo route to a `postgres` OS user","refs":["mendixlabs/mxcli#1485","#1342"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-22-watch-reload-drops-dist-pages-prebuilt-serve.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-22-watch-reload-drops-dist-pages-prebuilt-serve.json new file mode 100644 index 0000000000..09d0487a0b --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-22-watch-reload-drops-dist-pages-prebuilt-serve.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-22","symptom":"`run --local --watch` on Mendix 11.12.5: the first hot reload after a model change (one new microflow) reports \"applied: build #2 via reload in 4.3s\" but leaves deployment/web/dist holding only index.js and chunks/ — no pages/, no widgets.css — so every page fails with \"An error occurred while loading page\" until the loop restarts. A page edit in the same state FAILS instead, with \"Could not find a part of the path '…/web/pages/.js'\"","cause":"mxbuild --serve in the prebuilt shape (mxbuild writes web/dist itself): only the FIRST build in a serve process bundles completely; it leaves no web/pages, web/layouts or bundler config for its own incremental builds. Every later build replaces dist with a 683 KB monolithic index.js + one chunk (a non-page change, or NO change) or fails (a page change). mxcli's existing guards could not see it: ensureClientServed probes index.js and its static imports (both fine), and missingPageChunks needs web/pages source, which the prebuilt shape never has","file":"`cmd/mxcli/docker/serve_recovery.go` (`serveRecovery`, `readPrebuiltBundle`, `lostIn`); `mxserve.go` (`ServeServer.Restart`); `runlocal.go` (`watchAndApply` builds through it)","insight":"Settled by driving mxbuild's /build API from a 40-line harness, no watch loop: cold → 13 pages/5.6 KB index.js; second build, model UNCHANGED → 0 pages/683 KB; after rm web/dist → same; after rm web/ → Failure; fresh process → complete. The unchanged-model row is what rules out 'something in the change' and every deployment-side repair at once. A fresh process's first build reports restartRequired=true even for an unchanged model, so recovery keeps the INCREMENTAL build's answer — otherwise each change would restart the runtime and drop sessions. The wrong turn to skip: snapshot-and-restore of the old dist. The incomplete build's index.js is a different bundle (different chunk hashes), so restoring old pages/chunks next to it mixes two module graphs. This is the same defect the 11.14 hint in webclient_legacy_paths.go described as 'nothing outside the process fixes it' — true of the deployment, false of the process; that hint now only prints if the fresh process fails too. Controls: pre-fix binary on the same app → dist = chunks index.js, `mxcli playwright check` FAIL (error banner, 2 console errors, 2 failed requests); fixed → reload in ~29s, 13 pages, check OK, and a page edit (previously a hard failure) applies with the new title in the chunk","refs":["mendixlabs/mxcli#1580","#1485"],"test":"cmd/mxcli/docker/serve_recovery_test.go"} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-22-watch-stale-theme-error-ce6093.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-22-watch-stale-theme-error-ce6093.json new file mode 100644 index 0000000000..646521e576 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-22-watch-stale-theme-error-ce6093.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-22","symptom":"`run --local --watch`: after a Sass error in a theme module is fixed, every following build fails with \"[CE6093] The theme could not be compiled\" quoting the FIRST error's line, which is no longer in the file; changing to a different error still reports the first; `touch app.mpr` does not help, `mxcli run restart` does. The app keeps serving the last good CSS","cause":"mxbuild --serve keeps its first theme compile error as process state and re-reports it as CE6093 (0.3s, no recompile) on every later build in the same process. Not mxcli, not the project: the bundled Dart Sass compiles the generated entry point cleanly at the same moment","file":"`cmd/mxcli/docker/serve_recovery.go` (`serveRecovery.themeFailed`, `observe`, `isThemeFailure`)","insight":"Isolated in one harness run against mxbuild 11.12.5's /build API: broken → 'An error occurred while compiling Theme files'; fixed, same process → CE6093 quoting the old error (twice); fresh process → Success. The trigger is 'a theme failure following a theme failure in the same process' — not 'any theme failure' (a first one is genuine and must not cost a 25s restart) and not 'CE6093' alone (it is just the later builds' spelling). A failure that survives the restart is reported from the fresh process and is current, which also fixes 'a different error still reports the first'. The serve restart is shared with #1485's recovery (same class: state in a long-lived mxbuild --serve that only a fresh process clears — see local-loop-silence 'Stale state has more than one home'). Cost the reporter's step list paid: the theme-only fix applies via runtime RESTART, because the fresh build's restartRequired is always true and there is no successful incremental build to take it from. Controls on the same app: pre-fix binary → build #3 failed [CE6093] Undefined variable, served CSS without the fix; fixed → build #3 applied, served CSS has #123456","refs":["mendixlabs/mxcli#1577","#1482"],"test":"cmd/mxcli/docker/serve_recovery_test.go"} diff --git a/.claude/skills/fix-issue/findings/mdl-backend/2026-10-21-consumed-rest-list-markers-copied-not-measured.json b/.claude/skills/fix-issue/findings/mdl-backend/2026-10-21-consumed-rest-list-markers-copied-not-measured.json new file mode 100644 index 0000000000..af80abd8cb --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-backend/2026-10-21-consumed-rest-list-markers-copied-not-measured.json @@ -0,0 +1 @@ +{"area": "mdl/backend", "date": "2026-10-21", "symptom": "TestApp round trip: re-executing `describe consumed rest service Mappings.RestClient` (a Studio Pro-authored service, nothing changed) rewrote the unit: `/Operations[0]: 3 -> 2`, `/Operations[1](Request)/Headers[0]: 3 -> 2`, `/Operations[1](Request)/Parameters[0]: 3 -> 2` — law getput, surfaced the moment `Response: json` started to parse", "cause": "consumed_rest_write.go registered marker 2 for Rest$RestOperation, Rest$HeaderWithValueTemplate and Rest$OperationParameter lists (and the mandatory empty Operations/Headers/Parameters lists), copied from the legacy serializer in the parity port and never measured; Studio Pro writes 3 for those three and 2 only for QueryParameters", "file": "`mdl/backend/modelsdk/consumed_rest_write.go` (init registrations), `mdl/backend/modelsdk/consumed_rest_edit.go` (the ALTER splice keeps the stored Operations marker); test `mdl/backend/modelsdk/consumed_rest_markers_test.go`; `mdl/roundtrip/testapp_allowlist_test.go` (Mappings.RestClient struck)", "insight": "An allowlisted failure at one law can hide a failure at the next: Mappings.RestClient sat on the allowlist as 'breaks parse' since it joined the harness, so nobody ever executed its describe output, and the wrong markers had nothing to fail against. Fixing the parse is what exposed them — expect the outcome of a struck allowlist entry to move one law down, not to go green. Two independent Studio Pro measurements settle a marker cheaply when the census projects are not at hand: the stored bytes of a Studio Pro-authored fixture (TestApp, 11.14.0) and `mx convert -p` of an mxcli-written project (11.6.6), which re-serializes exactly the non-canonical units; they agreed field for field (3/3/3, QueryParameters 2, Tags 1). A comment saying a value 'mirrors the legacy serializer' is a flag that it was never measured", "refs": ["mendixlabs/mxcli#1608", "ako/mxcli#1536", "ako/mxcli#1551", "#721"]} diff --git a/.claude/skills/fix-issue/findings/mdl-backend/2026-10-21-mcp-create-drops-flow-sides-and-notes.json b/.claude/skills/fix-issue/findings/mdl-backend/2026-10-21-mcp-create-drops-flow-sides-and-notes.json new file mode 100644 index 0000000000..8c762dbd52 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-backend/2026-10-21-mcp-create-drops-flow-sides-and-notes.json @@ -0,0 +1 @@ +{"area": "mdl/backend", "date": "2026-10-21", "symptom": "A microflow created over --mcp (Studio Pro PED, 11.15.0-rc.4) read back with every sequence flow 1 -> 3: a CASE built with `@anchor(from: bottom, Approved: (from: right), (empty): (from: bottom, to: top))` lost all three anchors and the merge's bottom exit. A script with any `@annotation` failed the create outright: `microflow object type *microflows.Annotation is not yet supported by the MCP backend`.", "cause": "buildFlowDocContent mapped a SequenceFlow to originId/destinationId/caseValue only, so Studio Pro applied its right -> left default; the splice (flowPatchOps) already sent originConnectionSide/destinationConnectionSide. mapObjectTree had no Annotation case and ObjectCollection.AnnotationFlows was never read. The splice's note line used originId/destinationId, the SequenceFlow constructor's keys. PED's AnnotationFlow is an element type that takes origin/destination plus the required originConnectionIndex/destinationConnectionIndex, and refuses the constructor keys with `\"/origin\":\"Expected reference (string), got undefined\"`. That bug was hidden because the note was refused before its line was mapped.", "file": "mdl/backend/mcp/microflow.go (buildFlowDocContent, mapObjectTree, setPEDSides, pedAnnotationFlow, adaptMicroflowSkeleton, CreateMicroflow), mdl/backend/mcp/microflow_mutator.go (flowPatchOps), sdk/microflows/microflows.go (AnnotationFlow.SidesOrDefault)", "insight": "Compare engines, not shapes: run the same script through the modelsdk engine and over --mcp, then diff each flow's (origin, destination, sides) after resolving PED's $id(/objectCollection/objects/N) references to positions. That diff covered all 10 flows (identical after the fix, every one 1 -> 3 before) and settled the bug faster than reading either writer. Read ped_get_schema per element type: 'constructor type' and 'element type' differ in key names (originId vs origin) for what looks like the same flow list. PED's read-back omits a property at its default, so a missing connection index means 0 (Top), not absent. The pre-11.14 constructor cannot type a flow, so notes are refused there: an untyped note line would be read as a SequenceFlow. Within one update PED stores an AnnotationFlow after a SequenceFlow added later, so the splice adds its note lines last; otherwise its index-addressed sets on the new sequence flow would point at the wrong flow. Still open: ALTER insert over MCP on 11.15 fails whenever it sets line/ControlVector on a flow added in the same update (`undefined is not an object (evaluating 'u.$Type')`), with or without a note.", "refs": ["ako/mxcli#1528"]} diff --git a/.claude/skills/fix-issue/findings/mdl-backend/2026-10-21-mcp-splice-addresses-elements-it-adds.json b/.claude/skills/fix-issue/findings/mdl-backend/2026-10-21-mcp-splice-addresses-elements-it-adds.json new file mode 100644 index 0000000000..1fa0d67e30 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-backend/2026-10-21-mcp-splice-addresses-elements-it-adds.json @@ -0,0 +1 @@ +{"area": "mdl/backend", "date": "2026-10-21", "symptom": "On Studio Pro 11.15.0-rc.4 every `alter microflow … { insert after … begin … end; }` over --mcp failed: `ped_update_document … undefined is not an object (evaluating 'u.$Type')`, with or without a note; a plain one-activity insert reproduced it. With that set removed, the insert landed but ped_check_errors reported `No action defined` on the new activity, and an inserted IF/ELSE came back wired to the wrong objects (the activity before it led into the else-branch log).", "cause": "flowPatchOps sent the whole insert as one update and addressed the elements it added in that same update by index: /flows/N/line/ControlVector, /objectCollection/objects/N/action, …/relativeMiddlePoint, /flows/N/isErrorHandler. PED resolves a path against the document as it was before the update, so none of those reach the new element. A set below a new flow's line dereferences a line that is not there yet and fails the whole update. Any other set is accepted and lost, and so is the same property put in the constructor value (an ActionActivity's constructor takes only x/y/caption). Separately, several index-less appends to one array in one update are stored in reverse, and $id(/…/N) resolves against that final order, so with more than one new object every flow reference pointed at the wrong one.", "file": "mdl/backend/mcp/microflow_mutator.go (SaveUnit, flowPatchOps, newFlowLineOps)", "insight": "Two direct ped_update_document probes settle what an update can address, and they are cheaper than any reading of the code. First, add an element and set one of its properties in the same update, then read it back. Second, send the same set in a separate update. The difference is the whole bug. The separate-update set always landed: control vectors, action and isErrorHandler alike. So the fix is two updates. The first is all or nothing: the new elements, wired with $id(), and the stored elements moved to make room. The second fills in the new elements. A failed second update leaves an incomplete insert in Studio Pro, the same contract as CreateMicroflow's create-then-set. That is the price of 11.15, and the error says so. For ordering, give every add the array's stored length as its index; PED keeps adds at one shared index in listed order (its documented rule for inserting several elements at a position). The 1534 finding's 'an AnnotationFlow added before a SequenceFlow is stored after it' was this reversal, not something about annotation flows. A one-activity insert hides both bugs, because it adds one object and one flow, so verify with an insert of several elements (an IF/ELSE) and compare every flow's ends, sides and vectors against the same script run locally.", "refs": ["ako/mxcli#1534"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-21-create-or-modify-rest-client-drops-operations-silently.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-21-create-or-modify-rest-client-drops-operations-silently.json new file mode 100644 index 0000000000..96e21fbcd7 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-21-create-or-modify-rest-client-drops-operations-silently.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-21", "symptom": "`create or modify consumed rest service M.S (…) { operation GetBooking (…) operation PutBooking (…) }` on a service holding PostBooking deletes PostBooking, and the only output is `Modified rest client: M.S (2 operations)`; the session log shows one CreateRestClientStmt with errors_count 0", "cause": "createRestClient builds the service's operation list from the statement alone and reports the count it wrote; nothing compared it with the stored list", "file": "`mdl/executor/cmd_rest_clients.go` (warnDroppedRestOperations, called after the write); test `mdl/executor/issue1608_consumed_rest_dropped_ops_test.go`", "insight": "Same remedy as `create or modify entity` (findings #24): a full-replace statement that drops members is doing what it says, so warn rather than refuse — but name the members, and name the incremental statement, which has to exist first (the hint was held back until `alter consumed rest service` landed, or it would have pointed at a parse error). A success line that reports the NEW count reads like confirmation; the count of what was removed is what the reader needed. The control is the same statement restating every operation: it must print no warning", "refs": ["mendixlabs/mxcli#1608", "ako/mxcli#1536"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-21-no-alter-consumed-rest-service-add-operation.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-21-no-alter-consumed-rest-service-add-operation.json new file mode 100644 index 0000000000..c6388f9169 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-21-no-alter-consumed-rest-service-add-operation.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-21", "symptom": "No MDL way to add an operation to an existing consumed REST service: `alter consumed rest service M.S add operation X (…)` fails `no viable alternative at input 'alterconsumedrest'` (also `alter rest client`, `create operation … in`), and `create or modify` naming only the new operations deleted the stored ones with only `Modified rest client: M.S (2 operations)`", "cause": "Consumed REST services had CREATE [OR MODIFY] and DROP only, and their writer (UpdateConsumedRestService) rebuilds the whole document from the semantic model, so even a restatement of every operation resets what the reader does not carry", "file": "`mdl/executor/cmd_rest_clients_alter.go` (alterRestClient), `mdl/backend/modelsdk/consumed_rest_edit.go` (EditConsumedRestService — a raw splice via UpdateRawUnitPatch), grammar `alterConsumedRestServiceAction`; tests `mdl/executor/issue1608_consumed_rest_alter_test.go`", "insight": "An ALTER built as read-semantic → modify → UpdateX(model) is a whole-document rewrite in disguise: it resets every unmodelled property of every sibling, which is exactly the loss the report wanted to avoid (an operation made in Studio Pro). Build it as a splice of the stored bson.D instead — untouched list elements pass through as stored bytes, a modified element re-encodes only the keys the statement named (each taken from the CREATE encoder, so new and created elements stay identical), and save through UpdateRawUnitPatch. Prove it with a seed only Studio Pro writes (Rest$OptionalQueryParameterUsage plus a TestValue on a query parameter): byte-compare the sibling before/after, and keep the control — a CREATE OR MODIFY from describe output resets the seed — or the seed may be one no rewrite can lose. A temporary swap of the splice for UpdateConsumedRestService failed both 'left as stored' tests. `mx check` on the result needs its own control too: a copy with one operation's Accept header removed reports CE7062, so a clean check of the real output means something", "refs": ["mendixlabs/mxcli#1608", "ako/mxcli#1536"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-21-system-enumeration-flow-parameter-written-object-type.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-21-system-enumeration-flow-parameter-written-object-type.json new file mode 100644 index 0000000000..cc48819ac0 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-21-system-enumeration-flow-parameter-written-object-type.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-21", "symptom": "Over --mcp, `create microflow M.X ($T: System.UserType) begin … end;` fails: `ped_create_document … {\"/parameters/0/entity\":\"Reference with qualified name System.UserType of type DomainModels$Entity not found.\"}`. A module enumeration (`Rules.RuleResult`) works. On the local engines the same script reports success and DESCRIBE prints a plausible bare `$T: System.UserType`, but mx check gives `[CE1613] \"The selected entity 'System.UserType' no longer exists.\"` for the parameter and for a `returns System.DeviceType`. The explicit `$T: enum System.UserType` (what DESCRIBE emits for an enumeration) was refused: \"enumeration 'System.UserType' not found for parameter 'T'\"", "cause": "The visitor types every bare qualified name in a flow parameter or return position as TypeEntity. buildMicroflowFromStmt / buildNanoflowFromStmt / execCreateRule fall back from entity to enumeration only after the entity lookup misses, and skip that lookup entirely for the System module (isBuiltinModuleEntity: its entities are not stored in the project), so a System enumeration kept the entity guess and was written as DataTypes$ObjectType with Entity \"System.UserType\". The explicit form failed separately: findEnumeration reads ListEnumerations, which never contains the System module's enumerations", "fix": "settleSystemEnumeration retypes a TypeEntity whose qualified name is in meta.SystemEnumerations to TypeEnumeration before validation, at all six sites (microflow, nanoflow, rule × parameter, return type); isKnownEnumeration accepts a System enumeration where findEnumeration cannot see it. A System entity (System.User) stays an object type", "file": "mdl/executor/cmd_microflows_create.go (settleSystemEnumeration, isKnownEnumeration), mdl/executor/cmd_microflows_build.go, mdl/executor/cmd_rules_create.go", "insight": "It was never an MCP bug: the local engines wrote the same wrong object type and only mx check (CE1613) noticed; MCP surfaced it because PED resolves references at write time. DESCRIBE hid it — it renders an enumeration type as `enum X` and an object type as bare `X`, so a bare `System.UserType` in describe output IS the symptom, not proof of a correct write. Dump the stored DataTypes$*Type (`mxcli bson dump --type microflow`) rather than reading describe. The general lesson for references into System: 'do not judge what you cannot list' covers existence, not kind — whether a name is an entity or an enumeration changes the stored shape, so it must still be decided, and modelsdk/meta.SystemEnumerations is the list to decide it from. Verified live over --mcp on Studio Pro 11.15.0-rc.4 (ped_read_document reads back DataTypes$EnumerationType for parameter and return type), with a stubbed-fix control reproducing the PED error", "refs": ["ako/mxcli#1534"], "ce": ["CE1613"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-combobox-enumeration-mode-attribute-type-ce7247.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-combobox-enumeration-mode-attribute-type-ce7247.json new file mode 100644 index 0000000000..2a327521e0 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-combobox-enumeration-mode-attribute-type-ce7247.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-22", "symptom": "`combobox cb (Attribute: Flag)` on a Boolean, no data source, passed `mxcli check --references` and `exec`, then failed mxbuild with CE7247 \"Only attributes of type Enumeration are allowed here.\" at Combo box 'cb' — and so did the same binding on String, Integer, Long, Decimal, AutoNumber, DateTime and Binary", "cause": "MDL-WIDGET39's attribute-type matrix covered the built-in inputs (CE2421) only; the pluggable combo box in enumeration mode writes its attribute to `attributeEnumeration`, whose template AllowedTypes is Enum alone, and nothing judged it", "file": "`mdl/executor/validate_widget_attribute_type.go` (`comboBoxEnumerationRule`, `inputAttributeRuleFor` — a combo box with an options data source is association mode and is skipped)", "insight": "The pluggable template states what a slot takes: `attributeEnumeration`'s ValueType.AllowedTypes is [Enum], and the combo box has a SEPARATE `attributeBoolean` slot (optionsSourceType `boolean`) that MDL never writes — so a Boolean combo box is not unsupported by the widget, only by the writer. Read AllowedTypes first, then measure: one combo box per type, each with its own name, on fresh 11.14.0 and 11.13.0 projects gave the same 8 CE7247s and a clean Enumeration control. The error code differs from the built-in inputs' (CE7247 vs CE2421) with the same message text, so the rule carries the code per widget. Lives in the --references tier with the rest of MDL-WIDGET39, so its bug-test is plain .mdl.", "refs": ["mendixlabs/mxcli#1466", "ako/mxcli#1390"], "ce": ["CE7247"], "rules": ["MDL-WIDGET39"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-create-members-decode-every-domain-model.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-create-members-decode-every-domain-model.json new file mode 100644 index 0000000000..5726b0e13d --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-create-members-decode-every-domain-model.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-10-22","symptom":"`exec`/`check` of flows full of `create M.E (A1 = …, … A8 = …)` slowed with the number of modules in the project, not with the script: 200 creates of 8 members 14 s on a fresh app, 294 s after 30 modules × 20 entities were added; no member assignments 3 s. Cripples `mxcli test`, whose injected test flows are mostly such creates","cause":"Two factors multiplied. (1) `Backend.GetDomainModel(moduleID)` called `ListUnitsWithContainer[*DomainModel]`, decoding every module's domain model, then kept the one whose container matched; inherited persistability (#1600) also listed every module. (2) The flow builder calls `GetModuleByName` + `GetDomainModel` per member assignment (`resolveMemberChange` → `resolveAttributeInEntityHierarchy`, eight more sites), unmemoised, and the same build runs again in exec's preflight","file":"`modelsdk/mpr/reader_units.go` + `reader_units_v1.go` (`ListUnitsByTypeInContainer`, filters by container before reading contents), `modelsdk/mprread/generic.go` (`ListUnitsInContainer`), `mdl/backend/modelsdk/domainmodel.go` (`GetDomainModel`, lazy `persistabilityIndex`), `mdl/executor/cmd_microflows_derived_layout_backend.go` (`flowBuildBackend`, wired in `cmd_microflows_build.go` and `cmd_alter_flow.go`)","insight":"Fixing the backend alone was not enough and the counter showed it: with GetDomainModel narrowed, 160 member assignments still cost 4001 extra unit-file reads on pedapp, ~25 each, almost all module listings from `GetModuleByName`. The memo already existed (`layoutCheckBackend`, built for the describe layout rounds); a flow build is the same kind of span — the builder only reads, the flow is stored after — so wrapping its backend is safe and took it to 11. Measure with a call-free control (the same flow with the member assignments left out) and assert extra reads < assignments, not wall time. The narrowed lookup must agree with the full listing: `TestGetDomainModel_AgreesWithTheFullListing` deep-compares every module, and the cross-module generalization chain is the one path only the lazy index takes, so it gets its own test with a control (loader stubbed → reads persistent). Issue's script on pedapp + 30 modules: exec 142.6 s → 1.6 s, check 40.3 s → 1.8 s","refs":["mendixlabs/mxcli#1616","ako/mxcli#1549","mendixlabs/mxcli#1600"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-describe-folds-item-slots-into-body.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-describe-folds-item-slots-into-body.json new file mode 100644 index 0000000000..0682fd033b --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-describe-folds-item-slots-into-body.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-22", "symptom": "an Accordion group with widgets in headerContent and content describes them all as the group's body; regenerating the page from the description stores the header widget in content (headerContent -> [txtHead], content -> [txtBody] became content -> [txtHead, txtBody]); and `headercontent { … }` inside a group fails check with MDL-WIDGET25 \"not a widget in this project\"", "cause": "extractObjectListItem appended every Widgets-typed sub-property of an item to one Children list; the widget-kind validator resolved no parent definition for a child of an object-list ITEM, so the item's own slot keyword fell through to the unknown-widget error even though the item builder's slot pass routes it", "file": "`mdl/executor/cmd_pages_describe_objectlist.go` (Slots, promoteBodySlot, objectListItemWidgetSlotKeys), `cmd_pages_describe_output.go`, `widget_engine.go` (bodyItemSlotKey shared with defaultItemSlotKey), `validate_widgets.go` / `validate_widget_kind.go` (parentItem, itemDeclaresSlot)", "insight": "Updating the existing page reported 'Unchanged page' and hid the loss — the round trip has to be run against a FRESH app, which is the regenerate-from-describe case the bug actually hurts. Map each widget to its slot by TypePointer -> PropertyKey in the JSON dump; the ndsl view does not show which slot a widget is in. The body rule (content, else first widget slot) now lives in one function both CREATE and DESCRIBE call. The fix needed both halves: an honest describe whose output check refuses is no better than a silent one", "refs": ["mendixlabs/mxcli#1453", "ako/mxcli#1326"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-describe-module-all-large-module.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-describe-module-all-large-module.json new file mode 100644 index 0000000000..a9f4facf44 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-describe-module-all-large-module.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-10-22","symptom":"`describe module M with all` on a module of 229 microflows hit `statement timed out` after 5 minutes; with MXCLI_EXEC_TIMEOUT=40m it stopped at `output line limit exceeded (10000 lines) ... statement aborted`, truncated mid-flow","cause":"Three things. (1) The default 5-minute wall-clock guard and (2) the fixed 10,000-line output guard, both meant for runaway statements, applied to a dump that is bounded by the module. (3) The time: describing a microflow derives its layout by rebuilding it (`derivedFlowLayout` → `buildMicroflowFromStmt` → `microflowsNamed`) with `ctx.Backend` swapped for `layoutCheckBackend`, which embeds the `FullBackend` interface and so hid the modelsdk backend's optional `microflowLookup`; `microflowsNamed` fell back to `ListMicroflows`, decoding every microflow of the project once per described flow — quadratic in the module, linear in the project","file":"`mdl/executor/executor.go` (`isModuleDump`, `effectiveOutputLineLimit`, MXCLI_OUTPUT_LINE_LIMIT, per-describe `CacheUnitReads` scope), `mdl/executor/output_guard.go` (per-statement limit, 0 = none), `mdl/executor/cmd_microflows_derived_layout_backend.go` (memoised `ListMicroflowsNamed`/`ListMicroflowHeaders`)","insight":"Lifting the limits alone would have turned a timeout into a 40-minute success. The cost was found by CPU profile, not guessed: 63% of a 229-flow dump in `ListMicroflows` → `codec.Decode` under `derivedFlowLayout`, while the unit-file read count (2069 → 369 with a statement-scoped `CacheUnitReads`) moved wall time not at all — reads were not the cost. A wrapper that embeds an interface drops every optional capability of what it wraps: any `ctx.Backend.(X)` check inside the span silently takes the slow fallback, and nothing fails. Grep the type assertions against `ctx.Backend` whenever the backend is swapped for a decorator. Guard with a count (full listings per dump < flows, via a counting backend that embeds the concrete `*modelsdkbackend.Backend` so it keeps the optional lookups) and prove output unchanged: the 229-flow dump is byte-identical to main's. Real binaries on pedapp + 400 flows: 66.9 s exit 1 at 10,025 lines → 5.7 s, 12,810 lines","refs":["mendixlabs/mxcli#1624","ako/mxcli#1556","mendixlabs/mxcli#1425","mendixlabs/mxcli#651"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-page-rewrite-drops-unauthorable-widgets.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-page-rewrite-drops-unauthorable-widgets.json new file mode 100644 index 0000000000..08329417e8 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-page-rewrite-drops-unauthorable-widgets.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-22", "symptom": "`describe page M.P` prints Forms$DataGrid, Forms$ReferenceSelector, Forms$DropDown, Forms$InputReferenceSetSelector as `-- NOT re-executable` comments, and `exec` of that output replaces the page with the widgets gone — `Replaced page M.P`, no warning, mx check clean on both sides (found on a Mendix 10.24 MPR v1 app)", "cause": "execCreatePageV3 / execCreateSnippetV3 / execCreateLayout rebuild the document from the statement, and a widget MDL cannot author can never be in the statement; describe's NOT re-executable note was the only signal, and it is a comment the parser discards", "file": "`mdl/executor/cmd_pages_rewrite_guard.go` (checkNoUnauthorableWidgetsDropped, validateUnauthorableRewrites), `cmd_pages_describe_dangling.go` (describeDroppedWidget feeds unauthorableSink), `cmd_pages_create_v3.go`, `cmd_pages_layout_v3.go`, `cmd_pages_describe.go` (layout body through describeWidgetBody); tests `mdl/executor/issue1622_page_rewrite_unauthorable_test.go`", "insight": "A describe note that says 'NOT re-executable' is a promise the executor never read: the comment is dropped by the parser, so the warning lives only in the file the user is about to run. Make the describer the single source of what a rewrite loses — describeDroppedWidget already ran for exactly the unauthorable widgets — and have the rewrite describe the stored document into io.Discard and refuse on that list, rather than keeping a second list of authorable $Types that would drift from the describe switch (the staticimage finding is that drift). Exempt any widget the statement declares by name, or a widget the builder can write but describe cannot print refuses its own script. No fixture holds these widgets (grep of testdata finds none), so seed one into a real stored unit with UpdateRawUnit — the #1459 test's legacyDataGridBSON is the Studio Pro shape — and test at exec, check -p and the ALTER-drop escape hatch. Enumerate the rebuild call sites: snippets and layouts take the same builder; layout describe had to be routed through describeWidgetBody before the sink could see it", "refs": ["mendixlabs/mxcli#1622", "ako/mxcli#1554", "mendixlabs/mxcli#1459", "mendixlabs/mxcli#1415"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-pluggable-attribute-of-page-parameter-ce7247.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-pluggable-attribute-of-page-parameter-ce7247.json new file mode 100644 index 0000000000..6417dd5596 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-22-pluggable-attribute-of-page-parameter-ce7247.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-10-22","symptom":"A Custom chart's eventDataAttribute bound to an attribute of another page parameter ($Ctx) inside a data view over $Entry had no spelling that built: `eventDataAttribute: $Ctx.EventData` was MDL-WIDGET42 \"is a attribute property and cannot hold the expression $Ctx.EventData\"; the qualified `MyFirstModule.ChartContext.EventData` passed check and exec and mx check said CE7247 \"Attribute 'MyFirstModule.ChartContext.EventData' is not an attribute of entity 'MyFirstModule.ChartEntry'\"; DESCRIBE of the Studio Pro-authored binding printed the qualified name, re-executing into the CE7247 shape.","cause":"A generic pluggable key's value goes through the expression grammar, so `$Ctx.EventData` arrives as *ast.WidgetExpressionV3 and scalarPropertyText refused every expression on a non-Expression property; the engine's `$Param.Attr` -> AttributeRef + SourceVariable path existed only for mapped keys. The explicit-key pass stored any name with two dots as written, and misboundAttributeError skipped every qualified name. The first #1591 fix made describe print the qualified name back faithfully, which pinned the CE7247 shape in the bug-test MDL.","fix":"`$Param.Attr` (strict shape: one variable, one attribute name) is accepted on attribute properties by exec and check, mapped or authored by key, and written as AttributeRef + Forms$PageVariable{PageParameter/SnippetParameter}; check also lets it through on a key whose kind is unknown (no project), since it is no Mendix expression. A qualified attribute whose entity is not the bound entity or a generalization of it is refused by the shared misboundAttributeError (check and exec, word for word), naming CE7247 and `$Param.Attr`. The generic pluggable describer prints a parameter-sourced attribute as `$Param.Attr`. mx check 11.12.5: 0 errors; describe -> exec round trip: Unchanged.","insight":"A describe fix that makes the output 'faithful' to a stored shape is only half the question: run mx check on the shape it re-creates. Here the qualified name round-tripped byte for byte and still did not build, because what Studio Pro stores for the binding is a pair (AttributeRef + SourceVariable) and MDL had a spelling for the pair only on mapped keys. Wrong turn to skip: tightening the scope check alone would have left the reporter with no spelling at all; the refusal needs the spelling it points at. namedObjectBinding accepts `$Ctx.Name + 'x'` (it only cuts at the first dot), so the expression/attribute decision needs a strict shape test; the control expression case in the test caught it.","issue":"mendixlabs/mxcli#1591","refs":["ako/mxcli#1502"],"ce":["CE7247","CE1613"],"rules":["MDL-WIDGET42"],"file":"mdl/executor/widget_engine.go (explicit-key Attribute pass), mdl/executor/widget_attribute_scope.go (foreignQualifiedAttributeError), mdl/executor/widget_expression_value.go, mdl/executor/validate_widget_expression_value.go, mdl/executor/cmd_pages_describe_pluggable.go (extractExplicitProperties)","test":"mdl/executor/widget_parameter_attribute_1591_test.go, mdl-examples/bug-tests/1591-pluggable-attribute-outside-data-view-entity.mdl"} diff --git a/.claude/skills/fix-issue/findings/mdl-grammar/2026-10-21-consumed-rest-describe-response-json-does-not-reparse.json b/.claude/skills/fix-issue/findings/mdl-grammar/2026-10-21-consumed-rest-describe-response-json-does-not-reparse.json new file mode 100644 index 0000000000..0b834abeef --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-grammar/2026-10-21-consumed-rest-describe-response-json-does-not-reparse.json @@ -0,0 +1 @@ +{"area": "mdl/grammar", "date": "2026-10-21", "symptom": "`describe consumed rest service` of an operation created with `Response: json as $Result` prints `Response: json`, and running the output back fails `line 14:2 missing VARIABLE at ')'` plus `property 'Response' on a REST client operation takes none or json as $var …`; string/file/status responses described as `Response: string as ` with nothing after it", "cause": "The grammar required `(FROM | AS) VARIABLE` after the response kind, but the variable is not part of the stored operation (Rest$RestOperation has no response variable; the call activity names it), so the reader can never supply one. #192 fixed the empty `as ;` for json by printing the bare kind, which the grammar had never accepted", "file": "`mdl/grammar/domains/MDLService.g4` (restClientOpProp), `mdl/visitor/visitor_rest.go` (parseRestClientOpProp), `mdl/visitor/visitor_strict_properties.go` (Response shapes), `mdl/executor/cmd_rest_clients.go` (outputRestOperation); tests `mdl/executor/issue1608_consumed_rest_response_reparse_test.go`, `issue1608_consumed_rest_ops_test.go`", "insight": "A describe fix that changes what is PRINTED needs the round-trip asserted through the real parser, not a substring: #192's test checked the output no longer contained `as ;` and so passed while the new form was a parse error. The cheapest detector is a table over every enum value the reader can produce (here five response kinds) fed through outputX → visitor.Build → buildX, which found three more kinds with the same defect. When a property of the language is not stored at all, the fix is to make the language's bare form valid, not to invent a value on read", "refs": ["mendixlabs/mxcli#1608", "ako/mxcli#1536", "mendixlabs/mxcli#192"]} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-10-21-openapi-swagger2-parses-to-empty-contract.json b/.claude/skills/fix-issue/findings/mdl-other/2026-10-21-openapi-swagger2-parses-to-empty-contract.json new file mode 100644 index 0000000000..2c72b05471 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-10-21-openapi-swagger2-parses-to-empty-contract.json @@ -0,0 +1 @@ +{"area": "mdl/openapi", "date": "2026-10-21", "symptom": "`create consumed rest service … ( OpenAPI: '…swagger.json' )` / `describe contract operation from openapi` on a Swagger 2.0 spec reports success and produces `BaseUrl: ''`, no request bodies and `Response: none` on every operation, with no warning", "cause": "`openapi.ParseSpec` unmarshals into OpenAPI 3 types only; a 2.0 document has none of those fields where they are expected (host/basePath, response `schema`, `in: body`, `definitions`), so unmarshalling succeeds and yields an empty contract", "file": "`mdl/openapi/parser.go` (`Spec.Swagger`, `ParseSpec`)", "insight": "**A permissive unmarshal is a silent format detector that always answers yes**: JSON/YAML into a struct ignores every unknown field, so the wrong document VERSION parses cleanly and degrades into defaults. Read the version discriminator (`swagger` vs `openapi`) before trusting the parse. The reported issue (#280, missing response entities) hid this: the reporter's spec was 2.0, so even response TYPES were lost, which nobody had noticed because the doctype test only ever used the 3.0 Petstore. When testing an importer, run the reporter's actual input before reasoning about the feature gap. Refusal with a conversion hint (swagger2openapi / converter.swagger.io) beats a half-import; full 2.0 normalisation is Phase 4 of PROPOSAL_openapi_response_entities.md. Test `TestParseSpec_RefusesSwagger2` (JSON and YAML) with an OpenAPI 3.0/3.1 control", "refs": ["mendixlabs/mxcli#280", "ako/mxcli#1234"]} diff --git a/.claude/skills/fix-issue/findings/modelsdk/2026-10-22-v2-listing-opens-every-unit-file.json b/.claude/skills/fix-issue/findings/modelsdk/2026-10-22-v2-listing-opens-every-unit-file.json new file mode 100644 index 0000000000..ba76a77b5e --- /dev/null +++ b/.claude/skills/fix-issue/findings/modelsdk/2026-10-22-v2-listing-opens-every-unit-file.json @@ -0,0 +1 @@ +{"area":"modelsdk/mpr","date":"2026-10-22","symptom":"On MPR v2, `describe microflow` of one flow opened 393 of a blank app's 394 unit files (counted by atime), and the next listing after any write opened them all again; wall time is fine on a small app, the cost is per file open and scales with the project","cause":"`buildUnitCache` learnt each unit's `$Type` by `os.ReadFile` of its .mxunit and threw the bytes away, because the v2 Unit table has no type column; `InvalidateCache` (every write) dropped the whole cache, and nothing persisted between processes. The `listUnitTypes` doc comment claimed an index that did not exist","file":"`modelsdk/mpr/reader_units_v2.go` (ContentsHash → $Type index, persisted to `.mxcli/unit-types.idx`), `modelsdk/mpr/reader_units.go` (`buildUnitCache`)","insight":"Key the type index by ContentsHash, not by unit ID: a content-addressed entry can never be stale, so the index needs no invalidation and survives writes, re-types in place (#503) and other processes; every v2 writer, ours included (`writer_core.go`), keeps the hash in step with the file. The v1 counterpart (`reader_units_v1.go`) keys by blob length for the same reason. Persist on a reader's first rebuild and on Close, not after every write-triggered rebuild, or a 200-statement script rewrites the index 200 times. Measure with `strace -f -e trace=openat` counting distinct .mxunit paths, not wall time: on pedapp (375 files) 375 -> 117, and the remaining 117 are per-type reads the describe really makes (folders for the path, every microflow to find one by name), which is a different cost","refs":["mendixlabs/mxcli#1603","ako/mxcli#1513"]} diff --git a/.claude/skills/mendix/alter-page/SKILL.md b/.claude/skills/mendix/alter-page/SKILL.md index 91c1741caa..3538b76230 100644 --- a/.claude/skills/mendix/alter-page/SKILL.md +++ b/.claude/skills/mendix/alter-page/SKILL.md @@ -11,16 +11,16 @@ description: "Modify an existing page or snippet's widget tree in place with ALT | lines | section | |---|---| | 28-31 | Overview | -| 32-56 | When to Use | -| 57-107 | Syntax | -| 108-473 | Operations | -| 474-552 | Examples | -| 553-643 | DataGrid 2 columns: how to address them, and what you can set | -| 644-655 | Common Mistakes | -| 656-684 | Limitations — prefer binding at page creation (ledger finding #45) | -| 685-692 | Validation Checklist | -| 693-702 | Related Commands | -| 703-707 | Related Skills | +| 32-64 | When to Use | +| 65-115 | Syntax | +| 116-481 | Operations | +| 482-560 | Examples | +| 561-651 | DataGrid 2 columns: how to address them, and what you can set | +| 652-663 | Common Mistakes | +| 664-692 | Limitations — prefer binding at page creation (ledger finding #45) | +| 693-700 | Validation Checklist | +| 701-710 | Related Commands | +| 711-715 | Related Skills | # ALTER PAGE / ALTER SNIPPET - Modify Existing Pages and Snippets @@ -54,6 +54,14 @@ MDL scripts created and nobody has changed in Studio Pro since: on a Studio Pro round trip has dropped translations and filled an empty English caption from another language, and on a snippet it has dropped the snippet's type. +**A page holding a widget MDL cannot author refuses the rewrite.** `describe` prints a +classic data grid, a reference selector, a drop-down or a reference set selector as a +`-- NOT re-executable` comment, so `create or modify page` (also `create or replace`, and +the same for snippets and layouts) would delete it. The statement is refused, naming each +widget; `mxcli check -p` reports the same. Use `alter page` — it leaves those widgets in +place — or, to rebuild the page without them, drop them first on purpose: +`alter page M.P { drop grid1, referenceSelector1; };`. + ## Syntax ```sql diff --git a/.claude/skills/mendix/create-page/reference/widgets.md b/.claude/skills/mendix/create-page/reference/widgets.md index a8fddacc31..7d24ce3668 100644 --- a/.claude/skills/mendix/create-page/reference/widgets.md +++ b/.claude/skills/mendix/create-page/reference/widgets.md @@ -544,6 +544,9 @@ A text box binds no dates (CE2421), so `DateFormat` on it is dropped and warned ```sql -- Enumeration mode (attribute is an enum type): combobox cbCountry (label: 'Country', attribute: Country) +-- WRONG: enumeration mode on any other type — a Boolean included — is CE7247 +-- "Only attributes of type Enumeration are allowed here." (MDL-WIDGET39 with -p). +-- For a Boolean use `checkbox` or `radiobuttons`. -- Association mode: bind a reference. Requires the option DataSource (the target -- entity whose objects fill the dropdown) AND a CaptionAttribute (display value). diff --git a/.claude/skills/mendix/rest-client/SKILL.md b/.claude/skills/mendix/rest-client/SKILL.md index e13ca14111..d5525a77da 100644 --- a/.claude/skills/mendix/rest-client/SKILL.md +++ b/.claude/skills/mendix/rest-client/SKILL.md @@ -11,15 +11,15 @@ description: "Call external REST APIs from Mendix — the three approaches (inli | lines | section | |---|---| | 29-44 | Three Approaches | -| 45-91 | Approach 0: OpenAPI Import (Fastest) | -| 92-288 | Approach 1: REST Client (Manual) | -| 289-334 | Approach 2: REST CALL (Inline HTTP) | -| 335-367 | Data Transformers (JSLT — Mendix 11.9+) | -| 368-405 | JSON Structures & Mappings | -| 406-477 | Complete Pipeline Example | -| 478-508 | Published REST: Authentication Parameters | -| 509-518 | Mendix Validation Rules | -| 519-539 | BSON Types Reference | +| 45-110 | Approach 0: OpenAPI Import (Fastest) | +| 111-337 | Approach 1: REST Client (Manual) | +| 338-383 | Approach 2: REST CALL (Inline HTTP) | +| 384-416 | Data Transformers (JSLT — Mendix 11.9+) | +| 417-454 | JSON Structures & Mappings | +| 455-526 | Complete Pipeline Example | +| 527-557 | Published REST: Authentication Parameters | +| 558-567 | Mendix Validation Rules | +| 568-588 | BSON Types Reference | # REST Integration Skill @@ -44,7 +44,10 @@ No API to call against yet — or one you would rather not depend on while build ## Approach 0: OpenAPI Import (Fastest) -If the API has an OpenAPI 3.0 spec (JSON or YAML), generate the REST client in one command: +If the API has an OpenAPI 3.x spec (JSON or YAML), generate the REST client in one command. +A **Swagger 2.0** spec (`"swagger": "2.0"`, e.g. `petstore.swagger.io/v2/swagger.json`) is +refused — convert it first (`npx swagger2openapi in.json -o out.json`, or +https://converter.swagger.io) and import the converted file: ```sql mdl 1; @@ -87,6 +90,22 @@ describe contract operation from openapi 'specs/capital.json'; **After import:** the REST client is ready to use with `SEND REST REQUEST`. No manual operation definition needed. +**But every JSON response is imported as `response: json` — a raw string, not objects.** The +import does not generate response entities yet (ako/mxcli#1234). To get objects, create the +non-persistent entities yourself and give the operations you call a mapping (see *Response +Types* below): + +```sql +mdl 1; +alter consumed rest service PetStoreModule.PetStoreAPI + modify operation getPetById ( + response: mapping PetStoreModule.Pet { Id = id, Name = name } + ); +``` + +Writing the entities and an import-mapping document but leaving the operation at +`response: json` is the common mistake: nothing uses the mapping, and nothing reports that. + --- ## Approach 1: REST Client (Manual) @@ -178,7 +197,10 @@ call rest service post 'https://api.example.com/upload' ( ### Response Types ```sql --- Simple types (variable binding is at the call site, not stored on the operation) +-- Simple types. The variable is optional and is NOT stored — the call site +-- binds it — so `describe` prints the bare kind (`response: json`), and that +-- form runs again as written. +response: json response: json as $Result response: string as $text response: file as $Document @@ -281,9 +303,36 @@ end; list consumed rest services [in module]; describe consumed rest service Module.ClientName; drop consumed rest service Module.ClientName; -create or modify consumed rest service Module.ClientName ... -- idempotent +create or modify consumed rest service Module.ClientName ... -- idempotent; REPLACES the operation list ``` +### Change an Existing Client — `alter`, Not `create or modify` + +`create or modify` declares the client's whole operation list: an operation it +does not list is **deleted** (the output names it). To add, change or remove +one operation — above all on a client made in Studio Pro, whose operations carry +settings MDL cannot express — use `alter`. Every operation it does not name is +written back exactly as stored. + +```sql +mdl 1; + +alter consumed rest service Module.BookingApi + add operation GetBooking ( + method: get, + path: '/bookings/{id}', + parameters: ($id: string), + response: json + ), + modify operation PostBooking (timeout: 60), -- only the keys listed change + drop operation LegacyBooking, + set (BaseUrl: 'https://api.example.com/v2'); -- BaseUrl, Authentication +``` + +`add` refuses a name that exists (use `modify`); `modify` and `drop` refuse one +that does not, listing the operations there are. Actions apply in order, so +`drop operation X, add operation X (…)` replaces X whole. + --- ## Approach 2: REST CALL (Inline HTTP) diff --git a/.claude/skills/mendix/run-local/SKILL.md b/.claude/skills/mendix/run-local/SKILL.md index ccb7296a37..9c1d086e81 100644 --- a/.claude/skills/mendix/run-local/SKILL.md +++ b/.claude/skills/mendix/run-local/SKILL.md @@ -12,27 +12,27 @@ description: "The warm Docker-free dev loop, `mxcli run --local` — model chang |---|---| | 40-47 | Overview | | 48-60 | When to Use This Skill | -| 61-116 | Usage | -| 117-129 | How apply is chosen | -| 130-195 | Prerequisites | -| 196-213 | The intended loop | -| 214-238 | Editing themes (SCSS) — rebuild, never clear caches | -| 239-265 | `--watch` waits for a write to finish before it builds | -| 266-294 | "My edit didn't show up" — stale process, not stale cache | -| 295-324 | Flags | -| 325-378 | Metrics and OpenTelemetry | -| 379-436 | Pages render in the browser | -| 437-454 | Pixel-perfect page loop | -| 455-475 | Debugging a server-side error | -| 476-496 | "Sign in failed" that is not about the password | -| 497-529 | The same licence also stops the app after a few hours | -| 530-609 | External browser preview (`--hub`) | -| 610-618 | Validation checklist | -| 619-658 | Constant values come from a configuration | -| 659-662 | Warm local dev loop (`mxcli run --local [--watch] [--screenshot]`) | -| 663-666 | External browser preview (`mxcli run --hub ` + `mxcli tunnel-hub`) | -| 667-670 | Tunnel-hub GitHub authentication (opt-in, gated on `--github-oauth-client-id`; absent = today's open hub) | -| 671-673 | Runtime metrics + settings passthrough (`mxcli run --local --metrics` / `--runtime-setting Key=Value`) | +| 61-118 | Usage | +| 119-131 | How apply is chosen | +| 132-201 | Prerequisites | +| 202-219 | The intended loop | +| 220-247 | Editing themes (SCSS) — rebuild, never clear caches | +| 248-274 | `--watch` waits for a write to finish before it builds | +| 275-303 | "My edit didn't show up" — stale process, not stale cache | +| 304-334 | Flags | +| 335-388 | Metrics and OpenTelemetry | +| 389-446 | Pages render in the browser | +| 447-464 | Pixel-perfect page loop | +| 465-485 | Debugging a server-side error | +| 486-506 | "Sign in failed" that is not about the password | +| 507-539 | The same licence also stops the app after a few hours | +| 540-619 | External browser preview (`--hub`) | +| 620-628 | Validation checklist | +| 629-668 | Constant values come from a configuration | +| 669-672 | Warm local dev loop (`mxcli run --local [--watch] [--screenshot]`) | +| 673-676 | External browser preview (`mxcli run --hub ` + `mxcli tunnel-hub`) | +| 677-680 | Tunnel-hub GitHub authentication (opt-in, gated on `--github-oauth-client-id`; absent = today's open hub) | +| 681-683 | Runtime metrics + settings passthrough (`mxcli run --local --metrics` / `--runtime-setting Key=Value`) | # Warm Local Dev Loop — `mxcli run --local` @@ -104,8 +104,10 @@ What each one does, precisely: returns the first build made from that source or newer. A change the loop applied **before** you called `wait` is reported at once — it never waits for a "next" change that already happened. With nothing changed it reports the last build. - A failure prints the CE codes / SCSS error lines; on 11.14 the second-build defect - is reported as a failure with its explanation, never as a hang. A run started + A failure prints the CE codes / SCSS error lines, never a hang. Where mxbuild + writes the client itself (11.12.5, 11.14) its serve process only builds + completely once, so the loop restarts it per change (one line says so; ~25–30 s + instead of ~5 s, still a hot reload). A run started without `--watch` never applies a change: `wait` says so and exits 1. - **`run stop`** asks the run to shut down (it removes `--test-endpoint`, turns the debugger off), waits until it is gone, then sweeps the run's process session, so an @@ -143,6 +145,10 @@ association catalog only at startup; behavioural changes are hot-reloaded. permits only that target — the devcontainer default. Both are non-interactive, so a run never blocks on a password prompt; where sudo is unavailable the user-owned cluster still carries it. + On **Windows** the installer always gives `postgres` a password and there is no + sudo: pass `--db-admin-user postgres --db-admin-password ` (or set + `PGPASSWORD` for postgres). The admin login is used only to create the role and + database, and is refused without `--ensure-db`. The user-owned cluster persists across sessions; its server log is `~/.mxcli/postgres/server.log`. Stop it with `pg_ctl -D "$HOME/.mxcli/postgres/data" stop`. To remove it, stop it first and @@ -219,7 +225,10 @@ the loop rebuilds and hot-applies. Without `--watch`, nothing is watched, so the does nothing until you restart `run --local` (or re-run with `--watch`). `mxbuild --serve` recompiles the theme on its next build and correctly picks up SCSS content changes, so **never** `rm -rf theme-cache/ .mendix-cache/ deployment/` — clearing -caches is a red herring. +caches is a red herring. The one stale state is the serve process's own: after a +Sass error it reports `[CE6093]` with that *first* error on every later build, even +once fixed. `--watch` restarts `mxbuild --serve` when a theme failure follows +another (one line says so), so the error you see after that is the current one. A **theme compile error fails the build step, before the runtime starts** — the error text is in the serve `/build` output (printed by `run --local`), not a @@ -303,6 +312,7 @@ run's process session), and `run status` reports them. Do **not** | `--watch` | off | Rebuild + hot-apply on each change | | `--web-client-timeout` | `$MXCLI_WEB_CLIENT_TIMEOUT`, else `5m` | Limit for one web client bundle build; on timeout the tail of `deployment/log/web-client-build.log` is printed | | `--ensure-db` | off | Provision local Postgres + app database if missing | +| `--db-admin-user` / `--db-admin-password` | postgres / — | Superuser login for `--ensure-db` when it has a password (Windows); only with `--ensure-db` | | `--app-port` / `--admin-port` / `--serve-port` | 8080 / 8090 / 6543 | Ports | | `--listen-addr` | `127.0.0.1` | IP the app binds to. `0.0.0.0` (or `::`) makes it reachable from other machines; the network URLs are printed once it runs. If the app builds absolute URLs (SSO redirects, email links), pair it with `--runtime-setting ApplicationRootUrl=http://:/` (or a real host in the configuration): the runtime does not derive one from the bind address — a warning says so. The M2EE admin API always stays on loopback | | `--db-host` / `--db-name` / `--db-user` / `--db-password` | 127.0.0.1:5432 / derived / mendix / mendix | Database; bracket IPv6 endpoints (`[::1]:5432`) | diff --git a/.gitignore b/.gitignore index d8249d8b7a..5b15d750c9 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,6 @@ snap-bson # CI test-output capture (.github/workflows/push-test.yml); also created if you # run those step bodies locally. go-test-output.txt +# MPR v2 unit-type index, written next to any project mxcli opens — including +# the testdata/ fixtures tests open in place. +.mxcli/unit-types.idx diff --git a/CHANGELOG.md b/CHANGELOG.md index ea8ca9c59d..7304aa6130 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **Rewriting a page, snippet or layout no longer deletes the widgets MDL cannot author** (mendixlabs/mxcli#1622) — `describe page` prints a classic data grid, a reference selector, a drop-down or a reference set selector as a `-- NOT re-executable` comment, so executing that output rebuilt the page without them and said only `Replaced page M.P`, with `mx check` clean on both sides. `create or modify` / `create or replace` of a stored page, snippet or layout that holds such a widget — one the statement does not declare by name — is now refused, listing each widget, with the `alter` that edits the document while keeping them and the `alter … { drop … }` that removes them on purpose first. `mxcli check -p` reports the same before anything is written, following an earlier `alter` of the document in the same script. +- **A consumed REST service is written with Studio Pro's list markers** — `Operations`, an operation's `Headers` and `Parameters` are typed-array marker 3 (only `QueryParameters` is 2), measured on Studio Pro-authored services (11.14.0) and on `mx convert` output (11.6.6). mxcli wrote 2, copied from the legacy serializer, so running a Studio Pro-made service's `describe` output back rewrote it although nothing had changed — visible once that output parsed (mendixlabs/mxcli#1608). +- **`create or modify consumed rest service` names the operations it removes** (mendixlabs/mxcli#1608) — the statement declares the service's whole operation list, so one naming only the operations to add deleted the stored ones — often made in Studio Pro, with their headers, timeouts and mappings — and the output said only `Modified rest client: MyFirstModule.BookingApi (2 operations)`. It still applies, as `create or modify entity` does for attributes, and now prints `⚠ create or modify consumed rest service MyFirstModule.BookingApi drops 1 existing operation(s) not listed in this statement: PostBooking` with the `alter consumed rest service … add operation` that adds one without touching the rest. +- **`describe consumed rest service` output runs again: `Response: json` parses** (mendixlabs/mxcli#1608) — an operation created with `Response: json as $Result` describes as `Response: json`, because the variable is not stored (a consumed REST operation has none; the call activity names it), and the parser rejected that form with `missing VARIABLE at ')'`, so describe → edit → `create or modify` was not available for any such service. `Response: json | string | file | status` now takes `as $var` as optional, and describe prints `string`, `file` and `status` responses bare too instead of an unparseable `as ` with nothing after it. +- **`mxcli init` and `init --sync-skills` keep a bundled lint rule you edited, and do not recreate one you renamed** (mendixlabs/mxcli#1615) — bundled rules were recognised by file name alone, so an edited `example_microflow.star` (shipped to be customised) was overwritten by every `mxcli init` and, since v0.27.0, at every session start by the generated SessionStart hook; moving the edit to a new file name did not help, because the stock file was written back beside it and CUSTOM002 then ran twice, once from the stock list. A bundled file is now refreshed only while its content is a version mxcli shipped (`cmd/mxcli/lint_rules_shipped.txt`, seeded from the rules' whole git history); any other content is kept, and named once per mxcli version ("Kept 1 edited bundled lint rule(s) … delete the file to take it"). A missing bundled rule is not written while another file defines its rule ID. `add-tool` takes the same path. - **A specialization of a non-persistent entity reads as non-persistent** (mendixlabs/mxcli#1600) — `describe entity Probe.Child` printed `create or modify persistent entity Probe.Child extends Probe.Base (` when `Probe.Base` is non-persistent, so a reader took the child for a table and the output stated a combination Mendix does not have. A specialization stores no persistable flag of its own; it takes its root generalization's, and the reader defaulted it to persistent. The flag is now resolved along the generalization chain, across modules and into the System module (whose `HttpRequest`, `HttpResponse` and `SoapFault` read persistent for the same reason), and the fix reaches everything that reads it: `describe`, the catalog's `EntityType`, the domain-model diagrams, and `alter entities … where persistent`, which used to add its attribute to such a child as well. Re-running the describe output leaves the entity unchanged. - **`go mod tidy` runs again, and the expression round-trip test runs for the first time** — `mdl/exprcheck/roundtrip_test.go` imported `mdl/backend/mpr` (the legacy engine, deleted) plus `executor.GetHierarchyForMining` and `executor.DescribeMicroflowToString` (both removed). It never broke a build because it sat behind `//go:build roundtrip`, a tag no Makefile target and no CI job has ever passed — the only file in the tree using it. `go mod tidy` considers every build tag, so it was the one command that still saw the file, and it failed with "no matching versions for query latest" on the deleted package, which reads as a dependency problem rather than a dead test. The test is ported to `mdl/executor/exprcheck_roundtrip_test.go` under the `integration` tag, where `setupTestEnv`/`describeMDL` already provide the real-project harness and `make test-integration-executor` actually runs it; the removed hierarchy helper is replaced by building the container-to-module map from `backend.ListModules()`. Its assertion is worth keeping: describing every microflow and re-parsing each expression with the robust parser catches grammar gaps — a missing `parsePrimary` branch, a missing `funcTable` entry, a missing slot constraint — that the unit tests cannot, since those only feed the parser expressions someone thought to write down. It now also fails rather than passing silently when the fixture project yields no microflows or no expression slots. @@ -25,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **`alter consumed rest service` — add, change and remove operations without restating the service** (mendixlabs/mxcli#1608) — `alter consumed rest service M.Api add operation GetBooking ( … ), modify operation PostBooking (Timeout: 60), drop operation Old, set (BaseUrl: '…')`. The operation body is CREATE's; `modify` writes only the keys it lists; actions apply in order, so `drop operation X, add operation X (…)` replaces X. It edits the stored document in place rather than rebuilding it, so an operation no action names is written back byte for byte — including what Studio Pro stored that MDL cannot say (an optional query parameter, a test value), which a `create or modify` restating the service resets. Before this the statement did not parse (`no viable alternative at input 'alterconsumedrest'`) and the only MDL route was a `create or modify` that deleted any operation it did not list. - **Lint rule CONV020: an object created in a microflow and never committed, returned or passed on** (mendixlabs/mxcli#1518) — `$o = create Shop.Item (…)` without `commit` makes an object in memory only; if the microflow then never commits it or hands it to anything that could, the object is discarded when the microflow ends, and `check`, `mx check` and the build all pass. In the field an after-startup seed microflow logged "Seeded 20" and the table had 0 rows. CONV020 (warning) reports a create with commit No of a persistent entity in a microflow when nothing in the flow uses the variable except to read a member (`$o/Cnt`) or change it without commit. Any other mention — commit, change with commit, return, an argument to a microflow, Java/JavaScript action, workflow, mapping or page, a list, an association value, delete, rollback, or the bare variable in any expression — counts as passing it on. It is conservative by design: flow-insensitive (a commit on one branch or in an error handler counts), and nanoflows, non-persistent entities and System/Marketplace modules are not reported; anything it cannot classify keeps it quiet. Docs: "Built-in Rules" › CONV020 details. - **QUAL004's entry-point prefixes and CUSTOM002's valid prefixes are `lint-config.yaml` options** (mendixlabs/mxcli#1509) — both lists were constants in the rule files, and `mxcli init` rewrites those files, so a project whose convention names microflows `TEST_`, `GET_` or `PRS_` could not keep an edit and got one finding per such microflow (84 QUAL004 and 121 CUSTOM002 findings on the reporting project). `rules.QUAL004.options.entry_point_prefixes` and `rules.CUSTOM002.options.valid_prefixes` take a YAML list or comma-separated text and replace the built-in list; unset, the built-in list applies as before. Underneath, a YAML list or map under any rule's `options` reached a Starlark rule as `None` — `get_option` converted scalars only — so a list-valued option was dropped without a word; lists and maps now convert recursively, the new `get_list_option(key, default)` builtin reads a list of strings in either spelling, and an option value a rule cannot use is reported as a warning naming `rules..options.` (kept out of the score) while the rule runs on its default. QUAL004 still counts only callers in the model: a microflow called only from `tests/*.test.mdl` is reported, and its prefix belongs in `entry_point_prefixes`. Docs: "Starlark Rules" › QUAL004 and CUSTOM002 options. - **`layoutheader` and native layout parts — every Atlas layout copies whole** — `describe layout` → `create layout` of Atlas's Phone_ and Tablet_ layouts dropped their header bar (`Forms$Header`, printed as NOT re-executable), so pages bound to its `HeaderLeft` / `HeaderRight` placeholders failed CE1613 and a copy of Phone_Sidebar or Tablet_Sidebar failed CE0611 "A sidebar toggle button is required if a region can toggle". `layoutheader header1 { region left { … } region right { … } }` writes it. A native layout's copy silently lost its header placeholder, bottom bar and sidebar — pages bound to `Header` or a sidebar placeholder then failed CE1613; they are `region header { placeholder Header }`, `showbottombar: true` and `region sidebar { … }`, and the native content is written with the eight keys Studio Pro stores. Measured on a new 11.13.0 app: all 22 Atlas_Core layouts copied, with pages bound to the copied header and sidebar placeholders, build with 0 errors. diff --git a/cmd/mxcli/cmd_add_tool.go b/cmd/mxcli/cmd_add_tool.go index ed6fe27a28..deee9c5302 100644 --- a/cmd/mxcli/cmd_add_tool.go +++ b/cmd/mxcli/cmd_add_tool.go @@ -173,29 +173,7 @@ Supported Tools: fmt.Printf(" Created %d command files in .opencode/commands/\n", cmdCount) } - lintCount := 0 - if err := fs.WalkDir(lintRulesFS, "lint-rules", func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return err - } - content, err := lintRulesFS.ReadFile(path) - if err != nil { - return err - } - targetPath := filepath.Join(lintRulesDir, d.Name()) - if _, statErr := os.Stat(targetPath); statErr == nil { - return nil // skip existing - } - if err := os.WriteFile(targetPath, content, 0644); err != nil { - return err - } - lintCount++ - return nil - }); err != nil { - fmt.Fprintf(os.Stderr, " Error writing lint rules: %v\n", err) - } else if lintCount > 0 { - fmt.Printf(" Created %d lint rule files in .claude/lint-rules/\n", lintCount) - } + initBundledLintRules(lintRulesDir) skillCount := 0 if err := fs.WalkDir(skillsFS, "skills", func(path string, d fs.DirEntry, err error) error { diff --git a/cmd/mxcli/cmd_run.go b/cmd/mxcli/cmd_run.go index fd30a63041..bc31a39b98 100644 --- a/cmd/mxcli/cmd_run.go +++ b/cmd/mxcli/cmd_run.go @@ -203,6 +203,8 @@ Examples: dbName, _ := cmd.Flags().GetString("db-name") dbUser, _ := cmd.Flags().GetString("db-user") dbPassword, _ := cmd.Flags().GetString("db-password") + dbAdminUser, _ := cmd.Flags().GetString("db-admin-user") + dbAdminPassword, _ := cmd.Flags().GetString("db-admin-password") screenshot, _ := cmd.Flags().GetBool("screenshot") pageCheck, _ := cmd.Flags().GetBool("page-check") screenshotPath, _ := cmd.Flags().GetString("screenshot-path") @@ -275,11 +277,13 @@ Examples: TraceService: traceService, TraceOTLP: traceOTLP, DB: docker.DBConfig{ - Type: dbType, - Host: dbHost, - Name: dbName, - User: dbUser, - Password: dbPassword, + Type: dbType, + Host: dbHost, + Name: dbName, + User: dbUser, + Password: dbPassword, + AdminUser: dbAdminUser, + AdminPassword: dbAdminPassword, }, Stdout: os.Stdout, Stderr: os.Stderr, @@ -398,6 +402,8 @@ func init() { runCmd.Flags().String("db-name", "", "Database name (default derived from the project name)") runCmd.Flags().String("db-user", "", "Database user (default mendix)") runCmd.Flags().String("db-password", "", "Database password (default mendix)") + runCmd.Flags().String("db-admin-user", "", "PostgreSQL superuser --ensure-db creates the role and database with, when it has a password (e.g. every Windows install). Default postgres when --db-admin-password is given") + runCmd.Flags().String("db-admin-password", "", "Password for --db-admin-user, used only by --ensure-db and never passed to the runtime. PGPASSWORD or a .pgpass file for postgres works too, and keeps it off the command line") runCmd.Flags().Bool("screenshot", false, "Capture a Playwright screenshot after boot and each applied change") runCmd.Flags().Bool("page-check", false, "Print a text verdict for each page (title, headings, error banners, row count, console errors) instead of reading a screenshot — far cheaper, and it reports console errors a PNG cannot show") runCmd.Flags().String("screenshot-path", "", "Screenshot output PNG (default /.mxcli/run-local.png)") diff --git a/cmd/mxcli/docker/ensuredb.go b/cmd/mxcli/docker/ensuredb.go index 35eb21fd49..d4324a9d59 100644 --- a/cmd/mxcli/docker/ensuredb.go +++ b/cmd/mxcli/docker/ensuredb.go @@ -3,6 +3,7 @@ package docker import ( + "errors" "fmt" "io" "net" @@ -269,7 +270,10 @@ func EnsureDatabase(db *DBConfig, w io.Writer) error { } // Ensure the role and database exist (needs a local superuser). - su, err := resolveSuperuser(host, port) + su, err := resolveSuperuser(host, port, dbAdmin{User: db.AdminUser, Password: db.AdminPassword}) + if errors.Is(err, errNoSuperuser) { + return fmt.Errorf("%w%s", err, noSuperuserHint(*db)) + } if err != nil { return err } @@ -621,14 +625,27 @@ func waitTCP(hostPort string, timeout time.Duration) error { return fmt.Errorf("%s did not accept connections within %s", hostPort, timeout) } +// dbAdmin is a PostgreSQL superuser login supplied by the user +// (--db-admin-user/--db-admin-password). A server whose superuser has a password +// — every Windows install, where the installer always sets one, and there is no +// sudo either — can be provisioned no other way (mendixlabs/mxcli#1485). +type dbAdmin struct{ User, Password string } + +func (a dbAdmin) given() bool { return a.User != "" || a.Password != "" } + // superuser is how we reach a PostgreSQL superuser to provision the role and // database: a direct `psql -U postgres` against the user-owned cluster (over its -// private socket), or `sudo -u postgres psql` over the system cluster's default -// Unix socket. The latter preserves the peer-auth path used by Debian/Ubuntu. +// private socket), `sudo -u postgres psql` over the system cluster's default +// Unix socket (the peer-auth path used by Debian/Ubuntu), or the user's own +// admin login over TCP. type superuser struct { host, port string - sock string // Unix-socket dir for the user-owned cluster; preferred over TCP - sudo bool + // user is the role to connect as; empty means postgres. password, when set, + // reaches psql through PGPASSWORD, never argv, where any local user could + // read it from the process list. + user, password string + sock string // Unix-socket dir for the user-owned cluster; preferred over TCP + sudo bool // viaRoot reaches the postgres account through root instead of directly. // Devcontainers built from mcr.microsoft.com/devcontainers/base grant their // non-root user sudo to root ONLY (`vscode ALL=(root) NOPASSWD:ALL`), so @@ -674,7 +691,7 @@ func (s superuser) psql(args ...string) *exec.Cmd { if host != "" { base = append(base, "-h", host) } - base = append(base, "-p", s.port, "-U", "postgres", "-d", "postgres") + base = append(base, "-p", s.port, "-U", s.psqlUser(), "-d", "postgres") if s.sudo { // -n prevents sudo from opening /dev/tty for a password; psql's -w above // independently prevents a database-password prompt. Omit -h to retain the @@ -688,19 +705,36 @@ func (s superuser) psql(args ...string) *exec.Cmd { cmd.Env = withoutPostgresTargetEnv(os.Environ()) return cmd } - return exec.Command("psql", append(base, args...)...) + cmd := exec.Command("psql", append(base, args...)...) + if s.password != "" { + cmd.Env = append(os.Environ(), "PGPASSWORD="+s.password) + } + return cmd } -// resolveSuperuser picks a working superuser path: a direct connection (the -// user-owned initdb cluster over its trust socket, or a plain loopback cluster — -// the only paths that work without elevation on Arch) or `sudo -u postgres` for -// a system cluster. -func resolveSuperuser(host, port string) (superuser, error) { +// resolveSuperuser picks a working superuser path: the user's own admin login +// when one is given, otherwise a direct connection (the user-owned initdb cluster +// over its trust socket, or a plain loopback cluster — the only paths that work +// without elevation on Arch) or `sudo -u postgres` for a system cluster. +func resolveSuperuser(host, port string, admin dbAdmin) (superuser, error) { if !isLocalHost(host) { return superuser{}, fmt.Errorf("refusing to use a local PostgreSQL superuser for non-local host %q", host) } + if admin.given() { + // Explicit credentials are the only path tried: falling back to sudo + // after they fail would hide that the user's own login was rejected. + su := superuser{host: host, port: port, user: admin.User, password: admin.Password} + out, err := su.psql("-tAc", "select 1").CombinedOutput() + if err != nil { + return superuser{}, fmt.Errorf("connecting as PostgreSQL admin %q at %s: %w\n%s", + su.psqlUser(), net.JoinHostPort(host, port), err, strings.TrimSpace(string(out))) + } + return su, nil + } // Prefer our cluster's private socket (trust); fall back to a plain // loopback connection for a pre-existing cluster a user pointed us at. + // That one also succeeds with a password from PGPASSWORD or a .pgpass file, + // which psql reads on its own. candidates := []superuser{{host: host, port: port}} if _, _, sockDir, err := userClusterDirs(); err == nil { candidates = append([]superuser{{host: host, port: port, sock: sockDir}}, candidates...) @@ -720,9 +754,33 @@ func resolveSuperuser(host, port string) (superuser, error) { } } } - return superuser{}, fmt.Errorf("no local PostgreSQL superuser available to create the " + - "role/database (tried a direct 'psql -U postgres' connection over the cluster socket " + - "and TCP, and non-interactive 'sudo -u postgres' both directly and via root)") + return superuser{}, errNoSuperuser +} + +// errNoSuperuser is resolveSuperuser's refusal when nothing was given and no +// passwordless path works. EnsureDatabase adds the way out, which needs the +// app's database and role names. +var errNoSuperuser = fmt.Errorf("no local PostgreSQL superuser available to create the " + + "role/database (tried a direct 'psql -U postgres' connection over the cluster socket " + + "and TCP, and non-interactive 'sudo -u postgres' both directly and via root)") + +// noSuperuserHint is the way past errNoSuperuser: the superuser's password (a +// Windows install always has one), or the two statements to run by hand. The +// app password is not echoed — this text lands in logs. +func noSuperuserHint(db DBConfig) string { + return fmt.Sprintf("\n If the postgres superuser has a password (the Windows installer always sets one),\n"+ + " pass it: --db-admin-user postgres --db-admin-password (or set PGPASSWORD).\n"+ + " Or create the role and database yourself as a superuser, then rerun:\n"+ + " CREATE ROLE %s WITH LOGIN PASSWORD '<--db-password>' CREATEDB;\n"+ + " CREATE DATABASE %s OWNER %s;", db.User, db.Name, db.User) +} + +// psqlUser is the role this superuser connects as. +func (s superuser) psqlUser() string { + if s.user == "" { + return "postgres" + } + return s.user } // ensureRole creates the app login role if it does not already exist. diff --git a/cmd/mxcli/docker/ensuredb_test.go b/cmd/mxcli/docker/ensuredb_test.go index 6dde385441..f8dcba2567 100644 --- a/cmd/mxcli/docker/ensuredb_test.go +++ b/cmd/mxcli/docker/ensuredb_test.go @@ -367,7 +367,7 @@ func TestResolveSuperuser(t *testing.T) { dir, logPath := newStubPATH(t) writeStub(t, dir, "psql", `echo "$@" >> "`+logPath+`"`) writeStub(t, dir, "sudo", `echo sudo >> "`+logPath+`"`) - su, err := resolveSuperuser("127.0.0.1", "5432") + su, err := resolveSuperuser("127.0.0.1", "5432", dbAdmin{}) if err != nil || su.sudo { t.Fatalf("su=%+v err=%v", su, err) } @@ -387,7 +387,7 @@ func TestResolveSuperuser(t *testing.T) { case " $* " in *" -h "*) exit 1;; esac case " $* " in *" -p 5544 "*) exit 0;; esac exit 1`) - su, err := resolveSuperuser("127.0.0.1", "5544") + su, err := resolveSuperuser("127.0.0.1", "5544", dbAdmin{}) if err != nil || !su.sudo { t.Fatalf("su=%+v err=%v", su, err) } @@ -400,7 +400,7 @@ exit 1`) t.Run("unavailable", func(t *testing.T) { dir, _ := newStubPATH(t) writeStub(t, dir, "psql", "exit 1") - if _, err := resolveSuperuser("127.0.0.1", "5432"); err == nil { + if _, err := resolveSuperuser("127.0.0.1", "5432", dbAdmin{}); err == nil { t.Fatal("expected an error") } }) @@ -410,7 +410,7 @@ func TestResolveSuperuser_RefusesRemoteHost(t *testing.T) { dir, logPath := newStubPATH(t) writeStub(t, dir, "psql", `echo psql >> "`+logPath+`"; exit 0`) writeStub(t, dir, "sudo", `echo sudo >> "`+logPath+`"; exit 0`) - if _, err := resolveSuperuser("db.example.com", "5432"); err == nil || + if _, err := resolveSuperuser("db.example.com", "5432", dbAdmin{}); err == nil || !strings.Contains(err.Error(), "non-local") { t.Fatalf("expected remote provisioning to be refused, got %v", err) } @@ -691,7 +691,7 @@ func TestResolveSuperuser_PrefersClusterSocket(t *testing.T) { writeStub(t, dir, "psql", `echo "$@" >> "`+logPath+`"`) // exits 0 writeStub(t, dir, "sudo", `echo sudo >> "`+logPath+`"`) - su, err := resolveSuperuser("127.0.0.1", "5432") + su, err := resolveSuperuser("127.0.0.1", "5432", dbAdmin{}) if err != nil { t.Fatal(err) } @@ -953,7 +953,7 @@ case " $* " in esac exit 1`) - su, err := resolveSuperuser("127.0.0.1", "5432") + su, err := resolveSuperuser("127.0.0.1", "5432", dbAdmin{}) if err != nil { t.Fatalf("a root-only sudoers policy must still reach postgres: %v", err) } @@ -1015,3 +1015,103 @@ exit 1`) "actually said, got:\n%v", err) } } + +// The Windows PostgreSQL installer always gives `postgres` a password, and +// there is no sudo, so none of the passwordless paths can work +// (mendixlabs/mxcli#1485). Explicit admin credentials must be used — over TCP, +// as the named user, with the password in the environment rather than argv. +func TestResolveSuperuser_UsesAdminCredentials(t *testing.T) { + dir, logPath := newStubPATH(t) + t.Setenv("PGPASSWORD", "") + writeStub(t, dir, "psql", `echo "$@ PGPASSWORD=$PGPASSWORD" >> "`+logPath+`" +case " $* " in *" -U pgadmin "*) [ "$PGPASSWORD" = "s3cret" ] && exit 0;; esac +echo 'psql: error: password authentication failed for user "postgres"' >&2 +exit 2`) + + su, err := resolveSuperuser("127.0.0.1", "5432", dbAdmin{User: "pgadmin", Password: "s3cret"}) + if err != nil { + t.Fatalf("explicit admin credentials must provision a password-protected "+ + "Windows install: %v", err) + } + cmd := su.psql("-tAc", "select 1") + for _, a := range cmd.Args { + if strings.Contains(a, "s3cret") { + t.Fatalf("the admin password must not appear in argv: %#v", cmd.Args) + } + } + want := []string{"-h", "127.0.0.1", "-p", "5432", "-U", "pgadmin"} + if got := strings.Join(cmd.Args, " "); !strings.Contains(got, strings.Join(want, " ")) { + t.Fatalf("admin psql must connect over TCP as the admin user, got %q", got) + } +} + +// Without credentials the refusal is the reported symptom. It must at least say +// how to get past it, rather than only list what was tried. +func TestEnsureDatabase_NoSuperuserErrorNamesTheWayOut(t *testing.T) { + dir, _ := newStubPATH(t) + writeStub(t, dir, "psql", "exit 2") + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + db := DBConfig{ + Type: "PostgreSQL", Host: listener.Addr().String(), Name: "shopapp", User: "mendix", Password: "app-secret", + } + err = EnsureDatabase(&db, io.Discard) + if err == nil { + t.Fatal("expected an error") + } + for _, want := range []string{"--db-admin-user", "--db-admin-password", "PGPASSWORD", + "CREATE ROLE mendix", "CREATE DATABASE shopapp OWNER mendix"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("error must name %q as a way out, got:\n%v", want, err) + } + } + if strings.Contains(err.Error(), "app-secret") { + t.Fatalf("the hint must not echo the app password into logs:\n%v", err) + } +} + +// Wrong admin credentials must be reported as such, not folded into the generic +// "no superuser" text — the user supplied them and needs to know they failed. +func TestResolveSuperuser_ReportsRejectedAdminCredentials(t *testing.T) { + dir, _ := newStubPATH(t) + writeStub(t, dir, "psql", `echo 'psql: error: password authentication failed for user "pgadmin"' >&2; exit 2`) + _, err := resolveSuperuser("127.0.0.1", "5432", dbAdmin{User: "pgadmin", Password: "wrong"}) + if err == nil || !strings.Contains(err.Error(), "password authentication failed") || + !strings.Contains(err.Error(), "pgadmin") { + t.Fatalf("expected the admin connection's own failure, got: %v", err) + } +} + +// The admin password reaches EnsureDatabase through DBConfig, and is never the +// app's password: the role check must still run as the admin. +func TestEnsureDatabase_ProvisionsWithAdminCredentials(t *testing.T) { + dir, logPath := newStubPATH(t) + t.Setenv("PGPASSWORD", "") + // The app login fails until "provisioning" has happened; the admin login + // works only with its own password. + marker := filepath.Join(dir, "provisioned") + writeStub(t, dir, "psql", `echo "$@ PGPASSWORD=$PGPASSWORD" >> "`+logPath+`" +case " $* " in + *" -U pgadmin "*) [ "$PGPASSWORD" = "s3cret" ] || exit 2 + case " $* " in *" -f - "*) : > "`+marker+`";; esac + exit 0;; + *" -U mendix "*) [ -f "`+marker+`" ] && exit 0; exit 2;; +esac +exit 2`) + // A server is "running" — the Windows service is — so nothing is started. + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + db := DBConfig{ + Type: "PostgreSQL", Host: listener.Addr().String(), Name: "app", User: "mendix", Password: "mendix", + AdminUser: "pgadmin", AdminPassword: "s3cret", + } + if err := EnsureDatabase(&db, io.Discard); err != nil { + t.Fatalf("EnsureDatabase with admin credentials: %v\ncalls:\n%s", err, readCalls(t, logPath)) + } +} diff --git a/cmd/mxcli/docker/localapp_dbtype_test.go b/cmd/mxcli/docker/localapp_dbtype_test.go index 880dc9b945..7228632659 100644 --- a/cmd/mxcli/docker/localapp_dbtype_test.go +++ b/cmd/mxcli/docker/localapp_dbtype_test.go @@ -51,3 +51,16 @@ func TestPrepareLocalAppDatabase_RefusesEnsureDBForFileDatabase(t *testing.T) { t.Fatalf("prepareLocalAppDatabase err = %v, want a refusal naming the file database", err) } } + +// The admin login exists only to provision. Without --ensure-db nothing reads +// it, so it is refused rather than silently ignored (mendixlabs/mxcli#1485). +func TestApplyDatabaseDefaults_AdminLoginNeedsEnsureDB(t *testing.T) { + o := LocalRunOptions{ProjectPath: "/proj/App.mpr", DB: DBConfig{AdminPassword: "s3cret"}} + if err := o.applyDatabaseDefaults(); err == nil || !strings.Contains(err.Error(), "--ensure-db") { + t.Fatalf("applyDatabaseDefaults err = %v, want a refusal naming --ensure-db", err) + } + o = LocalRunOptions{ProjectPath: "/proj/App.mpr", EnsureDB: true, DB: DBConfig{AdminPassword: "s3cret"}} + if err := o.applyDatabaseDefaults(); err != nil { + t.Fatalf("with --ensure-db the admin login is accepted, got %v", err) + } +} diff --git a/cmd/mxcli/docker/localboot.go b/cmd/mxcli/docker/localboot.go index 3abf310e18..e7980e5a7c 100644 --- a/cmd/mxcli/docker/localboot.go +++ b/cmd/mxcli/docker/localboot.go @@ -42,6 +42,11 @@ type DBConfig struct { Name string User string Password string + // AdminUser/AdminPassword are a superuser login for --ensure-db to create + // the role and database with, when no passwordless path exists (mendixlabs/mxcli#1485). + // They are never passed to the runtime. + AdminUser string + AdminPassword string } // IsFileBased reports whether this is the runtime's built-in file database, which diff --git a/cmd/mxcli/docker/mxserve.go b/cmd/mxcli/docker/mxserve.go index 9366e76b64..66c4ecacf8 100644 --- a/cmd/mxcli/docker/mxserve.go +++ b/cmd/mxcli/docker/mxserve.go @@ -174,6 +174,9 @@ type ServeServer struct { Port int cmd *exec.Cmd log *syncBuffer + // opts are the resolved options it was started with, so Restart starts + // the same mxbuild with the same JDK on the same port. + opts ServeOptions } // verifyMxBuildCache checks the mxbuild cache is complete: the sibling runtime/ @@ -294,7 +297,9 @@ func StartServe(opts ServeOptions) (*ServeServer, error) { return nil, fmt.Errorf("starting mxbuild --serve: %w", err) } - s := &ServeServer{Host: host, Port: port, cmd: cmd, log: log} + resolved := opts + resolved.MxBuildPath, resolved.JavaHome, resolved.Host, resolved.Port = mxbuildPath, javaHome, host, port + s := &ServeServer{Host: host, Port: port, cmd: cmd, log: log, opts: resolved} readyTimeout := opts.ReadyTimeout if readyTimeout == 0 { @@ -405,6 +410,21 @@ func (s *ServeServer) alive() bool { return processAlive(s.cmd.Process) } +// Restart replaces the serve process with a fresh one started from the same +// options, in place, so every holder of this *ServeServer — including the +// caller's deferred Stop — follows it. Its first build is cold. See +// serve_recovery.go for why the watch loop needs one. +func (s *ServeServer) Restart() error { + _ = s.Stop() + s.cmd = nil // a failed start below must not leave Stop signalling a reaped pid + next, err := StartServe(s.opts) + if err != nil { + return err + } + s.cmd, s.log = next.cmd, next.log + return nil +} + // Log returns the captured mxbuild --serve output (for diagnostics). func (s *ServeServer) Log() string { return s.log.String() } diff --git a/cmd/mxcli/docker/runlocal.go b/cmd/mxcli/docker/runlocal.go index c8e813e14a..15ca890ff1 100644 --- a/cmd/mxcli/docker/runlocal.go +++ b/cmd/mxcli/docker/runlocal.go @@ -231,6 +231,12 @@ func (o *LocalRunOptions) applyDatabaseDefaults() error { if o.DB.Name == "" { o.DB.Name = deriveDBName(o.ProjectPath) } + // The admin login only provisions; nothing else would use it, so accepting + // it alone would suggest it had been applied. + if (o.DB.AdminUser != "" || o.DB.AdminPassword != "") && !o.EnsureDB { + return fmt.Errorf("--db-admin-user/--db-admin-password are used only by --ensure-db, " + + "to create the role and database — add --ensure-db") + } if IsFileBasedDBType(kind) { // The built-in database is a file: it has no host or credentials, and // accepting them would imply a connection that never happens. @@ -1397,6 +1403,8 @@ func watchAndApply(opts LocalRunOptions, serve *ServeServer, rt *LocalRuntime, b DeployDir: opts.DeployDir, MxBuildPath: mxbuildPath, Timeout: opts.WebClientTimeout, Stdout: opts.Stdout, }) + builds := &serveRecovery{serve: serve, deployDir: opts.DeployDir, out: w} + sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(sigCh) @@ -1478,8 +1486,11 @@ func watchAndApply(opts LocalRunOptions, serve *ServeServer, rt *LocalRuntime, b // change that makes mxbuild recreate web/ (a domain model change) fails // until the bundler lets go; the supervisor stops it, rebuilds, and // starts a fresh one on the new web/ (#1342). + // The build goes through serveRecovery, which restarts mxbuild --serve + // when a build leaves state only a fresh process clears — e.g. an + // "applied" reload whose dist/ lost every page (mendixlabs/mxcli#1580). build, freshBundle, err := bundler.BuildReleasingWebDir(func() (*BuildResult, error) { - return serve.Build(BuildRequest{Target: TargetDeploy, ProjectFilePath: opts.ProjectPath}) + return builds.Build(BuildRequest{Target: TargetDeploy, ProjectFilePath: opts.ProjectPath}) }) if err != nil { fmt.Fprintf(opts.Stderr, " build error: %v\n", err) diff --git a/cmd/mxcli/docker/serve_recovery.go b/cmd/mxcli/docker/serve_recovery.go new file mode 100644 index 0000000000..bef1a47205 --- /dev/null +++ b/cmd/mxcli/docker/serve_recovery.go @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// serve_recovery.go recovers the watch loop from state a long-lived +// `mxbuild --serve` process keeps and only a fresh process clears. +// +// On Mendix versions where mxbuild writes web/dist itself (the prebuilt shape: +// measured on 11.12.5, and 11.14), the FIRST build in a serve process is the +// only complete one. It leaves no web/pages, web/layouts or bundler config +// behind, so every later build in the same process either +// +// - fails, when the change regenerates a page or layout ("Could not find a part +// of the path '…/web/pages/X.js'") — the shape legacyClientBuildHint names; +// or +// - SUCCEEDS with an incomplete bundle, for every other change and for no +// change at all: dist/ is replaced by a monolithic index.js and one chunk, +// with no pages/ and no widgets.css, so the browser 404s on every page while +// the loop reports "applied via reload" (mendixlabs/mxcli#1580). +// +// Measured by driving mxbuild 11.12.5's /build API directly on a blank app: +// +// build restartRequired dist after it +// 1 cold true 13 pages, widgets.css, 5.6 KB index.js +// 2 same process, NO change false 0 pages, no widgets.css, 683 KB index.js +// 3 same process, rm web/dist false same as 2 +// 4 same process, rm web/ Failure - +// 5 fresh process true complete again +// +// So nothing done to the deployment from outside helps, and a fresh process +// always does. The cost of one is a ~5s start plus a cold ~18s build per change, +// against a 4s reload that serves no pages — correct beats fast. +// +// The same process also keeps a theme error past its fix, in every shape +// (mendixlabs/mxcli#1577). Measured on 11.12.5 over the same API: +// +// broken scss Failure An error occurred while compiling Theme files +// fixed scss, same process Failure CE6093 The theme could not be compiled, quoting the OLD error +// again, same process Failure the same +// fresh process Success +// +// Running the bundled Dart Sass on the generated entry point at the second step +// compiles cleanly, so the project is fine and the error is process state. + +// serveProcess is the part of *ServeServer the watch loop's recovery drives — a +// seam so the recovery is testable without an mxbuild. +type serveProcess interface { + Build(BuildRequest) (*BuildResult, error) + Restart() error +} + +// serveRecovery runs the watch loop's builds, restarting the serve process and +// building once more when a build's outcome is one only a fresh process fixes. +type serveRecovery struct { + serve serveProcess + deployDir string + out io.Writer + + // restartPending is set when a restart failed: the process it was meant to + // replace is gone, so the next build starts one first instead of failing + // against nothing on every later change. + restartPending bool + + // themeFailed is set while the CURRENT serve process's last theme compile + // failed. A theme failure then is not trustworthy: the process may be + // reporting the earlier error rather than the source on disk. + themeFailed bool +} + +func (r *serveRecovery) Build(req BuildRequest) (*BuildResult, error) { + if r.restartPending { + fmt.Fprintln(r.out, " starting mxbuild --serve (the last restart failed)...") + if err := r.restart(); err != nil { + return nil, err + } + } + before, prebuilt := readPrebuiltBundle(r.deployDir) + + res, err := r.serve.Build(req) + if err != nil { + return res, err + } + staleTheme := r.themeFailed && isThemeFailure(res) + r.observe(res) + + if staleTheme { + fmt.Fprintln(r.out, " the theme error may be stale — mxbuild --serve keeps reporting its first Sass error "+ + "(CE6093) after the SCSS is fixed; restarting mxbuild --serve to compile the theme afresh...") + // The fresh process's answer stands either way: a theme error it + // reports is the source's own, and current. + return r.rebuildFresh(req) + } + + if res.OK() { + if !prebuilt { + return res, nil + } + after, _ := readPrebuiltBundle(r.deployDir) + lost := before.lostIn(after) + if len(lost) == 0 { + return res, nil + } + fmt.Fprintf(r.out, " mxbuild --serve's incremental build left web/dist without %s "+ + "(this Mendix version bundles the client completely only in a serve process's first build); "+ + "restarting mxbuild --serve for a full build...\n", strings.Join(lost, " and ")) + fresh, err := r.rebuildFresh(req) + if err != nil || !fresh.OK() { + return fresh, err + } + // A fresh process's first build reports restartRequired even for an + // unchanged model (measured: true on every cold build), because it has + // no previous build to compare with. The incremental build that just + // succeeded does, and its model is the one the cold build rewrote + // byte-for-byte; taking the cold answer would restart the runtime — and + // drop every browser session — on every change. + fresh.RestartRequired = res.RestartRequired + return fresh, nil + } + + if legacyClientBuildHint(r.deployDir, res.Message, string(res.Raw)) != "" { + fmt.Fprintln(r.out, " mxbuild --serve cannot build this change incrementally (it wants client source its "+ + "first build does not leave behind); restarting mxbuild --serve for a full build...") + // No incremental build succeeded, so the cold build's restartRequired + // is the only answer there is. + return r.rebuildFresh(req) + } + return res, nil +} + +// rebuildFresh replaces the serve process and builds once on the new one. +func (r *serveRecovery) rebuildFresh(req BuildRequest) (*BuildResult, error) { + if err := r.restart(); err != nil { + return nil, err + } + res, err := r.serve.Build(req) + if err == nil { + r.observe(res) + } + return res, err +} + +func (r *serveRecovery) restart() error { + r.themeFailed = false // whatever the old process held went with it + if err := r.serve.Restart(); err != nil { + r.restartPending = true + return fmt.Errorf("restarting mxbuild --serve: %w", err) + } + r.restartPending = false + return nil +} + +// observe records what a build says about the current process's theme state. +// A non-theme failure leaves it as it was: the process still holds whatever it +// held. +func (r *serveRecovery) observe(res *BuildResult) { + switch { + case res == nil: + case res.OK(): + r.themeFailed = false + case isThemeFailure(res): + r.themeFailed = true + } +} + +// themeCompileMessage is mxbuild's message for a theme that failed to compile +// in this build; CE6093 is the consistency error a later build reports for it. +const themeCompileMessage = "An error occurred while compiling Theme files" + +// isThemeFailure reports whether a failed build failed on the theme. +func isThemeFailure(res *BuildResult) bool { + if res == nil || res.OK() { + return false + } + if strings.Contains(res.Message, themeCompileMessage) { + return true + } + for _, p := range res.Errors() { + if p.ErrorCode == "CE6093" || strings.Contains(p.Message, themeCompileMessage) { + return true + } + } + return false +} + +// prebuiltBundle is what a prebuilt-shape web/dist holds that the incremental +// serve build loses. +type prebuiltBundle struct { + pages int // page modules under dist/pages + widgetsCSS bool // dist/widgets.css +} + +// readPrebuiltBundle reads the bundle when the deployment is the prebuilt shape, +// and reports false for every other shape: in the rollup shape dist is written +// by mxcli's own bundler (recoverMissingPages' business), and the classic client +// has no dist at all. +func readPrebuiltBundle(deployDir string) (prebuiltBundle, bool) { + if planWebClient(deployDir) != webClientPrebuilt { + return prebuiltBundle{}, false + } + dist := filepath.Join(deployDir, "web", "dist") + var b prebuiltBundle + _ = filepath.WalkDir(filepath.Join(dist, "pages"), func(path string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && strings.HasSuffix(d.Name(), ".js") { + b.pages++ + } + return nil + }) + if fi, err := os.Stat(filepath.Join(dist, "widgets.css")); err == nil && !fi.IsDir() { + b.widgetsCSS = true + } + return b, true +} + +// lostIn names what the bundle had before a build and no longer has. Only a +// total loss counts: a change that deletes one page legitimately shrinks +// dist/pages, while every page vanishing at once is the incomplete bundle. +func (b prebuiltBundle) lostIn(after prebuiltBundle) []string { + var lost []string + if b.pages > 0 && after.pages == 0 { + lost = append(lost, fmt.Sprintf("dist/pages (%d page modules)", b.pages)) + } + if b.widgetsCSS && !after.widgetsCSS { + lost = append(lost, "dist/widgets.css") + } + // widgets.css alone going is not this defect's signature — an app whose + // last styled pluggable widget was just removed loses it legitimately. + if b.pages == 0 || after.pages > 0 { + return nil + } + return lost +} diff --git a/cmd/mxcli/docker/serve_recovery_test.go b/cmd/mxcli/docker/serve_recovery_test.go new file mode 100644 index 0000000000..3306e4bcc7 --- /dev/null +++ b/cmd/mxcli/docker/serve_recovery_test.go @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 + +// `run --local --watch` on Mendix 11.12.5: the first hot reload after any change +// leaves deployment/web/dist holding only index.js and chunks/, so every page +// 404s until the loop is restarted (mendixlabs/mxcli#1580). +// +// The defect is in mxbuild --serve, and the measurements that place it there +// were taken by driving its /build API directly, no watch loop involved, on a +// blank 11.12.5 app: +// +// build restartRequired dist after it +// 1 cold true 13 pages, widgets.css, 5.6 KB index.js +// 2 same process, NO change false 0 pages, no widgets.css, 683 KB index.js +// 3 same process, rm web/dist false same as 2 +// 4 fresh process true complete again +// +// So no in-process step recovers it; a fresh serve process does, and only its +// restartRequired is untrustworthy (true even for an unchanged model). +package docker + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +// prebuiltDeployment writes an 11.12.5/11.14-shaped deployment: mxbuild wrote +// web/dist itself, and there is no rollup config and no web/pages source. +func prebuiltDeployment(t *testing.T) string { + t.Helper() + deploy := t.TempDir() + writeCompleteBundle(t, deploy) + return deploy +} + +func writeCompleteBundle(t *testing.T, deploy string) { + t.Helper() + dist := filepath.Join(deploy, "web", "dist") + if err := os.RemoveAll(dist); err != nil { + t.Fatal(err) + } + for _, rel := range []string{"index.js", "widgets.css", "chunks/a.js", + "pages/MyFirstModule.Home_Web.js", "pages/Administration.Account_Overview.js"} { + p := filepath.Join(dist, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("//"), 0o600); err != nil { + t.Fatal(err) + } + } +} + +// writeIncompleteBundle is what mxbuild 11.12.5's second serve build leaves: +// dist replaced by a monolithic index.js and one chunk, no pages, no widgets.css. +func writeIncompleteBundle(t *testing.T, deploy string) { + t.Helper() + dist := filepath.Join(deploy, "web", "dist") + if err := os.RemoveAll(dist); err != nil { + t.Fatal(err) + } + for _, rel := range []string{"index.js", "chunks/b.js"} { + p := filepath.Join(dist, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("// monolithic"), 0o600); err != nil { + t.Fatal(err) + } + } +} + +// scriptedServe is a serve process whose builds are scripted. Each build is run by +// the process generation it lands on: proc 0 is the one the loop started with, +// and every Restart begins a new one. +type scriptedServe struct { + build func(proc, nth int) (*BuildResult, error) // nth build within proc, from 1 + proc int + nth int + restarts int + restartErr error + reqs []BuildRequest +} + +func (f *scriptedServe) Build(req BuildRequest) (*BuildResult, error) { + f.nth++ + f.reqs = append(f.reqs, req) + return f.build(f.proc, f.nth) +} + +func (f *scriptedServe) Restart() error { + f.restarts++ + if f.restartErr != nil { + return f.restartErr + } + f.proc++ + f.nth = 0 + return nil +} + +func buildOK(restart bool) *BuildResult { + return &BuildResult{Status: "Success", RestartRequired: restart} +} + +// The reported symptom: a successful reload that leaves dist without pages. +func TestServeRecovery_IncompletePrebuiltBundleRestartsServe(t *testing.T) { + deploy := prebuiltDeployment(t) + f := &scriptedServe{build: func(proc, nth int) (*BuildResult, error) { + if nth == 1 && proc == 0 { + writeIncompleteBundle(t, deploy) // a second build in the boot's process + return buildOK(false), nil + } + writeCompleteBundle(t, deploy) // the first build of a fresh process + return buildOK(true), nil + }} + var out strings.Builder + r := &serveRecovery{serve: f, deployDir: deploy, out: &out} + + res, err := r.Build(BuildRequest{ProjectFilePath: "/p/App.mpr"}) + if err != nil || !res.OK() { + t.Fatalf("Build = %+v, %v", res, err) + } + if _, err := os.Stat(filepath.Join(deploy, "web", "dist", "pages", "MyFirstModule.Home_Web.js")); err != nil { + t.Fatalf("dist/pages must be complete after the build — every page 404s otherwise: %v", err) + } + if f.restarts != 1 { + t.Fatalf("restarts = %d, want 1 (a fresh serve process is the only thing that rebuilds dist completely)", f.restarts) + } + // The fresh process says restartRequired for an unchanged model; the + // incremental build's answer is the one that knows what changed. Taking the + // cold one would restart the runtime — and drop every session — per change. + if res.RestartRequired { + t.Fatal("RestartRequired must be the incremental build's (false), not the cold build's") + } + if got := out.String(); !strings.Contains(got, "dist/pages") || !strings.Contains(got, "mxbuild --serve") { + t.Fatalf("the recovery must be reported in one line naming what was lost, got:\n%s", got) + } +} + +// Control: a build that keeps the bundle complete costs nothing extra. +func TestServeRecovery_CompleteBundleNoRestart(t *testing.T) { + deploy := prebuiltDeployment(t) + f := &scriptedServe{build: func(_, _ int) (*BuildResult, error) { return buildOK(false), nil }} + r := &serveRecovery{serve: f, deployDir: deploy, out: io.Discard} + if _, err := r.Build(BuildRequest{}); err != nil { + t.Fatal(err) + } + if f.restarts != 0 { + t.Fatalf("restarts = %d, want 0", f.restarts) + } +} + +// The rollup shape (Mendix 11.13 and earlier) has its dist written by mxcli's +// own bundler, not by the serve build; missing pages there are +// recoverMissingPages' business, and a serve restart would not help. +func TestServeRecovery_RollupShapeIsNotServesBusiness(t *testing.T) { + deploy := prebuiltDeployment(t) + if err := os.WriteFile(filepath.Join(deploy, "web", "rollup.config.mjs"), []byte("export default {}"), 0o600); err != nil { + t.Fatal(err) + } + f := &scriptedServe{build: func(_, _ int) (*BuildResult, error) { + writeIncompleteBundle(t, deploy) + return buildOK(false), nil + }} + r := &serveRecovery{serve: f, deployDir: deploy, out: io.Discard} + if _, err := r.Build(BuildRequest{}); err != nil { + t.Fatal(err) + } + if f.restarts != 0 { + t.Fatalf("restarts = %d, want 0 for the rollup shape", f.restarts) + } +} + +// The same serve process FAILS a change that regenerates a page: it wants +// web/pages/.js, which its own first build did not leave behind. Measured +// on 11.12.5 as well as 11.14. A fresh process builds it. +func TestServeRecovery_MissingClientSourceFailureRestartsServe(t *testing.T) { + deploy := prebuiltDeployment(t) + pagePath := filepath.Join(deploy, "web", "pages", "MyFirstModule.Home_Web.js") + f := &scriptedServe{build: func(proc, _ int) (*BuildResult, error) { + if proc == 0 { + msg := "One or more errors occurred. (Could not find a part of the path '" + pagePath + "'.)" + return &BuildResult{Status: "Failure", Message: msg}, nil + } + return buildOK(true), nil + }} + r := &serveRecovery{serve: f, deployDir: deploy, out: io.Discard} + res, err := r.Build(BuildRequest{}) + if err != nil || !res.OK() { + t.Fatalf("a page change must build on a fresh serve process, got %+v, %v", res, err) + } + if !res.RestartRequired { + t.Fatal("with no successful incremental build to say otherwise, the cold build's restartRequired stands") + } + if f.restarts != 1 { + t.Fatalf("restarts = %d, want 1", f.restarts) + } +} + +// A serve process that could not be restarted must not be left for every later +// change to fail against: the next build starts one first. +func TestServeRecovery_FailedRestartIsRetriedOnNextBuild(t *testing.T) { + deploy := prebuiltDeployment(t) + f := &scriptedServe{restartErr: errors.New("port in use"), build: func(proc, nth int) (*BuildResult, error) { + if proc == 0 { + writeIncompleteBundle(t, deploy) + return buildOK(false), nil + } + writeCompleteBundle(t, deploy) + return buildOK(false), nil + }} + r := &serveRecovery{serve: f, deployDir: deploy, out: io.Discard} + if _, err := r.Build(BuildRequest{}); err == nil || !strings.Contains(err.Error(), "port in use") { + t.Fatalf("a failed restart must be reported, got %v", err) + } + f.restartErr = nil + before := f.restarts + res, err := r.Build(BuildRequest{}) + if err != nil || !res.OK() { + t.Fatalf("second Build = %+v, %v", res, err) + } + if f.restarts != before+1 { + t.Fatalf("the next build must start a serve process first, restarts %d -> %d", before, f.restarts) + } +} + +// themeCompileFailure is the first Sass failure in a serve process, as +// mxbuild 11.12.5 reports it. +func themeCompileFailure() *BuildResult { + return &BuildResult{Status: "Failure", Message: "An error occurred while compiling Theme files", + Raw: []byte(`{"problems":{"errors":[{"message":"An error occurred while compiling Theme files","details":"Error: Undefined variable.\n .probe2 { color: $does-not-exist; }"}],"problems":[]}}`)} +} + +// staleThemeFailure is every later build in that process, after the SCSS is +// fixed: CE6093 quoting the FIRST error (mendixlabs/mxcli#1577). +func staleThemeFailure() *BuildResult { + return &BuildResult{Status: "Failure", Message: "The project cannot be deployed, because it contains errors.", + Problems: BuildProblems{Problems: []BuildProblem{{Severity: "Error", ErrorCode: "CE6093", + Message: "The theme could not be compiled:\nError: Undefined variable.\n .probe2 { color: $does-not-exist; }"}}}} +} + +// The reported symptom, measured on 11.12.5 over mxbuild's /build API: +// +// B broken scss Failure An error occurred while compiling Theme files +// C fixed scss, same process Failure CE6093, quoting B's error +// D again, same process Failure CE6093, quoting B's error +// E fresh process Success +func TestServeRecovery_StaleThemeErrorRestartsServe(t *testing.T) { + deploy := prebuiltDeployment(t) + f := &scriptedServe{build: func(proc, nth int) (*BuildResult, error) { + switch { + case proc == 0 && nth == 1: + return themeCompileFailure(), nil // the user broke the SCSS + case proc == 0: + return staleThemeFailure(), nil // ...and fixed it; the process still says no + } + return buildOK(true), nil + }} + var out strings.Builder + r := &serveRecovery{serve: f, deployDir: deploy, out: &out} + + res, err := r.Build(BuildRequest{}) + if err != nil || res.OK() { + t.Fatalf("the first theme failure is genuine and must be reported, got %+v, %v", res, err) + } + if f.restarts != 0 { + t.Fatalf("a first theme failure must not restart the serve process (restarts = %d)", f.restarts) + } + + res, err = r.Build(BuildRequest{}) + if err != nil || !res.OK() { + t.Fatalf("after the SCSS is fixed the build must succeed on a fresh serve process, got %+v, %v", res, err) + } + if f.restarts != 1 { + t.Fatalf("restarts = %d, want 1", f.restarts) + } + if got := out.String(); !strings.Contains(got, "theme") || !strings.Contains(got, "mxbuild --serve") { + t.Fatalf("the restart must be reported in one line, got:\n%s", got) + } +} + +// A theme error that is still there after the restart is the user's, and is +// reported from the fresh process — whose message is current, unlike the stale +// one. One restart per build, never a loop. +func TestServeRecovery_GenuineThemeErrorSurvivesOneRestart(t *testing.T) { + deploy := prebuiltDeployment(t) + f := &scriptedServe{build: func(proc, nth int) (*BuildResult, error) { + if proc == 0 && nth > 1 { + return staleThemeFailure(), nil + } + return themeCompileFailure(), nil + }} + r := &serveRecovery{serve: f, deployDir: deploy, out: io.Discard} + _, _ = r.Build(BuildRequest{}) + res, err := r.Build(BuildRequest{}) + if err != nil || res.OK() || res.Message != themeCompileFailure().Message { + t.Fatalf("the fresh process's own theme failure must be returned, got %+v, %v", res, err) + } + if f.restarts != 1 { + t.Fatalf("restarts = %d, want exactly 1", f.restarts) + } +} + +// Control: a theme failure after a SUCCESSFUL build is a fresh error, not a +// stale one — the process compiled the theme in between. +func TestServeRecovery_ThemeFailureAfterSuccessIsNotStale(t *testing.T) { + deploy := prebuiltDeployment(t) + f := &scriptedServe{build: func(_, nth int) (*BuildResult, error) { + switch nth { + case 1: + return themeCompileFailure(), nil + case 2: + return buildOK(false), nil + } + return themeCompileFailure(), nil + }} + r := &serveRecovery{serve: f, deployDir: deploy, out: io.Discard} + for i := 0; i < 3; i++ { + _, _ = r.Build(BuildRequest{}) + } + if f.restarts != 0 { + t.Fatalf("restarts = %d, want 0: no build here followed a theme failure in the same process", f.restarts) + } +} diff --git a/cmd/mxcli/docker/webclient_legacy_paths.go b/cmd/mxcli/docker/webclient_legacy_paths.go index fba0fb93c6..c826c00147 100644 --- a/cmd/mxcli/docker/webclient_legacy_paths.go +++ b/cmd/mxcli/docker/webclient_legacy_paths.go @@ -53,8 +53,11 @@ import ( // deployment state 11.14's cold build no longer produces, and supplying the // missing directories only moves the failure to exporting two pluggable widgets // that ship with a blank app. Across 2' and 3', web/dist/index.js never moved -// off its cold-build timestamp. Nothing outside the process fixes that half, -// which is why this file reports rather than repairs. +// off its cold-build timestamp. Nothing done to the deployment fixes that half. +// A fresh serve process does — its first build is the one that succeeds — so +// the watch loop now restarts mxbuild --serve on this shape (serve_recovery.go, +// which also measured it on 11.12.5), and this hint is only printed when the +// build on the fresh process fails as well. // // The controls that place this in mxbuild rather than here: a one-shot // `mxbuild --target=deploy` run TWICE into the same deployment directory @@ -138,11 +141,12 @@ func hintText(what string) string { return fmt.Sprintf( " This is not a problem with your model, and deployment/ is not corrupt:\n"+ " %s.\n"+ - " On Mendix 11.14 the first build in a serve process succeeds and every later\n"+ - " one fails this way — measured with the model untouched between two identical\n"+ - " builds, and with either app bundler, so switching bundlers does not help.\n"+ + " On Mendix 11.12.5 and 11.14 the first build in a serve process succeeds and\n"+ + " every later one fails this way — measured with the model untouched between two\n"+ + " identical builds, and with either app bundler, so switching bundlers does not help.\n"+ " Deleting deployment/ does not either: the next second build fails the same way.\n"+ - " Until mxbuild closes this, drop --watch and restart per change:\n"+ + " mxcli already retried on a fresh mxbuild --serve; if this persists, drop --watch\n"+ + " and restart per change:\n"+ " mxcli run --local --screenshot\n", what) } diff --git a/cmd/mxcli/exec_atomic_test.go b/cmd/mxcli/exec_atomic_test.go index ff7a7229e5..86b5e89c00 100644 --- a/cmd/mxcli/exec_atomic_test.go +++ b/cmd/mxcli/exec_atomic_test.go @@ -139,6 +139,12 @@ func hashTree(t *testing.T, dir string) map[string]string { return err } rel, _ := filepath.Rel(dir, p) + // .mxcli holds mxcli's own caches — the catalog, the unit-type index + // any listing may refresh — not the project, as mdl/scriptdiff's + // snapshot treats it. + if d.IsDir() && rel == ".mxcli" { + return filepath.SkipDir + } if d.IsDir() { // A directory a created unit needed must go when it is rolled back. out[rel+"/"] = "dir" diff --git a/cmd/mxcli/init.go b/cmd/mxcli/init.go index c7e0186174..f5426e308f 100644 --- a/cmd/mxcli/init.go +++ b/cmd/mxcli/init.go @@ -371,30 +371,7 @@ Container Runtime: fmt.Printf(" Created %d command files in .claude/commands/\n", cmdCount) } - lintRuleCount := 0 - err = fs.WalkDir(lintRulesFS, "lint-rules", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - return nil - } - content, err := lintRulesFS.ReadFile(path) - if err != nil { - return err - } - targetPath := filepath.Join(lintRulesDir, d.Name()) - if err := os.WriteFile(targetPath, content, 0644); err != nil { - return err - } - lintRuleCount++ - return nil - }) - if err != nil { - fmt.Fprintf(os.Stderr, " Error writing lint rules: %v\n", err) - } else { - fmt.Printf(" Created %d lint rule files in .claude/lint-rules/\n", lintRuleCount) - } + initBundledLintRules(lintRulesDir) } // OpenCode-specific: write commands, lint rules, and skills @@ -424,34 +401,11 @@ Container Runtime: fmt.Printf(" Created %d command files in .opencode/commands/\n", cmdCount) } - lintRuleCount := 0 // Only write lint rules from the OpenCode path when Claude is not also // being initialised — the Claude path already writes the same files to // .claude/lint-rules/ and we don't want duplicate log output or writes. if !slices.Contains(tools, "claude") { - err = fs.WalkDir(lintRulesFS, "lint-rules", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - return nil - } - content, err := lintRulesFS.ReadFile(path) - if err != nil { - return err - } - targetPath := filepath.Join(lintRulesDir, d.Name()) - if err := os.WriteFile(targetPath, content, 0644); err != nil { - return err - } - lintRuleCount++ - return nil - }) - if err != nil { - fmt.Fprintf(os.Stderr, " Error writing lint rules: %v\n", err) - } else { - fmt.Printf(" Created %d lint rule files in .claude/lint-rules/\n", lintRuleCount) - } + initBundledLintRules(lintRulesDir) } skillCount2, err := writeWrappedSkills(opencodeSkillsDir, wrapSkillContent) @@ -615,6 +569,17 @@ Container Runtime: }, } +// initBundledLintRules writes the bundled lint rules for an explicit init, +// keeping the ones the project edited or moved to a new name +// (mendixlabs/mxcli#1615), and says which. +func initBundledLintRules(lintRulesDir string) { + res, err := writeBundledLintRules(lintRulesDir) + if err != nil { + fmt.Fprintf(os.Stderr, " Error writing lint rules: %v\n", err) + } + printBundledLintRuleWrite(os.Stdout, res) +} + // defaultLintConfig is the lint configuration written into a freshly // initialised project. System is excluded because its contents are Mendix's, // not the developer's: you cannot document its entities, give them access diff --git a/cmd/mxcli/init_tooling_sync.go b/cmd/mxcli/init_tooling_sync.go index a4e1f3ade0..cb0ac5b80d 100644 --- a/cmd/mxcli/init_tooling_sync.go +++ b/cmd/mxcli/init_tooling_sync.go @@ -3,7 +3,6 @@ package main import ( - "bytes" "fmt" "io" "io/fs" @@ -168,48 +167,30 @@ func bundledLintRuleNames() ([]string, error) { return names, err } -// syncBundledLintRules rewrites the bundled rules in a project's lint-rules -// directory from the binary, returning the names that differed. A missing -// directory means the project was not initialised with a tool that uses the -// rules, and stays missing. -// -// An edit to a bundled rule file is overwritten — the file belongs to mxcli, -// like a skill. To change one, copy it under a new name (and a new rule ID), -// or override it in lint-config.yaml. -func syncBundledLintRules(lintRulesDir string) ([]string, error) { +// syncBundledLintRules brings the bundled rules in a project's lint-rules +// directory up to date with the binary, keeping the ones the project made its +// own (see writeBundledLintRules). A missing directory means the project was +// not initialised with a tool that uses the rules, and stays missing. +func syncBundledLintRules(lintRulesDir string) (bundledLintRuleWrite, error) { if st, err := os.Stat(lintRulesDir); err != nil || !st.IsDir() { - return nil, nil - } - names, err := bundledLintRuleNames() - if err != nil { - return nil, err - } - var changed []string - for _, name := range names { - want, err := lintRulesFS.ReadFile("lint-rules/" + name) - if err != nil { - return changed, err - } - target := filepath.Join(lintRulesDir, name) - if have, readErr := os.ReadFile(target); readErr == nil && bytes.Equal(have, want) { - continue - } - if err := os.WriteFile(target, want, 0o644); err != nil { - return changed, fmt.Errorf("writing %s: %w", target, err) - } - changed = append(changed, name) + return bundledLintRuleWrite{}, nil } - return changed, nil + return writeBundledLintRules(lintRulesDir) } // toolingSyncResult reports a full tooling refresh. type toolingSyncResult struct { - Skills skillSyncResult - Companion []string // companion mod files written or retired - LintRules []string // bundled rule files rewritten - Docs []string // owned docs whose mxcli section was rewritten - NoMarkers []string // owned docs left alone: no markers to refresh between - StampWritten bool + Skills skillSyncResult + Companion []string // companion mod files written or retired + LintRules []string // bundled rule files rewritten + // LintRulesKept are bundled rule files left alone because the project + // edited them; LintRulesShadowed are bundled rules not written because + // another file already defines their rule ID ("name (ID in file)"). + LintRulesKept []string + LintRulesShadowed []string + Docs []string // owned docs whose mxcli section was rewritten + NoMarkers []string // owned docs left alone: no markers to refresh between + StampWritten bool // RefusedFor is set when the sync did nothing because this binary is older // than the one that wrote the tooling. RefusedFor *toolingStamp @@ -240,9 +221,11 @@ func syncProjectTooling(projectDir string) (toolingSyncResult, error) { return res, fmt.Errorf("syncing the companion mod: %w", err) } - if res.LintRules, err = syncBundledLintRules(filepath.Join(projectDir, ".claude", "lint-rules")); err != nil { + lint, err := syncBundledLintRules(filepath.Join(projectDir, ".claude", "lint-rules")) + if err != nil { return res, fmt.Errorf("syncing lint rules: %w", err) } + res.LintRules, res.LintRulesKept, res.LintRulesShadowed = lint.Written, lint.Kept, lint.Shadowed // The docs embed the .mpr path, so without one there is nothing correct to // render; init would write a placeholder, a refresh must not. @@ -290,6 +273,12 @@ func reportToolingSync(w, errw io.Writer, res toolingSyncResult) { fmt.Fprintf(w, "Refreshed %d bundled lint rule(s) in .claude/lint-rules/ to match this mxcli: %s\n", len(res.LintRules), abridge(res.LintRules)) } + // What was kept is the project's choice and holds every session, so it is + // said once per mxcli — on the sync that stamps a new version, which is + // when the kept files first miss an update — not on every session start. + if res.StampWritten { + printKeptLintRules(w, "", res.LintRulesKept, res.LintRulesShadowed) + } if len(res.Docs) > 0 { fmt.Fprintf(w, "Refreshed the mxcli section of %s\n", strings.Join(res.Docs, ", ")) } diff --git a/cmd/mxcli/init_tooling_sync_test.go b/cmd/mxcli/init_tooling_sync_test.go index 3f3af049c6..71a385825e 100644 --- a/cmd/mxcli/init_tooling_sync_test.go +++ b/cmd/mxcli/init_tooling_sync_test.go @@ -48,15 +48,15 @@ func mustRead(t *testing.T, path string) string { // ako/mxcli#952: `init --sync-skills` refreshed only the skills, so a project // kept the lint rules of whichever mxcli first initialised it. The bundled // rules are refreshed now; a project's own rule beside them is not touched. +// +// The stale copy is one mxcli shipped (v0.27.0): any other content under a +// bundled name is the project's edit, and is kept (mendixlabs/mxcli#1615). func TestSyncProjectTooling_RefreshesBundledLintRulesOnly(t *testing.T) { dir := newSyncProject(t) - names, err := bundledLintRuleNames() - if err != nil || len(names) == 0 { - t.Fatalf("no bundled lint rules: %v", err) - } + names := []string{exampleRule} rulesDir := filepath.Join(dir, ".claude", "lint-rules") stale := filepath.Join(rulesDir, names[0]) - mustWrite(t, stale, "# yesterday's rule\n") + mustWrite(t, stale, string(v0270ExampleRule(t))) user := filepath.Join(rulesDir, "zz_my_project_rule.star") const userRule = "RULE_ID = \"PROJ001\"\ndef check():\n return []\n" mustWrite(t, user, userRule) diff --git a/cmd/mxcli/lint_rules_bundled.go b/cmd/mxcli/lint_rules_bundled.go new file mode 100644 index 0000000000..7213504db4 --- /dev/null +++ b/cmd/mxcli/lint_rules_bundled.go @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "bytes" + "crypto/sha256" + _ "embed" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "sync" +) + +// lint_rules_bundled.go tells a bundled lint rule mxcli may refresh from one a +// project has made its own (mendixlabs/mxcli#1615). +// +// Bundled rules were recognised by file name alone, so `mxcli init` and the +// session-start `init --sync-skills` overwrote every edit to one — including +// example_microflow.star, which ships to be customised — and wrote the stock +// file back beside a renamed copy, which then ran the same rule ID twice. + +// shippedLintRulesFile lists every version of every bundled rule mxcli has +// shipped. A file whose content is on it is the stock rule, however old; any +// other content under a bundled name is an edit. +const shippedLintRulesFile = "lint_rules_shipped.txt" + +//go:embed lint_rules_shipped.txt +var shippedLintRulesText string + +// shippedLintRuleHashes maps a bundled file name to the hashes of the +// versions of it mxcli has shipped. +var shippedLintRuleHashes = sync.OnceValue(func() map[string]map[string]bool { + out := map[string]map[string]bool{} + sc := bufio.NewScanner(strings.NewReader(shippedLintRulesText)) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + if len(fields) != 2 { + continue + } + if out[fields[1]] == nil { + out[fields[1]] = map[string]bool{} + } + out[fields[1]][fields[0]] = true + } + return out +}) + +// lintRuleHash is the hash the shipped list records: sha256 of the content +// with CRLF read as LF, so a Windows checkout of a stock rule is still stock. +func lintRuleHash(content []byte) string { + sum := sha256.Sum256(bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n"))) + return hex.EncodeToString(sum[:]) +} + +var ruleIDRe = regexp.MustCompile(`(?m)^RULE_ID\s*=\s*["']([^"'\n]+)["']`) + +// lintRuleID is the ID a rule file reports under: its RULE_ID, or the file +// name without .star when it has none — the same fallback the linter uses. +func lintRuleID(name string, content []byte) string { + if m := ruleIDRe.FindSubmatch(content); m != nil { + return string(m[1]) + } + return strings.TrimSuffix(name, ".star") +} + +// bundledLintRuleWrite is what bringing one directory's bundled rules up to +// date did. +type bundledLintRuleWrite struct { + Written []string // created, or refreshed from a version mxcli shipped + Kept []string // edited by the project, and left alone + // Shadowed are bundled rules not written because another file in the + // directory already defines their rule ID: "name (ID in other.star)". + Shadowed []string +} + +// writeBundledLintRules writes this binary's bundled rules into dir, which +// must exist. It is the one write path for `mxcli init`, `init --sync-skills` +// and `add-tool`, so the three cannot disagree about what is the project's: +// +// - a file that is a version mxcli shipped is brought up to date; +// - a file under a bundled name with any other content is the project's +// edit, and is kept — delete it to take this binary's version; +// - a missing bundled rule is written, unless another file already defines +// its rule ID: that is the edit moved to a new name, and the stock copy +// beside it would run the same rule a second time. To switch a bundled +// rule off, set `enabled: false` for its ID in lint-config.yaml. +func writeBundledLintRules(dir string) (bundledLintRuleWrite, error) { + var res bundledLintRuleWrite + names, err := bundledLintRuleNames() + if err != nil { + return res, err + } + shipped := shippedLintRuleHashes() + + var definedBy map[string][]string // rule ID -> files in dir; read on first need + for _, name := range names { + want, err := lintRulesFS.ReadFile("lint-rules/" + name) + if err != nil { + return res, err + } + target := filepath.Join(dir, name) + have, readErr := os.ReadFile(target) + switch { + case readErr == nil: + h := lintRuleHash(have) + if h == lintRuleHash(want) { + continue + } + if !shipped[name][h] { + res.Kept = append(res.Kept, name) + continue + } + case os.IsNotExist(readErr): + if definedBy == nil { + if definedBy, err = lintRuleIDsIn(dir); err != nil { + return res, err + } + } + id := lintRuleID(name, want) + if others := definedBy[id]; len(others) > 0 { + res.Shadowed = append(res.Shadowed, fmt.Sprintf("%s (%s in %s)", name, id, strings.Join(others, ", "))) + continue + } + default: + return res, readErr + } + if err := os.WriteFile(target, want, 0o644); err != nil { + return res, fmt.Errorf("writing %s: %w", target, err) + } + res.Written = append(res.Written, name) + } + return res, nil +} + +// lintRuleIDsIn maps each rule ID defined by a .star file in dir to the files +// that define it. +func lintRuleIDsIn(dir string) (map[string][]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + out := map[string][]string{} + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".star") { + continue + } + data, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + return nil, err + } + id := lintRuleID(e.Name(), data) + out[id] = append(out[id], e.Name()) + } + return out, nil +} + +// printBundledLintRuleWrite reports an explicit init's or add-tool's write. +func printBundledLintRuleWrite(w io.Writer, res bundledLintRuleWrite) { + if len(res.Written) > 0 { + fmt.Fprintf(w, " Created %d lint rule files in .claude/lint-rules/\n", len(res.Written)) + } + printKeptLintRules(w, " ", res.Kept, res.Shadowed) +} + +// printKeptLintRules names the bundled rules a write left to the project. +func printKeptLintRules(w io.Writer, indent string, kept, shadowed []string) { + if len(kept) > 0 { + fmt.Fprintf(w, "%sKept %d edited bundled lint rule(s) in .claude/lint-rules/ — this mxcli's version was not written; delete the file to take it: %s\n", + indent, len(kept), abridge(kept)) + } + if len(shadowed) > 0 { + fmt.Fprintf(w, "%sDid not add %d bundled lint rule(s) whose rule ID another file in .claude/lint-rules/ already defines: %s\n", + indent, len(shadowed), strings.Join(shadowed, "; ")) + } +} diff --git a/cmd/mxcli/lint_rules_bundled_test.go b/cmd/mxcli/lint_rules_bundled_test.go new file mode 100644 index 0000000000..ae1928ac0e --- /dev/null +++ b/cmd/mxcli/lint_rules_bundled_test.go @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// mendixlabs/mxcli#1615: `init` and `init --sync-skills` overwrote an edited +// bundled rule (example_microflow.star, shipped to be customised) and +// recreated it beside a renamed copy, so CUSTOM002 fired from the stock list +// again at every session start. + +const exampleRule = "example_microflow.star" + +// v0270ExampleRule is example_microflow.star exactly as mxcli v0.27.0 shipped +// it (md5 cec8b599…, the stock copy the report measured). +func v0270ExampleRule(t *testing.T) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "lint-rules", "example_microflow.v0.27.0.star")) + if err != nil { + t.Fatal(err) + } + return data +} + +func bundledRule(t *testing.T, name string) string { + t.Helper() + data, err := lintRulesFS.ReadFile("lint-rules/" + name) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +// editedExampleRule is the report's edit: the stock rule with extra prefixes. +func editedExampleRule(t *testing.T) string { + t.Helper() + stock := string(v0270ExampleRule(t)) + edited := strings.Replace(stock, `"OCH_"]`, `"OCH_", "TEST_", "GET_", "PRS_"]`, 1) + if edited == stock { + t.Fatal("fixture no longer has the VALID_PREFIXES line the edit targets") + } + return edited +} + +func TestSyncProjectTooling_KeepsEditedBundledLintRule(t *testing.T) { + dir := newSyncProject(t) + rule := filepath.Join(dir, ".claude", "lint-rules", exampleRule) + edited := editedExampleRule(t) + mustWrite(t, rule, edited) + + res, err := syncProjectTooling(dir) + if err != nil { + t.Fatal(err) + } + if got := mustRead(t, rule); got != edited { + t.Fatalf("sync overwrote the edited %s", exampleRule) + } + if strings.Contains(strings.Join(res.LintRules, ","), exampleRule) { + t.Errorf("sync reports the kept file as refreshed: %v", res.LintRules) + } + if !strings.Contains(strings.Join(res.LintRulesKept, ","), exampleRule) { + t.Errorf("sync does not report the kept file: %+v", res) + } +} + +// Control: a copy that is a version mxcli shipped is the stock rule, not an +// edit, and is brought up to date — or upgrades would stop reaching every +// project initialised by an older mxcli. +func TestSyncProjectTooling_RefreshesPreviouslyShippedLintRule(t *testing.T) { + dir := newSyncProject(t) + rule := filepath.Join(dir, ".claude", "lint-rules", exampleRule) + old := v0270ExampleRule(t) + if string(old) == bundledRule(t, exampleRule) { + t.Fatal("fixture equals the current rule; it cannot show a refresh") + } + mustWrite(t, rule, string(old)) + + res, err := syncProjectTooling(dir) + if err != nil { + t.Fatal(err) + } + if got := mustRead(t, rule); got != bundledRule(t, exampleRule) { + t.Errorf("the v0.27.0 stock %s was not refreshed", exampleRule) + } + if !strings.Contains(strings.Join(res.LintRules, ","), exampleRule) { + t.Errorf("refresh not reported: %v", res.LintRules) + } + if len(res.LintRulesKept) != 0 { + t.Errorf("stock copy reported as an edit: %v", res.LintRulesKept) + } +} + +// A stock copy checked out with CRLF line endings is the stock rule too. +func TestSyncProjectTooling_CRLFStockLintRuleIsCurrent(t *testing.T) { + dir := newSyncProject(t) + rule := filepath.Join(dir, ".claude", "lint-rules", exampleRule) + crlf := strings.ReplaceAll(bundledRule(t, exampleRule), "\n", "\r\n") + mustWrite(t, rule, crlf) + + res, err := syncProjectTooling(dir) + if err != nil { + t.Fatal(err) + } + if got := mustRead(t, rule); got != crlf { + t.Error("a CRLF checkout of the current rule was rewritten") + } + if len(res.LintRulesKept) != 0 || strings.Contains(strings.Join(res.LintRules, ","), exampleRule) { + t.Errorf("CRLF copy not treated as current: written %v, kept %v", res.LintRules, res.LintRulesKept) + } +} + +// The session-start sync says what it kept once per mxcli, not every session. +func TestReportToolingSync_KeptLintRulesSaidOncePerMxcli(t *testing.T) { + dir := newSyncProject(t) + mustWrite(t, filepath.Join(dir, ".claude", "lint-rules", exampleRule), editedExampleRule(t)) + mustWrite(t, filepath.Join(dir, ".claude", "lint-rules", "my_rules.star"), "RULE_ID = \"MPR001\"\n") + + report := func() string { + t.Helper() + res, err := syncProjectTooling(dir) + if err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + reportToolingSync(&out, &errOut, res) + return out.String() + errOut.String() + } + first := report() + if !strings.Contains(first, "Kept 1 edited bundled lint rule(s)") || !strings.Contains(first, exampleRule) { + t.Errorf("first sync does not name the kept rule:\n%s", first) + } + if second := report(); second != "" { + t.Errorf("second sync is not quiet:\n%s", second) + } +} + +// The report's third step: the edit moved to a new file name, keeping its +// rule ID. Recreating the stock file beside it runs CUSTOM002 twice, once +// from the stock list. +func TestSyncProjectTooling_DoesNotRecreateRenamedBundledLintRule(t *testing.T) { + dir := newSyncProject(t) + rulesDir := filepath.Join(dir, ".claude", "lint-rules") + renamed := filepath.Join(rulesDir, "my_prefixes.star") + mustWrite(t, renamed, editedExampleRule(t)) + + res, err := syncProjectTooling(dir) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(rulesDir, exampleRule)); !os.IsNotExist(err) { + t.Fatalf("sync recreated %s beside the renamed copy", exampleRule) + } + if got := strings.Join(res.LintRulesShadowed, ","); !strings.Contains(got, exampleRule) || !strings.Contains(got, "my_prefixes.star") { + t.Errorf("skipped rule not reported with the file that defines its ID: %q", got) + } +} + +// Control: a deleted bundled rule with no copy defining its ID is written +// again — deleting the file is how an edited one is reset to mxcli's version. +func TestSyncProjectTooling_RecreatesDeletedBundledLintRule(t *testing.T) { + dir := newSyncProject(t) + rule := filepath.Join(dir, ".claude", "lint-rules", exampleRule) + + if _, err := syncProjectTooling(dir); err != nil { + t.Fatal(err) + } + if got := mustRead(t, rule); got != bundledRule(t, exampleRule) { + t.Errorf("deleted %s was not written again", exampleRule) + } +} + +// The same rules hold for an explicit `mxcli init` re-run, which wrote every +// bundled file unconditionally ("Created 31 lint rule files"). +func TestInit_KeepsEditedAndRenamedBundledLintRules(t *testing.T) { + dir := t.TempDir() + rulesDir := filepath.Join(dir, ".claude", "lint-rules") + edited := editedExampleRule(t) + + // Edited in place. + mustWrite(t, filepath.Join(rulesDir, exampleRule), edited) + runInit(t, []string{"claude"}, dir) + if got := mustRead(t, filepath.Join(rulesDir, exampleRule)); got != edited { + t.Fatalf("init overwrote the edited %s", exampleRule) + } + + // Moved to a new name. + if err := os.Rename(filepath.Join(rulesDir, exampleRule), filepath.Join(rulesDir, "my_prefixes.star")); err != nil { + t.Fatal(err) + } + runInit(t, []string{"claude"}, dir) + if _, err := os.Stat(filepath.Join(rulesDir, exampleRule)); !os.IsNotExist(err) { + t.Errorf("init recreated %s beside the renamed copy", exampleRule) + } + // Every other bundled rule is still written. + if got := mustRead(t, filepath.Join(rulesDir, "orphaned_elements.star")); got != bundledRule(t, "orphaned_elements.star") { + t.Error("init did not write the other bundled rules") + } +} + +// Every rule this binary ships must be in the list of shipped versions, or the +// next mxcli will take an unedited copy of it for an edit and stop updating +// it. The list is append-only: an older version's line stays. +func TestShippedLintRuleHashes_CoverEmbeddedRules(t *testing.T) { + names, err := bundledLintRuleNames() + if err != nil || len(names) == 0 { + t.Fatalf("no bundled lint rules: %v", err) + } + shipped := shippedLintRuleHashes() + for _, name := range names { + h := lintRuleHash([]byte(bundledRule(t, name))) + if !shipped[name][h] { + t.Errorf("%s: this version is not in cmd/mxcli/%s; append the line\n%s %s", + name, shippedLintRulesFile, h, name) + } + } + // The v0.27.0 fixture has to be recognised as well. + if !shipped[exampleRule][lintRuleHash(v0270ExampleRule(t))] { + t.Errorf("the v0.27.0 %s is not a shipped version", exampleRule) + } +} + +func TestLintRuleID(t *testing.T) { + for _, c := range []struct{ name, src, want string }{ + {"x.star", "RULE_ID = \"CUSTOM002\"\n", "CUSTOM002"}, + {"x.star", "# RULE_ID = \"NOPE\"\nRULE_ID='Q1'\n", "Q1"}, + {"my_rule.star", "def check():\n return []\n", "my_rule"}, + } { + if got := lintRuleID(c.name, []byte(c.src)); got != c.want { + t.Errorf("lintRuleID(%q) = %q, want %q", c.src, got, c.want) + } + } +} diff --git a/cmd/mxcli/lint_rules_shipped.txt b/cmd/mxcli/lint_rules_shipped.txt new file mode 100644 index 0000000000..ba1c27c80b --- /dev/null +++ b/cmd/mxcli/lint_rules_shipped.txt @@ -0,0 +1,79 @@ +# Every version of every bundled lint rule mxcli has shipped, as +# sha256 (of the file with CRLF line endings read as LF) and file name. +# +# `mxcli init` and `init --sync-skills` refresh a bundled rule in a project +# only while its file is one of these versions; any other content is the +# project's edit, and is kept (mendixlabs/mxcli#1615). +# +# LINES ARE ONLY ADDED. When you change a rule in .claude/lint-rules/, add a +# line for the new version and leave the old one: removing it makes every +# project still on that version read as edited, and they stop receiving +# updates to the rule. TestShippedLintRuleHashes_CoverEmbeddedRules prints the +# line to add. Keep the lines sorted by file name. +# +# Seeded from the git history of .claude/lint-rules/ (every blob at every +# commit reachable from main, 2026-03-13 to 2026-10-21). +345ec51ebfb650b34a72b45be215bd97e895f09a4f2689e23c47185434630584 conv001_boolean_naming.star +7dc6f673fdd7e692a6c328e2b72da57b9223007f65a37a0a891f238ea701ad45 conv002_no_default_values.star +80e4d6b3cdcf34d8b4a521a32c4254b9dc61b0f2d091802b0969118264cfffa3 conv003_page_naming_suffix.star +c4d0ebd17284f6fb85b6acc751e6615c0c92f8d252c0d118846e95077a92624e conv004_enumeration_prefix.star +a873cb374e73d122c4aa7eb36cc6af3bb7df2695c1d22a5805b295193880c0cb conv005_snippet_prefix.star +a50597f64de8f41306891ef46f43c284d1be67fbbca400888465b96faaab671a conv006_no_create_delete_rights.star +bd116c2be0049cfc26e34e4919492fa2fca7bf126d2b42d544aecafff5d992e6 conv006_no_create_delete_rights.star +54a54e887fcd330dca55af39f3b604d394c0a69f5dbe4bf9b0986c4ff1d1a6c1 conv007_xpath_on_all_access.star +25484ba6d1f80a1c564ce7cb45a92badab5a706bc459566889a5050b7e9f7674 conv008_module_role_mapping.star +021306047eb98be739b2c3419b0f94b90879b206b556bd56bef4aff192ce7eaf conv009_max_microflow_objects.star +7f806b6303f31fb709e0712b8489ac32c725853da12b1ee7d5a5c7742ff7dce3 conv009_max_microflow_objects.star +041c04e7819df47c80c59251fa86799784c265b0c48253b94098f3b2174b5f1c conv010_act_microflow_content.star +652f771263ca868bf5a578a7aab856fc060df5b9552f089903c7738149d8f37b conv010_act_microflow_content.star +be977d5efbd40850f6e48ace82bdf4a36dfe76d0d74f2fc706d8e3b24102b8ba conv010_act_microflow_content.star +d4b81e3061daed1a75b6b16fda52e3e66d152db078881bdaf293d1b20b71b6f0 conv010_act_microflow_content.star +fb3018d5b7a7ee702d2cb949ccdba0ec011380e1e9feca6cfda9b4cbdd20fc4c conv010_act_microflow_content.star +a61a63be96c055581d16411423946d4235256d9d7e9369a67d98c87c04ce260c conv015_no_validation_rules.star +99cf64fc14aae67b80f9cc21338365b61e147fbfe0259deb8f937647d8d34891 conv016_no_event_handlers.star +b77bfc1debf60dde94567951d5855bfc5733bd374befd36dc38df07eb6fd2c8b conv017_no_calculated_attributes.star +732ed750c3944a5c6e8c6b4520ac3f6b682381ce138ffdc5a61a65826cd9b3bf conv018_module_folder_organization.star +f4756e3819e88f9fd9454fe614cdca46f86f800d84c00712e3265f9b8b2ff973 conv018_module_folder_organization.star +f42e2de397bf259bcb9e7958f2515b0420bde6ac0551eed7148abe6826be2cc8 conv019_navigation_page_url.star +151fa2f467aeeea18d6436e2da0014799774e42d4213c6b518f663318abaaf7c cross_module_data.star +05a46b0b13825ce4d7a992fb8ec51a77e9855f28eb0c2d76adfd275cccd5d9d5 data_change_microflows.star +bf98483f1c32f6f1d4ebb7c17326b0f9035c2fd9b7762b5bbc40ace8e04cc959 data_change_microflows.star +8422e11af9ab25efd1233dd90725b451a0f61812300735f85d7470cf6bfb1bec entity_attributes.star +842816ee411a8b2853ea9b21b5c2630735e84e18b97c3eeb04a4f26626b2adce entity_business_key.star +85c9f1db293c80df01d13b5dab8702ea93dc9f891bcb7d232cde0584e79f98f2 entity_business_key.star +049c3344414db812ebbf112965a7d260d508c5abda5201237250b454cb97eba1 example_microflow.star +2cf64067e3f61efcb1c73841af5de9eea04a46b25d0706765280f7145b14df8b example_microflow.star +347605fd8db0c33a2fbd315cfcdbd2eea12265a5e26546e14597a518723c94e6 example_microflow.star +576f4db4d07de5b04a6457be1ac5f85d9533950349400be22f318baf6c79d4e8 example_microflow.star +74076f3a022226abd581aebda83cd43ff53e290fd4f0a741187e2054e76b2c4f example_microflow.star +7afc00e60864ec0575eff7af82980f55c99482c3c22d8437c011232c2e1a0612 example_microflow.star +cb0806bf2637e04d7062188124032d9653c4e17577dba56caf106b7dedc8ae37 example_naming.star +a0ecc82b6c6ee9e8daa749c9bb435029943e550524d71b2eecb6853dc91faf46 long_microflows.star +fb66e3ae8521cce1b6f844570fccdd9e48d27a052f8c4fb7a290426e24d95b9b long_microflows.star +0ab09009807e6669f3948a3b96410e0e31f182e4899a446fe21c78df697c67d6 mccabe_complexity.star +4d000b7bf35df9abf3b357db27555b3f5b5b6f9db47b810da088be37bd6e2bb4 mccabe_complexity.star +81630881831a5f159b2cb1031c9c4b161eb5f206c39bf2b9e3c335f6a134d9a3 mccabe_complexity.star +d84aa6ad432839b369f3128a680302a678d530594adbb2170e02ee176bc8dcc8 mccabe_complexity.star +41d3301717995857d5c7bba45a4679142bdfdad0c743da8f5f0bb5e15be18b63 missing_documentation.star +814ec33d4686f3c76021d3b25f0ae4a2a405342eeea4a6cd616bf42eccb080a5 missing_documentation.star +902d073f90b8b6d8368f101d983991717c3ac17bae3e3280ae5be0e968989dd3 missing_documentation.star +bbd935d1ee0fe6c635572f19c030f511e1010f13b38f49622f28060c3b3e8305 missing_documentation.star +0e4f2f891e34e7428bcd037ed73a81b15903171cf617d6e4b995adb9029b67c4 orphaned_elements.star +0e5acc3343fdf45a8083a9d392f0f5866cc5f05fb829581bbc30cd5cb8f85495 orphaned_elements.star +16d69c2cfae987bc48d3333139b63ddf685a8a24648a5b589d35ddd8c216e8a1 orphaned_elements.star +2a0265053405affcdab9621f371e9bfaa0b6d99ef1a6125168a65b13a7512835 orphaned_elements.star +2aa15dd26aa9b45a352df70b53de62b3b17043977e0f58602a7bcb76cf01e328 orphaned_elements.star +553aef8d7acfa579dde8dfd43f18b14ffe7a495024a84df980c5c87a88b224d5 orphaned_elements.star +8cde2668fb193d7b7b5f30743c173e0c5f634793b53f5ad675c5635471abbf3a orphaned_elements.star +a5fa3f0f37f7fde0206df2880fdeb676d0db656957ce89d7b09033af8a7a4476 orphaned_elements.star +c27185493f9b8f0b26eff68634c522b61ddf41a59b614d14e046c28bbc60a7d3 orphaned_elements.star +e05678135cbc750a3cfef07ddf348b7f207196f0a7762907e416e1a3c973de79 orphaned_elements.star +4500272ad6d0ae8876081de0825c633117b471913079be08fb78311e10470071 sec_guest_access.star +922394f50a7d8b8fab406dcd623536f98dc0d01b9970053776cd609900b2dc39 sec_missing_member_restriction.star +46d8bf64c7a30453685c4c470d6864e9b8f7cfd69d33985a50e24a358c309324 sec_pii_attributes.star +611b1a6202d9c7b153195056993b400006a5e091c13c10578b3172bc493f1571 sec_strict_mode.star +d62de1d6e29cc8da4e4d4cad990b09a64b2859ee1bc975865192c1618c0a56ea sec_strict_mode.star +df118032d8e9554b860ed09401a06397efaff09738dfe403c77a2ee9f9cbe941 sec_strict_mode.star +6d58dfb79c07955a3608978e68c65e9ea13f1692e24d208c399700d33de8de25 sec_unconstrained_anon_read.star +72060544ea8fd6dd0d2d73b0cce09f3cb3802677acfa4f7cce02e5204bb7153f sec_unconstrained_pii_read.star +d926cf53fd54bc73473b567849bf866f045b5b2bed60eeacc66e982d5e21225f sec_unconstrained_pii_read.star diff --git a/cmd/mxcli/syntax/features_integration.go b/cmd/mxcli/syntax/features_integration.go index 1fae5af288..91583527a9 100644 --- a/cmd/mxcli/syntax/features_integration.go +++ b/cmd/mxcli/syntax/features_integration.go @@ -328,11 +328,12 @@ func init() { Summary: "Create consumed REST clients with operations, mappings, and authentication", Keywords: []string{ "create consumed rest service", "create rest client", "rest client", "consume rest", "rest operation", + "alter consumed rest service", "add operation", "modify operation", "drop operation", "get", "post", "put", "delete", "patch", "body", "response", "mapping", "authentication", "json structure", "import mapping", "export mapping", }, - Syntax: "CREATE [OR MODIFY] CONSUMED REST SERVICE Module.Name (\n BaseUrl: 'https://...',\n Authentication: NONE | BASIC (...)\n)\n{\n OPERATION Name (\n Method: GET|POST|PUT|DELETE|PATCH,\n Path: '/path/{param}',\n Parameters: ($param: Type),\n Query: ($param: Type),\n Headers: ('Key': 'Value'),\n Timeout: 30,\n Body: JSON FROM $var | MAPPING Entity { jsonField = Attribute, ... },\n Response: JSON AS $var | MAPPING Entity { Attribute = jsonField, ... }\n )\n};\n\n-- An operation is a child of the service, so its properties are in ( ).\n-- `OPERATION Name { ... }` is the deprecated spelling (MDL-DEPR070).\n\n-- MAPPING takes a target ENTITY plus a body listing the JSON fields; Mendix\n-- stores it inline on the operation. An existing import/export mapping\n-- document cannot be referenced here (rejected as MDL-REST01).\n-- There is no FILE request body: Mendix's consumed operation stores one of\n-- Rest$JsonBody, Rest$StringBody or Rest$ImplicitMappingBody, so a file\n-- document has nowhere to go. `Body: FILE FROM $Doc` is rejected as\n-- MDL-REST02 rather than sent as the literal text \"$Doc\" (it used to be,\n-- returning 200 with a 4-byte payload). Binary POST lives on the\n-- microflow activity: `call rest service post '' (Body: binary $Doc/Contents)`.\n-- `Response: FILE AS $Doc` is unaffected — downloads work.", + Syntax: "CREATE [OR MODIFY] CONSUMED REST SERVICE Module.Name (\n BaseUrl: 'https://...',\n Authentication: NONE | BASIC (...)\n)\n{\n OPERATION Name (\n Method: GET|POST|PUT|DELETE|PATCH,\n Path: '/path/{param}',\n Parameters: ($param: Type),\n Query: ($param: Type),\n Headers: ('Key': 'Value'),\n Timeout: 30,\n Body: JSON FROM $var | MAPPING Entity { jsonField = Attribute, ... },\n Response: NONE | JSON [AS $var] | STRING | FILE | STATUS | MAPPING Entity { Attribute = jsonField, ... }\n )\n};\n\n-- Change an existing service without restating it. An operation no action names\n-- is left exactly as stored (also one made in Studio Pro); MODIFY writes only the\n-- keys it lists. CREATE OR MODIFY instead replaces the whole operation list.\nALTER CONSUMED REST SERVICE Module.Name\n ADD OPERATION Name ( ... ),\n MODIFY OPERATION Name ( Timeout: 60 ),\n DROP OPERATION Name,\n SET ( BaseUrl: '...', Authentication: NONE | BASIC (...) );\n\n-- The response variable is not stored (the call site binds it), so describe\n-- prints `Response: JSON`; `AS $var` is optional.\n-- An operation is a child of the service, so its properties are in ( ).\n-- `OPERATION Name { ... }` is the deprecated spelling (MDL-DEPR070).\n\n-- MAPPING takes a target ENTITY plus a body listing the JSON fields; Mendix\n-- stores it inline on the operation. An existing import/export mapping\n-- document cannot be referenced here (rejected as MDL-REST01).\n-- There is no FILE request body: Mendix's consumed operation stores one of\n-- Rest$JsonBody, Rest$StringBody or Rest$ImplicitMappingBody, so a file\n-- document has nowhere to go. `Body: FILE FROM $Doc` is rejected as\n-- MDL-REST02 rather than sent as the literal text \"$Doc\" (it used to be,\n-- returning 200 with a 4-byte payload). Binary POST lives on the\n-- microflow activity: `call rest service post '' (Body: binary $Doc/Contents)`.\n-- `Response: FILE AS $Doc` is unaffected — downloads work.", Example: "CREATE CONSUMED REST SERVICE Module.PetStore (\n BaseUrl: 'https://petstore.example.com/api',\n Authentication: NONE\n)\n{\n OPERATION GetPet (\n Method: GET,\n Path: '/pets/{id}',\n Parameters: ($id: String),\n Query: ($verbose: String),\n Response: MAPPING Module.Pet {\n Name = name,\n Status = status\n }\n )\n};", SeeAlso: []string{"rest", "rest.published"}, }) diff --git a/cmd/mxcli/testdata/lint-rules/example_microflow.v0.27.0.star b/cmd/mxcli/testdata/lint-rules/example_microflow.v0.27.0.star new file mode 100644 index 0000000000..c76adbac9c --- /dev/null +++ b/cmd/mxcli/testdata/lint-rules/example_microflow.v0.27.0.star @@ -0,0 +1,66 @@ +# Example Starlark Lint Rule: Microflow Naming Prefixes +# +# This rule checks that microflows follow naming conventions with prefixes: +# ACT_ - Action microflows (called from UI) +# SUB_ - Sub-microflows (called from other microflows) +# DS_ - Data source microflows +# VAL_ - Validation microflows +# SCH_ - Scheduled event microflows +# +# Customize the prefixes and patterns to match your project standards. +# +# Microflow properties: +# .id - Document ID +# .name - Simple name (e.g., "ACT_ProcessOrder") +# .qualified_name - Full name (e.g., "MyModule.ACT_ProcessOrder") +# .module_name - Module name +# .microflow_type - "MICROFLOW", "NANOFLOW" or "RULE" +# .description - Documentation +# .return_type - Return type +# .parameter_count - Number of parameters +# .activity_count - Number of activities +# .document_noun - "microflow", "nanoflow" or "rule" (microflows() yields all three) +# .document_noun_title - the same, capitalised: use it for document_type= + +RULE_ID = "CUSTOM002" +RULE_NAME = "Microflow Prefix Convention" +DESCRIPTION = "Microflows should have standard naming prefixes (ACT_, SUB_, DS_, VAL_, SCH_)" +CATEGORY = "naming" +SEVERITY = "info" + +# Define valid prefixes - customize for your project +VALID_PREFIXES = ["ACT_", "SUB_", "DS_", "VAL_", "SCH_", "IVK_", "OCH_"] + +def check(): + """ + Check that microflows use standard naming prefixes. + """ + violations = [] + + for mf in microflows(): + name = mf.name # Simple name + + # Check if name starts with any valid prefix + has_valid_prefix = False + for prefix in VALID_PREFIXES: + if name.startswith(prefix): + has_valid_prefix = True + break + + if not has_valid_prefix: + loc = location( + module=mf.module_name, + document_type=mf.document_noun_title, + document_name=mf.qualified_name + ) + v = violation( + message="{} '{}' should start with a standard prefix ({})".format( + mf.document_noun_title, + name, + ", ".join(VALID_PREFIXES) + ), + location=loc + ) + violations.append(v) + + return violations diff --git a/docs-site/src/examples/rest-integration.md b/docs-site/src/examples/rest-integration.md index 465286b4dc..8644568dc8 100644 --- a/docs-site/src/examples/rest-integration.md +++ b/docs-site/src/examples/rest-integration.md @@ -219,7 +219,9 @@ CREATE IMPORT MAPPING Integration.IMM_Order ### From an OpenAPI Spec (Recommended) -If the API has an OpenAPI 3.0 spec (JSON or YAML), generate the REST client in one command: +If the API has an OpenAPI 3.x spec (JSON or YAML), generate the REST client in one command. +A Swagger 2.0 spec is refused; convert it to OpenAPI 3 first (for example +`npx swagger2openapi swagger.json -o openapi.json`) and import the converted file. ```sql mdl 1; @@ -236,6 +238,10 @@ CREATE OR MODIFY CONSUMED REST SERVICE PetStoreModule.PetStoreAPI ( Operations, path/query parameters, request bodies, response types, resource groups (from `tags`), and Basic auth are all derived automatically from the spec. +JSON responses are imported as `Response: JSON` (a string). The import does not yet create +entities for them: to get objects, create non-persistent entities and give the operation a +`Response: MAPPING Module.Entity { … }`, as shown under *Manual Definition* below. + **Preview without writing:** ```sql DESCRIBE CONTRACT OPERATION FROM OPENAPI 'specs/capital.json'; @@ -281,7 +287,7 @@ CREATE CONSUMED REST SERVICE Integration.OrdersApi ( A header value is a template, like the path: `{Token}` is the operation parameter `Token`, which the operation must declare (`Parameters: ($Token: String)`, else CE7056). An authorization header is `Headers: ('Authorization': 'Bearer {Token}')`. The expression spelling `'Bearer ' + $Token` is deprecated (MDL-DEPR711) and `mxcli fmt --upgrade` rewrites it; it used to store only `Bearer `. -Use `CREATE OR MODIFY CONSUMED REST SERVICE` to update an existing client without dropping it first: +Use `CREATE OR MODIFY CONSUMED REST SERVICE` to replace an existing client's definition without dropping it first. It declares the **whole** operation list: an operation it does not list is removed, and the output names it. The statement below keeps `GetOrder` and removes `CreateOrder`: ```sql CREATE OR MODIFY CONSUMED REST SERVICE Integration.OrdersApi ( @@ -300,8 +306,23 @@ CREATE OR MODIFY CONSUMED REST SERVICE Integration.OrdersApi ( }; ``` +To add, change or remove one operation and leave the rest as stored — including operations made in Studio Pro, whose settings MDL cannot always express — use `ALTER CONSUMED REST SERVICE`. `MODIFY OPERATION` writes only the keys it lists: + +```sql +ALTER CONSUMED REST SERVICE Integration.OrdersApi + ADD OPERATION CancelOrder ( + Method: POST, + Path: '/orders/{id}/cancel', + Parameters: ($id: String), + Response: NONE + ), + MODIFY OPERATION GetOrder (Timeout: 60), + DROP OPERATION LegacyOrder, + SET (BaseUrl: 'https://api.example.com/v2'); +``` + **Body types:** `JSON FROM $var`, `TEMPLATE '...'`, `MAPPING Entity { jsonField = Attr, ... }` -**Response types:** `JSON AS $var`, `STRING AS $var`, `FILE AS $var`, `STATUS AS $var`, `NONE`, `MAPPING Entity { Attr = jsonField, ... }` +**Response types:** `JSON`, `STRING`, `FILE`, `STATUS` (each optionally `AS $var` — the variable is not stored; the call site binds it), `NONE`, `MAPPING Entity { Attr = jsonField, ... }` **Authentication:** `NONE`, `BASIC (Username: '...', Password: '...')` ## Publishing a REST API diff --git a/docs-site/src/ide/syncing.md b/docs-site/src/ide/syncing.md index 76c061b6d2..017355ca5d 100644 --- a/docs-site/src/ide/syncing.md +++ b/docs-site/src/ide/syncing.md @@ -15,13 +15,22 @@ It is quiet when everything is already current. | Component | Refreshed by `--sync-skills` | What is never touched | |-----------|------------------------------|-----------------------| | Skills | `.ai-context/skills/` and `.claude/skills/` | skill directories mxcli does not ship | -| Lint rules | the bundled rules in `.claude/lint-rules/`, identified by file name | any rule file under another name — your own rules | +| Lint rules | the bundled rules in `.claude/lint-rules/`, while each file is a version mxcli shipped | a bundled rule you edited; any rule file under another name — your own rules | | `CLAUDE.md`, `AGENTS.md` | the part between the `` and `` markers | everything outside the markers; a file that does not exist is not created | | Version stamp | `.ai-context/mxcli-tooling.json` | — | -A bundled lint rule file belongs to mxcli, like a skill: an edit to it is -overwritten. To change a bundled rule, copy it under a new file name and rule -ID, or override its severity or options in `.claude/lint-config.yaml`. +A bundled lint rule is refreshed only while its file is a version mxcli itself +shipped. Once you edit it, it is yours: `--sync-skills` and `mxcli init` keep +it and say so on the first sync by a new mxcli, and it no longer receives +mxcli's updates to the rule. Delete the file to take the current version back. + +Moving an edited rule to a new file name works too. A bundled rule is not +written while another file in `.claude/lint-rules/` defines its rule ID, so the +stock copy does not come back beside yours and report the same rule twice. + +Prefer the options and overrides in `.claude/lint-config.yaml` where a rule has +them (`severity`, `options`, or `enabled: false` to switch it off): those keep +receiving the rule's fixes. ### The mxcli section of CLAUDE.md and AGENTS.md diff --git a/docs-site/src/reference/capabilities.md b/docs-site/src/reference/capabilities.md index 78e39865df..34984d86d1 100644 --- a/docs-site/src/reference/capabilities.md +++ b/docs-site/src/reference/capabilities.md @@ -84,7 +84,7 @@ Everything mxcli can do, organized by use case. | Capability | Command | Status | |---|---|---| | REST clients | `CREATE CONSUMED REST SERVICE` | Consumed REST services | -| OpenAPI import | `CREATE CONSUMED REST SERVICE (OpenAPI: '...')` | Generate REST client from OpenAPI 3.0 spec | +| OpenAPI import | `CREATE CONSUMED REST SERVICE (OpenAPI: '...')` | Generate REST client from an OpenAPI 3.x spec (Swagger 2.0 refused; responses imported as JSON, no entities yet) | | Business events | `CREATE BUSINESS EVENT SERVICE` | Event definitions | | Database connections | `CREATE DATABASE CONNECTION` | External SQL | | OData services | `LIST CONSUMED ODATA SERVICES/SERVICES` | Read-only browsing | diff --git a/docs-site/src/reference/page/alter-layout.md b/docs-site/src/reference/page/alter-layout.md index b352719b39..dfc42f6ae5 100644 --- a/docs-site/src/reference/page/alter-layout.md +++ b/docs-site/src/reference/page/alter-layout.md @@ -54,7 +54,9 @@ LAYOUT` rebuilds the document from your MDL, which means it can only reproduce what MDL can express; `ALTER LAYOUT` edits what is stored and leaves everything it was not asked about untouched — including widgets MDL has no syntax for, which a describe → rename → run copy drops (`describe` names each one in a -`-- NOT re-executable` comment). An `ALTER LAYOUT` against it does not. +`-- NOT re-executable` comment). An `ALTER LAYOUT` against it does not, and a +`CREATE OR REPLACE LAYOUT` of the stored layout itself is refused while it +holds one. Like `CREATE LAYOUT`, it refuses a Marketplace target and names the copy-then-repoint route in the error. diff --git a/docs-site/src/reference/page/create-page.md b/docs-site/src/reference/page/create-page.md index 2e30cfda60..cc13ba8ab3 100644 --- a/docs-site/src/reference/page/create-page.md +++ b/docs-site/src/reference/page/create-page.md @@ -21,6 +21,8 @@ Creates a new page in the specified module. A page defines a user interface scre If `OR REPLACE` is specified and a page with the same qualified name already exists, it is replaced. Otherwise, creating a page with an existing name is an error. +A replace (`OR REPLACE` or `OR MODIFY`) is refused when the stored page holds a widget MDL cannot author — a classic data grid, a reference selector, a drop-down, a reference set selector — that the statement does not declare by name, because the rebuilt page would not contain it. `DESCRIBE PAGE` marks each such widget with a `-- NOT re-executable` comment, and the error lists them. Change the page with [ALTER PAGE](alter-page.md), which keeps them, or remove them on purpose first (`ALTER PAGE M.P { DROP grid1; };`). The same applies to snippets and layouts, and `mxcli check -p` reports it before anything is written. + ### Page Properties The page declaration block `( ... )` contains comma-separated key-value properties: @@ -226,6 +228,16 @@ These properties are available on most widget types: | `Visible` | Visibility expression | `Visible: '$showField'` | | `DesignProperties` | Atlas design properties | `DesignProperties: ('Spacing top': 'Large')` | +### Attributes of a Page or Snippet Parameter + +A bare attribute name binds to the enclosing data container's object. To bind +an attribute of a page or snippet **parameter** instead, whatever container the +widget is in, write `$Param.Attr`. This works for a pluggable widget's +attribute property set by its own key too, such as a Custom chart's +`eventDataAttribute: $Ctx.EventData`. The qualified name of another entity's +attribute (`Module.Other.Attr`) is refused, because stored without the +parameter it is mx check CE7247. + ## Parameters `module.Name` diff --git a/docs-site/src/tools/run-local.md b/docs-site/src/tools/run-local.md index 2f6d6a71b7..afe42c28b4 100644 --- a/docs-site/src/tools/run-local.md +++ b/docs-site/src/tools/run-local.md @@ -43,9 +43,10 @@ stopped: pid 30133 in 300ms returns the first build made from that source or newer — so a change the loop applied *before* `wait` was called is reported at once rather than waited for, and `mxcli exec x.mdl && mxcli run wait` is a correct one-call chain. With nothing -changed it reports the last build. On Mendix 11.14, where every serve build after -the first fails ([below](#--watch-on-mendix-1114)), `wait` reports that failure with -its explanation; it never hangs on it. A run started without `--watch` never applies +changed it reports the last build. On Mendix versions whose serve process only +builds completely once ([below](#--watch-when-mxbuild-writes-the-client-11125-1114)), +the loop restarts that process per change; if even the fresh one fails, `wait` +reports the failure with its explanation — it never hangs on it. A run started without `--watch` never applies a change, and `wait` says so (exit 1). `run status|wait|stop` also work on a foreground `run --local` started in another @@ -100,6 +101,12 @@ again` when it happens, so a browser test knows to sign in before its next step. `~/.mxcli/postgres/server.log`. Stop it with `pg_ctl -D "$HOME/.mxcli/postgres/data" stop`. To remove it, stop it first and then delete `~/.mxcli/postgres` (this permanently deletes its databases). + A server whose `postgres` superuser has a password — **every Windows install**, + where the installer always sets one — needs that login: + `--ensure-db --db-admin-user postgres --db-admin-password `, or set + `PGPASSWORD` (or a `.pgpass` entry) for `postgres`, which keeps the password off + the command line. The admin login only creates the role and database; the app + still connects as `--db-user`. - Otherwise create it once yourself; without `--ensure-db`, `run --local` stops with an actionable message if the DB is unreachable: @@ -127,6 +134,7 @@ would make Studio Pro 11.13 fail to open it — a branch with no upstream, or | `--watch` | off | Rebuild + hot-apply on every project change | | `--web-client-timeout` | `$MXCLI_WEB_CLIENT_TIMEOUT`, else `5m` | Limit for one web client bundle build (e.g. `10m`); on timeout the tail of `deployment/log/web-client-build.log` is printed | | `--ensure-db` | off | Provision local Postgres + the app database if missing (fresh-session bootstrap) | +| `--db-admin-user` / `--db-admin-password` | postgres / — | Superuser login `--ensure-db` creates the role and database with when it has a password (Windows). Only with `--ensure-db`; never passed to the runtime | | `--setup` | off | Prepare prerequisites (cache MxBuild+runtime, ensure DB) and exit without booting — for a SessionStart hook | | `--app-port` | 8080 | App HTTP port | | `--admin-port` | 8090 | M2EE admin API port | @@ -359,6 +367,15 @@ clear (verified: one `/build` after an `main.scss` content edit changes - **With `--watch`** — just save the `.scss`; the theme source is watched and the loop rebuilds and hot-applies automatically. + One exception, in `mxbuild --serve` itself: after a theme **error**, the same serve + process keeps reporting that first Sass error — as `[CE6093] The theme could not be + compiled`, quoting a line no longer in the file — on every later build, even once the + SCSS compiles (measured on 11.12.5; the bundled Dart Sass compiles the same source + cleanly). So when a theme failure follows another one, the loop restarts + `mxbuild --serve` and builds once more, printing *"the theme error may be stale … + restarting mxbuild --serve"*. A theme error that survives the restart is the source's + own, and is the one reported. The fresh build cannot tell a theme-only change from a + model change, so that apply restarts the runtime. - **Without `--watch`** — nothing watches anything, so a save changes nothing in the running app. Trigger a rebuild: restart `run --local`, or use `--watch`. @@ -446,54 +463,52 @@ the deployment's own `web/index.html` rather than from the model setting, since the deployment is what gets served and the two disagree right after the setting is changed. -### `--watch` on Mendix 11.14 +### `--watch` when mxbuild writes the client (11.12.5, 11.14) -**`--watch` is not usable on 11.14 yet.** It starts and the app boots, but every -*rebuild* fails inside mxbuild: +Where mxbuild bundles the client itself, the **first** build in an `mxbuild --serve` +process is the only complete one: it does not leave behind the client source +(`web/pages/`, `web/layouts/`) or the bundler config its own incremental build +needs. Every later build in the same process then goes one of two ways: -``` -Could not find a part of the path '…/deployment/web/pages/MyModule.Home_Web.js' -``` - -or, when the change did not touch any page: - -``` -Compilation of the app bundle failed. -Cannot find module '…/deployment/web/rollup.config.mjs' - imported from …/modeler/tools/node/rollup-runner.mjs -``` +- **A change that regenerates a page or layout fails** inside mxbuild: -The first build in an `mxbuild --serve` process does not leave the deployment in a -state its own incremental build can continue from — neither the bundler's config -file nor `web/pages/`/`web/layouts/` survive it — so the first build succeeds and -every later one fails. Measured against mxbuild 11.14.0 over its own HTTP API with -no mxcli involved: the same `/build` request POSTed twice, model untouched between -them, goes Success then Failure. + ``` + Could not find a part of the path '…/deployment/web/pages/MyModule.Home_Web.js' + ``` -No remedy from outside the process works, and three were measured: + (on 11.14 a change that touches no page fails too, with + `Cannot find module '…/deployment/web/rollup.config.mjs'`). +- **Any other change — even no change at all — succeeds with an incomplete + bundle** (11.12.5): `deployment/web/dist` is replaced by a monolithic `index.js` + and one chunk, with no `pages/` and no `widgets.css`. The reload is reported as + applied and every page fails to load. -| Attempt | Result | -|---|---| -| Switch *App > Settings > Runtime > App bundler* to Rspack | Identical failure, naming `rspack.config.mjs` | -| Delete `deployment/` and start over | Next second build fails the same way | -| Restore the config (it exists for ~1.5s mid-build) | Rebuilds fine **while the model is unchanged**; fails as soon as a page changes | +Measured against mxbuild 11.12.5 over its own HTTP API, no watch loop involved: -That last one is the interesting near-miss: the config carries nothing -model-specific, so it can be captured and put back — but doing so only rescues the -case a warm loop never needs. There are two regressions here, and the second one -(the per-document client export) has no external fix. `run --local` prints an -explanation when it sees either, so the failure does not read as a corrupt -`deployment/`. - -For contrast, a one-shot `mxbuild --target=deploy` run twice into the same -deployment directory succeeds both times — it is the serve process, not the 11.14 -deployment shape. - -Until mxbuild closes this, use a restart per change: +| Build | `restartRequired` | `dist/` after it | +|---|---|---| +| 1, cold | true | 13 pages, `widgets.css`, 5.6 KB `index.js` | +| 2, same process, model unchanged | false | 0 pages, no `widgets.css`, 683 KB `index.js` | +| 3, same process, after `rm -rf web/dist` | false | same as 2 | +| 4, same process, after `rm -rf web/` | build fails | — | +| 5, fresh process | true | complete again | + +Nothing done to the deployment helps (also measured on 11.14: switching the app +bundler to Rspack, deleting `deployment/`, restoring the bundler config), and a +fresh process always does — a one-shot `mxbuild --target=deploy` run twice into +the same directory succeeds both times. So under `--watch` mxcli **restarts +`mxbuild --serve` and builds again** when a build fails this way or leaves +`dist/` without its pages, and says so in one line: -```bash -mxcli run --local --screenshot -p app.mpr ``` +mxbuild --serve's incremental build left web/dist without dist/pages (13 page modules) and dist/widgets.css (…); restarting mxbuild --serve for a full build... +``` + +A change then costs ~25–30 s instead of ~5 s. It is still hot-applied: the +incremental build's `restartRequired` is kept, because a fresh process's first +build reports `true` even for an unchanged model, and taking it would restart the +runtime — dropping every browser session — on every change. Measured end to end +on 11.12.5; on 11.14 the same restart applies to the same failure shape. ## Pixel-perfect page loop diff --git a/docs-wiki/bug-patterns/binding-context.md b/docs-wiki/bug-patterns/binding-context.md index 002061c324..03357345c3 100644 --- a/docs-wiki/bug-patterns/binding-context.md +++ b/docs-wiki/bug-patterns/binding-context.md @@ -97,7 +97,13 @@ refused it (there is no enclosing object) or wrote a binding with no source variable: a different document. The cure for a refusal that fires on Studio Pro-authored content is usually a **spelling for the stored shape**, not a looser guard — and the survey that finds them is over the slot combinations -the corpus actually stores, not over the ones the report mentions. +the corpus actually stores, not over the ones the report mentions. The spelling +has to reach every path that writes the slot: a pluggable property authored by +its own key goes through the expression grammar, so `$Ctx.EventData` there was +refused as an expression while the mapped keys took it. In the meantime the +qualified name stood in for it, and a describer that prints that "faithfully" +round-trips byte for byte into a binding that is CE7247. Faithful output is not +evidence until the shape it re-creates has been through mx check. **Both halves or neither.** A describer that emits a path the builder cannot consume turns silent corruption into a build error, and a builder taught a form diff --git a/docs-wiki/bug-patterns/cost-scales-with-the-project.md b/docs-wiki/bug-patterns/cost-scales-with-the-project.md index 3f9156fb03..acb14a43be 100644 --- a/docs-wiki/bug-patterns/cost-scales-with-the-project.md +++ b/docs-wiki/bug-patterns/cost-scales-with-the-project.md @@ -1,11 +1,15 @@ --- title: Cost That Scales With the Project, Not the Request category: bug-pattern -last-synced: a20932c1 +last-synced: f2d8dc9b covers: - mdl/executor + - modelsdk/mpr sources: - .claude/skills/fix-issue/findings/mdl-executor/ + - .claude/skills/fix-issue/findings/modelsdk/ + - modelsdk/mpr/reader_units_v2.go + - modelsdk/mpr/reader_units_v1.go - mdl/executor/untimed_work.go - mdl/executor/cmd_microflows_derived_layout_backend.go - mdl/executor/cmd_microflows_derived_layout.go @@ -63,6 +67,27 @@ and the search moves to per-construct work multiplied by something. A `SIGQUIT` goroutine dump at a few moments, showing the same frame each time, plus one hand-timed call, settled that case before any profile was taken. +It held a fourth time on a module dump that timed out: a statement-scoped read +cache cut its unit-file reads by five and its wall time by nothing. The reads +were not the cost; the profile put two thirds of it in decoding every microflow +of the project once per described flow. + +**A decorator that embeds an interface hides what it wraps.** The memo the +layout rounds read through embeds `FullBackend`, so every optional capability +of the real backend — a by-name lookup that decodes one flow instead of all of +them — failed its type assertion inside the span and took the slow fallback. +Nothing errs and nothing is wrong, only slow. Whenever `ctx.Backend` is swapped +for a wrapper, grep the `ctx.Backend.(…)` assertions the span can reach, and +forward the ones that matter. + +**An index keyed by content cannot go stale.** The cheapest cache for "what +type is this unit" is the one keyed by the hash the storage already keeps: +MPR v2's `ContentsHash`, v1's blob length. It needs no invalidation, survives +writes and processes, and turns "every listing reads the project" into "every +listing reads what changed". A cache keyed by identity needs an invalidation +story for every write path and every other process; one keyed by content needs +only that the key is kept honest, which the storage already promises. + **Assert counts, not wall time.** Every fix in this class is guarded by a counter — split/merge analyses per describe, builds per plan, `IsRule` calls per describe, project listings per lookup, backend `List*` calls per check — because @@ -108,7 +133,7 @@ backend once. ## See also -- [fix-issue findings](../../.claude/skills/fix-issue/findings/) — the four +- [fix-issue findings](../../.claude/skills/fix-issue/findings/) — the multiplier chains, with their before/after counts - [[scripts-that-cannot-rerun]] — the `create or modify` design whose comparisons are what repeats the build diff --git a/docs-wiki/bug-patterns/local-loop-silence.md b/docs-wiki/bug-patterns/local-loop-silence.md index 8165fdf299..89e4534824 100644 --- a/docs-wiki/bug-patterns/local-loop-silence.md +++ b/docs-wiki/bug-patterns/local-loop-silence.md @@ -11,6 +11,7 @@ sources: - cmd/mxcli/docker/webclient_supervisor.go - cmd/mxcli/docker/webclient_watch.go - cmd/mxcli/docker/webclient_pages.go + - cmd/mxcli/docker/serve_recovery.go - cmd/mxcli/devloop_admin.go --- @@ -88,6 +89,20 @@ it covers a dead watcher, missing pages, dangling chunks and a `dist/` someone else deleted. Reproducing it needs no patience — kill the runner process and the dead-watcher state is there in seconds. +**The same holds for `mxbuild --serve`, and its state is worse hidden.** On the +versions where mxbuild writes `web/dist` itself, only a serve process's first build +bundles completely — every later one, *even with the model unchanged*, replaced the +bundle with one that has no pages, and reported success. The same process also +kept its first Sass error as CE6093 after the SCSS was fixed. Both were settled by +driving the serve `/build` API from a small harness with no watch loop: the +unchanged-model second build is the row that rules out "something in the change" +and every repair applied to the deployment at once, and a fresh process is the row +that fixes it. So the loop restarts the serve process on those shapes — but must +keep the *incremental* build's `restartRequired`, because a cold build always says +"restart" and taking it would drop every session per change. An earlier hint had +concluded "nothing outside the process fixes it" — true of the deployment, false of +the process; check which "outside" a negative result actually covered. + **The tools the loop drives have their own idea of what is being watched.** mxbuild's page plugin globs `web/pages` once, at bundler start, and matches later events against that snapshot — so a page added while the loop runs is never diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index 1f72217340..0f33572935 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -1184,7 +1184,11 @@ Icon collections (`CustomIcons$CustomIconCollection`, e.g. `Atlas_Core.Atlas_Fil | Show clients | `list consumed rest services [in module];` | List all or filter by module | | Describe client | `describe consumed rest service Module.Name;` | Re-executable CREATE | | Create client | See syntax below | Property-based `{}` syntax | -| Create or modify | `create or modify consumed rest service ...` | Replaces existing | +| Create or modify | `create or modify consumed rest service ...` | Replaces existing — an operation it omits is removed (and named in the output) | +| Add operation | `alter consumed rest service Module.Name add operation Name ( ... );` | Operation body uses CREATE syntax; other operations stay as stored | +| Modify operation | `alter consumed rest service Module.Name modify operation Name ( Timeout: 60 );` | Writes only the keys listed | +| Drop operation | `alter consumed rest service Module.Name drop operation Name;` | | +| Set service properties | `alter consumed rest service Module.Name set ( BaseUrl: '...', Authentication: none );` | BaseUrl, Authentication | | Drop client | `drop consumed rest service [if exists] Module.Name;` | | | Import from OpenAPI | See OpenAPI import below | Auto-generate from spec | | Preview OpenAPI | `describe contract operation from openapi 'path';` | Preview without writing | @@ -1222,7 +1226,7 @@ create consumed rest service Module.Api ( ``` **Body types:** `json from $var`, `template '...'`, `mapping entity { jsonField = attr, ... }` -**Response types:** `json as $var`, `string as $var`, `file as $var`, `status as $var`, `none`, `mapping entity { attr = jsonField, ... }` +**Response types:** `json`, `string`, `file`, `status` (each optionally `as $var` — the variable is not stored), `none`, `mapping entity { attr = jsonField, ... }` **Authentication:** `none`, `basic (username: '...', password: '...')` ### OpenAPI Import @@ -1967,7 +1971,9 @@ Name the widget the way you write it in a page body. The target is stored as the | Setup mxcli | `mxcli setup mxcli [--os linux]` | Download platform-specific mxcli binary | | LSP server | `mxcli lsp --stdio` | Language server for VS Code | -Set `MXCLI_EXEC_TIMEOUT` to override the per-statement execution timeout (default 5m) used by `mxcli exec` (for example `MXCLI_EXEC_TIMEOUT=12m` or `MXCLI_EXEC_TIMEOUT=900`). `REFRESH CATALOG` is exempt from the default guard — it runs without a wall-clock limit on large projects — unless you set `MXCLI_EXEC_TIMEOUT` explicitly (which then applies to it too, letting you cap it). +Set `MXCLI_EXEC_TIMEOUT` to override the per-statement execution timeout (default 5m) used by `mxcli exec` (for example `MXCLI_EXEC_TIMEOUT=12m` or `MXCLI_EXEC_TIMEOUT=900`). `REFRESH CATALOG` and `DESCRIBE MODULE … WITH ALL` are exempt from the default guard — they run without a wall-clock limit on large projects and modules — unless you set `MXCLI_EXEC_TIMEOUT` explicitly (which then applies to them too, letting you cap them). + +Set `MXCLI_OUTPUT_LINE_LIMIT` to override the per-statement output limit (default 10,000 lines; `0` for none). A statement that passes it is aborted with its output truncated. `DESCRIBE MODULE … WITH ALL` is exempt from the default limit — its output is bounded by the module — unless you set `MXCLI_OUTPUT_LINE_LIMIT` explicitly. ## ANTLR4 Parser Architecture diff --git a/docs/03-development/PED_MCP_CAPABILITIES.md b/docs/03-development/PED_MCP_CAPABILITIES.md index 37c726b2d6..b2e24eae81 100644 --- a/docs/03-development/PED_MCP_CAPABILITIES.md +++ b/docs/03-development/PED_MCP_CAPABILITIES.md @@ -572,6 +572,96 @@ behaviour change, not an error. read the `page-gen-common` skill first); page create and ALTER ran clean on both servers. +### Microflow canvas: flow sides and notes (measured live 2026-10-21, 11.15.0-rc.4) + +- **A flow created without sides is drawn right → left.** The `SequenceFlow` + constructor takes `originConnectionSide` / `destinationConnectionSide` + (`Top | Right | Bottom | Left`). The create path sent neither, so every `@anchor`, + CASE outcome anchor and builder default was lost: a CASE built with + `@anchor(from: bottom, Approved: (from: right), (empty): (from: bottom, to: top))` + read back `1 -> 3` on all eight flows. **Fixed:** `buildFlowDocContent` maps the + connection indexes the way the splice already did (`setPEDSides`). +- **A note is a `Microflows$Annotation` object** (`x`, `y`, `caption`, `size`) and + its line a **`Microflows$AnnotationFlow` element, not a constructor.** Its ends + are `origin` / `destination`, not `originId` / `destinationId`, and its sides are + the required `originConnectionIndex` / `destinationConnectionIndex` themselves. + The constructor keys are refused: `"/origin":"Expected reference (string), got + undefined"`. Before the fix, a microflow with a note failed the whole create + (`mapObjectTree` had no case for it). The splice had the same wrong keys too, + hidden because the note was refused before its line was mapped. +- Verified: the same script read back from Studio Pro and from the modelsdk engine + gives **identical sides on all 10 flows**, both note lines included (an unplaced + note `2 -> 0`, a note placed below its activity `0 -> 2`). It also gives the same + note positions, sizes and captions, and `ped_check_errors` is clean on two + successive calls. A read-back leaves out a connection index of 0, its default. +- The pre-11.14 (full) constructor's flows carry no `$Type`, so a note's line could + only be read as a `SequenceFlow`. The backend refuses notes there rather than + risk an execution edge out of a note. +- ~~Within one `ped_update_document`, an `AnnotationFlow` added *before* a + `SequenceFlow` is stored *after* it.~~ This was the general reversal described + below, not something about note lines. + +### Microflow canvas: what one update can address (measured live 2026-10-21, 11.15.0-rc.4) + +**Fixed: ALTER MICROFLOW insert over MCP failed on 11.15.** Every insert failed +with `undefined is not an object (evaluating 'u.$Type')`, with or without a note. +Direct `ped_update_document` probes on bare test documents found two rules behind +it. Both follow from the tool's own description: *"every `index` you provide refers +to the array as it exists BEFORE this call"*. + +- **An update cannot address an element it adds.** A path resolves against the + document as it was before the update, so `/flows/N/…` or `/objectCollection/objects/N/…` + for a new N does not reach the new element: + + | Same update as the add | Result | + |---|---| + | set `/flows/N/line/originControlVector` | **fails the whole update** (`u.$Type`): the line is not there yet | + | set `/flows/N/line` (whole `BezierCurve`), then a vector | fails the same way | + | set `/flows/N/line` (whole `BezierCurve`) alone | accepted, **not applied** | + | `line` or `isErrorHandler` in the `SequenceFlow` constructor value | accepted, **ignored** (the constructor has neither) | + | set `/flows/N/isErrorHandler` | accepted, **not applied**: the flow reads back as an ordinary one | + | set `/objectCollection/objects/N/action` or `…/relativeMiddlePoint` | accepted, **not applied**: "No action defined"; the position is the constructor's `x`/`y` | + | `action` in the `ActionActivity` constructor value | accepted, **ignored** (the constructor takes `x`, `y`, `caption` only) | + | add the flow in its element shape (`origin`, `line`, `isErrorHandler`) | refused: `NodeUnableToConvertElement, String, System.Guid`, so `$id()` is not resolved on that path | + + **Every one of these sets lands in a later update**: control vectors, + `isErrorHandler` and `action` alike, on a flow or activity the previous update + created. `$id(/…/N)` references *do* resolve to elements added earlier in the + same update. +- **Several adds to one array without an `index` are stored in reverse**, and + `$id()` resolves against that final order. Three objects added in one update came + back as 3, 2, 1, and two flows came back as 2, 1. An inserted IF/ELSE was + therefore wired to the wrong objects, while a one-activity insert (one object, one + flow) hid the problem. Adds given **one shared `index`** (the array's length before + the update) are stored in listed order, which is PED's documented rule for adding + several elements at one position. + +**The fix** (`flowPatchOps` / `SaveUnit`) sends an insert as **two updates**. The +first adds the new elements, constructor properties only, each at the stored length +as its `index` and wired with `$id()`, and moves the stored elements to make room. +The second fills in the new elements: actions, split conditions, loop sources, +`isErrorHandler` and control vectors (a `0;0` vector is the schema default and is +not sent). + +This gives up some of the one-update atomicity the splice relied on. The first +update is still all or nothing, but **a failed second update leaves the insert in +Studio Pro incomplete**. That is the same contract as `CreateMicroflow`, which +creates and then sets activities. The error says so, and `ped_check_errors` names +what is missing. There is no write that avoids it on 11.15: nothing in the update +that adds an activity can give it its action. A retry is refused by the live/local +index check, so it cannot insert twice. + +Verified live: plain, IF/ELSE and note inserts, each compared with the same script +run with the default engine. All flows matched on ends, sides and control vectors +(4, 9 and 6 flows), as did every object's position. The fills read back: actions, +the condition, `true`/`false` cases, and the note's caption and size. +`ped_check_errors` was clean on two successive calls. The script is +`mdl-examples/bug-tests/mcp-alter-insert-addresses-new-elements.mdl`. + +Still refused upstream: a custom error handler (`on error … begin … end error`) +over MCP. `isErrorHandler` is settable in a later update, so the splice marks such a +flow in its second update, but MDL cannot reach that path yet. + ## Capability gaps (established 11.11, status re-checked each release) These are the *absences* that bound what the backend can do. They are as diff --git a/docs/11-proposals/PROPOSAL_alter_object_list_item_address.md b/docs/11-proposals/PROPOSAL_alter_object_list_item_address.md new file mode 100644 index 0000000000..716008b437 --- /dev/null +++ b/docs/11-proposals/PROPOSAL_alter_object_list_item_address.md @@ -0,0 +1,213 @@ +--- +title: ALTER PAGE Addressing of Pluggable Object-List Items +status: proposed +date: 2026-10-17 +--- + +# Proposal: ALTER PAGE Addressing of Pluggable Object-List Items + +**Status:** Proposed +**Date:** 2026-10-17 + +Refs ako/mxcli#1326, mendixlabs/mxcli#1453. + +## Problem Statement + +`ALTER PAGE` can reach a widget *inside* an Accordion group (set, insert +before/after, drop — e94a6df2), but not the group itself: + +``` +alter page M.P { insert into grp1 { actionbutton btnNew (…) }; }; +-- widget "grp1" not found +``` + +The name never existed. Mendix stores no `Name` on a pluggable widget's +object-list item (`CustomWidgets$WidgetObject`); `create … group grp1 (…)` +parses the name and drops it, and describe prints `group ( … ) { … }`. So an +**empty** group — or a group's header — cannot be filled, and a group cannot be +removed, without `create or replace` of the whole page. The same holds for every +widget with an object list of items holding widgets: Pop-up menu items, and any +container-list style widget the field reported (9 of 12 non-MDL change scripts +on one production app were BSON patches for this class). + +The class is the one `docs-wiki/bug-patterns/mutator-addressing.md` describes: +a node the model does not name needs an address that does not depend on a name. + +## BSON Structure + +Measured on a blank Mendix 11.13.0 app, Accordion +(`com.mendix.widget.web.accordion.Accordion`) authored by mxcli and checked with +`mx check` (0 errors): + +``` +CustomWidgets$CustomWidget Name: "acc1" + Type.ObjectType.PropertyTypes[] + PropertyKey "groups" ValueType { Type: "Object", IsList: true, + ObjectType.PropertyTypes[ … ] } + headerRenderMode Enumeration + headerText TextTemplate <- the group's visible title + headerHeading Enumeration + headerContent Widgets <- second widget slot + content Widgets <- the body + visible / dynamicClass / initiallyCollapsed Expression + loadContent / initialCollapsedState Enumeration + collapsed Attribute + onToggleCollapsed Action + Object.Properties[ TypePointer -> "groups" ] + Value.Objects [marker=2]: + - CustomWidgets$WidgetObject (no Name — one per group) + Properties[ TypePointer -> "content" ].Value.Widgets [marker=2]: [ … ] + Properties[ TypePointer -> "headerContent" ].Value.Widgets [2]: [ … ] +``` + +- An item is a `CustomWidgets$WidgetObject` in `Value.Objects`, list marker 2. +- Each widget slot is a `Widgets` property of the item; its `Widgets` key is + always present (mandatory list, `[2]` when empty), so a write never has to add + it. +- The item's identity for addressing must therefore be **position** or + **content**; there is nothing else stored. + +## Proposed MDL Syntax + +Reuse the navigation-list item address (#1388) for every widget whose children +are unnamed items, and the slot suffix `#1462` introduced (`dataGrid1.controlbar`): + +```sql +-- the group's body (bare item = body slot) +alter page M.P { insert into acc1 item@1 { actionbutton btnNew (Caption: 'New', Action: close page) }; }; +alter page M.P { insert into acc1 item('One') { … }; }; -- matched on the item's text (headerText) +alter page M.P { insert into acc1 item(btnInGroup) { … }; }; -- the item holding btnInGroup + +-- another slot of the item +alter page M.P { insert into acc1 item@1.headercontent { dynamictext t (Content: 'Hi') }; }; +alter page M.P { replace acc1 item@1.headercontent with { … }; }; +alter page M.P { drop acc1 item@1.headercontent; }; -- empties the slot + +-- the item itself +alter page M.P { drop acc1 item@2; }; -- removes the group +alter page M.P { set (headerText: 'Renamed') on acc1 item@1; }; -- an item property +``` + +Rules, decided with the maintainer: + +1. **One word, `item`, for every list** — not the per-widget describe keyword + (`group`). The address is about the list's structure, which is the same for + all of them; a per-widget word would be a second vocabulary to learn and to + keep in step with describe. +2. **A bare item address means the item's body slot**: the slot CREATE routes + loose children to — `content` when the item has one, else its first `Widgets` + property (`defaultItemSlotKey`, `widget_engine.go`). Another slot needs + `.slot`, spelled as its property key (case-insensitive) or the item-slot + keyword CREATE accepts for it (`headercontent`). +3. **Legacy table cells are deferred** (`table1 cell(r, c)` sketched below). + +One rule follows from the widget's own definition: **a slot the item's +configuration hides is refused, naming the property that enables it.** The +Accordion's editor config hides `headerContent` while `headerRenderMode` is +`'text'` — the default (`hidePropertyIn(…, "groups", …, "headerContent")`, +pinned in `editorconfig_nested_test.go`). Writing widgets into it would store +content Studio Pro hides and does not render, the same hazard #574 settled for +hidden text templates. So `insert into acc1 item@1.headercontent { … }` on a +text-mode group answers "headerContent is hidden while headerRenderMode is +'text' — `set (headerRenderMode: 'custom') on acc1 item@1` first" rather than +writing it. + +Shared with the existing forms, so nothing new to learn: +- `@n` is 1-based and, after a selector, chooses among its matches. +- A miss lists the addresses that do resolve; an ambiguous selector is refused + with the `@n` forms, never resolved to the first match. +- BEFORE/AFTER a **slot** is refused (as for `grid.controlbar`); BEFORE/AFTER an + **item** inserts an item (`insert after acc1 item@1 { group (…) { … } }`), + mirroring navigation-list items. + +## Implementation Plan + +Order matters: describe has to stop misplacing header widgets before an address +that edits them is useful, or describe → edit → exec moves them. + +1. **Describe fix (prerequisite, own PR).** `cmd_pages_describe_objectlist.go` + folds *every* `Widgets` sub-property of an item into `item.Children`, so a + group's `headerContent` widgets are printed as body and re-created in + `content` on round-trip. Print the body slot as children and any other + non-empty slot as a nested ` { … }` block — the form CREATE + already parses. Test: round-trip a group with both slots filled. +2. **Shared rule.** Move `defaultItemSlotKey`'s choice into `mdl/types` (as + `WidgetSlotKeyword` was in #1462) so CREATE, describe and the mutator pick the + same body slot from one function. +3. **Grammar.** `alterTarget`'s two `ITEM` alternatives gain + `(DOT identifierOrKeyword)?`. No new keyword. +4. **Selector.** Generalise `backend.NavItemSelector` / `ParseNavItemSelector` + to carry the optional slot; keep the type name or rename it in a separate + refactor commit. +5. **Resolver.** Generalise `resolveNavItem` (`navlist.go`) to any object-list + item: on a CustomWidget, the list is the `Object`-kind, `IsList` property + (refuse when a widget has more than one such list, naming them, until a + use case needs a list qualifier). `item('text')` matches the item's + `TextTemplate` / `String` properties; `item(w)` the item whose slots hold + widget `w`. +6. **Operations.** `InsertWidget` / `ReplaceWidget` / `DropWidget` / + `SetWidgetProperty` and `ResolveAlterTarget` route an item address to the + item and an item.slot address to the slot (reusing `pluggableSlot` from + `pluggableslot.go`). Inserting an *item* serializes it through the CREATE + object-list builder, the way navigation-list items are serialized. +7. **Data context.** Widgets inserted into a slot are built in the item's + enclosing data context — the three child walkers already descend into + object-list items (#891), so this is a test, not new code. + +### Files to modify/create + +| File | Change | +|------|--------| +| `mdl/executor/cmd_pages_describe_objectlist.go` | Step 1: print non-body slots as nested blocks | +| `mdl/types/` | Step 2: shared body-slot rule | +| `mdl/executor/widget_engine.go` | Use the shared rule | +| `mdl/grammar/MDLParser.g4`, `mdl/visitor/` | Step 3: `.slot` after an `item` selector | +| `mdl/backend/navitem_selector.go` | Step 4: slot in the selector | +| `mdl/backend/pagemutator/navlist.go`, new `objectlist_items.go` | Steps 5–6 | +| `mdl/backend/pagemutator/alter_target.go`, `mutator.go` | Route the new addresses | +| `docs-site/src/reference/page/alter-page.md`, `.claude/skills/mendix/alter-page/SKILL.md` | Document | + +## Version Compatibility + +No gate. The storage shape is the pluggable widget framework's +`CustomWidgets$WidgetObject` list, unchanged across the Mendix 10 and 11 versions +mxcli supports, and the address is resolved against the widget's own stored +schema (`ObjectType.PropertyTypes`), not a version table. + +## Test Plan + +- Mutator unit tests on a fixture with a two-group Accordion: insert into an + empty body, into `headercontent`, replace/drop a slot, drop an item, set an + item property, each selector form, `@n` among matches, ambiguity and miss + messages — and a **control**: each test fails with today's `widget "…" not + found` before the change. +- Describe round-trip test for step 1 (both slots filled → identical BSON slot + contents after exec of the describe output). +- `mdl-examples/bug-tests/1453-alter-insert-into-accordion-group.mdl`, executed + on a blank app and checked with `mxcli docker check` (0 errors) — run on the + unfixed and fixed builds. +- Pop-up menu: one test that the same address reaches a `basicItem`, proving the + generalisation rather than an Accordion special case. + +## Deferred: Legacy Table Cells + +A `Forms$Table` cell is positional (`TopRowIndex`, `LeftColumnIndex`), so its +address would be `table1 cell(row, column)`, 1-based — not `item@n`, which +would depend on storage order. Deferred until (a) a Studio Pro-authored table is +available to confirm the stored shape (a Mendix 11 cell is `Forms$DbTableCell`, +which gen does not model, and `mx convert` refuses `Forms$TableCell`), and (b) +describe prints tables at all. #1463 already lets ALTER reach widgets inside +cells by name. + +## Open Questions + +1. **Text selector scope.** `item('One')` matches any `TextTemplate`/`String` + property of the item. For an Accordion that is only `headerText`; a widget + whose items carry two texts could match on either. Narrow it to the first + such property in schema order if that turns out confusing. +2. **Hidden-slot refusal vs. auto-enable.** The proposal refuses a write into a + hidden slot. Switching `headerRenderMode` to `'custom'` automatically would + be friendlier but changes a property the statement never named; the refusal + keeps every change explicit. +3. **Widgets with two object lists.** Refused (named) for now; a qualifier such + as `w groups item@1` can be added if a real widget needs it. diff --git a/docs/11-proposals/PROPOSAL_openapi_response_entities.md b/docs/11-proposals/PROPOSAL_openapi_response_entities.md new file mode 100644 index 0000000000..d7501c87da --- /dev/null +++ b/docs/11-proposals/PROPOSAL_openapi_response_entities.md @@ -0,0 +1,297 @@ +--- +title: OpenAPI import — generate response entities and wire operations to them +status: draft +date: 2026-10-21 +related: + - PROPOSAL_mapping_coverage.md +--- + +# Proposal: OpenAPI import generates response entities and mappings + +**Status:** Draft +**Date:** 2026-10-21 +**Issue:** ako/mxcli#1234 (mendixlabs/mxcli#280) + +## Problem Statement + +`create consumed rest service M.Svc ( OpenAPI: '…' )` imports a spec's +operations, parameters and authentication, but not what the operations return. +Every JSON response becomes `Response: json`, which gives a microflow a raw +string. To get objects, the user has to write the non-persistent entities, the +associations and a `Response: mapping` tree for each operation by hand. That is +the part of a REST integration that takes longest and goes wrong most often. + +The report (mendixlabs/mxcli#280) came from the second route into this problem. +An agent asked to "generate the Consume Rest Client from +https://petstore.swagger.io/v2/swagger.json" wrote the entities, JSON +structures and import-mapping documents correctly. It then wired every +operation as `Response: JSON AS $PetJson`, so none of them was used. The tool +gave the agent no way to generate them and no guidance on how to wire them. + +Measured on `main` (d8f0371b9) against both Petstore specs, the gap is larger +than the issue describes: + +| Spec | What `describe contract operation from openapi` produces today | +|---|---| +| OpenAPI 3.0 (`petstore3.swagger.io/api/v3/openapi.json`) | Operations and parameters correct; every response `json`; request bodies `json from $body`; `BaseUrl: ''` with a warning (relative server URL) | +| **Swagger 2.0** (`petstore.swagger.io/v2/swagger.json`, the reporter's spec) | `BaseUrl: ''`, request bodies dropped, **every response `none`**, and **no warning**. The parser reads only OpenAPI 3 fields, so a 2.0 document parses "successfully" into an empty contract | + +The Swagger 2.0 row is a silent downgrade in its own right. It is fixed in +Phase 0 whatever happens to the rest of this proposal. + +**Who benefits:** anyone who consumes a third-party API, and agents most of +all, because generating this scaffolding is exactly what they get wrong. + +## What exists to build on + +The **storage side is already done**. Mendix 11 does not use an import-mapping +document for a consumed REST operation. It stores the mapping **inline** on the +operation as `Rest$ImplicitMappingResponseHandling`, whose `RootMappingElement` +is an `ImportMappings$ObjectMappingElement` tree. mxcli writes and reads that +today (`mdl/backend/modelsdk/consumed_rest_write.go`, +`restImplicitMappingResponseToGen`). `describe` round-trips it (#843), and +`mdl-examples/bug-tests/843-rest-response-mapping.mdl` builds with 0 errors on +11.13.0: + +```sql +operation "SearchRoutes" ( …, + Response: mapping ZZB.Routing { + RoutingCode = routing_code, + RoutingName = routing_name + } +) +``` + +So the issue's framing ("generate JSON structures and import mappings") is the +wrong target for this document type. The target is **non-persistent entities +plus an inline `Response: mapping`**, which no extra documents need to back. + +What does **not** exist: + +| Gap | Where | +|---|---| +| No schema model: `openapi.Schema` is `{type, format}` only. There is no `$ref`, `properties`, `items`, `required`, `enum`, `allOf`, `components.schemas`, or Swagger 2.0 `definitions` | `mdl/openapi/parser.go` | +| `resolveResponseType` reads only `content`, so a 2.0 response (`schema` directly on the response) resolves to `NONE` | `mdl/openapi/parser.go` | +| The inline response writer hardcodes an `(Object)` root with `MaxOccurs 1`. An **array-rooted** response (`findPetsByStatus` returns `Pet[]`) and a nested array (`Pet.tags`) cannot be expressed | `consumed_rest_write.go` | +| Value elements are always written with `DataTypes$StringType`, whatever the attribute's type. This is untested for non-String attributes | `consumed_rest_write.go` | +| Status code fixed at 200 | `consumed_rest_write.go` | + +## BSON Structure + +**Known, and implemented:** `Rest$ImplicitMappingResponseHandling` with +`ContentType`, `StatusCode` and `RootMappingElement` (an +`ImportMappings$ObjectMappingElement` with `Entity`, `ExposedName`, +`JsonPath` such as `(Object)|field`, `ObjectHandling: Create`, `Association`, +`MinOccurs`/`MaxOccurs` and `Children` of +`ImportMappings$ValueMappingElement`). Measured fixes already recorded: +- no `IsDefaultType` on value elements (#1097); +- multi-segment paths (`rest-inline-mapping-paths.mdl`). + +**Unknown. These must be measured from a Studio Pro reference before Phase 2:** + +1. **Array root.** What Studio Pro stores for an operation whose response is a + JSON array: the root element's `JsonPath` (`(Array)`? `(Array)|(Object)`?), + whether there is a wrapper element, and its `MaxOccurs`. Neither the demo + corpus nor mxcli has an example. A related finding recorded zero + `Rest$ImplicitMappingBody` across 9 packages, which suggests the corpus is + thin on inline REST mappings generally. +2. **Nested array** (`Pet.tags: Tag[]`): the element shape and `MaxOccurs -1`, + by analogy with message definitions and JSON-structure mappings. Verify it, + don't assume it. +3. **Value element `Type`** for Integer, Long, Decimal, Boolean and DateTime + attributes: whether `DataTypes$StringType` is accepted, or whether the type + must match the attribute. `mx check` may tolerate a mismatch that fails at + runtime, so this needs a runtime check (see Test Plan). +4. **Association direction and name** that Studio Pro generates when it creates + entities from a JSON response, so generated models match what a Studio Pro + user would get. +5. **Primitive arrays** (`Pet.photoUrls: string[]`): whether Studio Pro maps + them at all (helper entity with a value attribute, as JSON structures do?) + or skips them. + +**Requested reference project:** a Mendix 11.x app with one consumed REST +service holding operations whose responses are: +(a) an object with nested object, object array and primitive array; +(b) a top-level array; +(c) an object with Integer, Long, Decimal, Boolean and DateTime fields. +Entities should be created through Studio Pro's own response-to-entity flow. +`mxcli bson dump --type Rest$ConsumedRestService` on it settles items 1–5. + +## Proposed MDL Syntax + +No new statement. One property on the existing statement, which is off by +default so existing scripts keep their behaviour: + +```sql +create or modify consumed rest service Shop.Petstore ( + OpenAPI: 'https://petstore3.swagger.io/api/v3/openapi.json', + BaseUrl: 'https://petstore3.swagger.io/api/v3', + GenerateEntities: Yes +); +``` + +With `GenerateEntities: Yes`, the statement: + +1. resolves each operation's success response schema (`200`, `201`, then the + first `2xx`) for `application/json`; +2. creates or modifies one **non-persistent entity per named schema** + (`#/components/schemas/Pet` → `Shop.Pet`). Anonymous inline schemas are + named `Response`, or `` when nested; +3. creates an association for each nested object and object array; +4. writes each operation's response as `Response: mapping Shop.Pet { … }`. + +The output is ordinary MDL. `describe contract operation from openapi '…'` +already prints the MDL an import would run, so it gains the entity and +association statements too. The user (or agent) can review the import, or +paste it and edit it instead of running it: + +```sql +create or modify non-persistent entity Shop.Pet ( + Id: Long, + Name: String, + Status: String +); +create or modify non-persistent entity Shop.Category ( Id: Long, Name: String ); +create or modify association Shop.Pet_Category from Shop.Pet to Shop.Category; -- direction: BSON item 4 + +create or modify consumed rest service Shop.Petstore ( … ) +{ + operation getPetById ( Method: get, Path: '/pet/{petId}', Parameters: ($petId: Long), + Response: mapping Shop.Pet { + Id = id, Name = name, Status = status, + create Shop.Pet_Category/Shop.Category = category { Id = id, Name = name } + } + ) +}; +``` + +| OpenAPI | Mendix attribute | +|---|---| +| `integer` / `int32` | Integer | +| `integer` / `int64` | Long | +| `number` | Decimal | +| `boolean` | Boolean | +| `string` / `date-time`, `date` | DateTime | +| `string` (incl. `enum`) | String | +| `$ref` to object | associated entity | +| `array` of object | associated entity, list (BSON item 2) | +| `array` of primitive | BSON item 5 — until measured, reported as not imported | +| `oneOf` / `anyOf` / `additionalProperties` / `not` | not imported, reported by name | +| `allOf` | properties merged into one entity | + +Every schema member the import cannot represent is **listed in the output with +its reason**, never dropped silently. This is the lesson of mendixlabs/mxcli#1118 +and of the Swagger 2.0 row above. + +Attribute names follow the existing rules: Mendix-reserved names are renamed +and reported (`id` → `Id`, `type` → `TypeValue`; CE7247). The mapping keeps +the JSON name, so the wire contract is unaffected. + +## Implementation Plan + +Each phase is its own PR. + +**Phase 0: stop the silent downgrades.** Small, independent, can ship now. +- Detect `swagger: "2.0"` and refuse it with a clear message until Phase 4 + ("Swagger 2.0 is not supported yet; convert with …"), instead of returning an + empty contract. +- Add skill guidance (`.claude/skills/mendix/` REST client skill) to wire + responses with `Response: mapping Entity { … }`, and to remember that an + import-mapping document is not referenced from a consumed REST operation. + This closes the agent route the reporter took. + +**Phase 1: schema model.** Model `Schema` with `$ref` (local `#/` only), +`properties`, `items`, `required`, `enum`, `format`, `allOf`, `nullable`, and +`components.schemas`. Add a resolver with cycle detection: a recursive schema +reuses its entity rather than recursing. Pure unit tests in `mdl/openapi/`. + +**Phase 2: generation for object-rooted responses** (needs BSON items 3–4). +Plan entities, associations and mapping trees from Phase 1. Emit them as **AST +statements executed through the existing handlers** rather than writing +documents directly. Entity creation then goes through the normal create/modify +path, so `canon.Reconcile`, GUID preservation on re-import and validation come +for free, and `describe contract … from openapi` prints the same statements. +Wire `GenerateEntities` through grammar, visitor and AST. + +**Phase 3: array roots and nested arrays** (needs BSON items 1, 2, 5). Extend +`model.RestResponseMapping` and `restInlineMappingElementToGen` with the +measured array shapes, and describe/round-trip them. This is the first change +to the writer. + +**Phase 4: Swagger 2.0.** Normalise 2.0 into the 3.0 model (`definitions` → +`components.schemas`, `host`/`basePath`/`schemes` → server URL, response +`schema` → `content["application/json"]`, `in: body` → request body). Then +remove the Phase 0 refusal. + +**Out of scope:** request bodies as `Body: mapping` (export direction; same +generator, separate PR once responses are proven); non-`2xx` responses; XML +responses; generating enumerations from `enum` (String first, an enumeration +is a later opt-in); external `$ref` URLs. + +### Files to modify/create + +| File | Change | +|------|--------| +| `mdl/openapi/parser.go` | Schema model, `components.schemas`, `$ref` resolver; Swagger 2.0 detection (P0) and normalisation (P4) | +| `mdl/openapi/entities.go` (new) | Schema → planned entities, associations and mapping trees; type table; naming; unsupported-member report | +| `mdl/grammar/MDLParser.g4`, `mdl/visitor/visitor_rest.go`, `restClientSchema` | `GenerateEntities` property | +| `mdl/ast/` | `CreateRestClientStmt.GenerateEntities` | +| `mdl/executor/cmd_rest_clients.go` | `createRestClientFromSpec` and `describeContractFromOpenAPI` run or print the generated statements before the service | +| `model/types.go`, `mdl/backend/modelsdk/consumed_rest_write.go` + read | Array roots and nested arrays (P3), value element types (if item 3 requires it) | +| `.claude/skills/mendix/…` (REST client skill) | P0 guidance | +| `docs-site/src/` REST client reference | `GenerateEntities` and the type table | + +## Version Compatibility + +The OpenAPI import already requires Mendix 11.0+ (`Rest$JsonBody`; see the +`@version: 11.0+` marker in `20-openapi-import-examples.mdl`). +`Rest$ImplicitMappingResponseHandling` is already written under the same gate. +`GenerateEntities` inherits the gate; no new version row is expected. Confirm +against the reference project that array-rooted mappings exist in the +earliest 11.x supported. + +## Test Plan + +- **Unit, `mdl/openapi/`:** `$ref` resolution, `allOf` merge, recursive + schema, unsupported-member report, Swagger 2.0 refusal (P0) and normalisation + (P4). Petstore 2.0 and 3.0 specs as **local fixtures**: the doctype test + today fetches over the network. +- **Executor, `MockBackend`:** `GenerateEntities: Yes` creates the expected + entities and associations and wires each operation. Re-running `create or + modify` creates nothing new and changes no entity's GUID. Include a control + with an entity created by Studio Pro, where `GUID != $ID`, because only that + subject can fail a GUID test (CLAUDE.md). +- **Doctype test:** extend `20-openapi-import-examples.mdl` with + `GenerateEntities: Yes` over the local fixtures. +- **Build, two variants:** `mx check` 0 errors on the generated Petstore 3 + project, and the same project with each unmeasured shape forced in (array + root, a typed value element). The second variant is what shows whether + mxbuild catches a wrong shape at all. +- **Runtime (`verify-in-runtime.md`):** a valid-looking mapping can pass + `mx check` and still fail on the first call. Boot with `run --local` against + a stub server returning Petstore payloads. Assert that a microflow calling + `findPetsByStatus` gets a list of `Pet` with `Category` associated and typed + attributes filled. This is the only test that proves item 3. + +## Open Questions + +1. **BSON items 1–5** above. Can you provide the Studio Pro reference project? + Phase 2 can start on items 3–4 alone; Phase 3 waits for 1, 2 and 5. +2. **What does Studio Pro's own OpenAPI import do with responses?** If it + generates entities, mxcli should copy its naming and shape exactly rather + than invent its own. If it does not, `GenerateEntities` is mxcli's own + addition and the naming below is ours to choose. +3. **Property name and default.** Is `GenerateEntities: Yes` right, and opt-in + rather than default? Opt-in keeps existing scripts from suddenly creating + domain-model entities. A default would make the reporter's one-liner just + work. +4. **Where generated entities go:** always the service's module, or an + `EntitiesModule:`/folder option? Proposed: the service's module, nothing + else until asked. +5. **Name collisions** with an existing entity that the import did not create + (`Shop.Order` already exists and is persistent). Proposed: refuse and name + the conflict, rather than overwrite or suffix. An import must not convert a + user's entity. +6. **Shared schemas across services:** two specs that both declare `Error` in + one module. Proposed: the second import reuses the entity only if its + attributes match; otherwise it refuses as in (5). diff --git a/docs/11-proposals/README.md b/docs/11-proposals/README.md index ac4edf60d3..eb01524dd5 100644 --- a/docs/11-proposals/README.md +++ b/docs/11-proposals/README.md @@ -27,7 +27,7 @@ for display in this README): -## Active Proposals (116) +## Active Proposals (118) ### In Progress (partial) (15) @@ -59,10 +59,11 @@ for display in this README): | [MDL Language Critique and Beta Syntax Freeze](PROPOSAL_mdl_beta_syntax_freeze.md) | Accepted | Related: ADR-0003; resulting decisions ADR-0010, ADR-0011, ADR-0012; design-mdl-syntax skill, PROPOSAL_workflow_microflow_syntax_alignment.m | | [Size containers from their contents, not from a statement count](PROPOSAL_container_autosize.md) | Accepted | Closes the last open item of upstream #884 — | -### Proposed (40) +### Proposed (41) | Proposal | Status | Summary | |----------|--------|---------| +| [ALTER PAGE Addressing of Pluggable Object-List Items](PROPOSAL_alter_object_list_item_address.md) | Proposed | Refs ako/mxcli#1326, mendixlabs/mxcli#1453. | | [Bootstrap hooks that fetch the mxcli the project was built with](PROPOSAL_bootstrap_source.md) | Proposed | mxcli init writes .claude/bootstrap-mxcli.sh with a hard-coded download of | | [Built-in file database for run --local (HSQLDB)](PROPOSAL_run_local_builtin_database.md) | Proposed | Depends on: the Windows process-helper fix (PR #1147) — this builds on that branch. | | [Comprehensive MDL Syntax and Grammar Improvements (v2)](PROPOSAL_mdl_syntax_improvements_v2.md) | Proposed | This document consolidates and expands upon previous proposals, offering a unified set of syntax and grammar improvements for all major Mend | @@ -104,7 +105,7 @@ for display in this README): | [VS Code Search — Quick Pick + Workspace Symbol](PROPOSAL_vscode_search.md) | Proposed | Full-text search exists in mxcli (mxcli search) but is only accessible via the terminal. | | [Workflow Improvements: ALTER WORKFLOW + Cross-References](PROPOSAL_workflow_improvements.md) | Proposed | Workflow support in mxcli has full CREATE/DESCRIBE/DROP/SHOW coverage with 13 activity types and BSON round-trip fidelity. | -### Draft (48) +### Draft (49) | Proposal | Status | Summary | |----------|--------|---------| @@ -145,6 +146,7 @@ for display in this README): | [mxcli marketplace diff — detect local modification and plan a GUID-preserving module upgrade](PROPOSAL_marketplace_module_upgrade.md) | Draft | 2026-08-11 (DESCRIBE coverage measured — §7; GUID = database identity measured — §8) | | [mxcli Playground](mxcli-playground.md) | Draft | A public GitHub repository (mendixlabs/mxcli-playground) containing a ready-to-use Mendix project pre-configured with mxcli, Claude Code ski | | [Offline synchronization configuration — the one thing an offline profile needs that MDL cannot say](PROPOSAL_offline_sync_configuration.md) | Draft | mxcli can author an offline navigation profile — PhoneOffline, | +| [OpenAPI import — generate response entities and wire operations to them](PROPOSAL_openapi_response_entities.md) | Draft | Issue: ako/mxcli#1234 (mendixlabs/mxcli#280) | | [Owning the modelsdk/gen codegen — why the vendored generator cannot be adopted as-is](PROPOSAL_codegen_ownership.md) | Draft | Constraint set by the maintainer (2026-08-13): the reflection data is an | | [Playwright Session Reuse and Lifecycle Control](PROPOSAL_playwright_session_reuse.md) | Draft | Builds on proposal-playwright-cli.md, which | | [Project brain — an opt-in store, in a user's Mendix project, for what mxcli cannot compute](PROPOSAL_project_brain.md) | Draft | Audience: users of mxcli building Mendix projects — not mxcli's own | diff --git a/mdl-examples/bug-tests/1453-describe-accordion-headercontent-slot.mdl b/mdl-examples/bug-tests/1453-describe-accordion-headercontent-slot.mdl new file mode 100644 index 0000000000..6ebc11e668 --- /dev/null +++ b/mdl-examples/bug-tests/1453-describe-accordion-headercontent-slot.mdl @@ -0,0 +1,32 @@ +mdl 1; +-- mendixlabs/mxcli#1453 (prerequisite of +-- PROPOSAL_alter_object_list_item_address.md): an Accordion group has two +-- widget slots, headerContent and content. DESCRIBE folded both into the +-- group's body, so a page regenerated from the description stored the header +-- widget in content (blank 11.13.0: headerContent -> [txtHead], +-- content -> [txtBody] became content -> [txtHead, txtBody]). And `check` +-- refused the block form CREATE builds from — `headercontent { … }` inside a +-- group was MDL-WIDGET25 "not a widget in this project". +-- +-- Expected: the describe below prints +-- group (headerRenderMode: 'custom', …) { +-- headercontent { +-- dynamictext txtHead (Content: 'Header') +-- } +-- dynamictext txtBody (Content: 'Body') +-- } +-- and executing that description on a fresh app stores txtHead in +-- headerContent and txtBody in content (`mxcli bson dump --format json`). + +create module Bug1453; + +create page Bug1453.P (title: 'P', layout: Atlas_Core.Atlas_Default) { + accordion acc1 (advancedMode: false, collapsible: true, expandBehavior: 'singleExpanded', animate: true, showIcon: 'right', animateIcon: true) { + group (headerRenderMode: 'custom') { + headercontent { dynamictext txtHead (Content: 'Header') } + dynamictext txtBody (Content: 'Body') + } + } +}; + +describe page Bug1453.P; diff --git a/mdl-examples/bug-tests/1485-ensure-db-password-protected-superuser.mdl b/mdl-examples/bug-tests/1485-ensure-db-password-protected-superuser.mdl new file mode 100644 index 0000000000..9187aa32a9 --- /dev/null +++ b/mdl-examples/bug-tests/1485-ensure-db-password-protected-superuser.mdl @@ -0,0 +1,32 @@ +-- ============================================================================ +-- Bug test: mendixlabs/mxcli#1485 — `run --ensure-db` fails on Windows +-- PostgreSQL installs (postgres always has a password) +-- ============================================================================ +-- +-- Reported: +-- Error: ensuring database: no local PostgreSQL superuser available to create +-- the role/database (tried a direct 'psql -U postgres' connection over the +-- cluster socket and TCP, and non-interactive 'sudo -u postgres' ...) +-- +-- The Windows installer always sets a password for `postgres` and there is no +-- sudo, so every passwordless path fails. Fixed by --db-admin-user / +-- --db-admin-password (the password reaches psql through PGPASSWORD, never +-- argv), and by an error that names them, PGPASSWORD, and the two statements +-- to run by hand. The script is incidental: the defect is in provisioning. +-- +-- To verify (any PostgreSQL whose superuser needs a password; on Linux, +-- initdb -U postgres --auth=scram-sha-256 --pwfile=pw reproduces the shape): +-- mxcli run --local --ensure-db -p App.mpr +-- -> the error above, now followed by the --db-admin-* hint +-- mxcli run --local --ensure-db --db-admin-password -p App.mpr +-- -> Creating role "mendix"... Creating database ... Database ready +-- mxcli exec mdl-examples/bug-tests/1485-ensure-db-password-protected-superuser.mdl -p App.mpr +-- ============================================================================ + +mdl 1; + +create module Bug1485; + +create persistent entity Bug1485.Probe ( + "Label": String(100) +); diff --git a/mdl-examples/bug-tests/1577-watch-stale-theme-error-ce6093.mdl b/mdl-examples/bug-tests/1577-watch-stale-theme-error-ce6093.mdl new file mode 100644 index 0000000000..fc53c8f085 --- /dev/null +++ b/mdl-examples/bug-tests/1577-watch-stale-theme-error-ce6093.mdl @@ -0,0 +1,31 @@ +-- ============================================================================ +-- Bug test: mendixlabs/mxcli#1577 — `run --local --watch`: after a SCSS error +-- is fixed, every following build still fails with CE6093 and the old Sass +-- error until a restart +-- ============================================================================ +-- +-- Reported: build #2 fails "An error occurred while compiling Theme files" +-- (correct); after the fix, build #3 fails "[CE6093] The theme could not be +-- compiled: Error: Undefined variable." quoting the removed line, and so does +-- every later build until `mxcli run restart`. +-- +-- Cause, in mxbuild --serve (measured on 11.12.5 over its /build API): the +-- process keeps its first theme error; a fresh process compiles the same source. +-- Fixed in serveRecovery (cmd/mxcli/docker/serve_recovery.go): a theme failure +-- that follows another in the same serve process restarts it and builds once +-- more; a failure that survives the restart is reported as the current one. +-- +-- To verify (Mendix 11.12.5; a blank app from +-- `mxcli new App --version 11.12.5 --theme none --layout none --skip-init`): +-- mxcli exec mdl-examples/bug-tests/1577-watch-stale-theme-error-ce6093.mdl -p App.mpr +-- mxcli run --local --watch --detach --db-type hsqldb -p App.mpr +-- printf '.probe2 { color: $does-not-exist; }\n' > themesource/bug1577/web/main.scss +-- mxcli run wait -p App.mpr # failed: ... compiling Theme files (correct) +-- printf '.probe2 { color: #123456; }\n' > themesource/bug1577/web/main.scss +-- mxcli run wait -p App.mpr # applied: build #3 (was: failed, [CE6093]) +-- curl -s http://127.0.0.1:8080/theme.compiled.css | grep -c 123456 # 1 +-- ============================================================================ + +mdl 1; + +create module Bug1577; diff --git a/mdl-examples/bug-tests/1580-watch-reload-drops-dist-pages.mdl b/mdl-examples/bug-tests/1580-watch-reload-drops-dist-pages.mdl new file mode 100644 index 0000000000..328fe8576f --- /dev/null +++ b/mdl-examples/bug-tests/1580-watch-reload-drops-dist-pages.mdl @@ -0,0 +1,39 @@ +-- ============================================================================ +-- Bug test: mendixlabs/mxcli#1580 — `run --local --watch`: a hot reload after a +-- model change leaves deployment/web/dist without pages/ and widgets.css, so +-- every page 404s +-- ============================================================================ +-- +-- Reported: after `create or modify microflow …`, `mxcli run wait` prints +-- "applied: build #2 via reload in 4.3s", `deployment/web/dist` holds only +-- `chunks` and `index.js`, and GET /dist/pages/..js returns 404. +-- +-- Cause, in mxbuild --serve (measured on 11.12.5 over its HTTP API): only the +-- first build in a serve process bundles the client completely. Every later +-- one — even with the model unchanged — replaces dist/ with a monolithic +-- index.js and no pages/widgets.css, or, when a page changes, fails with +-- "Could not find a part of the path '…/web/pages/…'". A fresh process always +-- builds completely. Fixed in serveRecovery (cmd/mxcli/docker/serve_recovery.go): +-- the watch loop restarts mxbuild --serve and builds once more, keeping the +-- incremental build's restartRequired so the change is still a reload. +-- +-- To verify (Mendix 11.12.5; a blank app from +-- `mxcli new App --version 11.12.5 --theme none --layout none --skip-init`): +-- mxcli run --local --watch --detach --db-type hsqldb -p App.mpr +-- mxcli exec mdl-examples/bug-tests/1580-watch-reload-drops-dist-pages.mdl -p App.mpr +-- mxcli run wait -p App.mpr # applied: build #2 via reload +-- ls deployment/web/dist # pages/ and widgets.css present +-- mxcli playwright check -p App.mpr # OK, 0 console errors +-- The run log shows "restarting mxbuild --serve for a full build...". +-- Before the fix: dist/ = chunks index.js, and the page check FAILs with an +-- error banner and failed requests. Re-running adds nothing; to repeat, change +-- the returned string. +-- ============================================================================ + +mdl 1; + +create module Bug1580; + +create or modify microflow Bug1580.MF_Watch () returns String as $R begin + return 'x'; +end; diff --git a/mdl-examples/bug-tests/1591-pluggable-attribute-outside-data-view-entity.mdl b/mdl-examples/bug-tests/1591-pluggable-attribute-outside-data-view-entity.mdl index c898a68508..a15c7f4fef 100644 --- a/mdl-examples/bug-tests/1591-pluggable-attribute-outside-data-view-entity.mdl +++ b/mdl-examples/bug-tests/1591-pluggable-attribute-outside-data-view-entity.mdl @@ -23,12 +23,25 @@ mdl 1; -- view over a page parameter put the parameter's NAME in scope instead of -- its entity, so the two could not be compared. -- +-- Follow-up: the qualified form was itself the wrong shape. Stored as the +-- AttributeRef alone, mx check reports +-- [CE7247] "Attribute 'MyFirstModule.ChartContext.EventData' is not an +-- attribute of entity 'MyFirstModule.ChartEntry'." at Custom chart 'chart1' +-- What Studio Pro stores for an attribute of another page parameter is the +-- AttributeRef plus a SourceVariable naming the parameter — spelled +-- `$Ctx.EventData`, which a generic attribute key refused as MDL-WIDGET42. +-- It is now written as that pair (mx check: 0 errors on 11.12.5), DESCRIBE +-- prints it back as `$Ctx.EventData`, and the qualified name of an entity the +-- property does not bind to is refused, naming CE7247 and `$Param.EventData`. +-- -- Usage: -- mxcli exec mdl-examples/bug-tests/1591-pluggable-attribute-outside-data-view-entity.mdl -p app.mpr +-- mxcli docker check -p app.mpr → 0 errors -- mxcli -p app.mpr -c "describe page BugTest1591.ChartPage" --- → eventDataAttribute: BugTest1591.ChartContext.EventData --- Changing that line to `eventDataAttribute: EventData` is now refused by --- check: "`EventData` is not an attribute of BugTest1591.ChartEntry …". +-- → eventDataAttribute: $Ctx.EventData +-- Changing that line to `eventDataAttribute: EventData` is refused by check: +-- "`EventData` is not an attribute of BugTest1591.ChartEntry …"; to +-- `BugTest1591.ChartContext.EventData`, by check as CE7247. -- ============================================================================ create module BugTest1591; @@ -45,7 +58,7 @@ create page BugTest1591.ChartPage ( customchart chart1 ( dataAttribute: JsonData, layoutAttribute: JsonLayout, - eventDataAttribute: BugTest1591.ChartContext.EventData + eventDataAttribute: $Ctx.EventData ) } }; diff --git a/mdl-examples/bug-tests/1603-describe-opens-every-unit-file.mdl b/mdl-examples/bug-tests/1603-describe-opens-every-unit-file.mdl new file mode 100644 index 0000000000..f6d677f895 --- /dev/null +++ b/mdl-examples/bug-tests/1603-describe-opens-every-unit-file.mdl @@ -0,0 +1,27 @@ +-- mendixlabs/mxcli#1603 (ako/mxcli#1513): on MPR v2, `describe` of one +-- document opened every unit file in mprcontents/ first. +-- +-- Symptom: on a blank Mendix 10.24.18 app (394 unit files after this script), +-- mxcli exec this.mdl -p App/App.mpr +-- touch -a -t 200001010000 $(find App/mprcontents -type f) +-- mxcli -p App/App.mpr -c "describe microflow Probe.ACT_Do" +-- find App/mprcontents -type f -atime -1 | wc -l +-- printed 393: every listing typed each unit by reading its file, and every +-- write made the next listing do it again. +-- +-- Fix: unit types are indexed by the Unit table's ContentsHash and persisted +-- to .mxcli/unit-types.idx, so only a unit whose hash is new is opened. Measured +-- on testdata/pedapp (375 unit files) with strace: distinct .mxunit files +-- opened by the describe 375 -> 117, the rest being the folders, microflows, +-- modules and domain models the describe itself reads. +-- +-- Expected: both statements succeed; the count above is far below the total. + +mdl 1; + +create module Probe; + +create microflow Probe.ACT_Do() returns Boolean as $r +begin + return true; +end; diff --git a/mdl-examples/bug-tests/1608-alter-consumed-rest-service.mdl b/mdl-examples/bug-tests/1608-alter-consumed-rest-service.mdl new file mode 100644 index 0000000000..50ecdc0b78 --- /dev/null +++ b/mdl-examples/bug-tests/1608-alter-consumed-rest-service.mdl @@ -0,0 +1,50 @@ +-- mendixlabs/mxcli#1608 (ako/mxcli#1536): no way to add an operation to an +-- existing consumed REST service. +-- +-- Symptom: `alter consumed rest service … add operation` did not parse +-- line 2:15 no viable alternative at input 'alterconsumedrest' +-- and the only alternative, CREATE OR MODIFY naming the new operations, +-- deleted every operation it did not list — often ones made in Studio Pro — +-- reporting only "Modified rest client: … (2 operations)". +-- +-- Fix: ALTER CONSUMED REST SERVICE with ADD / MODIFY / DROP OPERATION and +-- SET ( … ). It splices the stored document: an operation no action names is +-- written back as stored, and MODIFY rewrites only the keys it lists. +-- (The CREATE OR MODIFY first puts the service back to two operations, so the +-- script runs again; on its own, a second ADD of GetBooking is refused.) + +mdl 1; + +create or modify consumed rest service MyFirstModule.BookingApi ( + BaseUrl: 'https://api.example.com', + Authentication: none +) +{ + operation PostBooking ( + Method: post, + Path: '/bookings', + Body: json from $Payload, + Response: json as $Result + ) + + operation LegacyBooking (Method: get, Path: '/legacy', Response: none) +}; + +alter consumed rest service MyFirstModule.BookingApi add operation GetBooking ( + Method: get, + Path: '/bookings/{id}', + Parameters: ($id: String), + Response: json as $Result +); + +alter consumed rest service MyFirstModule.BookingApi + add operation PutBooking ( + Method: put, + Path: '/bookings/{id}', + Parameters: ($id: String), + Body: json from $Payload, + Response: json + ), + modify operation PostBooking (Timeout: 60), + drop operation LegacyBooking, + set (BaseUrl: 'https://api.example.com/v2'); diff --git a/mdl-examples/bug-tests/1608-consumed-rest-response-without-variable.mdl b/mdl-examples/bug-tests/1608-consumed-rest-response-without-variable.mdl new file mode 100644 index 0000000000..15d9de0852 --- /dev/null +++ b/mdl-examples/bug-tests/1608-consumed-rest-response-without-variable.mdl @@ -0,0 +1,37 @@ +-- mendixlabs/mxcli#1608 (ako/mxcli#1536): DESCRIBE of a consumed REST service +-- did not re-parse. +-- +-- Symptom: an operation created with `Response: json as $Result` describes as +-- `Response: json`, and running that output back failed: +-- line 14:2 missing VARIABLE at ')' +-- A consumed REST operation stores no response variable (the call activity +-- names it), so there is nothing else describe could print. +-- +-- Fix: `Response: json | string | file | status` takes the variable as +-- optional, and describe prints the bare kind. Every form below must pass +-- `mxcli check`. + +mdl 1; + +create or modify consumed rest service MyFirstModule.BookingApi ( + BaseUrl: 'https://api.example.com', + Authentication: none +) +{ + operation PostBooking ( + Method: post, + Path: '/bookings', + Headers: ('Accept': '*/*'), + Body: json from $Payload, + Timeout: 300, + Response: json + ) + + operation GetNote (Method: get, Path: '/notes/{id}', Parameters: ($id: String), Response: string) + + operation GetScan (Method: get, Path: '/scans/{id}', Parameters: ($id: String), Response: file) + + operation Ping (Method: get, Path: '/ping', Response: status) + + operation GetBooking (Method: get, Path: '/bookings/{id}', Parameters: ($id: String), Response: json as $Result) +}; diff --git a/mdl-examples/bug-tests/1616-create-members-scan-every-domain-model.mdl b/mdl-examples/bug-tests/1616-create-members-scan-every-domain-model.mdl new file mode 100644 index 0000000000..0ff5c28e07 --- /dev/null +++ b/mdl-examples/bug-tests/1616-create-members-scan-every-domain-model.mdl @@ -0,0 +1,32 @@ +-- mendixlabs/mxcli#1616 (ako/mxcli#1549): every member assignment of a +-- `create` action made exec and check decode the domain model of every module. +-- +-- Symptom: 10 microflows of 20 `create MyFirstModule.Sale (A1 = …, … A8 = …)` +-- took 14.2 s on a fresh app and 293.8 s after 30 modules × 20 entities were +-- added (`check --references` alone 72 s); the same flows with no member +-- assignments took 3.0 s. Each `create microflow` logged 19–29 s. +-- +-- Fix: GetDomainModel(module) reads and decodes that module's domain model +-- only (and follows a generalization chain into another module only when it +-- has to), and one flow build reads modules and domain models through a memo, +-- so a flow's member assignments cost one lookup per module, not one project +-- scan each. Measured on testdata/pedapp: 160 member assignments cost 4001 extra +-- unit-file reads before, 11 after. +-- +-- Expected: both statements succeed; the second costs about what the first +-- does with the member assignments left out. + +mdl 1; + +create persistent entity MyFirstModule.Sale ( + A1: String(100), A2: String(100), A3: String(100), A4: String(100), + A5: Integer, A6: Integer, A7: Boolean, A8: DateTime +); + +create microflow MyFirstModule.ACT_CreateSales () returns Boolean as $r +begin + $S1 = create MyFirstModule.Sale (A1 = 'a', A2 = 'b', A3 = 'c', A4 = 'd', A5 = 1, A6 = 2, A7 = true, A8 = [%CurrentDateTime%]); + $S2 = create MyFirstModule.Sale (A1 = 'a', A2 = 'b', A3 = 'c', A4 = 'd', A5 = 1, A6 = 2, A7 = true, A8 = [%CurrentDateTime%]); + $S3 = create MyFirstModule.Sale (A1 = 'a', A2 = 'b', A3 = 'c', A4 = 'd', A5 = 1, A6 = 2, A7 = true, A8 = [%CurrentDateTime%]); + return true; +end; diff --git a/mdl-examples/bug-tests/1622-rewrite-keeps-unauthorable-widgets.mdl b/mdl-examples/bug-tests/1622-rewrite-keeps-unauthorable-widgets.mdl new file mode 100644 index 0000000000..69d945bfb3 --- /dev/null +++ b/mdl-examples/bug-tests/1622-rewrite-keeps-unauthorable-widgets.mdl @@ -0,0 +1,50 @@ +-- mendixlabs/mxcli#1622 (ako/mxcli#1554): `describe` writes widgets it cannot +-- execute, and `exec` then dropped them silently. +-- +-- Symptom: on a page built in Studio Pro with a classic data grid and a +-- reference selector, +-- describe page Shop.Orders > p.mdl ; exec p.mdl ; describe page Shop.Orders +-- printed the two widgets as +-- -- Forms$DataGrid (grid1) -- NOT re-executable: mxcli cannot author this widget, … +-- and exec answered "Replaced page Shop.Orders" with both widgets gone and no +-- warning; mx check was clean before and after. +-- +-- Fix: the rewrite is refused — "page Shop.Orders holds 2 widget(s) MDL cannot +-- author, and rewriting it from this statement would delete them: grid1 +-- (Forms$DataGrid), referenceSelector1 (Forms$ReferenceSelector)" — and +-- `mxcli check -p` says so first. The same holds for snippets and layouts. +-- +-- To verify in Studio Pro: build Shop.Orders with a classic Data grid named +-- grid1 and a Reference selector named referenceSelector1, then run the +-- describe output (refused), then this script: the ALTER below edits the page +-- and keeps both widgets; the CREATE OR MODIFY after the commented-out DROP +-- would be refused until the DROP is run. + +mdl 1; + +create module Shop; + +create persistent entity Shop.Order ( + Number: autonumber default 1, + Note: string(200) +); + +create page Shop.Orders (Title: 'Orders', Layout: Atlas_Core.Atlas_Default) { + container ctn1 { + dynamictext txtHeading (Content: 'Orders') + } +}; + +-- Editing a page that holds such widgets: ALTER changes only what it names. +alter page Shop.Orders { + set (Content: 'All orders') on txtHeading; +}; + +-- Rebuilding it without them: drop them on purpose first. +-- alter page Shop.Orders { drop grid1, referenceSelector1; }; + +create or modify page Shop.Orders (Title: 'Orders', Layout: Atlas_Core.Atlas_Default) { + container ctn1 { + dynamictext txtHeading (Content: 'All orders') + } +}; diff --git a/mdl-examples/bug-tests/1624-describe-module-all-large-module.mdl b/mdl-examples/bug-tests/1624-describe-module-all-large-module.mdl new file mode 100644 index 0000000000..2c33f61fc9 --- /dev/null +++ b/mdl-examples/bug-tests/1624-describe-module-all-large-module.mdl @@ -0,0 +1,36 @@ +-- mendixlabs/mxcli#1624 (ako/mxcli#1556): `describe module --all` could not +-- cover a large module. +-- +-- Symptom: for a module with 229 microflows, after 5 minutes +-- `statement timed out`; with MXCLI_EXEC_TIMEOUT=40m it stopped at 10,000 +-- output lines with "output line limit exceeded ... statement aborted", +-- truncated mid-flow. +-- +-- Fix: a module dump is bounded by the module, so neither the default +-- wall-clock limit nor the default line limit applies to it (an explicit +-- MXCLI_EXEC_TIMEOUT / MXCLI_OUTPUT_LINE_LIMIT still does, and the line limit +-- is now configurable for every statement). The time went into describing each +-- microflow: its layout derivation rebuilt the flow through a read memo that +-- hid the backend's by-name lookup, so every described flow decoded every +-- microflow in the project. Measured on testdata/pedapp with this module grown +-- to 400 flows: 66.9 s and exit 1 at 10,025 lines before, 5.7 s and all 12,810 +-- lines after, the first 10,000 byte-identical. +-- +-- To reproduce at scale, repeat the microflow below a few hundred times under +-- different names, then: +-- mxcli -p app.mpr -c "describe module Dump with all" + +mdl 1; + +create module Dump; + +create persistent entity Dump.Item (Name: String(100), Qty: Integer, Done: Boolean); + +create microflow Dump.MF_1 ($Name: String) returns Boolean as $r +begin + $I1 = create Dump.Item (Name = $Name, Qty = 1, Done = false); + if $I1/Qty > 3 then + change $I1 (Done = true); + end if; + return true; +end; diff --git a/mdl-examples/bug-tests/280-openapi-swagger2-refused.mdl b/mdl-examples/bug-tests/280-openapi-swagger2-refused.mdl new file mode 100644 index 0000000000..9c015b5717 --- /dev/null +++ b/mdl-examples/bug-tests/280-openapi-swagger2-refused.mdl @@ -0,0 +1,20 @@ +mdl 1; +-- Bug mendixlabs/mxcli#280 (ako/mxcli#1234), Phase 0 of +-- docs/11-proposals/PROPOSAL_openapi_response_entities.md. +-- +-- The reporter imported the Swagger 2.0 Petstore +-- (https://petstore.swagger.io/v2/swagger.json). The OpenAPI importer reads only +-- OpenAPI 3 fields, so a 2.0 document parsed "successfully" into an empty +-- contract: BaseUrl '', request bodies dropped, every response `none`, no warning. +-- +-- It is now refused at execution time. `mxcli check` cannot see that, so this is +-- a plain .mdl. The guard is pinned by TestParseSpec_RefusesSwagger2 +-- (mdl/openapi/parser_test.go). Run it to see the message: +-- +-- mxcli exec mdl-examples/bug-tests/280-openapi-swagger2-refused.mdl +-- +-- Expect: "this is a Swagger 2.0 document; only OpenAPI 3 is supported. Convert +-- it to OpenAPI 3 first ...". Before the fix it printed a service whose every +-- operation had `Response: none`. + +describe contract operation from openapi 'mdl-examples/openapi-local-specs/petstore-swagger2.json'; diff --git a/mdl-examples/bug-tests/combobox-enumeration-mode-attribute-type-ce7247.mdl b/mdl-examples/bug-tests/combobox-enumeration-mode-attribute-type-ce7247.mdl new file mode 100644 index 0000000000..2cd99fcc62 --- /dev/null +++ b/mdl-examples/bug-tests/combobox-enumeration-mode-attribute-type-ce7247.mdl @@ -0,0 +1,50 @@ +mdl 1; +-- ============================================================================ +-- Combo box in enumeration mode bound to a Boolean — CE7247 +-- (the gap left open by ako/mxcli#1390, found while measuring mendixlabs/mxcli#1466) +-- ============================================================================ +-- +-- `combobox cb (Attribute: Flag)` with no data source is enumeration mode: the +-- attribute is written to the widget's `attributeEnumeration`, which takes an +-- Enumeration only. `check --references` and `exec` passed it; on fresh 11.14.0 +-- and 11.13.0 projects mxbuild reported +-- +-- [error] [CE7247] "Only attributes of type Enumeration are allowed here." at Combo box 'cbFlag' +-- +-- and the same for String, Integer, Long, Decimal, AutoNumber, DateTime and +-- Binary. Enumeration builds clean. +-- +-- mxcli check this-file -p app.mpr --references +-- before: no error +-- after: MDL-WIDGET39 on cbFlag, naming CE7247 and `checkbox` / `radiobuttons` +-- +-- Plain .mdl, not .fail.mdl: the rule needs the project (it runs in the +-- --references walk with the built-in inputs' CE2421 matrix), and `make +-- check-mdl` runs without one. The unit tests in +-- mdl/executor/validate_widget_attribute_type_test.go cover the guard. +-- ============================================================================ + +create module BugCbBool; +create enumeration BugCbBool.Kind (Small 'Small', Large 'Large'); + +@position(100, 100) +create persistent entity BugCbBool.Thing ( + Flag: boolean default false, + Size: enumeration(BugCbBool.Kind) +); + +create page BugCbBool.Thing_Edit (title: 'Thing', layout: Atlas_Core.Atlas_Default, params: ($Thing: BugCbBool.Thing)) { + layoutgrid lg { + row { + column (desktopwidth: autofill) { + dataview dvThing (datasource: $Thing) { + -- The bug: enumeration mode on a Boolean. CE7247. + combobox cbFlag (label: 'Flag', attribute: Flag) + -- Controls, both build clean. + combobox cbSize (label: 'Size', attribute: Size) + checkbox chkFlag (label: 'Flag', attribute: Flag) + } + } + } + } +}; diff --git a/mdl-examples/bug-tests/mcp-alter-insert-addresses-new-elements.mdl b/mdl-examples/bug-tests/mcp-alter-insert-addresses-new-elements.mdl new file mode 100644 index 0000000000..a896bea361 --- /dev/null +++ b/mdl-examples/bug-tests/mcp-alter-insert-addresses-new-elements.mdl @@ -0,0 +1,73 @@ +mdl 1; +-- ============================================================================ +-- ALTER MICROFLOW insert over --mcp lands on Studio Pro 11.15 +-- ============================================================================ +-- +-- The insert reads the stored flow from the local project, so the local .mpr +-- must hold the same base document as Studio Pro. Run part 1 over --mcp +-- (`--mcp http://localhost/mcp --mcp-dial :`, with -p pointing at +-- the local project) BEFORE running it locally: the MCP create checks the +-- local project for an existing document. Then run part 2 the same two ways +-- and compare every flow's ends, connection sides and control vectors (the +-- local .mpr via `mxcli bson dump`, Studio Pro's document via +-- ped_read_document). They must match, and ped_check_errors must be clean. +-- +-- Before the fix, on 11.15.0-rc.4: +-- * every insert failed with `undefined is not an object (evaluating +-- 'u.$Type')` — the update set a control vector on a flow it added itself; +-- * without that set, the new activity had "No action defined": PED drops a +-- set on an element added in the same update; +-- * the IF/ELSE was wired to the wrong objects, because several adds to one +-- array in one update are stored in reverse. +-- After the fix, all 4 + 9 + 6 flows below match the default engine. +-- +-- Studio Pro's MCP server has no delete, so the documents stay in the open +-- project after a run. +-- ============================================================================ + +-- Part 1: the base documents. + +create microflow MyFirstModule.MCPA_XPlain () +begin + log info node 'MCPA' 'one'; + log info node 'MCPA' 'two'; +end; + +create microflow MyFirstModule.MCPA_XIf () +begin + log info node 'MCPA' 'one'; + log info node 'MCPA' 'two'; +end; + +create microflow MyFirstModule.MCPA_XNote () +begin + log info node 'MCPA' 'one'; + log info node 'MCPA' 'two'; +end; + +-- Part 2: the inserts. + +alter microflow MyFirstModule.MCPA_XPlain { + insert after log node 'MCPA' 'one' begin + log info node 'MCPA' 'inserted'; + end; +}; + +alter microflow MyFirstModule.MCPA_XIf { + insert after log node 'MCPA' 'one' begin + declare $N INTEGER = 1; + if $N > 0 then + log info node 'MCPA' 'positive'; + else + log info node 'MCPA' 'not positive'; + end if; + end; +}; + +alter microflow MyFirstModule.MCPA_XNote { + insert after log node 'MCPA' 'one' begin + @annotation 'Inserted with a note' + log info node 'MCPA' 'inserted'; + log info node 'MCPA' 'and another'; + end; +}; diff --git a/mdl-examples/bug-tests/mcp-create-flow-sides-and-notes.mdl b/mdl-examples/bug-tests/mcp-create-flow-sides-and-notes.mdl new file mode 100644 index 0000000000..468488bbd8 --- /dev/null +++ b/mdl-examples/bug-tests/mcp-create-flow-sides-and-notes.mdl @@ -0,0 +1,43 @@ +mdl 1; +-- ============================================================================ +-- A microflow created over --mcp keeps its flow sides and its notes +-- ============================================================================ +-- +-- Run it twice: once with the default engine, once against Studio Pro's MCP +-- server (`--mcp http://localhost/mcp --mcp-dial :`). Then compare +-- each flow's connection sides (the default engine's .mpr via +-- `mxcli bson dump`, Studio Pro's document via ped_read_document). They must +-- be the same. +-- +-- Before the fix, the MCP create sent no sides, so Studio Pro drew every flow +-- right -> left (1 -> 3): the three outcome anchors and the merge's bottom exit +-- were lost. Any @annotation failed the create outright. Measured on +-- 11.15.0-rc.4: after the fix, all 10 flows match the default engine, both +-- note lines included (2 -> 0 for the note above, 0 -> 2 for the one placed +-- below). +-- +-- Studio Pro's MCP server has no delete, so the documents stay in the open +-- project after a run. +-- ============================================================================ + +CREATE MODULE BugMcpSides; + +CREATE OR MODIFY ENUMERATION BugMcpSides.Verdict ( + Approved 'Approved', + Rejected 'Rejected' +); + +CREATE MICROFLOW BugMcpSides.MCPS_CaseNotes ($T: BugMcpSides.Verdict) +BEGIN + @anchor(from: bottom, Approved: (from: right), (empty): (from: bottom, to: top)) + CASE $T + WHEN Approved THEN + @annotation 'Approved takes this branch' + LOG INFO NODE 'MCPS' 'approved'; + WHEN Rejected THEN + @annotation(text: 'Below the rejected log', position: (600, 420)) + LOG INFO NODE 'MCPS' 'rejected'; + WHEN (empty) THEN + LOG WARNING NODE 'MCPS' 'no verdict'; + END CASE; +END; diff --git a/mdl-examples/bug-tests/mcp-system-enumeration-parameter.mdl b/mdl-examples/bug-tests/mcp-system-enumeration-parameter.mdl new file mode 100644 index 0000000000..4afde05542 --- /dev/null +++ b/mdl-examples/bug-tests/mcp-system-enumeration-parameter.mdl @@ -0,0 +1,48 @@ +mdl 1; +-- ============================================================================ +-- A System enumeration named bare in a flow signature is an enumeration +-- ============================================================================ +-- +-- `$T: System.UserType` used to be written as DataTypes$ObjectType with Entity +-- "System.UserType" on every engine. The visitor types every bare qualified +-- name in a parameter or return position as an entity; the executor's +-- entity-then-enumeration fallback was skipped for the System module, whose +-- entities are not in the project to look up. The local engines wrote the +-- object type without complaint, and DESCRIBE rendered it as a plausible bare +-- `System.UserType` (an enumeration renders as `enum System.UserType`). +-- Studio Pro's PED refused it over --mcp: +-- +-- {"/parameters/0/entity":"Reference with qualified name System.UserType +-- of type DomainModels$Entity not found."} +-- +-- The explicit `enum System.UserType` spelling — what DESCRIBE emits — was +-- refused outright ("enumeration 'System.UserType' not found"), because the +-- System module's enumerations are not in ListEnumerations. +-- +-- Verified over --mcp against Studio Pro 11.15.0-rc.4: both microflows create, +-- and ped_read_document reads back DataTypes$EnumerationType for the +-- parameters and the return type. The System.User parameter is the control: it +-- must stay an object type. +-- ============================================================================ + +CREATE MODULE BugSysEnumParam; + +CREATE OR MODIFY MICROFLOW BugSysEnumParam.ACT_ByUserType ($T: System.UserType) +RETURNS System.DeviceType +BEGIN + IF $T = System.UserType.Internal THEN + RETURN System.DeviceType.Desktop; + ELSE + RETURN System.DeviceType.Phone; + END IF; +END; + +CREATE OR MODIFY MICROFLOW BugSysEnumParam.ACT_ExplicitEnum ($T: enum System.UserType) +BEGIN + LOG INFO NODE 'BugSysEnumParam' 'explicit enum parameter'; +END; + +CREATE OR MODIFY MICROFLOW BugSysEnumParam.ACT_SystemEntity ($U: System.User) +BEGIN + LOG INFO NODE 'BugSysEnumParam' 'System entity parameter stays an object'; +END; diff --git a/mdl-examples/doctype-tests/06-rest-client-examples.mdl b/mdl-examples/doctype-tests/06-rest-client-examples.mdl index 001bf8244a..cc861d6811 100644 --- a/mdl-examples/doctype-tests/06-rest-client-examples.mdl +++ b/mdl-examples/doctype-tests/06-rest-client-examples.mdl @@ -1515,6 +1515,44 @@ begin log info node 'RestTest' 'Round-tripped: ' + $PetResponseJson; end; +-- ############################################################################ +-- PART 16b: CHANGING AN EXISTING CLIENT (ALTER) +-- ############################################################################ + +-- MARK: Alter + +-- `create or modify` declares the whole operation list, so an operation it does +-- not list is removed. `alter` changes the stored client in place: an operation +-- no action names is written back as stored (mendixlabs/mxcli#1608). + +create or modify consumed rest service RestTest.RC022_BookingAPI ( + BaseUrl: 'https://api.example.com', + Authentication: none +) +{ + operation PostBooking ( + Method: post, + Path: '/bookings', + Body: json from $Payload, + Response: json + ) + + operation LegacyBooking (Method: get, Path: '/legacy', Response: none) +}; + +alter consumed rest service RestTest.RC022_BookingAPI + add operation GetBooking ( + Method: get, + Path: '/bookings/{id}', + Parameters: ($id: String), + Response: json + ), + modify operation PostBooking (Timeout: 60), + drop operation LegacyBooking, + set (BaseUrl: 'https://api.example.com/v2'); + +describe consumed rest service RestTest.RC022_BookingAPI; + -- ############################################################################ -- PART 17: VERIFICATION COMMANDS -- ############################################################################ diff --git a/mdl-examples/openapi-local-specs/petstore-swagger2.json b/mdl-examples/openapi-local-specs/petstore-swagger2.json new file mode 100644 index 0000000000..1822c26c7f --- /dev/null +++ b/mdl-examples/openapi-local-specs/petstore-swagger2.json @@ -0,0 +1,18 @@ +{ + "swagger": "2.0", + "info": {"title": "Swagger Petstore", "version": "1.0.7"}, + "host": "petstore.swagger.io", + "basePath": "/v2", + "schemes": ["https"], + "paths": { + "/pet/{petId}": { + "get": { + "operationId": "getPetById", + "produces": ["application/json"], + "parameters": [{"name": "petId", "in": "path", "required": true, "type": "integer", "format": "int64"}], + "responses": {"200": {"description": "ok", "schema": {"$ref": "#/definitions/Pet"}}} + } + } + }, + "definitions": {"Pet": {"type": "object", "properties": {"id": {"type": "integer"}}}} +} diff --git a/mdl/ast/ast_rest.go b/mdl/ast/ast_rest.go index 6ce15e8a33..2bafbc3cc9 100644 --- a/mdl/ast/ast_rest.go +++ b/mdl/ast/ast_rest.go @@ -22,6 +22,58 @@ type CreateRestClientStmt struct { func (s *CreateRestClientStmt) isStatement() {} +// AlterRestClientStmt represents: +// +// ALTER CONSUMED REST SERVICE Module.Name [, ...] +// +// It edits the stored service in place: an operation no action names is +// written back exactly as stored (mendixlabs/mxcli#1608). +type AlterRestClientStmt struct { + Name QualifiedName + Actions []RestClientAlterAction +} + +func (s *AlterRestClientStmt) isStatement() {} + +// RestClientAlterAction is one action of an ALTER CONSUMED REST SERVICE. +type RestClientAlterAction interface { + isRestClientAlterAction() +} + +// RestClientSetAction represents SET ( BaseUrl: '…', Authentication: … ). +// A property the list does not state keeps its stored value. +type RestClientSetAction struct { + BaseUrl string + BaseUrlSet bool + Authentication *RestAuthDef // nil with AuthenticationSet = Authentication: none + AuthenticationSet bool +} + +func (a *RestClientSetAction) isRestClientAlterAction() {} + +// RestClientAddOperationAction represents ADD OPERATION Name ( … ). +type RestClientAddOperationAction struct { + Operation *RestOperationDef +} + +func (a *RestClientAddOperationAction) isRestClientAlterAction() {} + +// RestClientModifyOperationAction represents MODIFY OPERATION Name ( … ): +// the keys in Operation.Keys are written, every other one keeps its stored +// value. +type RestClientModifyOperationAction struct { + Operation *RestOperationDef +} + +func (a *RestClientModifyOperationAction) isRestClientAlterAction() {} + +// RestClientDropOperationAction represents DROP OPERATION Name. +type RestClientDropOperationAction struct { + Name string +} + +func (a *RestClientDropOperationAction) isRestClientAlterAction() {} + // RestAuthDef represents authentication configuration in a CREATE REST CLIENT statement. type RestAuthDef struct { Scheme string // "BASIC" @@ -45,6 +97,10 @@ type RestOperationDef struct { ResponseVariable string // e.g. "$CreatedItem" ResponseMapping *RestMappingDef // for MAPPING response Timeout int + // Keys are the properties the statement wrote, as their canonical names + // (Method, Path, Parameters, Query, Headers, Body, Timeout, Response) — + // what MODIFY OPERATION rewrites, leaving the others as stored. + Keys []string } // RestMappingDef represents an inline import/export mapping in a REST operation. diff --git a/mdl/backend/mcp/microflow.go b/mdl/backend/mcp/microflow.go index 6fcbf8a31c..791da254de 100644 --- a/mdl/backend/mcp/microflow.go +++ b/mdl/backend/mcp/microflow.go @@ -39,6 +39,10 @@ func (b *Backend) CreateMicroflow(mf *microflows.Microflow) error { var ops []pedOpEntry if b.microflowConstructorTakesSkeleton() { ops = adaptMicroflowSkeleton(content) + } else if containsNote(content) { + // The full constructor's flows carry no $Type, so a note's line could + // only be read as a SequenceFlow — an execution edge out of the note. + return fmt.Errorf("microflow %q has a note, which Studio Pro's MCP API takes only from the microflow constructor 11.14 introduced; this Studio Pro has the older one — remove the note or run without --mcp", mf.Name) } if err := b.ensureSchema(microflowDocType); err != nil { return err @@ -104,7 +108,8 @@ func (b *Backend) microflowConstructorTakesSkeleton() bool { // - action, returnValue, the split condition and the loop source become `set` // operations on the stored document (/objectCollection/objects/N/…, nesting // through a loop's objectCollection); -// - each flow gets its $Type, the return type flattens to an enum plus +// - each sequence flow gets its $Type (a note's line already has one), the +// return type flattens to an enum plus // returnTypeEntity/returnTypeEnumeration, and returnVariableName becomes an // operation. func adaptMicroflowSkeleton(content map[string]any) []pedOpEntry { @@ -140,8 +145,10 @@ func adaptMicroflowSkeleton(content map[string]any) []pedOpEntry { if !ok { continue } - fm["$Type"] = "Microflows$SequenceFlow" - for _, k := range []string{"originId", "destinationId"} { + if _, typed := fm["$Type"]; !typed { + fm["$Type"] = "Microflows$SequenceFlow" + } + for _, k := range []string{"originId", "destinationId", "origin", "destination"} { if ref, ok := fm[k].(string); ok { fm[k] = shiftObjectRef(ref, len(params)) } @@ -303,6 +310,7 @@ func (b *Backend) buildFlowDocContent(kind, name string, params []*microflows.Mi "originId": fmt.Sprintf("$id(%s)", op), "destinationId": fmt.Sprintf("$id(%s)", dp), } + setPEDSides(pf, f) cv, err := mapCaseValue(f.CaseValue) if err != nil { return nil, fmt.Errorf("%s %q: %w", kind, name, err) @@ -312,6 +320,16 @@ func (b *Backend) buildFlowDocContent(kind, name string, params []*microflows.Mi } flows = append(flows, pf) } + // A note's line shares the flows list, after the sequence flows, as the + // stored document has it. + for _, af := range oc.AnnotationFlows { + op, ok1 := idPath[af.OriginID] + dp, ok2 := idPath[af.DestinationID] + if !ok1 || !ok2 { + return nil, fmt.Errorf("%s %q: a note is attached to an object that is not supported yet", kind, name) + } + flows = append(flows, pedAnnotationFlow(af, fmt.Sprintf("$id(%s)", op), fmt.Sprintf("$id(%s)", dp))) + } } returnTypeName, rtEntity, rtEnum, err := mfDataType(returnType) @@ -416,6 +434,55 @@ func pedMiddlePoint(pt model.Point) map[string]int { return map[string]int{"x": pt.X, "y": pt.Y} } +var pedSides = []string{"Top", "Right", "Bottom", "Left"} + +// setPEDSides names a sequence flow's connection sides on its PED constructor. +// Left out, Studio Pro draws every flow right -> left (measured on +// 11.15.0-rc.4), whatever the builder or @anchor chose. +func setPEDSides(v map[string]any, f *microflows.SequenceFlow) { + if f.OriginConnectionIndex >= 0 && f.OriginConnectionIndex < 4 { + v["originConnectionSide"] = pedSides[f.OriginConnectionIndex] + } + if f.DestinationConnectionIndex >= 0 && f.DestinationConnectionIndex < 4 { + v["destinationConnectionSide"] = pedSides[f.DestinationConnectionIndex] + } +} + +// pedAnnotationFlow maps a note's line onto PED's Microflows$AnnotationFlow. +// Unlike a SequenceFlow it is an element, not a constructor: its ends are +// origin/destination (not originId/destinationId) and its sides are the +// required connection indexes themselves. +func pedAnnotationFlow(af *microflows.AnnotationFlow, origin, destination string) map[string]any { + sides := af.SidesOrDefault() + return map[string]any{ + "$Type": "Microflows$AnnotationFlow", + "origin": origin, + "destination": destination, + "originConnectionIndex": sides.Origin, + "destinationConnectionIndex": sides.Destination, + } +} + +// containsNote reports whether mapped content has a note anywhere on its +// canvas, loop bodies included. +func containsNote(content map[string]any) bool { + var walk func(objs []any) bool + walk = func(objs []any) bool { + for _, o := range objs { + m, _ := o.(map[string]any) + if m["$Type"] == "Microflows$Annotation" { + return true + } + if body, ok := m["objects"].([]any); ok && walk(body) { + return true + } + } + return false + } + objects, _ := content["objects"].([]any) + return walk(objects) +} + // mapObjectTree maps one executor microflow object onto a PED /objects entry, // registering its path in idPath and recursing into loop bodies (whose objects // nest at /objects/M). Positions come from the executor's layout engine. @@ -520,6 +587,16 @@ func (b *Backend) mapObjectTree(o microflows.MicroflowObject, path string, idPat } m["objects"] = body return m, nil + case *microflows.Annotation: + m := map[string]any{ + "$Type": "Microflows$Annotation", + "relativeMiddlePoint": pos, + "caption": obj.Caption, + } + if obj.Size.Width > 0 && obj.Size.Height > 0 { + m["size"] = map[string]int{"width": obj.Size.Width, "height": obj.Size.Height} + } + return m, nil default: return nil, fmt.Errorf("microflow object type %T is not yet supported by the MCP backend", o) } diff --git a/mdl/backend/mcp/microflow_constructor_test.go b/mdl/backend/mcp/microflow_constructor_test.go index 9b5d9cf927..23e253706d 100644 --- a/mdl/backend/mcp/microflow_constructor_test.go +++ b/mdl/backend/mcp/microflow_constructor_test.go @@ -3,6 +3,8 @@ package mcp import ( + "fmt" + "strings" "testing" "github.com/mendixlabs/mxcli/model" @@ -275,3 +277,194 @@ func TestShiftObjectRef(t *testing.T) { } } } + +// notedMicroflow is a parameter, start -> activity -> end with sides that are +// not PED's default right -> left, and two notes on the activity: one below it +// whose line the builder oriented (top of the note, bottom of the activity), and +// one nothing oriented, which is drawn as for a note above its element. +func notedMicroflow(containerID model.ID) *microflows.Microflow { + obj := func(id string, x, y int) microflows.BaseMicroflowObject { + return microflows.BaseMicroflowObject{BaseElement: model.BaseElement{ID: model.ID(id)}, Position: model.Point{X: x, Y: y}} + } + act := µflows.ActionActivity{ + BaseActivity: microflows.BaseActivity{BaseMicroflowObject: obj("act", 250, 150)}, + Action: µflows.LogMessageAction{ + LogLevel: microflows.LogLevelInfo, LogNodeName: "'Probe'", + MessageTemplate: &model.Text{Translations: map[string]string{"en_US": "hello"}}, + }, + } + below := µflows.Annotation{BaseMicroflowObject: obj("below", 250, 300), Caption: "Explains the log"} + below.Size = model.Size{Width: 220, Height: 60} + above := µflows.Annotation{BaseMicroflowObject: obj("above", 250, 30), Caption: "Unoriented"} + above.Size = model.Size{Width: 200, Height: 50} + return µflows.Microflow{ + ContainerID: containerID, + Name: "ZzNoted", + Parameters: []*microflows.MicroflowParameter{{Name: "N", Type: µflows.IntegerType{}}}, + ObjectCollection: µflows.MicroflowObjectCollection{ + Objects: []microflows.MicroflowObject{ + µflows.StartEvent{BaseMicroflowObject: obj("start", 100, 150)}, + act, + µflows.EndEvent{BaseMicroflowObject: obj("end", 400, 300)}, + below, above, + }, + Flows: []*microflows.SequenceFlow{ + {OriginID: "start", DestinationID: "act", OriginConnectionIndex: 1, DestinationConnectionIndex: 3}, + {OriginID: "act", DestinationID: "end", OriginConnectionIndex: 2, DestinationConnectionIndex: 0}, + }, + AnnotationFlows: []*microflows.AnnotationFlow{ + {OriginID: "below", DestinationID: "act", Sides: µflows.ConnectionSides{Origin: 0, Destination: 2}}, + {OriginID: "above", DestinationID: "act"}, + }, + }, + ReturnType: µflows.VoidType{}, + } +} + +// createdContent creates mf against a fake Studio Pro whose microflow +// constructor schema is ctorSchema ("" for the full, pre-11.14 shape) and +// returns the ped_create_document content it was sent. +func createdContent(t *testing.T, ctorSchema string, mf func(model.ID) *microflows.Microflow) (map[string]any, *fakePED, error) { + t.Helper() + ped := newFakePED(t, func(name string, _ map[string]any) (string, bool) { + switch name { + case "ped_get_schema": + if ctorSchema != "" { + return ctorSchema, false + } + case "ped_create_document": + return "SUCCESS: Creating documents (1)", false + case "ped_update_document": + return "SUCCESS: All operations have been performed successfully.", false + case "ped_check_errors": + return "No errors found.", false + } + return "{}", false + }) + b := backendFor(ped) + if err := b.Connect(localProject(t)); err != nil { + t.Fatalf("Connect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + mod, err := b.GetModuleByName("MyFirstModule") + if err != nil { + t.Fatalf("GetModuleByName: %v", err) + } + if err := b.CreateMicroflow(mf(mod.ID)); err != nil { + return nil, ped, err + } + create, ok := ped.callByName("ped_create_document") + if !ok { + t.Fatal("no ped_create_document") + } + return create.Args["documents"].([]any)[0].(map[string]any)["documentContent"].(map[string]any), ped, nil +} + +// A flow's sides are the builder's (and @anchor's) to choose. Measured against +// 11.15.0-rc.4: a create that names none gets Studio Pro's right -> left on +// every flow, so a CASE built with `@anchor(from: bottom, …)` read back 1 -> 3 +// throughout. The constructor takes them as originConnectionSide / +// destinationConnectionSide, as the splice already sends them. +func TestCreateMicroflow_FlowSides(t *testing.T) { + for _, shape := range []struct{ name, schema string }{{"skeleton", microflowCtorSchema1114}, {"full", ""}} { + t.Run(shape.name, func(t *testing.T) { + mf := func(id model.ID) *microflows.Microflow { + m := notedMicroflow(id) + m.ObjectCollection.Objects = m.ObjectCollection.Objects[:3] + m.ObjectCollection.AnnotationFlows = nil + return m + } + content, _, err := createdContent(t, shape.schema, mf) + if err != nil { + t.Fatalf("CreateMicroflow: %v", err) + } + flows, _ := content["flows"].([]any) + want := [][2]string{{"Right", "Left"}, {"Bottom", "Top"}} + if len(flows) != len(want) { + t.Fatalf("flows = %v, want %d", flows, len(want)) + } + for i, f := range flows { + fm, _ := f.(map[string]any) + if fm["originConnectionSide"] != want[i][0] || fm["destinationConnectionSide"] != want[i][1] { + t.Errorf("flows[%d] sides = %v -> %v, want %s -> %s", i, fm["originConnectionSide"], fm["destinationConnectionSide"], want[i][0], want[i][1]) + } + } + }) + } +} + +// A note is a canvas object and its line a typed flow; the skeleton constructor +// takes both in the create. Before, the create path mapped neither: an +// Annotation reached mapObjectTree's default case. +func TestCreateMicroflow_Notes(t *testing.T) { + content, ped, err := createdContent(t, microflowCtorSchema1114, notedMicroflow) + if err != nil { + t.Fatalf("CreateMicroflow: %v", err) + } + objects, _ := content["objects"].([]any) + if len(objects) != 5 { + t.Fatalf("objects = %d, want 5 (the parameter is not one of them)", len(objects)) + } + for i, want := range []struct { + caption string + x, y, w, h int + }{{"Explains the log", 250, 300, 220, 60}, {"Unoriented", 250, 30, 200, 50}} { + note, _ := objects[3+i].(map[string]any) + if note["$Type"] != "Microflows$Annotation" || note["caption"] != want.caption { + t.Errorf("objects[%d] = %v, want the note %q", 3+i, note, want.caption) + } + if fmt.Sprint(note["x"], note["y"]) != fmt.Sprint(want.x, want.y) { + t.Errorf("note %q at %v,%v, want %d,%d", want.caption, note["x"], note["y"], want.x, want.y) + } + if size, _ := note["size"].(map[string]any); fmt.Sprint(size["width"], size["height"]) != fmt.Sprint(want.w, want.h) { + t.Errorf("note %q size = %v, want %dx%d", want.caption, note["size"], want.w, want.h) + } + if _, ok := note["relativeMiddlePoint"]; ok { + t.Errorf("note %q still carries relativeMiddlePoint, which the skeleton drops silently", want.caption) + } + } + + flows, _ := content["flows"].([]any) + if len(flows) != 4 { + t.Fatalf("flows = %d, want 2 sequence flows and 2 note lines: %v", len(flows), flows) + } + for i, want := range []struct { + origin string + sides [2]int + }{{"$id(/objects/3)", [2]int{0, 2}}, {"$id(/objects/4)", [2]int{2, 0}}} { + af, _ := flows[2+i].(map[string]any) + if af["$Type"] != "Microflows$AnnotationFlow" { + t.Errorf("flows[%d] $Type = %v, want Microflows$AnnotationFlow", 2+i, af["$Type"]) + } + // References shift past the parameter like a sequence flow's. + if af["origin"] != want.origin || af["destination"] != "$id(/objects/1)" { + t.Errorf("flows[%d] = %v -> %v, want %s -> $id(/objects/1)", 2+i, af["origin"], af["destination"], want.origin) + } + if fmt.Sprint(af["originConnectionIndex"], af["destinationConnectionIndex"]) != fmt.Sprint(want.sides[0], want.sides[1]) { + t.Errorf("flows[%d] sides = %v -> %v, want %d -> %d", 2+i, af["originConnectionIndex"], af["destinationConnectionIndex"], want.sides[0], want.sides[1]) + } + } + + // The caption and size are constructor properties: nothing about a note is + // left for the update. + update, _ := ped.callByName("ped_update_document") + ops, _ := update.Args["operations"].([]any) + for _, o := range ops { + if p, _ := o.(map[string]any)["path"].(string); strings.HasPrefix(p, "/objectCollection/objects/4") || strings.HasPrefix(p, "/objectCollection/objects/5") { + t.Errorf("update touches a note: %v", o) + } + } +} + +// The full constructor's flows carry no $Type, so a note's line could only be +// read as a sequence flow — an execution edge from the note. It is refused, not +// sent. +func TestCreateMicroflow_NotesRefusedOnFullConstructor(t *testing.T) { + _, ped, err := createdContent(t, "", notedMicroflow) + if err == nil || !strings.Contains(err.Error(), "note") { + t.Fatalf("err = %v, want a refusal naming the note", err) + } + if _, ok := ped.callByName("ped_create_document"); ok { + t.Error("a refused create must send nothing") + } +} diff --git a/mdl/backend/mcp/microflow_mutator.go b/mdl/backend/mcp/microflow_mutator.go index 3d637a2b09..d8cf0b029a 100644 --- a/mdl/backend/mcp/microflow_mutator.go +++ b/mdl/backend/mcp/microflow_mutator.go @@ -25,7 +25,8 @@ import ( // takes no unit bytes, so Save diffs the spliced document against the stored // one and sends the difference as ped_update_document path operations on the // live document: positions set, new objects and flows added, rewired flow ends -// set, removed elements removed. PED addresses list entries by index, and those +// set, removed elements removed — and, in a second update, the new elements +// filled in (flowPatchOps says why). PED addresses list entries by index, and those // indexes are the stored order — so before anything is sent, the live document // is read back and compared with the local one, and a mismatch (the .mpr is // behind Studio Pro) refuses the write rather than patching the wrong element. @@ -203,7 +204,7 @@ func (d *mcpFlowDeps) SaveUnit(_ string, contents []byte) error { if err := bson.Unmarshal(contents, &spliced); err != nil { return err } - ops, err := d.b.flowPatchOps(d, spliced) + ops, fill, err := d.b.flowPatchOps(d, spliced) if err != nil { return err } @@ -221,10 +222,21 @@ func (d *mcpFlowDeps) SaveUnit(_ string, contents []byte) error { return fmt.Errorf("%s: refusing to send a removal over MCP", d.qn) } } - // One update: PED applies it all or, on a failure, nothing. + // Two updates, because PED cannot address an element in the update that + // adds it (see flowPatchOps). The first is all or nothing: the new + // elements, wired in, and the stored ones moved to make room. The second + // fills in the new elements. If it fails, the first is not rolled back: + // the insert is left in Studio Pro without what the second carried, as a + // create is when setting its activities fails (CreateMicroflow), and + // ped_check_errors names what is missing. if err := d.b.pedUpdateDoc(d.docType, d.qn, ops...); err != nil { return err } + if len(fill) > 0 { + if err := d.b.pedUpdateDoc(d.docType, d.qn, fill...); err != nil { + return fmt.Errorf("the insert into %s was added in Studio Pro, but setting up its new activities and flows failed (the insert is left in Studio Pro without them): %w", d.qn, err) + } + } return d.b.pedCheckDocument(d.docType, d.qn) } @@ -280,7 +292,21 @@ func pedVector(s string) map[string]int { return map[string]int{"width": p["x"], "height": p["y"]} } -var pedSides = []string{"Top", "Right", "Bottom", "Left"} +// newFlowLineOps sets the control vectors of a sequence flow the insert added, +// at fp. A 0;0 vector is the line's schema default and is left out. +func newFlowLineOps(fp string, f *microflows.SequenceFlow) []pedOpEntry { + var ops []pedOpEntry + for _, v := range []struct{ key, vec string }{ + {"originControlVector", f.OriginControlVector}, + {"destinationControlVector", f.DestinationControlVector}, + } { + if v.vec == "" || v.vec == "0;0" { + continue + } + ops = append(ops, pedOpEntry{Path: fp + "/line/" + v.key, Operation: pedOperation{Type: "set", Value: pedVector(v.vec)}}) + } + return ops +} // pedQualifiedFlowType maps the storage $Type of a flow object to the // qualified name PED reports for it, where the two differ (CLAUDE.md, "BSON @@ -294,8 +320,26 @@ var pedQualifiedFlowType = map[string]string{ // everything that uses a stored index (positions, rewired flows) and every // append goes first, while indexes are still the stored ones; removals go // last, highest index first. -func (b *Backend) flowPatchOps(d *mcpFlowDeps, spliced bson.D) ([]pedOpEntry, error) { - var ops []pedOpEntry +// +// They come in two updates. PED resolves a path against the document as it +// was before the update — its own rule for indexes — so an element the update +// adds cannot be addressed in it (measured on 11.15.0-rc.4). A set below a new +// flow's line fails the whole update with `undefined is not an object +// (evaluating 'u.$Type')`; any other set on a new element — an activity's +// action, its position, a flow's isErrorHandler or whole line — is accepted +// and then missing on read-back, and so is the same property put in the +// constructor value. The same sets land in a later update. So ops adds the new +// elements (constructor properties only, references as $id(), which do +// resolve) and changes the stored ones; fill sets the rest on the new elements, +// for SaveUnit to send once they exist. +// +// Every add carries the array's stored length as its index. Appended without +// one, several elements added to the same array in one update are stored in +// reverse (measured on 11.15.0-rc.4: three objects, two flows), and $id() +// resolves against that final order — so an insert of more than one element +// was wired to the wrong objects. Added at one shared index they are stored as +// listed, which is PED's own rule for adding several elements at a position. +func (b *Backend) flowPatchOps(d *mcpFlowDeps, spliced bson.D) (ops, fill []pedOpEntry, err error) { set := func(path string, v any) { ops = append(ops, pedOpEntry{Path: path, Operation: pedOperation{Type: "set", Value: v}}) } @@ -327,25 +371,25 @@ func (b *Backend) flowPatchOps(d *mcpFlowDeps, spliced bson.D) ([]pedOpEntry, er for j, id := range added { path[id] = fmt.Sprintf("/objectCollection/objects/%d", len(oldObjIDs)+j) } + objectsEnd, flowsEnd := len(oldObjIDs), len(oldFlowIDs) for _, id := range added { obj, ok := d.objects[id] if !ok { - return nil, fmt.Errorf("internal: spliced object %s was not serialized", id) + return nil, nil, fmt.Errorf("internal: spliced object %s was not serialized", id) } idPath := map[model.ID]string{} m, err := b.mapObjectTree(obj, path[id], idPath) if err != nil { - return nil, err + return nil, nil, err } - var sets []pedOpEntry skeletonObject(m, path[id], func(p string, v any) { - sets = append(sets, pedOpEntry{Path: p, Operation: pedOperation{Type: "set", Value: v}}) + fill = append(fill, pedOpEntry{Path: p, Operation: pedOperation{Type: "set", Value: v}}) }) - ops = append(ops, pedOpEntry{Path: "/objectCollection/objects", Operation: pedOperation{Type: "add", Value: m}}) - ops = append(ops, sets...) - // The skeleton constructor ignores a position; set it on the stored element. + ops = append(ops, pedOpEntry{Path: "/objectCollection/objects", Operation: pedOperation{Type: "add", Value: m, Index: &objectsEnd}}) + // The constructor's x/y place it on 11.15; the set is for a server + // whose skeleton constructor ignores them. p := obj.GetPosition() - set(path[id]+"/relativeMiddlePoint", map[string]int{"x": p.X, "y": p.Y}) + fill = append(fill, pedOpEntry{Path: path[id] + "/relativeMiddlePoint", Operation: pedOperation{Type: "set", Value: map[string]int{"x": p.X, "y": p.Y}}}) for nested, np := range idPath { path[string(nested)] = np } @@ -378,7 +422,7 @@ func (b *Backend) flowPatchOps(d *mcpFlowDeps, spliced bson.D) ([]pedOpEntry, er if pointerID(o, end+"Pointer") != pointerID(n, end+"Pointer") { r, err := ref(pointerID(n, end+"Pointer")) if err != nil { - return nil, err + return nil, nil, err } set(fp+"/"+key, r) } @@ -392,57 +436,55 @@ func (b *Backend) flowPatchOps(d *mcpFlowDeps, spliced bson.D) ([]pedOpEntry, er } } } - for j, id := range addedFlows { - fp := fmt.Sprintf("/flows/%d", len(oldFlowIDs)+j) + // Sequence flows are added before note lines, the order a stored document + // has them in, so fill addresses the new sequence flows from the stored + // length up. + var addedNotes []*microflows.AnnotationFlow + next := len(oldFlowIDs) + for _, id := range addedFlows { if af, ok := d.annotations[id]; ok { - o, err := ref(string(af.OriginID)) - if err != nil { - return nil, err - } - dst, err := ref(string(af.DestinationID)) - if err != nil { - return nil, err - } - ops = append(ops, pedOpEntry{Path: "/flows", Operation: pedOperation{Type: "add", Value: map[string]any{ - "$Type": "Microflows$AnnotationFlow", "originId": o, "destinationId": dst}}}) + addedNotes = append(addedNotes, af) continue } + fp := fmt.Sprintf("/flows/%d", next) + next++ f, ok := d.flows[id] if !ok { - return nil, fmt.Errorf("internal: spliced flow %s was not serialized", id) + return nil, nil, fmt.Errorf("internal: spliced flow %s was not serialized", id) } o, err := ref(string(f.OriginID)) if err != nil { - return nil, err + return nil, nil, err } dst, err := ref(string(f.DestinationID)) if err != nil { - return nil, err + return nil, nil, err } v := map[string]any{"$Type": "Microflows$SequenceFlow", "originId": o, "destinationId": dst} - if f.OriginConnectionIndex >= 0 && f.OriginConnectionIndex < 4 { - v["originConnectionSide"] = pedSides[f.OriginConnectionIndex] - } - if f.DestinationConnectionIndex >= 0 && f.DestinationConnectionIndex < 4 { - v["destinationConnectionSide"] = pedSides[f.DestinationConnectionIndex] - } + setPEDSides(v, f) cv, err := mapCaseValue(f.CaseValue) if err != nil { - return nil, err + return nil, nil, err } if cv != nil { v["caseValue"] = cv } - ops = append(ops, pedOpEntry{Path: "/flows", Operation: pedOperation{Type: "add", Value: v}}) + ops = append(ops, pedOpEntry{Path: "/flows", Operation: pedOperation{Type: "add", Value: v, Index: &flowsEnd}}) if f.IsErrorHandler { - set(fp+"/isErrorHandler", true) + fill = append(fill, pedOpEntry{Path: fp + "/isErrorHandler", Operation: pedOperation{Type: "set", Value: true}}) } - if f.OriginControlVector != "" { - set(fp+"/line/originControlVector", pedVector(f.OriginControlVector)) + fill = append(fill, newFlowLineOps(fp, f)...) + } + for _, af := range addedNotes { + o, err := ref(string(af.OriginID)) + if err != nil { + return nil, nil, err } - if f.DestinationControlVector != "" { - set(fp+"/line/destinationControlVector", pedVector(f.DestinationControlVector)) + dst, err := ref(string(af.DestinationID)) + if err != nil { + return nil, nil, err } + ops = append(ops, pedOpEntry{Path: "/flows", Operation: pedOperation{Type: "add", Value: pedAnnotationFlow(af, o, dst), Index: &flowsEnd}}) } for i := len(oldFlowIDs) - 1; i >= 0; i-- { @@ -457,7 +499,7 @@ func (b *Backend) flowPatchOps(d *mcpFlowDeps, spliced bson.D) ([]pedOpEntry, er ops = append(ops, pedOpEntry{Path: "/objectCollection/objects", Operation: pedOperation{Type: "remove", Index: &idx}}) } } - return ops, nil + return ops, fill, nil } // checkLiveFlowMatches compares the live document's objects and flows with the diff --git a/mdl/backend/mcp/microflow_mutator_test.go b/mdl/backend/mcp/microflow_mutator_test.go index df389411de..afce262f30 100644 --- a/mdl/backend/mcp/microflow_mutator_test.go +++ b/mdl/backend/mcp/microflow_mutator_test.go @@ -103,7 +103,8 @@ func logFragment() *backend.MicroflowFragment { // The splice becomes PED path operations addressed by the stored indexes: // the moved end, the new activity (added bare, then its action and position // set, as PED's skeleton constructor requires), the rewired flow's end, and -// the new flow — and no removal. +// the new flow — and no removal. Which of them wait for the second update is +// TestFlowPatchOps_NewElementsAreFilledInOnceTheyExist. func TestFlowPatchOps_InsertAfter(t *testing.T) { var sent []any f := newFakePED(t, func(name string, args map[string]any) (string, bool) { @@ -131,10 +132,10 @@ func TestFlowPatchOps_InsertAfter(t *testing.T) { if err := m.Save(); err != nil { t.Fatalf("save: %v", err) } - if len(sent) != 1 { - t.Fatalf("want one ped_update_document, got %d", len(sent)) + if len(sent) != 2 { + t.Fatalf("want the insert and its fill, got %d updates", len(sent)) } - js, _ := json.Marshal(sent[0]) + js, _ := json.Marshal(append(sent[0].([]any), sent[1].([]any)...)) got := string(js) var ops []pedOpEntry if err := json.Unmarshal(js, &ops); err != nil { @@ -166,6 +167,269 @@ func TestFlowPatchOps_InsertAfter(t *testing.T) { } } +// A note inserted with an activity is added as an object and its line as a +// Microflows$AnnotationFlow element, whose ends are origin/destination — not a +// SequenceFlow constructor's originId/destinationId. Measured on 11.15.0-rc.4: +// the old keys fail the whole update with `"/origin":"Expected reference +// (string), got undefined"`. +func TestFlowPatchOps_InsertNote(t *testing.T) { + var sent []any + f := newFakePED(t, func(name string, args map[string]any) (string, bool) { + switch name { + case "ped_read_document": + return `{"results":[{"path":"/objectCollection/objects","result":[` + + `{"$Type":"Microflows$StartEvent","relativeMiddlePoint":{"x":-98,"y":200}},` + + `{"$Type":"Microflows$EndEvent","relativeMiddlePoint":{"x":564,"y":200}},` + + `{"$Type":"Microflows$ActionActivity","relativeMiddlePoint":{"x":214,"y":200}},` + + `{"$Type":"Microflows$ActionActivity","relativeMiddlePoint":{"x":404,"y":200}}]},` + + `{"path":"/flows","result":[{},{},{}]}]}`, false + case "ped_update_document": + sent = append(sent, args["operations"]) + return "SUCCESS: All operations have been performed successfully.", false + case "ped_check_errors": + return "No errors found.", false + } + return "SUCCESS", false + }) + b := &Backend{client: f.connectClient(t)} + m, _ := openFlowMutator(t, b) + frag := logFragment() + note := µflows.Annotation{BaseMicroflowObject: microflows.BaseMicroflowObject{ + BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, + Position: model.Point{X: 360, Y: 100}, + Size: model.Size{Width: 200, Height: 50}, + }, Caption: "Why we log"} + frag.Objects = append(frag.Objects, note) + frag.AnnotationFlows = []*microflows.AnnotationFlow{{ + BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, + OriginID: note.ID, DestinationID: frag.Entry, + }} + if err := m.InsertAfter(model.ID(mfUUID("a")), frag); err != nil { + t.Fatal(err) + } + if err := m.Save(); err != nil { + t.Fatalf("save: %v", err) + } + if len(sent) != 2 { + t.Fatalf("want the insert and its fill, got %d updates", len(sent)) + } + js, _ := json.Marshal(sent[0]) + var ops []pedOpEntry + if err := json.Unmarshal(js, &ops); err != nil { + t.Fatal(err) + } + var noteAdd, line map[string]any + lineAt, lastSequenceOp := -1, -1 + for i, op := range ops { + v, _ := op.Operation.Value.(map[string]any) + switch { + case op.Operation.Type == "add" && v["$Type"] == "Microflows$Annotation": + noteAdd = v + case op.Operation.Type == "add" && v["$Type"] == "Microflows$AnnotationFlow": + line, lineAt = v, i + case strings.HasPrefix(op.Path, "/flows/3"), op.Operation.Type == "add" && v["$Type"] == "Microflows$SequenceFlow": + lastSequenceOp = i + } + } + // PED stores a note line after a sequence flow added later in the same + // update, so the new sequence flow must come first: the fill that follows + // addresses it as /flows/3. + if lineAt < lastSequenceOp { + t.Errorf("the note line (op %d) is added before the new sequence flow's ops (last at %d):\n%s", lineAt, lastSequenceOp, js) + } + if noteAdd == nil || noteAdd["caption"] != "Why we log" { + t.Errorf("note add = %v, want the Annotation with its caption in:\n%s", noteAdd, js) + } + if line == nil { + t.Fatalf("no AnnotationFlow add in:\n%s", js) + } + // The activity is the first object added (index 4), the note the second. + if line["origin"] != "$id(/objectCollection/objects/5)" || line["destination"] != "$id(/objectCollection/objects/4)" { + t.Errorf("note line = %v -> %v, want objects/5 -> objects/4", line["origin"], line["destination"]) + } + if _, ok := line["originId"]; ok { + t.Errorf("note line carries originId, which PED's AnnotationFlow does not take: %v", line) + } + if fmt.Sprint(line["originConnectionIndex"], line["destinationConnectionIndex"]) != "2 0" { + t.Errorf("note line sides = %v -> %v, want the unoriented 2 -> 0", line["originConnectionIndex"], line["destinationConnectionIndex"]) + } +} + +// reducePED fakes Studio Pro holding storedReduce, and records each +// ped_update_document's operations. +func reducePED(t *testing.T, sent *[][]pedOpEntry) *Backend { + t.Helper() + f := newFakePED(t, func(name string, args map[string]any) (string, bool) { + switch name { + case "ped_read_document": + return `{"results":[{"path":"/objectCollection/objects","result":[` + + `{"$Type":"Microflows$StartEvent","relativeMiddlePoint":{"x":-98,"y":200}},` + + `{"$Type":"Microflows$EndEvent","relativeMiddlePoint":{"x":564,"y":200}},` + + `{"$Type":"Microflows$ActionActivity","relativeMiddlePoint":{"x":214,"y":200}},` + + `{"$Type":"Microflows$ActionActivity","relativeMiddlePoint":{"x":404,"y":200}}]},` + + `{"path":"/flows","result":[{},{},{}]}]}`, false + case "ped_update_document": + js, _ := json.Marshal(args["operations"]) + var ops []pedOpEntry + if err := json.Unmarshal(js, &ops); err != nil { + t.Fatal(err) + } + *sent = append(*sent, ops) + return "SUCCESS: All operations have been performed successfully.", false + case "ped_check_errors": + return "No errors found.", false + } + return "SUCCESS", false + }) + return &Backend{client: f.connectClient(t)} +} + +// opsAt maps the paths an update sets to their JSON values. +func opsAt(ops []pedOpEntry) map[string]string { + got := map[string]string{} + for _, op := range ops { + if op.Operation.Type == "set" { + v, _ := json.Marshal(op.Operation.Value) + got[op.Path] = string(v) + } + } + return got +} + +// PED resolves a path against the document as it was before the update, so +// an element the update adds cannot be addressed in it (measured on +// 11.15.0-rc.4): a set below a new flow's line fails the whole update with +// `undefined is not an object (evaluating 'u.$Type')`, which made every insert +// fail, and a set of a new activity's action is accepted and lost, which left +// it with "No action defined". So the update that adds an element addresses +// nothing on it, and a second update fills it in. +func TestFlowPatchOps_NewElementsAreFilledInOnceTheyExist(t *testing.T) { + var sent [][]pedOpEntry + m, _ := openFlowMutator(t, reducePED(t, &sent)) + if err := m.InsertAfter(model.ID(mfUUID("a")), logFragment()); err != nil { + t.Fatal(err) + } + if err := m.Save(); err != nil { + t.Fatalf("save: %v", err) + } + if len(sent) != 2 { + t.Fatalf("want the insert and then its fill, got %d updates: %v", len(sent), sent) + } + // The stored document has four objects and three flows, so the insert + // adds /objectCollection/objects/4 and /flows/3. + for _, op := range sent[0] { + if strings.HasPrefix(op.Path, "/objectCollection/objects/4/") || strings.HasPrefix(op.Path, "/flows/3/") { + t.Errorf("the update that adds the element also sets %s on it", op.Path) + } + } + got := opsAt(sent[1]) + if !strings.Contains(got["/objectCollection/objects/4/action"], `"Microflows$LogMessageAction"`) { + t.Errorf("the fill does not set the new activity's action: %v", got) + } + for p, v := range map[string]string{ + // The new flow leaves the activity on the right… + "/flows/3/line/originControlVector": `{"height":0,"width":15}`, + // …and keeps the end the rewired flow had at b. + "/flows/3/line/destinationControlVector": `{"height":0,"width":-30}`, + } { + if got[p] != v { + t.Errorf("%s = %q, want %s", p, got[p], v) + } + } + for p := range got { + if !strings.HasPrefix(p, "/objectCollection/objects/4/") && !strings.HasPrefix(p, "/flows/3/") { + t.Errorf("the fill sets %s, which the first update could address", p) + } + } +} + +// Several elements appended to one array in one update are stored in reverse, +// and $id() resolves against that final order (measured on 11.15.0-rc.4): an +// IF inserted over MCP was wired to the wrong objects. Every add therefore +// carries the stored length as its index, which keeps them in listed order. +func TestFlowPatchOps_AddsKeepTheirListedOrder(t *testing.T) { + var sent [][]pedOpEntry + m, _ := openFlowMutator(t, reducePED(t, &sent)) + frag := logFragment() + second := logFragment().Objects[0] + second.SetPosition(model.Point{X: 520, Y: 200}) + frag.Objects = append(frag.Objects, second) + frag.Flows = []*microflows.SequenceFlow{{ + BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, + OriginID: frag.Entry, DestinationID: second.GetID(), + OriginConnectionIndex: 1, DestinationConnectionIndex: 3, + }} + frag.Exit = second.GetID() + if err := m.InsertAfter(model.ID(mfUUID("a")), frag); err != nil { + t.Fatal(err) + } + if err := m.Save(); err != nil { + t.Fatalf("save: %v", err) + } + adds := map[string]int{} + for _, op := range sent[0] { + if op.Operation.Type != "add" { + continue + } + adds[op.Path]++ + want := map[string]int{"/objectCollection/objects": 4, "/flows": 3}[op.Path] + if op.Operation.Index == nil || *op.Operation.Index != want { + t.Errorf("add to %s has index %v, want the stored length %d", op.Path, op.Operation.Index, want) + } + } + if adds["/objectCollection/objects"] != 2 || adds["/flows"] != 2 { + t.Errorf("adds = %v, want two objects and two flows", adds) + } +} + +// A 0;0 control vector is the line's schema default, which PED asks not to be +// sent. +func TestNewFlowLineOps_LeavesOutDefaults(t *testing.T) { + ops := newFlowLineOps("/flows/4", µflows.SequenceFlow{OriginControlVector: "0;0", DestinationControlVector: "0;-15"}) + if len(ops) != 1 || ops[0].Path != "/flows/4/line/destinationControlVector" { + t.Errorf("ops = %+v, want only the destination vector", ops) + } + if ops := newFlowLineOps("/flows/4", µflows.SequenceFlow{OriginControlVector: "0;0"}); len(ops) != 0 { + t.Errorf("default vectors produced %+v", ops) + } +} + +// An error handler flow is marked in the fill too. Set in the update that adds +// the flow, isErrorHandler is accepted and the flow reads back as an ordinary +// one (measured on 11.15.0-rc.4) — a second normal exit from the activity. +func TestFlowPatchOps_NewErrorHandlerFlowIsMarkedOnceItExists(t *testing.T) { + var sent [][]pedOpEntry + m, _ := openFlowMutator(t, reducePED(t, &sent)) + frag := logFragment() + handler := logFragment().Objects[0] + handler.SetPosition(model.Point{X: 360, Y: 300}) + frag.Objects = append(frag.Objects, handler) + frag.Flows = []*microflows.SequenceFlow{{ + BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, + OriginID: frag.Entry, DestinationID: handler.GetID(), + OriginConnectionIndex: 2, IsErrorHandler: true, + }} + if err := m.InsertAfter(model.ID(mfUUID("a")), frag); err != nil { + t.Fatal(err) + } + if err := m.Save(); err != nil { + t.Fatalf("save: %v", err) + } + if len(sent) != 2 { + t.Fatalf("want the insert and then its fill, got %d updates", len(sent)) + } + for _, op := range sent[0] { + if strings.HasSuffix(op.Path, "/isErrorHandler") { + t.Errorf("the update that adds the flow also marks it at %s", op.Path) + } + } + // The fragment's own flow is added first (/flows/3), then the one leaving + // the fragment (/flows/4). + if got := opsAt(sent[1]); got["/flows/3/isErrorHandler"] != "true" { + t.Errorf("the error handler flow is not marked in the fill: %v", got) + } +} + // PED addresses by index, so a live document that differs from the stored one // must refuse the write before anything is sent. func TestFlowPatchOps_StaleLiveDocumentIsRefused(t *testing.T) { diff --git a/mdl/backend/mcp/unsupported_gen.go b/mdl/backend/mcp/unsupported_gen.go index 17d7a1a01e..d89f5e34a1 100644 --- a/mdl/backend/mcp/unsupported_gen.go +++ b/mdl/backend/mcp/unsupported_gen.go @@ -494,6 +494,11 @@ func (unsupportedBackend) Disconnect() (err0 error) { return } +func (unsupportedBackend) EditConsumedRestService(_ model.ID, _ *backend.ConsumedRestServiceEdit) (err0 error) { + err0 = errUnsupported("EditConsumedRestService") + return +} + func (unsupportedBackend) ExportJSON() (r0 []uint8, err1 error) { err1 = errUnsupported("ExportJSON") return diff --git a/mdl/backend/mock/backend.go b/mdl/backend/mock/backend.go index 96125b5f8f..9d7bc1418d 100644 --- a/mdl/backend/mock/backend.go +++ b/mdl/backend/mock/backend.go @@ -202,6 +202,7 @@ type MockBackend struct { ListPublishedRestServicesFunc func() ([]*model.PublishedRestService, error) CreateConsumedRestServiceFunc func(svc *model.ConsumedRestService) error UpdateConsumedRestServiceFunc func(svc *model.ConsumedRestService) error + EditConsumedRestServiceFunc func(id model.ID, edit *backend.ConsumedRestServiceEdit) error DeleteConsumedRestServiceFunc func(id model.ID) error CreatePublishedRestServiceFunc func(svc *model.PublishedRestService) error UpdatePublishedRestServiceFunc func(svc *model.PublishedRestService) error diff --git a/mdl/backend/mock/mock_service.go b/mdl/backend/mock/mock_service.go index cb37e74fdf..ee422eabec 100644 --- a/mdl/backend/mock/mock_service.go +++ b/mdl/backend/mock/mock_service.go @@ -5,6 +5,7 @@ package mock import ( "errors" + "github.com/mendixlabs/mxcli/mdl/backend" "github.com/mendixlabs/mxcli/model" "github.com/mendixlabs/mxcli/sdk/javaactions" ) @@ -93,6 +94,13 @@ func (m *MockBackend) UpdateConsumedRestService(svc *model.ConsumedRestService) return nil } +func (m *MockBackend) EditConsumedRestService(id model.ID, edit *backend.ConsumedRestServiceEdit) error { + if m.EditConsumedRestServiceFunc != nil { + return m.EditConsumedRestServiceFunc(id, edit) + } + return nil +} + func (m *MockBackend) DeleteConsumedRestService(id model.ID) error { if m.DeleteConsumedRestServiceFunc != nil { return m.DeleteConsumedRestServiceFunc(id) diff --git a/mdl/backend/modelsdk/consumed_rest_edit.go b/mdl/backend/modelsdk/consumed_rest_edit.go new file mode 100644 index 0000000000..8f59fb8789 --- /dev/null +++ b/mdl/backend/modelsdk/consumed_rest_edit.go @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "fmt" + "strings" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/backend/bsonnav" + "github.com/mendixlabs/mxcli/model" +) + +// EditConsumedRestService is `alter consumed rest service`: a splice of the +// stored unit, not a rebuild (mendixlabs/mxcli#1608). +// +// UpdateConsumedRestService rebuilds the whole document from the semantic +// model, so everything the reader does not carry — a query parameter's +// optional usage, a parameter's test value, a response's status code — comes +// back as the writer's constant, on every operation, whichever one the +// statement was about. Adding one operation that way would reset all the +// others. Here an operation the edit does not name is copied as its stored +// bytes; a modified one has only the keys the statement wrote re-encoded; and +// a new one is encoded exactly as CREATE encodes it. +func (b *Backend) EditConsumedRestService(id model.ID, edit *backend.ConsumedRestServiceEdit) error { + if edit == nil { + return fmt.Errorf("EditConsumedRestService: nil edit") + } + if b.writer == nil { + return fmt.Errorf("EditConsumedRestService: not connected for writing") + } + doc, err := b.rawUnitDoc(id) + if err != nil { + return fmt.Errorf("EditConsumedRestService: %w", err) + } + forms := b.restBodyForms() + + if edit.BaseUrl != nil { + v, err := encodeFlowElementToD(valueTemplateElem(*edit.BaseUrl)) + if err != nil { + return fmt.Errorf("EditConsumedRestService: encode BaseUrl: %w", err) + } + doc = setDocKey(doc, "BaseUrl", v) + } + if edit.SetAuthentication { + var v any // BSON null: no authentication, as CREATE writes it + if edit.Authentication != nil { + if v, err = encodeFlowElementToD(restAuthSchemeToGen(edit.Authentication)); err != nil { + return fmt.Errorf("EditConsumedRestService: encode AuthenticationScheme: %w", err) + } + } + doc = setDocKey(doc, "AuthenticationScheme", v) + } + + drop := map[string]bool{} + for _, n := range edit.Drop { + drop[strings.ToLower(n)] = true + } + modify := map[string]backend.ConsumedRestOperationEdit{} + for _, m := range edit.Modify { + modify[strings.ToLower(m.Name)] = m + } + + var ops []any + for _, el := range bsonnav.DGetArrayElements(bsonnav.DGet(doc, "Operations")) { + od, ok := el.(bson.D) + if !ok { + ops = append(ops, el) + continue + } + name := strings.ToLower(bsonnav.DGetString(od, "Name")) + if drop[name] { + delete(drop, name) + continue + } + if m, ok := modify[name]; ok { + delete(modify, name) + if od, err = editRestOperationDoc(od, m, forms); err != nil { + return fmt.Errorf("EditConsumedRestService: operation %s: %w", m.Name, err) + } + } + ops = append(ops, od) + } + // The executor checks the names first; this is the backstop that keeps a + // misnamed edit from writing a document it did not describe. + for _, n := range edit.Drop { + if drop[strings.ToLower(n)] { + return fmt.Errorf("EditConsumedRestService: no operation %q to drop", n) + } + } + for _, m := range edit.Modify { + if _, ok := modify[strings.ToLower(m.Name)]; ok { + return fmt.Errorf("EditConsumedRestService: no operation %q to modify", m.Name) + } + } + for _, op := range edit.Add { + od, err := encodeFlowElementToD(restOperationToGen(op, forms)) + if err != nil { + return fmt.Errorf("EditConsumedRestService: encode operation %s: %w", op.Name, err) + } + ops = append(ops, od) + } + // The list keeps the marker it was stored with; one written without a list + // gets Studio Pro's. + marker := any(int32(3)) + if stored, ok := bsonnav.DGet(doc, "Operations").(bson.A); ok && len(stored) > 0 { + marker = stored[0] + } + doc = setDocKey(doc, "Operations", append(bson.A{marker}, ops...)) + + contents, err := bson.Marshal(doc) + if err != nil { + return fmt.Errorf("EditConsumedRestService: marshal: %w", err) + } + return b.writer.UpdateRawUnitPatch(string(id), contents) +} + +// restOperationKeys maps an MDL operation property to the stored keys it is +// written to. +var restOperationKeys = map[string][]string{ + "Method": {"Method"}, + "Body": {"Method"}, // the body lives in the method element + "Path": {"Path"}, + "Parameters": {"Parameters"}, + "Query": {"QueryParameters"}, + "Headers": {"Headers"}, + "Timeout": {"Timeout"}, + "Response": {"ResponseHandling"}, +} + +// editRestOperationDoc rewrites the stored keys behind the MDL properties in +// m.Keys, each taken from the operation encoded as CREATE would encode it, and +// copies every other key of the stored operation unchanged. +// +// A method change that keeps the stored method element's shape — with a body +// to with a body, or without to without — sets only its HttpMethod, so the +// stored body survives a `Method:` that does not mention it. +func editRestOperationDoc(stored bson.D, m backend.ConsumedRestOperationEdit, forms restBodyForms) (bson.D, error) { + encoded, err := encodeFlowElementToD(restOperationToGen(m.Operation, forms)) + if err != nil { + return nil, err + } + keys := map[string]bool{} + bodyWritten := false + for _, k := range m.Keys { + for _, sk := range restOperationKeys[k] { + keys[sk] = true + } + bodyWritten = bodyWritten || k == "Body" + } + out := stored + for key := range keys { + v := bsonnav.DGet(encoded, key) + if key == "Method" && !bodyWritten { + if kept, ok := withHttpMethodOnly(bsonnav.DGetDoc(stored, "Method"), v); ok { + v = kept + } + } + out = setDocKey(out, key, v) + } + return out, nil +} + +// withHttpMethodOnly returns the stored method element with the encoded one's +// HttpMethod, when both have the same $Type (so the stored body, if any, is +// still the right shape). +func withHttpMethodOnly(stored bson.D, encoded any) (bson.D, bool) { + enc, ok := encoded.(bson.D) + if !ok || stored == nil || bsonnav.DGetString(stored, "$Type") != bsonnav.DGetString(enc, "$Type") { + return nil, false + } + out := append(bson.D(nil), stored...) + bsonnav.DSet(out, "HttpMethod", bsonnav.DGet(enc, "HttpMethod")) + return out, true +} + +// setDocKey returns doc with key set to v, appended when doc lacks it. +func setDocKey(doc bson.D, key string, v any) bson.D { + if bsonnav.DSet(doc, key, v) { + return doc + } + return append(doc, bson.E{Key: key, Value: v}) +} diff --git a/mdl/backend/modelsdk/consumed_rest_markers_test.go b/mdl/backend/modelsdk/consumed_rest_markers_test.go new file mode 100644 index 0000000000..064572fab6 --- /dev/null +++ b/mdl/backend/modelsdk/consumed_rest_markers_test.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/model" +) + +// The typed-array markers of a consumed REST service are the ones Studio Pro +// writes: Operations, an operation's Headers and Parameters 3, QueryParameters +// 2, Tags 1. Measured on the Studio Pro-authored services in TestApp (11.14.0) +// and on `mx convert -p` of mxcli-written ones (11.6.6), which agree. The +// writer had 2 for the first three — copied from the legacy serializer, never +// measured — so re-executing describe output rewrote a Studio Pro-authored +// service it had not changed (`/Operations[0]: 3 -> 2`, mendixlabs/mxcli#1608 +// once its describe output parsed). +func TestConsumedRestService_ListMarkersMatchStudioPro(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + mod, err := b.GetModuleByName("MyFirstModule") + if err != nil || mod == nil { + t.Fatalf("GetModuleByName: %v", err) + } + svc := &model.ConsumedRestService{ + ContainerID: mod.ID, + Name: "MarkerProbe", + BaseUrl: "https://api.example.com", + Operations: []*model.RestClientOperation{ + {Name: "WithParams", HttpMethod: "GET", Path: "/items/{id}", ResponseType: "JSON", + Parameters: []*model.RestClientParameter{{Name: "id", DataType: "String"}}, + QueryParameters: []*model.RestClientParameter{{Name: "q", DataType: "String"}}}, + {Name: "Bare", HttpMethod: "GET", Path: "/ping", ResponseType: "NONE"}, + }, + } + if err := b.CreateConsumedRestService(svc); err != nil { + t.Fatalf("CreateConsumedRestService: %v", err) + } + raw, err := b.GetRawUnitBytes(svc.ID) + if err != nil { + t.Fatal(err) + } + var doc bson.D + if err := bson.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + if got := markerOf(t, doc, "Operations"); got != 3 { + t.Errorf("Operations marker = %d, want 3", got) + } + var ops bson.A + for _, e := range doc { + if e.Key == "Operations" { + ops = e.Value.(bson.A) + } + } + for i, el := range ops[1:] { + op := el.(bson.D) + // Both the filled and the empty lists: an empty one is still written. + for key, want := range map[string]int32{"Headers": 3, "Parameters": 3, "QueryParameters": 2, "Tags": 1} { + if got := markerOf(t, op, key); got != want { + t.Errorf("Operations[%d].%s marker = %d, want %d", i, key, got, want) + } + } + } +} diff --git a/mdl/backend/modelsdk/consumed_rest_write.go b/mdl/backend/modelsdk/consumed_rest_write.go index 79f97b3c90..2987ae3ec8 100644 --- a/mdl/backend/modelsdk/consumed_rest_write.go +++ b/mdl/backend/modelsdk/consumed_rest_write.go @@ -13,20 +13,25 @@ import ( ) func init() { - // A consumed REST service's Operations list and each operation's Headers / - // Parameters / QueryParameters use the typed-array marker 2; Tags is a marker-1 - // string array. BaseUrlParameter is always null; AuthenticationScheme is null - // when there is no authentication. - codec.RegisterListMarker("Rest$RestOperation", 2) - codec.RegisterListMarker("Rest$HeaderWithValueTemplate", 2) - codec.RegisterListMarker("Rest$OperationParameter", 2) + // A consumed REST service's Operations list and each operation's Headers and + // Parameters use the typed-array marker 3, its QueryParameters 2, and Tags is + // a marker-1 string array — measured on the Studio Pro-authored services in + // TestApp (11.14.0) and on `mx convert -p` of mxcli-written ones (11.6.6), + // which agree. The first three were 2, copied from the legacy serializer and + // never measured, so a rewrite of a Studio Pro-authored service changed its + // markers even when nothing else changed (mendixlabs/mxcli#1608). + // BaseUrlParameter is always null; AuthenticationScheme is null when there is + // no authentication. + codec.RegisterListMarker("Rest$RestOperation", 3) + codec.RegisterListMarker("Rest$HeaderWithValueTemplate", 3) + codec.RegisterListMarker("Rest$OperationParameter", 3) codec.RegisterListMarker("Rest$QueryParameter", 2) codec.RegisterTypeDefaults("Rest$ConsumedRestService", codec.TypeDefaults{ - MandatoryListMarkers: map[string]int32{"Operations": 2}, + MandatoryListMarkers: map[string]int32{"Operations": 3}, NullFields: []string{"BaseUrlParameter", "AuthenticationScheme"}, }) codec.RegisterTypeDefaults("Rest$RestOperation", codec.TypeDefaults{ - MandatoryListMarkers: map[string]int32{"Tags": 1, "Headers": 2, "Parameters": 2, "QueryParameters": 2}, + MandatoryListMarkers: map[string]int32{"Tags": 1, "Headers": 3, "Parameters": 3, "QueryParameters": 2}, }) } diff --git a/mdl/backend/modelsdk/domainmodel.go b/mdl/backend/modelsdk/domainmodel.go index 42b8bf3c37..ec4af54ea2 100644 --- a/mdl/backend/modelsdk/domainmodel.go +++ b/mdl/backend/modelsdk/domainmodel.go @@ -41,7 +41,7 @@ func (b *Backend) ListDomainModels() ([]*domainmodel.DomainModel, error) { // The System module is virtual (not stored in the project); inject its domain // model so platform entities (System.WorkflowUserTask, User, …) resolve. out = append(out, buildSystemDomainModel()) - b.resolveInheritedPersistability(units, out...) + b.resolveInheritedPersistability(b.persistabilityIndexOf(units), out...) return out, nil } @@ -107,11 +107,11 @@ type persistabilityLink struct { // … extends` for it, and `show entities where non-persistent`, the catalog and // the microflow builder all treated it as a table (mendixlabs/mxcli#1600). // -// The chain may cross modules and end in the virtual System module, so the index -// is built over every stored domain model in units plus the System entities. A -// chain that leaves the project or loops keeps the default: there is no root to -// read, and the model is already unloadable (CE1613). -func (b *Backend) resolveInheritedPersistability(units []mprread.Unit[*genDm.DomainModel], dms ...*domainmodel.DomainModel) { +// The chain may cross modules and end in the virtual System module; index says +// what each link's own generalization element holds. A chain that leaves the +// project or loops keeps the default: there is no root to read, and the model +// is already unloadable (CE1613). +func (b *Backend) resolveInheritedPersistability(index *persistabilityIndex, dms ...*domainmodel.DomainModel) { need := false for _, dm := range dms { for _, e := range dm.Entities { @@ -124,40 +124,11 @@ func (b *Backend) resolveInheritedPersistability(units []mprread.Unit[*genDm.Dom return } - index := make(map[string]persistabilityLink) - for _, e := range meta.ModelerSystemEntities() { - index["System."+e.Name] = persistabilityLink{persistable: e.Persistable, parent: e.Generalization} - } - moduleName := make(map[model.ID]string) - if mods, err := b.reader.ListModules(); err == nil { - for _, m := range mods { - moduleName[model.ID(m.ID)] = m.Name - } - } - for _, u := range units { - mod := moduleName[u.ContainerID] - if mod == "" { - continue - } - for _, el := range u.Element.EntitiesItems() { - e, ok := el.(*genDm.Entity) - if !ok { - continue - } - switch g := e.Generalization().(type) { - case *genDm.NoGeneralization: - index[mod+"."+e.Name()] = persistabilityLink{persistable: g.Persistable()} - case *genDm.Generalization: - index[mod+"."+e.Name()] = persistabilityLink{parent: g.GeneralizationQualifiedName()} - } - } - } - rootPersistable := func(qn string) (bool, bool) { seen := make(map[string]bool) for !seen[qn] { seen[qn] = true - link, ok := index[qn] + link, ok := index.link(qn) if !ok { return false, false } @@ -180,25 +151,128 @@ func (b *Backend) resolveInheritedPersistability(units []mprread.Unit[*genDm.Dom } } -// GetDomainModel returns the domain model whose container is moduleID. +// persistabilityIndex maps an entity's qualified name to what its own +// generalization element says about persistability. It starts with the System +// entities; a module's stored entities are added either up front (a listing +// that has decoded every domain model anyway) or on first reference, so a +// lookup of one module's domain model decodes only the modules its +// generalization chains pass through — usually none but System's, which is +// not stored (mendixlabs/mxcli#1616). +type persistabilityIndex struct { + links map[string]persistabilityLink + load func(module string) // adds a module's entities; nil once all are in + loaded map[string]bool +} + +func newPersistabilityIndex() *persistabilityIndex { + ix := &persistabilityIndex{links: make(map[string]persistabilityLink), loaded: map[string]bool{"System": true}} + for _, e := range meta.ModelerSystemEntities() { + ix.links["System."+e.Name] = persistabilityLink{persistable: e.Persistable, parent: e.Generalization} + } + return ix +} + +// persistabilityIndexOf indexes every unit in units up front. +func (b *Backend) persistabilityIndexOf(units []mprread.Unit[*genDm.DomainModel]) *persistabilityIndex { + ix := newPersistabilityIndex() + moduleName := make(map[model.ID]string) + if mods, err := b.reader.ListModules(); err == nil { + for _, m := range mods { + moduleName[model.ID(m.ID)] = m.Name + } + } + for _, u := range units { + ix.add(moduleName[u.ContainerID], u.Element) + } + return ix +} + +// lazyPersistabilityIndex indexes a module's units on first reference. own is +// the already-decoded domain model of module ownID, reused rather than read +// again when a chain comes back into its own module. +func (b *Backend) lazyPersistabilityIndex(ownID model.ID, own *genDm.DomainModel) *persistabilityIndex { + ix := newPersistabilityIndex() + var moduleID map[string]model.ID + ix.load = func(module string) { + if moduleID == nil { + moduleID = make(map[string]model.ID) + if mods, err := b.reader.ListModules(); err == nil { + for _, m := range mods { + moduleID[m.Name] = model.ID(m.ID) + } + } + } + id, ok := moduleID[module] + if !ok { + return + } + if id == ownID && own != nil { + ix.add(module, own) + return + } + units, err := mprread.ListUnitsInContainer[*genDm.DomainModel](b.reader, id) + if err != nil { + return + } + for _, u := range units { + ix.add(module, u.Element) + } + } + return ix +} + +func (ix *persistabilityIndex) add(module string, dm *genDm.DomainModel) { + if module == "" { + return + } + for _, el := range dm.EntitiesItems() { + e, ok := el.(*genDm.Entity) + if !ok { + continue + } + switch g := e.Generalization().(type) { + case *genDm.NoGeneralization: + ix.links[module+"."+e.Name()] = persistabilityLink{persistable: g.Persistable()} + case *genDm.Generalization: + ix.links[module+"."+e.Name()] = persistabilityLink{parent: g.GeneralizationQualifiedName()} + } + } +} + +func (ix *persistabilityIndex) link(qn string) (persistabilityLink, bool) { + if l, ok := ix.links[qn]; ok { + return l, true + } + module, _, ok := strings.Cut(qn, ".") + if !ok || ix.load == nil || ix.loaded[module] { + return persistabilityLink{}, false + } + ix.loaded[module] = true + ix.load(module) + l, ok := ix.links[qn] + return l, ok +} + +// GetDomainModel returns the domain model whose container is moduleID. It +// reads and decodes that one unit: the microflow builder calls it once per +// member assignment, and decoding every module's domain model each time made +// a script's cost members × domain models (mendixlabs/mxcli#1616). func (b *Backend) GetDomainModel(moduleID model.ID) (*domainmodel.DomainModel, error) { - units, err := mprread.ListUnitsWithContainer[*genDm.DomainModel](b.reader) + units, err := mprread.ListUnitsInContainer[*genDm.DomainModel](b.reader, moduleID) if err != nil { return nil, err } for _, u := range units { - if u.ContainerID == moduleID { - dm := domainModelFromGen(u.Element, u.ContainerID) - b.populateViewEntityOql([]*domainmodel.DomainModel{dm}) - b.resolveInheritedPersistability(units, dm) - return dm, nil - } + dm := domainModelFromGen(u.Element, u.ContainerID) + b.populateViewEntityOql([]*domainmodel.DomainModel{dm}) + b.resolveInheritedPersistability(b.lazyPersistabilityIndex(moduleID, u.Element), dm) + return dm, nil } // The System module is virtual — its domain model is not stored in the project, // so serve the injected one (matching ListDomainModels) so DESCRIBE System.* and // any lookup of a platform entity's domain model resolves instead of erroring. if sys := buildSystemDomainModel(); sys.ContainerID == moduleID { - b.resolveInheritedPersistability(units, sys) + b.resolveInheritedPersistability(b.lazyPersistabilityIndex("", nil), sys) return sys, nil } // Match the legacy contract: a missing domain model is an error, not (nil, nil). diff --git a/mdl/backend/modelsdk/domainmodel_get_cost_test.go b/mdl/backend/modelsdk/domainmodel_get_cost_test.go new file mode 100644 index 0000000000..b43c69d881 --- /dev/null +++ b/mdl/backend/modelsdk/domainmodel_get_cost_test.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// mendixlabs/mxcli#1616: GetDomainModel(moduleID) listed and decoded the domain +// model of every module and then kept the one whose container matched. The +// microflow builder calls it once per member assignment of a `create` action, +// so 200 creates of 8 members went from 14 s to 294 s when 30 modules were +// added. It now reads and decodes only the module's own domain model. + +func TestGetDomainModel_ReadsOnlyItsModulesDomainModel(t *testing.T) { + b := New() + if err := b.Connect(copyPedApp(t)); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + all, err := b.ListDomainModels() + if err != nil { + t.Fatalf("list domain models: %v", err) + } + stored := 0 + for _, dm := range all { + if dm.ContainerID != buildSystemDomainModel().ContainerID { + stored++ + } + } + if stored < 4 { + t.Fatalf("control: the fixture has %d stored domain models; the bound below would prove nothing", stored) + } + + mod, err := b.GetModuleByName("Administration") + if err != nil || mod == nil { + t.Fatalf("GetModuleByName(Administration) = %v, %v", mod, err) + } + b.GetDomainModel(mod.ID) //nolint:errcheck // warm the unit-type index + before := b.UnitFileReads() + if _, err := b.GetDomainModel(mod.ID); err != nil { + t.Fatalf("GetDomainModel: %v", err) + } + if reads := b.UnitFileReads() - before; reads != 1 { + t.Errorf("GetDomainModel of one module read %d unit files; the project has %d domain models, and it needs one", reads, stored) + } +} + +// The narrowing must not change the answer: every module's domain model is +// what the full listing — which still decodes them all — says it is, +// including the persistability a specialization inherits across modules +// (mendixlabs/mxcli#1600) and view entities' OQL. +func TestGetDomainModel_AgreesWithTheFullListing(t *testing.T) { + for _, fixture := range []struct { + name string + path func(*testing.T) string + }{ + {"pedapp", copyPedApp}, + {"expr-checker", copyFixture}, + {"v1", copyV1Project}, // the container filter on MPR v1's listing + } { + t.Run(fixture.name, func(t *testing.T) { + b := New() + if err := b.Connect(fixture.path(t)); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + all, err := b.ListDomainModels() + if err != nil { + t.Fatalf("list domain models: %v", err) + } + if len(all) < 2 { + t.Fatalf("control: the fixture has no stored domain model, only System's") + } + specializations := 0 + for _, want := range all { + got, err := b.GetDomainModel(want.ContainerID) + if err != nil { + t.Fatalf("GetDomainModel(%s): %v", want.ContainerID, err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("GetDomainModel(%s) differs from the listing's domain model for that module", want.ContainerID) + } + for _, e := range want.Entities { + if e.GeneralizationRef != "" { + specializations++ + } + } + } + if fixture.name == "pedapp" && specializations == 0 { + t.Errorf("control: no specializations in the fixture; inherited persistability is not exercised") + } + }) + } +} + +func copyV1Project(t *testing.T) string { + t.Helper() + dst := t.TempDir() + if err := os.CopyFS(dst, os.DirFS("../../../modelsdk/mpr/testdata/v1-project")); err != nil { + t.Fatalf("copy v1 fixture: %v", err) + } + return filepath.Join(dst, "App.mpr") +} + +// A generalization chain into another stored module is the one case where the +// narrowed lookup must still read a second domain model: the root's flag is +// stored there. +func TestGetDomainModel_InheritsPersistabilityAcrossModules(t *testing.T) { + b := New() + if err := b.Connect(copyPedApp(t)); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + dmOf := func(module string) (*domainmodel.DomainModel, model.ID) { + t.Helper() + mod, err := b.GetModuleByName(module) + if err != nil || mod == nil { + t.Fatalf("GetModuleByName(%s) = %v, %v", module, mod, err) + } + dm, err := b.GetDomainModel(mod.ID) + if err != nil { + t.Fatalf("GetDomainModel(%s): %v", module, err) + } + return dm, mod.ID + } + base, _ := dmOf("Administration") + if err := b.CreateEntity(base.ID, &domainmodel.Entity{Name: "ZzNpBase", Persistable: false}); err != nil { + t.Fatalf("create base: %v", err) + } + child, childModule := dmOf("MyFirstModule") + // Declared persistent: a reader that does not follow the chain reads that. + if err := b.CreateEntity(child.ID, &domainmodel.Entity{Name: "ZzNpChild", Persistable: true, GeneralizationRef: "Administration.ZzNpBase"}); err != nil { + t.Fatalf("create child: %v", err) + } + + got, err := b.GetDomainModel(childModule) + if err != nil { + t.Fatalf("GetDomainModel: %v", err) + } + e := got.FindEntityByName("ZzNpChild") + if e == nil { + t.Fatal("ZzNpChild not found") + } + if e.Persistable { + t.Errorf("MyFirstModule.ZzNpChild reads persistent; its root Administration.ZzNpBase is not (mendixlabs/mxcli#1600)") + } +} diff --git a/mdl/backend/modelsdk/mcp_bypass_reads.go b/mdl/backend/modelsdk/mcp_bypass_reads.go index 59d0da873a..eeddd17a75 100644 --- a/mdl/backend/modelsdk/mcp_bypass_reads.go +++ b/mdl/backend/modelsdk/mcp_bypass_reads.go @@ -41,7 +41,7 @@ import ( // unlike in GetDomainModel: it is injected rather than stored, so it has no unit // id to look up and any id that reached this function came from a real document. func (b *Backend) GetDomainModelByID(id model.ID) (*domainmodel.DomainModel, error) { - units, err := mprread.ListUnitsWithContainer[*genDm.DomainModel](b.reader) + units, err := mprread.ListUnitsWithContainerWhere[*genDm.DomainModel](b.reader, mprread.HasID(id)) if err != nil { return nil, err } @@ -49,7 +49,7 @@ func (b *Backend) GetDomainModelByID(id model.ID) (*domainmodel.DomainModel, err if model.ID(u.Element.ID()) == id { dm := domainModelFromGen(u.Element, u.ContainerID) b.populateViewEntityOql([]*domainmodel.DomainModel{dm}) - b.resolveInheritedPersistability(units, dm) + b.resolveInheritedPersistability(b.lazyPersistabilityIndex(u.ContainerID, u.Element), dm) return dm, nil } } diff --git a/mdl/backend/modelsdk/microflow_write.go b/mdl/backend/modelsdk/microflow_write.go index fe9686c0d7..080686bf85 100644 --- a/mdl/backend/modelsdk/microflow_write.go +++ b/mdl/backend/modelsdk/microflow_write.go @@ -499,12 +499,9 @@ func annotationFlowToGen(af *microflows.AnnotationFlow, v flowVersion) element.E // the note's bottom edge, into the element's top. Both indexes were once 0 // (top), which drew the line out of the top of the note and back down // around it. - origin, destination := int32(2), int32(0) - if af.Sides != nil { - origin, destination = int32(af.Sides.Origin), int32(af.Sides.Destination) - } - g.SetOriginConnectionIndex(origin) - g.SetDestinationConnectionIndex(destination) + sides := af.SidesOrDefault() + g.SetOriginConnectionIndex(int32(sides.Origin)) + g.SetDestinationConnectionIndex(int32(sides.Destination)) // An annotation flow is a Flow like a sequence flow, with the same floors: // the Line was written from 10.0 on, six minors before it existed. if v.hasLine() { diff --git a/mdl/backend/modelsdk/unimplemented_gen.go b/mdl/backend/modelsdk/unimplemented_gen.go index 83166e5cc1..efb57ff1a2 100644 --- a/mdl/backend/modelsdk/unimplemented_gen.go +++ b/mdl/backend/modelsdk/unimplemented_gen.go @@ -405,6 +405,10 @@ func (unimplemented) Disconnect() error { return errUnimplemented("Disconnect") } +func (unimplemented) EditConsumedRestService(_ model.ID, _ *backend.ConsumedRestServiceEdit) error { + return errUnimplemented("EditConsumedRestService") +} + func (unimplemented) ExportJSON() ([]uint8, error) { var r0 []uint8 return r0, errUnimplemented("ExportJSON") diff --git a/mdl/backend/service.go b/mdl/backend/service.go index 2b61be15cf..c524fb4cd5 100644 --- a/mdl/backend/service.go +++ b/mdl/backend/service.go @@ -37,6 +37,10 @@ type RESTBackend interface { ListPublishedRestServices() ([]*model.PublishedRestService, error) CreateConsumedRestService(svc *model.ConsumedRestService) error UpdateConsumedRestService(svc *model.ConsumedRestService) error + // EditConsumedRestService edits a stored consumed REST service in place + // (`alter consumed rest service`): what the edit does not name is written + // back as stored. + EditConsumedRestService(id model.ID, edit *ConsumedRestServiceEdit) error DeleteConsumedRestService(id model.ID) error CreatePublishedRestService(svc *model.PublishedRestService) error UpdatePublishedRestService(svc *model.PublishedRestService) error @@ -67,3 +71,34 @@ type DataTransformerBackend interface { UpdateDataTransformer(dt *model.DataTransformer) error DeleteDataTransformer(id model.ID) error } + +// ConsumedRestServiceEdit is an in-place edit of a stored consumed REST +// service. Every operation it does not drop or modify is written back as its +// stored bytes, keeping whatever Studio Pro gave it that MDL has no words for +// (mendixlabs/mxcli#1608). +type ConsumedRestServiceEdit struct { + // BaseUrl, when non-nil, replaces the service's base URL. + BaseUrl *string + // SetAuthentication replaces the authentication scheme with + // Authentication; nil is none. + SetAuthentication bool + Authentication *model.RestAuthentication + // Drop names the stored operations to remove. + Drop []string + // Modify rewrites some properties of stored operations. + Modify []ConsumedRestOperationEdit + // Add appends new operations after the stored ones, in order. + Add []*model.RestClientOperation +} + +// ConsumedRestOperationEdit rewrites some properties of one stored operation. +type ConsumedRestOperationEdit struct { + // Name is the stored operation's name. + Name string + // Operation is the operation as it reads after the edit. + Operation *model.RestClientOperation + // Keys are the MDL properties to write from Operation — Method, Path, + // Parameters, Query, Headers, Body, Timeout, Response. Every other stored + // property of the operation is kept. + Keys []string +} diff --git a/mdl/executor/cmd_alter_flow.go b/mdl/executor/cmd_alter_flow.go index 365e18f446..6c9cc2d219 100644 --- a/mdl/executor/cmd_alter_flow.go +++ b/mdl/executor/cmd_alter_flow.go @@ -551,7 +551,7 @@ func (a *alterFlowContext) fragmentBuilder(ctx *ExecContext) *flowBuilder { varTypes: varTypes, declaredVars: declared, measurer: &layoutMeasurer{varTypes: varTypes}, - backend: ctx.Backend, + backend: flowBuildBackend(ctx.Backend), hierarchy: hierarchy, restServices: restServices, isNanoflow: a.stmt.Nanoflow, diff --git a/mdl/executor/cmd_microflows_build.go b/mdl/executor/cmd_microflows_build.go index 43f0b7d978..f37093ccad 100644 --- a/mdl/executor/cmd_microflows_build.go +++ b/mdl/executor/cmd_microflows_build.go @@ -318,6 +318,8 @@ func buildMicroflowFromStmt(ctx *ExecContext, s *ast.CreateMicroflowStmt, opts b // Validate and add parameters for i, p := range s.Parameters { + p.Type = settleSystemEnumeration(p.Type) + s.Parameters[i].Type = p.Type // Validate entity references for List and Entity types. // Built-in modules (e.g. System) are not stored in the MPR domain models; // their types are serialized by qualified name and resolved at runtime. @@ -337,7 +339,7 @@ func buildMicroflowFromStmt(ctx *ExecContext, s *ast.CreateMicroflowStmt, opts b } // Validate enumeration references for Enumeration types if p.Type.Kind == ast.TypeEnumeration && p.Type.EnumRef != nil { - if found := findEnumeration(ctx, p.Type.EnumRef.Module, p.Type.EnumRef.Name); found == nil { + if !isKnownEnumeration(ctx, p.Type.EnumRef.Module, p.Type.EnumRef.Name) { return nil, mdlerrors.NewNotFoundMsg("enumeration", p.Type.EnumRef.Module+"."+p.Type.EnumRef.Name, fmt.Sprintf("enumeration '%s.%s' not found for parameter '%s'", p.Type.EnumRef.Module, p.Type.EnumRef.Name, p.Name)) } @@ -356,6 +358,7 @@ func buildMicroflowFromStmt(ctx *ExecContext, s *ast.CreateMicroflowStmt, opts b // Validate and set return type if s.ReturnType != nil { + s.ReturnType.Type = settleSystemEnumeration(s.ReturnType.Type) // Validate entity references for return type. // Built-in modules (e.g. System) are not stored in the MPR domain models; // their types are serialized by qualified name and resolved at runtime. @@ -373,7 +376,7 @@ func buildMicroflowFromStmt(ctx *ExecContext, s *ast.CreateMicroflowStmt, opts b } // Validate enumeration references for return type if s.ReturnType.Type.Kind == ast.TypeEnumeration && s.ReturnType.Type.EnumRef != nil { - if found := findEnumeration(ctx, s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name); found == nil { + if !isKnownEnumeration(ctx, s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name) { return nil, mdlerrors.NewNotFoundMsg("enumeration", s.ReturnType.Type.EnumRef.Module+"."+s.ReturnType.Type.EnumRef.Name, fmt.Sprintf("enumeration '%s.%s' not found for return type", s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name)) } @@ -431,7 +434,7 @@ func buildMicroflowFromStmt(ctx *ExecContext, s *ast.CreateMicroflowStmt, opts b declaredVars: declaredVars, measurer: &layoutMeasurer{varTypes: varTypes}, allowWrap: true, - backend: ctx.Backend, + backend: flowBuildBackend(ctx.Backend), quiet: opts.Quiet, hierarchy: hierarchy, restServices: restServices, @@ -630,6 +633,8 @@ func buildNanoflowFromStmt(ctx *ExecContext, s *ast.CreateNanoflowStmt, opts bui // Validate and add parameters for i, p := range s.Parameters { + p.Type = settleSystemEnumeration(p.Type) + s.Parameters[i].Type = p.Type if p.Type.EntityRef != nil && !isBuiltinModuleEntity(p.Type.EntityRef.Module) { entityID := entityResolver(*p.Type.EntityRef) if entityID == "" { @@ -645,7 +650,7 @@ func buildNanoflowFromStmt(ctx *ExecContext, s *ast.CreateNanoflowStmt, opts bui } } if p.Type.Kind == ast.TypeEnumeration && p.Type.EnumRef != nil { - if found := findEnumeration(ctx, p.Type.EnumRef.Module, p.Type.EnumRef.Name); found == nil { + if !isKnownEnumeration(ctx, p.Type.EnumRef.Module, p.Type.EnumRef.Name) { return nil, mdlerrors.NewNotFoundMsg("enumeration", p.Type.EnumRef.Module+"."+p.Type.EnumRef.Name, fmt.Sprintf("enumeration '%s.%s' not found for parameter '%s'", p.Type.EnumRef.Module, p.Type.EnumRef.Name, p.Name)) } @@ -664,6 +669,7 @@ func buildNanoflowFromStmt(ctx *ExecContext, s *ast.CreateNanoflowStmt, opts bui // Validate and set return type if s.ReturnType != nil { + s.ReturnType.Type = settleSystemEnumeration(s.ReturnType.Type) if s.ReturnType.Type.EntityRef != nil && !isBuiltinModuleEntity(s.ReturnType.Type.EntityRef.Module) { entityID := entityResolver(*s.ReturnType.Type.EntityRef) if entityID == "" { @@ -677,7 +683,7 @@ func buildNanoflowFromStmt(ctx *ExecContext, s *ast.CreateNanoflowStmt, opts bui } } if s.ReturnType.Type.Kind == ast.TypeEnumeration && s.ReturnType.Type.EnumRef != nil { - if found := findEnumeration(ctx, s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name); found == nil { + if !isKnownEnumeration(ctx, s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name) { return nil, mdlerrors.NewNotFoundMsg("enumeration", s.ReturnType.Type.EnumRef.Module+"."+s.ReturnType.Type.EnumRef.Name, fmt.Sprintf("enumeration '%s.%s' not found for return type", s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name)) } @@ -730,7 +736,7 @@ func buildNanoflowFromStmt(ctx *ExecContext, s *ast.CreateNanoflowStmt, opts bui declaredVars: declaredVars, measurer: &layoutMeasurer{varTypes: varTypes}, allowWrap: true, - backend: ctx.Backend, + backend: flowBuildBackend(ctx.Backend), hierarchy: hierarchy, restServices: restServices, isNanoflow: true, diff --git a/mdl/executor/cmd_microflows_create.go b/mdl/executor/cmd_microflows_create.go index 664416679f..313e6056ec 100644 --- a/mdl/executor/cmd_microflows_create.go +++ b/mdl/executor/cmd_microflows_create.go @@ -10,6 +10,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/modelsdk/meta" ) // isBuiltinModuleEntity returns true for modules whose entities are defined @@ -20,6 +21,38 @@ func isBuiltinModuleEntity(moduleName string) bool { return moduleName == "System" } +// systemEnumerations holds the qualified name of every enumeration the System +// module declares. Like its entities they are not stored in the project, so +// ListEnumerations never returns them and findEnumeration cannot see them. +var systemEnumerations = func() map[string]bool { + set := make(map[string]bool, len(meta.SystemEnumerations)) + for _, e := range meta.SystemEnumerations { + set[e.Name] = true + } + return set +}() + +// isKnownEnumeration reports whether module.name is an enumeration — one of the +// project's, or one of the System module's. +func isKnownEnumeration(ctx *ExecContext, module, name string) bool { + return systemEnumerations[module+"."+name] || findEnumeration(ctx, module, name) != nil +} + +// settleSystemEnumeration retypes a bare System enumeration in a flow +// parameter or return type. The visitor types every bare qualified name there +// as an entity, and the entity-then-enumeration fallback the callers apply is +// skipped for the System module (its entities are not in the project to look +// up), so without this `$T: System.UserType` was written as an ObjectType with +// Entity "System.UserType" — accepted by the local engines, refused by Studio +// Pro ("Reference with qualified name System.UserType of type +// DomainModels$Entity not found"). +func settleSystemEnumeration(dt ast.DataType) ast.DataType { + if dt.Kind == ast.TypeEntity && dt.EntityRef != nil && systemEnumerations[dt.EntityRef.Module+"."+dt.EntityRef.Name] { + return ast.DataType{Kind: ast.TypeEnumeration, EnumRef: dt.EntityRef} + } + return dt +} + // execCreateMicroflow handles CREATE MICROFLOW statements. // loadRestServices returns all consumed REST services, or nil if no backend. func loadRestServices(ctx *ExecContext) ([]*model.ConsumedRestService, error) { diff --git a/mdl/executor/cmd_microflows_derived_layout_backend.go b/mdl/executor/cmd_microflows_derived_layout_backend.go index e9bddb3749..184cd5c722 100644 --- a/mdl/executor/cmd_microflows_derived_layout_backend.go +++ b/mdl/executor/cmd_microflows_derived_layout_backend.go @@ -20,6 +20,11 @@ import ( // // It lives only for one derivation, which writes nothing, so nothing it caches // can go stale. Everything it does not override is the real backend. +// +// One `create microflow` build is the same kind of span (flowBuildBackend): +// the builder only reads, and asked for a module and its domain model once per +// member assignment of every `create` action, each a project listing +// (mendixlabs/mxcli#1616). type layoutCheckBackend struct { backend.FullBackend @@ -28,12 +33,15 @@ type layoutCheckBackend struct { nanoflows *memoResult[[]*microflows.Nanoflow] domainModels *memoResult[[]*domainmodel.DomainModel] - moduleByName map[string]memoResult[*model.Module] - domainModel map[model.ID]memoResult[*domainmodel.DomainModel] - rawUnit map[[2]string]memoResult[*types.RawUnitInfo] - javaAction map[string]memoResult[*javaactions.JavaAction] - jsAction map[string]memoResult[*types.JavaScriptAction] - isRule map[string]memoResult[bool] + microflowHeaders *memoResult[[]types.DocumentHeader] + + moduleByName map[string]memoResult[*model.Module] + microflowsNamed map[string]memoResult[[]*microflows.Microflow] + domainModel map[model.ID]memoResult[*domainmodel.DomainModel] + rawUnit map[[2]string]memoResult[*types.RawUnitInfo] + javaAction map[string]memoResult[*javaactions.JavaAction] + jsAction map[string]memoResult[*types.JavaScriptAction] + isRule map[string]memoResult[bool] } type memoResult[T any] struct { @@ -41,15 +49,30 @@ type memoResult[T any] struct { err error } +// flowBuildBackend is the backend one flow build reads through: memoised for +// that build, which writes nothing (the flow is stored after the builder is +// done). A nil backend stays nil — the builder checks for it — and one that is +// already memoised is not wrapped again. +func flowBuildBackend(b backend.FullBackend) backend.FullBackend { + if b == nil { + return nil + } + if _, ok := b.(*layoutCheckBackend); ok { + return b + } + return newLayoutCheckBackend(b) +} + func newLayoutCheckBackend(b backend.FullBackend) *layoutCheckBackend { return &layoutCheckBackend{ - FullBackend: b, - moduleByName: map[string]memoResult[*model.Module]{}, - domainModel: map[model.ID]memoResult[*domainmodel.DomainModel]{}, - rawUnit: map[[2]string]memoResult[*types.RawUnitInfo]{}, - javaAction: map[string]memoResult[*javaactions.JavaAction]{}, - jsAction: map[string]memoResult[*types.JavaScriptAction]{}, - isRule: map[string]memoResult[bool]{}, + FullBackend: b, + moduleByName: map[string]memoResult[*model.Module]{}, + microflowsNamed: map[string]memoResult[[]*microflows.Microflow]{}, + domainModel: map[model.ID]memoResult[*domainmodel.DomainModel]{}, + rawUnit: map[[2]string]memoResult[*types.RawUnitInfo]{}, + javaAction: map[string]memoResult[*javaactions.JavaAction]{}, + jsAction: map[string]memoResult[*types.JavaScriptAction]{}, + isRule: map[string]memoResult[bool]{}, } } @@ -86,6 +109,48 @@ func (b *layoutCheckBackend) ListDomainModels() ([]*domainmodel.DomainModel, err return memoOnce(&b.domainModels, b.FullBackend.ListDomainModels) } +// ListMicroflowsNamed and ListMicroflowHeaders keep the real backend's +// microflowLookup visible through the memo. The memo embeds the FullBackend +// interface, so without them a type assertion on it fails and microflowsNamed +// falls back to ListMicroflows: describing a microflow rebuilds it to derive +// its layout, and that decoded every microflow of the project once per +// described flow — the bulk of a module dump's time (mendixlabs/mxcli#1624). +func (b *layoutCheckBackend) ListMicroflowsNamed(name string) ([]*microflows.Microflow, error) { + return memoKeyed(b.microflowsNamed, name, func() ([]*microflows.Microflow, error) { + if l, ok := b.FullBackend.(microflowLookup); ok { + return l.ListMicroflowsNamed(name) + } + all, err := b.ListMicroflows() + if err != nil { + return nil, err + } + var out []*microflows.Microflow + for _, mf := range all { + if mf.Name == name { + out = append(out, mf) + } + } + return out, nil + }) +} + +func (b *layoutCheckBackend) ListMicroflowHeaders() ([]types.DocumentHeader, error) { + return memoOnce(&b.microflowHeaders, func() ([]types.DocumentHeader, error) { + if l, ok := b.FullBackend.(microflowLookup); ok { + return l.ListMicroflowHeaders() + } + all, err := b.ListMicroflows() + if err != nil { + return nil, err + } + out := make([]types.DocumentHeader, len(all)) + for i, mf := range all { + out[i] = types.DocumentHeader{ID: mf.ID, ContainerID: mf.ContainerID, Name: mf.Name, Excluded: mf.Excluded} + } + return out, nil + }) +} + func (b *layoutCheckBackend) GetModuleByName(name string) (*model.Module, error) { return memoKeyed(b.moduleByName, name, func() (*model.Module, error) { return b.FullBackend.GetModuleByName(name) }) } diff --git a/mdl/executor/cmd_pages_create_v3.go b/mdl/executor/cmd_pages_create_v3.go index b67c7b7e0c..48b7827095 100644 --- a/mdl/executor/cmd_pages_create_v3.go +++ b/mdl/executor/cmd_pages_create_v3.go @@ -93,6 +93,13 @@ func execCreatePageV3(ctx *ExecContext, s *ast.CreatePageStmtV3) error { pagesToDelete = append(pagesToDelete, p.ID) } + // A rewrite must not delete widgets MDL cannot author (mendixlabs/mxcli#1622). + if len(pagesToDelete) > 0 { + if err := checkNoUnauthorableWidgetsDropped(ctx, "page", s.Name, pagesToDelete[0], allPageWidgets(s)); err != nil { + return err + } + } + // Build the page BEFORE deleting the old one (atomic: if build fails, old page is preserved) pb := &pageBuilder{ ctx: ctx, @@ -270,6 +277,14 @@ func execCreateSnippetV3(ctx *ExecContext, s *ast.CreateSnippetStmtV3) error { tolerateDanglingRefs: existingExcluded, } + // As for pages (mendixlabs/mxcli#1622). describe finds a snippet by name, + // so only a name with a single stored document can be described. + if matches == 1 && len(snippetsToDelete) == 1 { + if err := checkNoUnauthorableWidgetsDropped(ctx, "snippet", s.Name, snippetsToDelete[0], s.Widgets); err != nil { + return err + } + } + // As for pages (#721 L4). describe finds a snippet by name, so only a name // with a single stored document is described. if matches == 1 && len(snippetsToDelete) == 1 { diff --git a/mdl/executor/cmd_pages_describe.go b/mdl/executor/cmd_pages_describe.go index c900af47c1..aa439bdeb0 100644 --- a/mdl/executor/cmd_pages_describe.go +++ b/mdl/executor/cmd_pages_describe.go @@ -377,8 +377,12 @@ func describeLayout(ctx *ExecContext, name ast.QualifiedName) error { fmt.Fprintf(ctx.Output, "create or modify layout %s.%s (\n%s\n) {\n", modName, mdlIdent(foundLayout.Name), header) - outputLayoutBodyMDL(ctx, rawData) - outputNativeLayoutPartsMDL(ctx, rawData) + // Through describeWidgetBody like a page's body, so a widget MDL cannot + // author is known to the rewrite guard (mendixlabs/mxcli#1622). + describeWidgetBody(ctx, func(c *ExecContext) { + outputLayoutBodyMDL(c, rawData) + outputNativeLayoutPartsMDL(c, rawData) + }) fmt.Fprint(ctx.Output, "};\n\n") return nil diff --git a/mdl/executor/cmd_pages_describe_dangling.go b/mdl/executor/cmd_pages_describe_dangling.go index 48bd5746ce..38ba162df8 100644 --- a/mdl/executor/cmd_pages_describe_dangling.go +++ b/mdl/executor/cmd_pages_describe_dangling.go @@ -34,6 +34,15 @@ type describeBodyState struct { // inComment is set while a commented-out widget's subtree is rendered: // every widget in it is dropped too. inComment bool + // unauthorable are the widgets of this pass MDL cannot author at all. + unauthorable []unauthorableWidget +} + +// unauthorableWidget is a stored widget describe can only print as a comment +// because MDL has no way to write it. +type unauthorableWidget struct { + Type string // its $Type, e.g. Forms$DataGrid + Name string // "" when the stored widget has none } // describeWidgetBody renders a document's widget body through render. A data @@ -51,11 +60,15 @@ func describeWidgetBody(ctx *ExecContext, render func(c *ExecContext)) { c.Output = &buf state.selected = map[string]bool{} state.inComment = false + state.unauthorable = nil render(&c) if !state.selectsADroppedWidget() { break } } + if ctx.unauthorableSink != nil { + *ctx.unauthorableSink = append(*ctx.unauthorableSink, state.unauthorable...) + } _, _ = ctx.Output.Write(buf.Bytes()) } @@ -132,8 +145,15 @@ func describeDanglingSelection(ctx *ExecContext, w rawWidget, indent int) bool { } // describeDroppedWidget records that w is emitted only as a comment. +// +// It is called for exactly the widgets MDL cannot author, so it is also where +// the rewrite guard learns what a rewrite would delete (unauthorableSink). func describeDroppedWidget(ctx *ExecContext, w rawWidget) { - if ctx.describeBody != nil && w.Name != "" { + if ctx.describeBody == nil { + return + } + if w.Name != "" { ctx.describeBody.dropped[w.Name] = true } + ctx.describeBody.unauthorable = append(ctx.describeBody.unauthorable, unauthorableWidget{Type: w.Type, Name: w.Name}) } diff --git a/mdl/executor/cmd_pages_describe_objectlist.go b/mdl/executor/cmd_pages_describe_objectlist.go index ee212bac75..044e49f0bf 100644 --- a/mdl/executor/cmd_pages_describe_objectlist.go +++ b/mdl/executor/cmd_pages_describe_objectlist.go @@ -18,11 +18,14 @@ type rawObjectList struct { type rawObjectListItem struct { Props []rawExplicitProp DataSource *rawDataSource - // Children are the widgets nested in a Widgets-typed sub-property of the - // item — an Accordion group's `content` / `headerContent` slot. Without - // these an accordion described as an empty group, and re-executing that - // description deleted whatever was inside it (#891). + // Children are the widgets in the item's BODY slot (bodyItemSlotKey — an + // Accordion group's `content`). Without these an accordion described as an + // empty group, and re-executing that description deleted whatever was + // inside it (#891). Children []rawWidget + // Slots are the item's other non-empty widget slots — an Accordion group's + // `headerContent` — printed as ` { … }` blocks inside the item. + Slots []rawChildSlot } // extractObjectLists reconstructs every object-list property of a pluggable @@ -72,7 +75,8 @@ func extractObjectLists(ctx *ExecContext, w map[string]any) []rawObjectList { continue } item := extractObjectListItem(ctx, om, nestedMap) - if len(item.Props) > 0 || item.DataSource != nil || len(item.Children) > 0 { + item.promoteBodySlot(bodyItemSlotKey(objectListItemWidgetSlotKeys(w, listKey))) + if len(item.Props) > 0 || item.DataSource != nil || len(item.Children) > 0 || len(item.Slots) > 0 { items = append(items, item) } } @@ -179,17 +183,22 @@ func extractObjectListItem(ctx *ExecContext, itemObj map[string]any, nestedMap m // change: an EMPTY map means the field is simply unset. continue } - // Child widgets (an Accordion group's `content` slot). A Widgets-typed - // sub-property holds a widget tree, not a scalar, so it is parsed with the - // same recursion the rest of DESCRIBE uses rather than stringified (#891). + // Child widgets. A Widgets-typed sub-property holds a widget tree, not a + // scalar, so it is parsed with the same recursion the rest of DESCRIBE + // uses rather than stringified (#891). Each slot is kept apart: folding + // them all into one body put an Accordion group's headerContent widgets + // into its content on re-exec (#1453). promoteBodySlot then moves the + // body slot's widgets into Children. if childElems := getBsonArrayElements(value["Widgets"]); len(childElems) > 0 { + slot := rawChildSlot{Keyword: strings.ToLower(key), PropertyKey: key} for _, ce := range childElems { cm, ok := ce.(map[string]any) if !ok { continue } - item.Children = append(item.Children, parseRawWidget(ctx, cm)...) + slot.Widgets = append(slot.Widgets, parseRawWidget(ctx, cm)...) } + item.Slots = append(item.Slots, slot) continue } // Action sub-property (a chart series' staticOnClickAction, a popupmenu @@ -272,3 +281,47 @@ func extractObjectListItem(ctx *ExecContext, itemObj map[string]any, nestedMap m func objectListMDLKey(schemaKey string) string { return schemaKey } + +// promoteBodySlot moves the widgets of the item's body slot (bodyKey) into +// Children, leaving its other slots as named blocks. With no body key — a +// schema that cannot be read — every slot stays a block, which CREATE routes by +// its keyword just as well. +func (item *rawObjectListItem) promoteBodySlot(bodyKey string) { + if bodyKey == "" { + return + } + for i, s := range item.Slots { + if s.PropertyKey == bodyKey { + item.Children = append(item.Children, s.Widgets...) + item.Slots = append(item.Slots[:i:i], item.Slots[i+1:]...) + return + } + } +} + +// objectListItemWidgetSlotKeys lists the Widgets-typed property keys of one +// object-list's item type, in schema order — the input bodyItemSlotKey needs. +func objectListItemWidgetSlotKeys(w map[string]any, listKey string) []string { + widgetType, _ := w["Type"].(map[string]any) + objType, _ := widgetType["ObjectType"].(map[string]any) + for _, pt := range getBsonArrayElements(objType["PropertyTypes"]) { + ptMap, ok := pt.(map[string]any) + if !ok || extractString(ptMap["PropertyKey"]) != listKey { + continue + } + valueType, _ := ptMap["ValueType"].(map[string]any) + itemObjType, _ := valueType["ObjectType"].(map[string]any) + var keys []string + for _, ipt := range getBsonArrayElements(itemObjType["PropertyTypes"]) { + iptMap, ok := ipt.(map[string]any) + if !ok { + continue + } + if vt, _ := iptMap["ValueType"].(map[string]any); extractString(vt["Type"]) == "Widgets" { + keys = append(keys, extractString(iptMap["PropertyKey"])) + } + } + return keys + } + return nil +} diff --git a/mdl/executor/cmd_pages_describe_output.go b/mdl/executor/cmd_pages_describe_output.go index 828db1468f..443e90d4a6 100644 --- a/mdl/executor/cmd_pages_describe_output.go +++ b/mdl/executor/cmd_pages_describe_output.go @@ -960,8 +960,11 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { // An item holding child widgets (an Accordion group's `content` // slot) needs a body, or the children have nowhere to go and the // description silently drops them on re-exec (#891). - if len(item.Children) > 0 { + if len(item.Children) > 0 || len(item.Slots) > 0 { formatWidgetProps(ctx.Output, childPrefix, itemHeader, itemProps, " {\n") + // The item's other slots as blocks (headercontent { … }), + // then its body — CREATE routes each by keyword (#1453). + outputChildSlots(ctx, item.Slots, childPrefix+" ", indent+2) for _, child := range item.Children { outputWidgetMDLV3(ctx, child, indent+2) } diff --git a/mdl/executor/cmd_pages_describe_pluggable.go b/mdl/executor/cmd_pages_describe_pluggable.go index e788056184..0260f2aeaf 100644 --- a/mdl/executor/cmd_pages_describe_pluggable.go +++ b/mdl/executor/cmd_pages_describe_pluggable.go @@ -1301,14 +1301,19 @@ func extractExplicitProperties(ctx *ExecContext, w map[string]any, enclosingEnti // Check for AttributeRef (attribute binding) if attrRef, ok := value["AttributeRef"].(map[string]any); ok && attrRef != nil { if attr := extractString(attrRef["Attribute"]); attr != "" { - value := describeAttr(ctx, attr) - if schemas[typePointerID].DataSourceProperty == "" && strings.Contains(enclosing, ".") && + described := describeAttr(ctx, attr) + if param := parameterSourceName(value["SourceVariable"]); param != "" && attrRef["EntityRef"] == nil { + // Read from a page or snippet parameter: `$Param.Attr`. The + // qualified name re-executes without the SourceVariable — + // CE7247 (mendixlabs/mxcli#1591). + described = parameterAttributeMDL(param, attr) + } else if schemas[typePointerID].DataSourceProperty == "" && strings.Contains(enclosing, ".") && !strings.EqualFold(attributeEntity(attr), enclosing) { - value = attr + described = attr } result = append(result, rawExplicitProp{ Key: propKey, - Value: value, + Value: described, IsRef: true, }) continue diff --git a/mdl/executor/cmd_pages_layout_v3.go b/mdl/executor/cmd_pages_layout_v3.go index 94056efd97..925fc5be35 100644 --- a/mdl/executor/cmd_pages_layout_v3.go +++ b/mdl/executor/cmd_pages_layout_v3.go @@ -233,6 +233,15 @@ func execCreateLayout(ctx *ExecContext, s *ast.CreateLayoutStmt) error { duplicates = append(duplicates, l.ID) } + // A rewrite must not delete widgets MDL cannot author (mendixlabs/mxcli#1622). + // describe finds a layout by name, so only a name with one stored layout + // can be described. + if stored != nil && len(duplicates) == 0 { + if err := checkNoUnauthorableWidgetsDropped(ctx, "layout", s.Name, stored.ID, s.AllWidgets()); err != nil { + return err + } + } + pb := &pageBuilder{ ctx: ctx, backend: ctx.Backend, diff --git a/mdl/executor/cmd_pages_parameter_binding.go b/mdl/executor/cmd_pages_parameter_binding.go index f5faeaf1b9..c3cdcc5b21 100644 --- a/mdl/executor/cmd_pages_parameter_binding.go +++ b/mdl/executor/cmd_pages_parameter_binding.go @@ -20,7 +20,10 @@ package executor import ( + "regexp" "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" ) // namedObjectBinding splits `$name.rest` into name (without the "$") and rest. @@ -38,6 +41,20 @@ func namedObjectBinding(s string) (name, rest string, ok bool) { return name, rest, true } +// parameterAttributeValue is the `$Param.Attr` text of a pluggable widget +// property authored by its own key. Unlike `Attribute:`, a generic key's value +// goes through the expression grammar, so `eventDataAttribute: $Ctx.EventData` +// arrives as an *ast.WidgetExpressionV3 — which every attribute property used +// to refuse as MDL-WIDGET42 (mendixlabs/mxcli#1591). Anything other than that +// one shape is still an expression. +func parameterAttributeValue(raw any) (string, bool) { + e, ok := raw.(*ast.WidgetExpressionV3) + if !ok || !isParameterAttributeText(e.Text) { + return "", false + } + return e.Text, true +} + // namedObjectBindingKinds are the widgets whose `Attribute: $name.Attr` may // name an enclosing data view as well as a parameter (resolveInputBinding). // Every other widget that takes the form — the combo box — reads a parameter @@ -97,3 +114,13 @@ func customWidgetPropertyParameter(w map[string]any, propertyKey string) string func parameterAttributeMDL(name, attrQN string) string { return "$" + name + "." + mdlIdent(shortAttributeName(attrQN)) } + +// parameterAttributeTextRe is `$Param.Attr` and nothing more: one variable, +// one attribute name (quoted when it is a keyword). namedObjectBinding alone +// takes `$Ctx.Name + 'x'` too, which is an expression. +var parameterAttributeTextRe = regexp.MustCompile(`^\$[A-Za-z_][A-Za-z0-9_]*\.(?:[A-Za-z_][A-Za-z0-9_]*|"[A-Za-z_][A-Za-z0-9_]*")$`) + +// isParameterAttributeText reports whether text is the `$Param.Attr` shape. +func isParameterAttributeText(text string) bool { + return parameterAttributeTextRe.MatchString(text) +} diff --git a/mdl/executor/cmd_pages_rewrite_guard.go b/mdl/executor/cmd_pages_rewrite_guard.go new file mode 100644 index 0000000000..169946b857 --- /dev/null +++ b/mdl/executor/cmd_pages_rewrite_guard.go @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "io" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/model" +) + +// A rewrite that would delete widgets MDL cannot author (mendixlabs/mxcli#1622). +// +// `describe page` prints a widget MDL has no way to write — a classic data +// grid, a reference selector, a drop-down, a reference set selector — as a +// `-- NOT re-executable` comment, so executing the description rebuilt the page +// without it and reported "Replaced page", with mx check clean on both sides. +// The statement cannot have asked for that: it has no words for the widget. +// So the rewrite is refused (guard-don't-drop, ADR-0005), naming each widget, +// and ALTER PAGE — which edits the stored document and leaves these widgets in +// place — is the way to change such a page; dropping them with it first makes +// the rewrite go through. +// +// The widgets are learnt from describe itself (describeDroppedWidget), not +// from a list of authorable types kept here: what a rewrite loses is exactly +// what the description cannot say, and a second list would drift from the +// describer's switch. A widget the statement does declare by name is the +// author writing it, so it does not count — that is what keeps a widget the +// builder can write but describe cannot print from refusing its own script. + +// unauthorableStoredWidgets describes the stored document through describe, +// output discarded, and returns the widgets the statement does not declare +// that the description could only print as a comment. +func unauthorableStoredWidgets(ctx *ExecContext, declared map[string]bool, describe func(c *ExecContext) error) []unauthorableWidget { + var found []unauthorableWidget + c := *ctx + c.Output = io.Discard + c.unauthorableSink = &found + if err := describe(&c); err != nil { + // A document describe cannot read is not this guard's business; the + // rewrite reports its own errors. + return nil + } + var out []unauthorableWidget + for _, w := range found { + if w.Name != "" && declared[w.Name] { + continue + } + out = append(out, w) + } + return out +} + +// checkNoUnauthorableWidgetsDropped refuses a rewrite of the stored page, +// snippet or layout unitID that would delete widgets MDL cannot author. kind +// is "page", "snippet" or "layout"; widgets are the statement's. +func checkNoUnauthorableWidgetsDropped(ctx *ExecContext, kind string, name ast.QualifiedName, unitID model.ID, widgets []*ast.WidgetV3) error { + if ctx == nil || ctx.Backend == nil || unitID == "" { + return nil + } + declared := map[string]bool{} + collectWidgetNames(widgets, declared) + lost := unauthorableStoredWidgets(ctx, declared, func(c *ExecContext) error { + switch kind { + case "snippet": + return describeSnippet(c, name) + case "layout": + return describeLayout(c, name) + } + c.describeID = unitID + return describePage(c, name) + }) + if len(lost) == 0 { + return nil + } + return mdlerrors.NewUnsupported(unauthorableRewriteMessage(kind, name, lost)) +} + +// unauthorableRewriteMessage explains the refusal: every widget, and the two +// ways forward. +func unauthorableRewriteMessage(kind string, name ast.QualifiedName, lost []unauthorableWidget) string { + var b strings.Builder + fmt.Fprintf(&b, "%s %s holds %d widget(s) MDL cannot author, and rewriting it from this statement would delete them:\n", + kind, name, len(lost)) + var named []string + for _, w := range lost { + if w.Name == "" { + fmt.Fprintf(&b, " - an unnamed %s\n", w.Type) + continue + } + fmt.Fprintf(&b, " - %s (%s)\n", w.Name, w.Type) + named = append(named, w.Name) + } + fmt.Fprintf(&b, " `describe %s` prints them as `-- NOT re-executable` comments, which is why executing its output drops them.\n", kind) + fmt.Fprintf(&b, " To change the %s and keep them, use ALTER, which edits only what it names:\n", kind) + fmt.Fprintf(&b, " alter %s %s { set (Caption: '…') on ; insert after { … }; replace with { … }; };\n", kind, name) + if len(named) > 0 { + fmt.Fprintf(&b, " To rebuild it without them, remove them on purpose first:\n") + fmt.Fprintf(&b, " alter %s %s { drop %s; };", kind, name, strings.Join(named, ", ")) + } else { + fmt.Fprintf(&b, " Unnamed widgets cannot be dropped by name: give them a name in Studio Pro, or drop the container that holds them.") + } + return b.String() +} + +// validateUnauthorableRewrites is the check -p half: every `create or modify` +// or `create or replace` of a stored page or snippet that exec would refuse +// for the widgets it deletes. It walks the script in order and skips a +// document an earlier statement already creates, alters or drops, since what +// is stored by then is not what the project holds now. +func validateUnauthorableRewrites(ctx *ExecContext, prog *ast.Program) []error { + if !ctx.Connected() { + return nil + } + touched := map[string]bool{} + key := func(kind string, qn ast.QualifiedName) string { return kind + " " + qn.String() } + var errs []error + for i, stmt := range prog.Statements { + switch s := stmt.(type) { + case *ast.CreatePageStmtV3: + k := key("page", s.Name) + if (s.IsModify || s.IsReplace) && !touched[k] { + if id := storedRewriteTarget(ctx, "page", s.Name); id != "" { + if err := checkNoUnauthorableWidgetsDropped(ctx, "page", s.Name, id, allPageWidgets(s)); err != nil { + errs = append(errs, fmt.Errorf("statement %d: %w", i+1, err)) + } + } + } + touched[k] = true + case *ast.CreateSnippetStmtV3: + k := key("snippet", s.Name) + if (s.IsModify || s.IsReplace) && !touched[k] { + if id := storedRewriteTarget(ctx, "snippet", s.Name); id != "" { + if err := checkNoUnauthorableWidgetsDropped(ctx, "snippet", s.Name, id, s.Widgets); err != nil { + errs = append(errs, fmt.Errorf("statement %d: %w", i+1, err)) + } + } + } + touched[k] = true + case *ast.CreateLayoutStmt: + k := key("layout", s.Name) + if (s.IsModify || s.IsReplace) && !touched[k] { + if id := storedRewriteTarget(ctx, "layout", s.Name); id != "" { + if err := checkNoUnauthorableWidgetsDropped(ctx, "layout", s.Name, id, s.AllWidgets()); err != nil { + errs = append(errs, fmt.Errorf("statement %d: %w", i+1, err)) + } + } + } + touched[k] = true + case *ast.AlterPageStmt: + touched[key(strings.ToLower(s.ContainerType), s.PageName)] = true + case *ast.DropPageStmt: + touched[key("page", s.Name)] = true + case *ast.DropSnippetStmt: + touched[key("snippet", s.Name)] = true + } + } + return errs +} + +// storedRewriteTarget is the stored document a rewrite of qn writes over, as +// execCreatePageV3 / execCreateSnippetV3 / execCreateLayout choose it: the +// first live document of the name, else the first excluded one; for a snippet +// or a layout only when the name has a single document, which is all describe +// can tell apart. "" when none. +func storedRewriteTarget(ctx *ExecContext, kind string, qn ast.QualifiedName) model.ID { + h, err := getHierarchy(ctx) + if err != nil { + return "" + } + inModule := func(container model.ID) bool { + return h.GetModuleName(h.FindModuleID(container)) == qn.Module + } + var live, excluded []model.ID + switch kind { + case "page": + ps, err := ctx.Backend.ListPages() + if err != nil { + return "" + } + for _, p := range ps { + if p.Name != qn.Name || !inModule(p.ContainerID) { + continue + } + if p.Excluded { + excluded = append(excluded, p.ID) + } else { + live = append(live, p.ID) + } + } + case "layout": + ls, err := ctx.Backend.ListLayouts() + if err != nil { + return "" + } + for _, l := range ls { + if l.Name == qn.Name && inModule(l.ContainerID) { + live = append(live, l.ID) + } + } + if len(live) != 1 { + return "" + } + case "snippet": + ss, err := ctx.Backend.ListSnippets() + if err != nil { + return "" + } + for _, sn := range ss { + if sn.Name != qn.Name || !inModule(sn.ContainerID) { + continue + } + if sn.Excluded { + excluded = append(excluded, sn.ID) + } else { + live = append(live, sn.ID) + } + } + if len(live)+len(excluded) != 1 { + return "" + } + } + if len(live) > 0 { + return live[0] + } + if len(excluded) > 0 { + return excluded[0] + } + return "" +} diff --git a/mdl/executor/cmd_rest_clients.go b/mdl/executor/cmd_rest_clients.go index 5bc763ad1e..ad4f24f1d9 100644 --- a/mdl/executor/cmd_rest_clients.go +++ b/mdl/executor/cmd_rest_clients.go @@ -212,22 +212,18 @@ func outputRestOperation(w io.Writer, op *model.RestClientOperation) { fmt.Fprintf(w, " Timeout: %d,\n", op.Timeout) } - // Response - switch strings.ToLower(op.ResponseType) { + // Response. An operation stores no response variable (the call activity + // names it), so the reader never fills one in and the bare kind is what + // describe prints (mendixlabs/mxcli#1608). + switch kind := strings.ToLower(op.ResponseType); kind { case "none": fmt.Fprintln(w, " Response: none") - case "json": + case "json", "string", "file", "status": if op.ResponseVariable != "" { - fmt.Fprintf(w, " Response: json as %s\n", op.ResponseVariable) + fmt.Fprintf(w, " Response: %s as %s\n", kind, op.ResponseVariable) } else { - fmt.Fprintln(w, " Response: json") + fmt.Fprintf(w, " Response: %s\n", kind) } - case "string": - fmt.Fprintf(w, " Response: string as %s\n", op.ResponseVariable) - case "file": - fmt.Fprintf(w, " Response: file as %s\n", op.ResponseVariable) - case "status": - fmt.Fprintf(w, " Response: status as %s\n", op.ResponseVariable) case "mapping": if op.ResponseEntity != "" && len(op.ResponseMappings) > 0 { fmt.Fprintf(w, " Response: mapping %s {\n", op.ResponseEntity) @@ -384,6 +380,7 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { // module root (#932). var preservedContainerID model.ID var preservedDocumentation string + var storedOperations []*model.RestClientOperation wasModified := false for _, existing := range existingServices { existModID := h.FindModuleID(existing.ContainerID) @@ -396,6 +393,7 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { // A rewrite that carried no doc comment keeps the stored one, so // it has to be read before anything replaces the unit (#1018). preservedDocumentation = existing.Documentation + storedOperations = existing.Operations wasModified = true } else { return mdlerrors.NewAlreadyExistsMsg("rest client", moduleName+"."+stmt.Name.Name, fmt.Sprintf("rest client already exists: %s.%s (use create or modify to overwrite)", moduleName, stmt.Name.Name)) @@ -430,42 +428,10 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { stmt.DocumentationSet, stmt.Documentation, preservedDocumentation) } - // Authentication — Mendix requires Rest$ConstantValue for BASIC auth credentials - // (Rest$StringValue causes InvalidCastException in Studio Pro). When literal - // strings are provided, auto-create constants to hold them. if stmt.Authentication != nil { - auth := &model.RestAuthentication{ - Scheme: stmt.Authentication.Scheme, - } - // Username — must use Rest$ConstantValue pointing to a real constant. - // $Variable refs pass through; literal strings auto-create constants. - if strings.HasPrefix(stmt.Authentication.Username, "$") { - // Already a $Constant ref — resolve to qualified name for BY_NAME lookup - name := strings.TrimPrefix(stmt.Authentication.Username, "$") - if !strings.Contains(name, ".") { - name = moduleName + "." + name - } - auth.Username = "$" + name - } else if stmt.Authentication.Username != "" { - constName := stmt.Name.Name + "_Username" - if err := ensureConstant(ctx, moduleName, containerID, constName, stmt.Authentication.Username); err != nil { - return fmt.Errorf("failed to create username constant: %w", err) - } - auth.Username = "$" + moduleName + "." + constName - } - // Password - if strings.HasPrefix(stmt.Authentication.Password, "$") { - name := strings.TrimPrefix(stmt.Authentication.Password, "$") - if !strings.Contains(name, ".") { - name = moduleName + "." + name - } - auth.Password = "$" + name - } else if stmt.Authentication.Password != "" { - constName := stmt.Name.Name + "_Password" - if err := ensureConstant(ctx, moduleName, containerID, constName, stmt.Authentication.Password); err != nil { - return fmt.Errorf("failed to create password constant: %w", err) - } - auth.Password = "$" + moduleName + "." + constName + auth, err := restAuthFromDef(ctx, stmt.Name, containerID, stmt.Authentication) + if err != nil { + return err } svc.Authentication = auth } @@ -490,9 +456,83 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { } ctx.ReportMutation(verb, "rest client: %s.%s (%d operations)", moduleName, stmt.Name.Name, len(svc.Operations)) + if wasModified { + warnDroppedRestOperations(ctx, stmt.Name, storedOperations, svc.Operations) + } return nil } +// warnDroppedRestOperations names the stored operations a CREATE OR MODIFY +// removed. The statement declares the service's whole operation list, so one +// that names only the operations to add deletes the rest — often ones made in +// Studio Pro, with headers and mappings — and used to report only +// "Modified rest client: … (2 operations)" (mendixlabs/mxcli#1608). It still +// applies, as `create or modify entity` does for attributes; it says so, and +// names the statement that adds an operation without touching the others. +func warnDroppedRestOperations(ctx *ExecContext, name ast.QualifiedName, stored, written []*model.RestClientOperation) { + kept := map[string]bool{} + for _, op := range written { + kept[strings.ToLower(op.Name)] = true + } + var dropped []string + for _, op := range stored { + if !kept[strings.ToLower(op.Name)] { + dropped = append(dropped, op.Name) + } + } + if len(dropped) == 0 { + return + } + fmt.Fprintf(ctx.Output, + "⚠ create or modify consumed rest service %s drops %d existing operation(s) not listed in this statement: %s\n", + name, len(dropped), strings.Join(dropped, ", ")) + fmt.Fprintf(ctx.Output, + " To add an operation without touching the others, use: alter consumed rest service %s add operation ( … );\n", name) +} + +// restAuthFromDef resolves a statement's basic authentication for service +// name. Mendix requires Rest$ConstantValue for BASIC auth credentials +// (Rest$StringValue causes InvalidCastException in Studio Pro), so a literal +// string gets a constant to hold it, created in containerID; a `$Constant` +// reference is qualified with the service's module. +func restAuthFromDef(ctx *ExecContext, name ast.QualifiedName, containerID model.ID, def *ast.RestAuthDef) (*model.RestAuthentication, error) { + moduleName := name.Module + auth := &model.RestAuthentication{ + Scheme: def.Scheme, + } + // Username — must use Rest$ConstantValue pointing to a real constant. + // $Variable refs pass through; literal strings auto-create constants. + if strings.HasPrefix(def.Username, "$") { + // Already a $Constant ref — resolve to qualified name for BY_NAME lookup + n := strings.TrimPrefix(def.Username, "$") + if !strings.Contains(n, ".") { + n = moduleName + "." + n + } + auth.Username = "$" + n + } else if def.Username != "" { + constName := name.Name + "_Username" + if err := ensureConstant(ctx, moduleName, containerID, constName, def.Username); err != nil { + return nil, fmt.Errorf("failed to create username constant: %w", err) + } + auth.Username = "$" + moduleName + "." + constName + } + // Password + if strings.HasPrefix(def.Password, "$") { + n := strings.TrimPrefix(def.Password, "$") + if !strings.Contains(n, ".") { + n = moduleName + "." + n + } + auth.Password = "$" + n + } else if def.Password != "" { + constName := name.Name + "_Password" + if err := ensureConstant(ctx, moduleName, containerID, constName, def.Password); err != nil { + return nil, fmt.Errorf("failed to create password constant: %w", err) + } + auth.Password = "$" + moduleName + "." + constName + } + return auth, nil +} + // saveConsumedRestService writes a consumed REST client, rewriting the stored // unit IN PLACE when the statement is a modification that leaves it where it is. // diff --git a/mdl/executor/cmd_rest_clients_alter.go b/mdl/executor/cmd_rest_clients_alter.go new file mode 100644 index 0000000000..3bfbb699fe --- /dev/null +++ b/mdl/executor/cmd_rest_clients_alter.go @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/model" +) + +// alterRestClient handles ALTER CONSUMED REST SERVICE (mendixlabs/mxcli#1608). +// +// Before it, the only MDL way to add an operation to an existing consumed REST +// service was CREATE OR MODIFY restating every operation, and an operation +// left out was deleted. The edit goes to the backend as a splice of the stored +// unit: an operation no action names is written back as stored, including the +// properties of a Studio Pro-made operation that MDL has no words for. +// +// The actions apply in order, so `drop operation X, add operation X (…)` +// replaces X, and `modify operation X` may follow `add operation X`. +func alterRestClient(ctx *ExecContext, stmt *ast.AlterRestClientStmt) error { + if !ctx.ConnectedForWrite() { + return mdlerrors.NewNotConnectedWrite() + } + if err := checkVersionGates(ctx, stmt); err != nil { + return err + } + svc, err := findRestClient(ctx, stmt.Name) + if err != nil { + return err + } + + // The operations as they will be, in order: the stored ones first, then + // the added ones. + type pendingOp struct { + op *model.RestClientOperation // what is written + stored string // the stored name; "" for an added operation + keys []string // MDL properties a modify wrote (stored operations) + } + var ops []*pendingOp + for _, op := range svc.Operations { + cp := *op + ops = append(ops, &pendingOp{op: &cp, stored: op.Name}) + } + find := func(name string) int { + for i, p := range ops { + if strings.EqualFold(p.op.Name, name) { + return i + } + } + return -1 + } + notFound := func(name string) error { + var have []string + for _, p := range ops { + have = append(have, p.op.Name) + } + return mdlerrors.NewNotFoundMsg("operation", name, fmt.Sprintf( + "consumed rest service %s has no operation %s (it has: %s)", stmt.Name, name, strings.Join(have, ", "))) + } + + edit := &backend.ConsumedRestServiceEdit{} + for _, action := range stmt.Actions { + switch a := action.(type) { + case *ast.RestClientSetAction: + if a.BaseUrlSet { + v := a.BaseUrl + edit.BaseUrl = &v + } + if a.AuthenticationSet { + edit.SetAuthentication = true + edit.Authentication = nil + if a.Authentication != nil { + if edit.Authentication, err = restAuthFromDef(ctx, stmt.Name, svc.ContainerID, a.Authentication); err != nil { + return err + } + } + } + + case *ast.RestClientAddOperationAction: + if find(a.Operation.Name) >= 0 { + return mdlerrors.NewAlreadyExistsMsg("operation", a.Operation.Name, fmt.Sprintf( + "consumed rest service %s already has an operation %s — change it with 'modify operation %s ( … )'", + stmt.Name, a.Operation.Name, a.Operation.Name)) + } + op, err := buildRestClientOperation(a.Operation) + if err != nil { + return fmt.Errorf("operation %q: %w", a.Operation.Name, err) + } + ops = append(ops, &pendingOp{op: op}) + + case *ast.RestClientModifyOperationAction: + i := find(a.Operation.Name) + if i < 0 { + return notFound(a.Operation.Name) + } + if err := applyRestOperationKeys(ops[i].op, a.Operation); err != nil { + return fmt.Errorf("operation %q: %w", a.Operation.Name, err) + } + ops[i].keys = append(ops[i].keys, a.Operation.Keys...) + + case *ast.RestClientDropOperationAction: + i := find(a.Name) + if i < 0 { + return notFound(a.Name) + } + ops = append(ops[:i], ops[i+1:]...) + + default: + return mdlerrors.NewUnsupported(fmt.Sprintf("unsupported alter action: %T", action)) + } + } + + kept := map[string]bool{} + for _, p := range ops { + switch { + case p.stored == "": + edit.Add = append(edit.Add, p.op) + case len(p.keys) > 0: + kept[p.stored] = true + edit.Modify = append(edit.Modify, backend.ConsumedRestOperationEdit{Name: p.stored, Operation: p.op, Keys: p.keys}) + default: + kept[p.stored] = true + } + } + for _, op := range svc.Operations { + if !kept[op.Name] { + edit.Drop = append(edit.Drop, op.Name) + } + } + + if err := ctx.Backend.EditConsumedRestService(svc.ID, edit); err != nil { + return mdlerrors.NewBackend("alter consumed rest service", err) + } + ctx.ReportMutation("Altered", "consumed rest service: %s (%d operations)", stmt.Name, len(ops)) + return nil +} + +// findRestClient returns the stored consumed REST service named name. +func findRestClient(ctx *ExecContext, name ast.QualifiedName) (*model.ConsumedRestService, error) { + services, err := ctx.Backend.ListConsumedRestServices() + if err != nil { + return nil, mdlerrors.NewBackend("list consumed rest services", err) + } + h, err := getHierarchy(ctx) + if err != nil { + return nil, mdlerrors.NewBackend("build hierarchy", err) + } + for _, svc := range services { + if strings.EqualFold(h.GetModuleName(h.FindModuleID(svc.ContainerID)), name.Module) && strings.EqualFold(svc.Name, name.Name) { + return svc, nil + } + } + return nil, mdlerrors.NewNotFound("consumed rest service", name.String()) +} + +// applyRestOperationKeys writes the properties def states onto op, leaving +// every other one as it was: `modify operation X (Timeout: 60)` changes the +// timeout and nothing else. +func applyRestOperationKeys(op *model.RestClientOperation, def *ast.RestOperationDef) error { + built, err := buildRestClientOperation(def) + if err != nil { + return err + } + for _, k := range def.Keys { + switch k { + case "Method": + op.HttpMethod = built.HttpMethod + case "Path": + op.Path = built.Path + case "Parameters": + op.Parameters = built.Parameters + case "Query": + op.QueryParameters = built.QueryParameters + case "Headers": + op.Headers = built.Headers + case "Body": + op.BodyType, op.BodyVariable, op.BodyMappings = built.BodyType, built.BodyVariable, built.BodyMappings + case "Timeout": + op.Timeout = built.Timeout + case "Response": + op.ResponseType, op.ResponseVariable = built.ResponseType, built.ResponseVariable + op.ResponseEntity, op.ResponseMappings = built.ResponseEntity, built.ResponseMappings + } + } + return nil +} diff --git a/mdl/executor/cmd_rules_create.go b/mdl/executor/cmd_rules_create.go index d7af12b895..30c20d41e4 100644 --- a/mdl/executor/cmd_rules_create.go +++ b/mdl/executor/cmd_rules_create.go @@ -163,6 +163,8 @@ func execCreateRule(ctx *ExecContext, s *ast.CreateRuleStmt) error { // Validate and add parameters for i, p := range s.Parameters { + p.Type = settleSystemEnumeration(p.Type) + s.Parameters[i].Type = p.Type if p.Type.EntityRef != nil && !isBuiltinModuleEntity(p.Type.EntityRef.Module) { entityID := entityResolver(*p.Type.EntityRef) if entityID == "" { @@ -178,7 +180,7 @@ func execCreateRule(ctx *ExecContext, s *ast.CreateRuleStmt) error { } } if p.Type.Kind == ast.TypeEnumeration && p.Type.EnumRef != nil { - if found := findEnumeration(ctx, p.Type.EnumRef.Module, p.Type.EnumRef.Name); found == nil { + if !isKnownEnumeration(ctx, p.Type.EnumRef.Module, p.Type.EnumRef.Name) { return mdlerrors.NewNotFoundMsg("enumeration", p.Type.EnumRef.Module+"."+p.Type.EnumRef.Name, fmt.Sprintf("enumeration '%s.%s' not found for parameter '%s'", p.Type.EnumRef.Module, p.Type.EnumRef.Name, p.Name)) } @@ -197,6 +199,7 @@ func execCreateRule(ctx *ExecContext, s *ast.CreateRuleStmt) error { // Validate and set return type if s.ReturnType != nil { + s.ReturnType.Type = settleSystemEnumeration(s.ReturnType.Type) if s.ReturnType.Type.EntityRef != nil && !isBuiltinModuleEntity(s.ReturnType.Type.EntityRef.Module) { entityID := entityResolver(*s.ReturnType.Type.EntityRef) if entityID == "" { @@ -210,7 +213,7 @@ func execCreateRule(ctx *ExecContext, s *ast.CreateRuleStmt) error { } } if s.ReturnType.Type.Kind == ast.TypeEnumeration && s.ReturnType.Type.EnumRef != nil { - if found := findEnumeration(ctx, s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name); found == nil { + if !isKnownEnumeration(ctx, s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name) { return mdlerrors.NewNotFoundMsg("enumeration", s.ReturnType.Type.EnumRef.Module+"."+s.ReturnType.Type.EnumRef.Name, fmt.Sprintf("enumeration '%s.%s' not found for return type", s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name)) } diff --git a/mdl/executor/create_member_cost_pedapp_test.go b/mdl/executor/create_member_cost_pedapp_test.go new file mode 100644 index 0000000000..cc444f6fea --- /dev/null +++ b/mdl/executor/create_member_cost_pedapp_test.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "strings" + "testing" +) + +// mendixlabs/mxcli#1616: every member assignment of a `create` action +// (`$S = create M.E (A1 = …, A2 = …)`) made exec and check list the modules +// and decode every domain model of the project, so 200 creates of 8 members +// went from 14 s to 294 s once 30 modules were added — and `mxcli test`, whose +// generated flows are mostly such creates, took minutes per test file. The +// member assignments of one flow must cost what one lookup of the entity's +// domain model costs, not one project scan each. + +// createsFlow is the issue's generator: one microflow of n `create` actions, +// each assigning members (or none, for the control). +func createsFlow(name string, n int, members bool) string { + var sb strings.Builder + fmt.Fprintf(&sb, "mdl 1;\ncreate microflow MyFirstModule.%s () returns Boolean as $r\nbegin\n", name) + for j := 1; j <= n; j++ { + if members { + fmt.Fprintf(&sb, " $S%d = create MyFirstModule.Sale (A1 = 'a', A2 = 'b', A3 = 'c', A4 = 'd', A5 = 1, A6 = 2, A7 = true, A8 = [%%CurrentDateTime%%]);\n", j) + } else { + fmt.Fprintf(&sb, " $S%d = create MyFirstModule.Sale;\n", j) + } + } + sb.WriteString(" return true;\nend;\n") + return sb.String() +} + +func TestCreateMemberAssignments_DoNotScanTheProjectEach(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + if err := agreeExec(t, exec, `mdl 1; +create persistent entity MyFirstModule.Sale ( + A1: String(100), A2: String(100), A3: String(100), A4: String(100), + A5: Integer, A6: Integer, A7: Boolean, A8: DateTime +);`); err != nil { + t.Fatalf("setup: %v\n%s", err, out.String()) + } + + const creates = 20 + const assignments = creates * 8 + reads := func(src string) int64 { + t.Helper() + before := unitFileReads(t, exec) + if err := agreeExec(t, exec, src); err != nil { + t.Fatalf("exec: %v\n%s", err, out.String()) + } + return unitFileReads(t, exec) - before + } + plain := reads(createsFlow("Plain", creates, false)) + withMembers := reads(createsFlow("WithMembers", creates, true)) + t.Logf("unit-file reads: %d without members, %d with %d member assignments", plain, withMembers, assignments) + + if extra := withMembers - plain; extra >= assignments { + t.Errorf("%d member assignments cost %d extra unit-file reads (%d without members, %d with): at least one per assignment, so the cost grows with the project, not the flow", + assignments, extra, plain, withMembers) + } +} diff --git a/mdl/executor/describe_item_slots_1453_test.go b/mdl/executor/describe_item_slots_1453_test.go new file mode 100644 index 0000000000..7906bebdda --- /dev/null +++ b/mdl/executor/describe_item_slots_1453_test.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 + +// An object-list item can hold widgets in more than one slot — an Accordion +// group has `headerContent` and `content`. DESCRIBE folded every slot into the +// item's body, so a header widget was printed beside the body widgets, and a +// page regenerated from that description stored it in `content` (measured on a +// blank 11.13.0 app: headerContent -> [txtHead], content -> [txtBody] became +// content -> [txtHead, txtBody]). And `check` refused the block form CREATE +// builds from — `headercontent { … }` inside a group was MDL-WIDGET25 "not a +// widget in this project" — so the honest description could not run either. +// (ako/mxcli#1326, mendixlabs/mxcli#1453; prerequisite of +// PROPOSAL_alter_object_list_item_address.md.) +package executor + +import ( + "bytes" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// accordionWithBothSlots is a group holding txtHead in headerContent and +// txtBody in content, with the schema declaring headerContent FIRST — so a +// "first widget slot is the body" rule would pick the wrong one. +func accordionWithBothSlots() map[string]any { + const ( + idGroups = "type-id-groups" + idHead = "type-id-headerContent" + idBody = "type-id-content" + ) + text := func(name, s string) map[string]any { + return map[string]any{"$Type": "Forms$StaticText", "Name": name, + "Text": map[string]any{"Items": []any{map[string]any{"Text": s}}}} + } + return map[string]any{ + "Name": "acc1", + "Type": map[string]any{ + "WidgetId": "com.mendix.widget.web.accordion.Accordion", + "ObjectType": map[string]any{"PropertyTypes": []any{ + map[string]any{"$ID": idGroups, "PropertyKey": "groups", + "ValueType": map[string]any{"ObjectType": map[string]any{"PropertyTypes": []any{ + map[string]any{"$ID": idHead, "PropertyKey": "headerContent", + "ValueType": map[string]any{"Type": "Widgets"}}, + map[string]any{"$ID": idBody, "PropertyKey": "content", + "ValueType": map[string]any{"Type": "Widgets"}}, + }}}}, + }}, + }, + "Object": map[string]any{"Properties": []any{ + map[string]any{"TypePointer": idGroups, "Value": map[string]any{"Objects": []any{ + map[string]any{"Properties": []any{ + map[string]any{"TypePointer": idHead, "Value": map[string]any{"Widgets": []any{text("txtHead", "Header")}}}, + map[string]any{"TypePointer": idBody, "Value": map[string]any{"Widgets": []any{text("txtBody", "Body")}}}, + }}, + }}}, + }}, + } +} + +func TestObjectListItem_BodyAndOtherSlotsKeptApart_1453(t *testing.T) { + lists := extractObjectLists(nil, accordionWithBothSlots()) + if len(lists) != 1 || len(lists[0].Items) != 1 { + t.Fatalf("want one group, got %+v", lists) + } + item := lists[0].Items[0] + if len(item.Children) != 1 || item.Children[0].Name != "txtBody" { + t.Fatalf("body = %+v, want only txtBody — headerContent was folded into the body", item.Children) + } + if len(item.Slots) != 1 || item.Slots[0].Keyword != "headercontent" || + len(item.Slots[0].Widgets) != 1 || item.Slots[0].Widgets[0].Name != "txtHead" { + t.Fatalf("slots = %+v, want headercontent: [txtHead]", item.Slots) + } +} + +func TestObjectListItem_EmitsHeaderContentBlock_1453(t *testing.T) { + var buf bytes.Buffer + ctx := &ExecContext{Output: &buf} + outputWidgetMDLV3(ctx, rawWidget{ + Type: "CustomWidgets$CustomWidget", + RenderMode: "accordion", + Name: "acc1", + WidgetID: "com.mendix.widget.web.accordion.Accordion", + ObjectLists: extractObjectLists(nil, accordionWithBothSlots()), + }, 0) + out := buf.String() + block := strings.Index(out, "headercontent {") + head := strings.Index(out, "txtHead") + body := strings.Index(out, "txtBody") + if block < 0 || head < block || body < 0 { + t.Fatalf("want txtHead inside a `headercontent { … }` block of the group:\n%s", out) + } + if end := strings.Index(out[block:], "}"); block+end > body { + t.Fatalf("txtBody must not be inside the headercontent block:\n%s", out) + } +} + +// With no `content` slot the body is the item's first widget slot, as CREATE's +// defaultItemSlotKey routes loose children. +func TestBodyItemSlotKey_1453(t *testing.T) { + for _, c := range []struct { + keys []string + want string + }{ + {[]string{"headerContent", "content"}, "content"}, + {[]string{"children", "extra"}, "children"}, + {nil, ""}, + } { + if got := bodyItemSlotKey(c.keys); got != c.want { + t.Errorf("bodyItemSlotKey(%v) = %q, want %q", c.keys, got, c.want) + } + } +} + +// The block describe now prints must pass check: an item's own slot keyword is +// a container of that item, not an unknown widget. +func TestCheckAcceptsItemSlotBlock_1453(t *testing.T) { + prog, errs := visitor.Build(`create page M.P (title: 'P', layout: Atlas_Core.Atlas_Default) { + pluggablewidget 'com.mendix.widget.web.accordion.Accordion' acc1 { + group (headerRenderMode: 'custom') { + headercontent { dynamictext txtHead (content: 'Header') } + dynamictext txtBody (content: 'Body') + } + } +}`) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + for _, v := range ValidateWidgetPropertiesForStatement(prog.Statements[0], accordionRegistry(t)) { + if v.Severity == linter.SeverityError { + t.Errorf("an item slot block was refused: %s: %s", v.RuleID, v.Message) + } + } +} + +// The control for the test above: a keyword that is NOT one of the item's +// slots is still refused, so the fix did not just switch the rule off. +func TestCheckStillRefusesUnknownBlockInItem_1453(t *testing.T) { + prog, errs := visitor.Build(`create page M.P (title: 'P', layout: Atlas_Core.Atlas_Default) { + pluggablewidget 'com.mendix.widget.web.accordion.Accordion' acc1 { + group (headerRenderMode: 'custom') { + headercontnt { dynamictext txtHead (content: 'Header') } + } + } +}`) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + for _, v := range ValidateWidgetPropertiesForStatement(prog.Statements[0], accordionRegistry(t)) { + if v.RuleID == "MDL-WIDGET25" || v.RuleID == "MDL-WIDGET26" { + return + } + } + t.Fatal("a misspelt slot block inside a group was not refused") +} diff --git a/mdl/executor/describe_module_all_limits_test.go b/mdl/executor/describe_module_all_limits_test.go new file mode 100644 index 0000000000..0f14cf95ba --- /dev/null +++ b/mdl/executor/describe_module_all_limits_test.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend" + modelsdkbackend "github.com/mendixlabs/mxcli/mdl/backend/modelsdk" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// mendixlabs/mxcli#1624: `describe module M with all` could not cover a large +// module. On a module of 229 microflows it hit the default 5-minute statement +// timeout, and with MXCLI_EXEC_TIMEOUT=40m it stopped at "output line limit +// exceeded ... statement aborted", truncated mid-flow. Both guards exist to +// stop a runaway statement; a module dump is bounded by the module, the way a +// catalog build (#651) is bounded by the project, so neither default applies +// to it. An explicit MXCLI_EXEC_TIMEOUT / MXCLI_OUTPUT_LINE_LIMIT still does. + +var moduleDump = &ast.DescribeStmt{ObjectType: ast.DescribeModule, WithAll: true} + +// executeLines runs stmt through a handler that writes n lines, as a describer +// does (ignoring Fprintf's error). +func executeLines(t *testing.T, stmt *ast.DescribeStmt, n int) (string, error) { + t.Helper() + var out bytes.Buffer + e := New(&out) + e.registry.handlers[reflect.TypeOf(&ast.DescribeStmt{})] = func(ctx *ExecContext, _ ast.Statement) error { + for i := 0; i < n; i++ { + fmt.Fprintf(ctx.Output, "line %d\n", i) + } + return nil + } + err := e.Execute(stmt) + return out.String(), err +} + +func TestDescribeModuleWithAll_NotCutAtTheDefaultLineLimit(t *testing.T) { + t.Setenv("MXCLI_OUTPUT_LINE_LIMIT", "") + n := maxOutputLines + 500 + + // Control: any other statement is still cut at the default. + if _, err := executeLines(t, &ast.DescribeStmt{ObjectType: ast.DescribeModule}, n); err == nil || !strings.Contains(err.Error(), "line limit") { + t.Fatalf("control: a plain describe of %d lines returned %v, want the line-limit error", n, err) + } + + out, err := executeLines(t, moduleDump, n) + if err != nil { + t.Fatalf("describe module … with all of %d lines failed: %v", n, err) + } + if got := strings.Count(out, "\n"); got != n { + t.Errorf("describe module … with all wrote %d of %d lines", got, n) + } +} + +func TestOutputLineLimit_ExplicitSettingWins(t *testing.T) { + t.Setenv("MXCLI_OUTPUT_LINE_LIMIT", "100") + for _, stmt := range []*ast.DescribeStmt{moduleDump, {ObjectType: ast.DescribeMicroflow}} { + if _, err := executeLines(t, stmt, 150); err == nil || !strings.Contains(err.Error(), "(100 lines)") { + t.Errorf("with MXCLI_OUTPUT_LINE_LIMIT=100, 150 lines of %v returned %v, want the 100-line limit error", stmt.ObjectType, err) + } + if _, err := executeLines(t, stmt, 100); err != nil { + t.Errorf("with MXCLI_OUTPUT_LINE_LIMIT=100, 100 lines of %v failed: %v", stmt.ObjectType, err) + } + } + + t.Setenv("MXCLI_OUTPUT_LINE_LIMIT", "0") // no limit + if _, err := executeLines(t, &ast.DescribeStmt{ObjectType: ast.DescribeMicroflow}, maxOutputLines+500); err != nil { + t.Errorf("with MXCLI_OUTPUT_LINE_LIMIT=0, a long describe failed: %v", err) + } + + t.Setenv("MXCLI_OUTPUT_LINE_LIMIT", "lots") // unparseable: the default + if _, err := executeLines(t, &ast.DescribeStmt{ObjectType: ast.DescribeMicroflow}, maxOutputLines+500); err == nil { + t.Errorf("with an unparseable MXCLI_OUTPUT_LINE_LIMIT, a describe past the default limit succeeded") + } +} + +func TestDescribeModuleWithAll_NotUnderTheDefaultTimeout(t *testing.T) { + t.Setenv("MXCLI_EXEC_TIMEOUT", "") + if d := effectiveExecuteTimeout(moduleDump); d != 0 { + t.Errorf("describe module … with all runs under a %v wall-clock limit, want none (it is bounded by the module)", d) + } + if d := effectiveExecuteTimeout(&ast.DescribeStmt{ObjectType: ast.DescribeModule}); d != defaultExecuteTimeout { + t.Errorf("control: describe module without `with all` gets %v, want the default %v", d, defaultExecuteTimeout) + } + t.Setenv("MXCLI_EXEC_TIMEOUT", "40m") + if d := effectiveExecuteTimeout(moduleDump); d != 40*time.Minute { + t.Errorf("with MXCLI_EXEC_TIMEOUT=40m, describe module … with all gets %v, want 40m", d) + } +} + +// The time half of the report: the dump describes every document of the module +// one after another, and each describe listed what it needed from disk again — +// the folders for a path, every microflow to find one by name, the domain +// models. A describe writes nothing, so it now reads through the unit cache +// for its whole statement: each file at most once. +func TestDescribeModuleWithAll_ReadsEachUnitFileOnce(t *testing.T) { + exec, out, dir := openPedAppCopy(t) + units := 0 + _ = filepath.Walk(filepath.Join(dir, "mprcontents"), func(p string, info os.FileInfo, err error) error { + if err == nil && !info.IsDir() && strings.HasSuffix(p, ".mxunit") { + units++ + } + return nil + }) + before := unitFileReads(t, exec) + if err := exec.Execute(&ast.DescribeStmt{ObjectType: ast.DescribeModule, Name: ast.QualifiedName{Module: "Administration"}, WithAll: true}); err != nil { + t.Fatalf("describe: %v\n%s", err, out.String()) + } + reads := unitFileReads(t, exec) - before + t.Logf("describe module Administration with all: %d unit-file reads; the project has %d", reads, units) + if reads > int64(units) { + t.Errorf("describe module … with all read %d unit files; the project has %d, so some were read more than once", reads, units) + } +} + +// listCountingBackend is the modelsdk backend, counting full microflow +// listings — each one decodes the whole flow of every microflow in the +// project. Embedding the concrete type keeps its optional lookups. +type listCountingBackend struct { + *modelsdkbackend.Backend + listMicroflows int +} + +func (b *listCountingBackend) ListMicroflows() ([]*microflows.Microflow, error) { + b.listMicroflows++ + return b.Backend.ListMicroflows() +} + +// Where the dump's time actually went, by CPU profile: describing a microflow +// derives its layout by rebuilding it, and the rebuild runs through a read +// memo that hid the backend's by-name lookup, so every described flow decoded +// every microflow of the project — quadratic in the module, linear in the +// project. 229 generated flows took 20 s on pedapp, 63% of it in that decode. +func TestDescribeModuleWithAll_DoesNotDecodeEveryMicroflowPerFlow(t *testing.T) { + exec, out, dir := openPedAppCopy(t) + const flows = 8 + var sb strings.Builder + sb.WriteString("mdl 1;\ncreate module Dump;\n") + for i := 1; i <= flows; i++ { + fmt.Fprintf(&sb, "create microflow Dump.MF_%d ($N: Integer) returns Boolean as $r\nbegin\n if $N > %d then\n return true;\n end if;\n return false;\nend;\n", i, i) + } + if err := agreeExec(t, exec, sb.String()); err != nil { + t.Fatalf("setup: %v\n%s", err, out.String()) + } + + counting := &listCountingBackend{Backend: modelsdkbackend.New()} + e := New(io.Discard) + e.SetBackendFactory(func() backend.FullBackend { return counting }) + if err := e.Execute(&ast.ConnectStmt{Path: filepath.Join(dir, "PedApp.mpr")}); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = e.Execute(&ast.DisconnectStmt{}) }) + if err := e.Execute(&ast.DescribeStmt{ObjectType: ast.DescribeModule, Name: ast.QualifiedName{Module: "Dump"}, WithAll: true}); err != nil { + t.Fatalf("describe: %v", err) + } + if counting.listMicroflows >= flows { + t.Errorf("describing a module of %d microflows listed (and decoded) every microflow of the project %d times", flows, counting.listMicroflows) + } +} diff --git a/mdl/executor/exec_context.go b/mdl/executor/exec_context.go index c813e1e3c2..5bc0087951 100644 --- a/mdl/executor/exec_context.go +++ b/mdl/executor/exec_context.go @@ -53,6 +53,11 @@ type ExecContext struct { // outside a describe body. describeBody *describeBodyState + // unauthorableSink, when set, receives every widget a describe body could + // only print as a comment because MDL cannot author it: what a rewrite of + // the document from its description would delete (mendixlabs/mxcli#1622). + unauthorableSink *[]unauthorableWidget + // scriptFlows are the flows earlier statements of a script declare, // keyed by scriptFlowKey, set only while check predicts a script's flow // verdicts (CheckFlowVerdicts): exec builds a flow after those statements diff --git a/mdl/executor/executor.go b/mdl/executor/executor.go index f5c6c6bc4d..558d4e6f5c 100644 --- a/mdl/executor/executor.go +++ b/mdl/executor/executor.go @@ -182,8 +182,9 @@ func getPageNames(ctx *ExecContext, h *ContainerHierarchy) map[model.ID]string { } const ( - // maxOutputLines is the per-statement line limit. Statements that produce more - // lines than this are aborted to prevent runaway output from infinite loops. + // maxOutputLines is the default per-statement line limit. Statements that + // produce more lines than this are aborted to prevent runaway output from + // infinite loops. MXCLI_OUTPUT_LINE_LIMIT overrides it (effectiveOutputLineLimit). maxOutputLines = 10_000 // defaultExecuteTimeout is the maximum wall-clock time allowed for a single // statement when MXCLI_EXEC_TIMEOUT is not set. @@ -218,11 +219,11 @@ func configuredExecuteTimeout() time.Duration { // effectiveExecuteTimeout returns the wall-clock timeout to apply to a single // statement. An explicit MXCLI_EXEC_TIMEOUT always wins (so users can both raise -// and cap the limit). Otherwise, known bounded-but-slow statements — currently -// REFRESH CATALOG, which on a large project legitimately runs past the default -// 5m — are exempt from the guard (returns 0 = no wall-clock limit); the per- -// statement output-line guard still bounds runaway output. The guard exists to -// abort runaway loops, which a deterministic catalog build is not. See #651. +// and cap the limit). Otherwise, known bounded-but-slow statements — REFRESH +// CATALOG, which on a large project legitimately runs past the default 5m, and +// DESCRIBE MODULE … WITH ALL on a large module (#1624) — are exempt from the +// guard (returns 0 = no wall-clock limit). The guard exists to abort runaway +// loops, which a deterministic catalog build or module dump is not. See #651. func effectiveExecuteTimeout(stmt ast.Statement) time.Duration { if os.Getenv("MXCLI_EXEC_TIMEOUT") != "" { return configuredExecuteTimeout() @@ -236,14 +237,40 @@ func effectiveExecuteTimeout(stmt ast.Statement) time.Duration { // isExemptFromExecuteTimeout reports whether a statement is a known long-running // but bounded operation that should not be capped by the default wall-clock guard. func isExemptFromExecuteTimeout(stmt ast.Statement) bool { - switch stmt.(type) { + switch s := stmt.(type) { case *ast.RefreshCatalogStmt: return true default: - return false + return isModuleDump(s) } } +// isModuleDump reports whether stmt is `describe module M with all`. Its work +// and its output are bounded by the module, not by anything that can run away, +// so neither default guard applies to it — on a module of 229 microflows it +// hit both, and could not be described at all (mendixlabs/mxcli#1624). +func isModuleDump(stmt ast.Statement) bool { + d, ok := stmt.(*ast.DescribeStmt) + return ok && d.ObjectType == ast.DescribeModule && d.WithAll +} + +// effectiveOutputLineLimit returns the line limit for one statement; 0 means +// none. An explicit MXCLI_OUTPUT_LINE_LIMIT — a number of lines, 0 for no +// limit — applies to every statement, as MXCLI_EXEC_TIMEOUT does; otherwise a +// module dump has no limit and everything else the default. +func effectiveOutputLineLimit(stmt ast.Statement) int { + if raw := os.Getenv("MXCLI_OUTPUT_LINE_LIMIT"); raw != "" { + if n, err := strconv.Atoi(raw); err == nil && n >= 0 { + return n + } + return maxOutputLines + } + if isModuleDump(stmt) { + return 0 + } + return maxOutputLines +} + // BackendFactory creates a new backend instance for connecting to a project. type BackendFactory func() backend.FullBackend @@ -368,7 +395,16 @@ func (e *Executor) Execute(stmt ast.Statement) error { // Reset per-statement line counter. if e.guard != nil { - e.guard.reset() + e.guard.reset(effectiveOutputLineLimit(stmt)) + } + + // A describe writes nothing, and the describers look the same documents up + // again and again — a module dump describes every document of the module, + // each one listing the folders, the flows and the domain models anew — so + // the statement reads each unit file once (mendixlabs/mxcli#1624). A write + // would empty the cache, as it does in check's passes. + if _, ok := stmt.(*ast.DescribeStmt); ok { + defer e.CacheUnitReads()() } // Enforce the wall-clock timeout with a timer that skips untimed work. diff --git a/mdl/executor/issue1608_consumed_rest_alter_test.go b/mdl/executor/issue1608_consumed_rest_alter_test.go new file mode 100644 index 0000000000..c3d67fecdc --- /dev/null +++ b/mdl/executor/issue1608_consumed_rest_alter_test.go @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "strings" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/backend/bsonnav" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// mendixlabs/mxcli#1608 (ako/mxcli#1536): `alter consumed rest service` adds, +// changes and removes operations of an existing service, and leaves every +// operation it does not name as stored — including what Studio Pro wrote that +// MDL cannot say. + +const bookingAPIWithQuery = `create consumed rest service MyFirstModule.BookingApi ( + BaseUrl: 'https://api.example.com', + Authentication: none +) +{ + operation PostBooking ( + Method: post, + Path: '/bookings', + Query: ($dryRun: String), + Body: json from $Payload, + Response: json + ) +};` + +// rawRestService reads the stored service unit. +func rawRestService(t *testing.T, exec *Executor, name string) (string, bson.D) { + t.Helper() + svc := restService(t, exec, name) + raw, err := exec.backend.GetRawUnitBytes(svc.ID) + if err != nil { + t.Fatal(err) + } + var doc bson.D + if err := bson.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + return string(svc.ID), doc +} + +// rawRestOperation returns the stored operation named name, marshalled, so two +// reads compare byte for byte. +func rawRestOperation(t *testing.T, doc bson.D, name string) bson.D { + t.Helper() + for _, el := range bsonnav.DGetArrayElements(bsonnav.DGet(doc, "Operations")) { + if od, ok := el.(bson.D); ok && bsonnav.DGetString(od, "Name") == name { + return od + } + } + t.Fatalf("no stored operation %s", name) + return nil +} + +func marshalD(t *testing.T, d bson.D) []byte { + t.Helper() + b, err := bson.Marshal(d) + if err != nil { + t.Fatal(err) + } + return b +} + +// seedStudioProQueryParameter gives PostBooking's query parameter what Studio +// Pro writes and mxcli never does: optional usage, and a test value. An +// element mxcli wrote holds the writer's constants, so it cannot show a +// rewrite that resets to them; this one can. +func seedStudioProQueryParameter(t *testing.T, exec *Executor) { + t.Helper() + id, doc := rawRestService(t, exec, "BookingApi") + op := rawRestOperation(t, doc, "PostBooking") + params := bsonnav.DGetArrayElements(bsonnav.DGet(op, "QueryParameters")) + if len(params) != 1 { + t.Fatalf("fixture has %d query parameters", len(params)) + } + q := params[0].(bson.D) + usage := bsonnav.DGetDoc(q, "ParameterUsage") + bsonnav.DSet(usage, "$Type", "Rest$OptionalQueryParameterUsage") + usage = append(usage, bson.E{Key: "Included", Value: true}) + bsonnav.DSet(q, "ParameterUsage", usage) + if !bsonnav.DSet(q, "TestValue", "studio-pro") { + q = append(q, bson.E{Key: "TestValue", Value: "studio-pro"}) + } + bsonnav.DSet(op, "QueryParameters", bson.A{int32(2), q}) + if err := exec.backend.UpdateRawUnit(id, marshalD(t, doc)); err != nil { + t.Fatal(err) + } + if got := queryUsage(t, exec); got != "Rest$OptionalQueryParameterUsage" { + t.Fatalf("seed did not take: %s", got) + } +} + +func queryUsage(t *testing.T, exec *Executor) string { + t.Helper() + _, doc := rawRestService(t, exec, "BookingApi") + op := rawRestOperation(t, doc, "PostBooking") + q := bsonnav.DGetArrayElements(bsonnav.DGet(op, "QueryParameters"))[0].(bson.D) + return bsonnav.DGetString(bsonnav.DGetDoc(q, "ParameterUsage"), "$Type") +} + +// Step 2 of the report: the statement it proposed, which did not parse. +func TestIssue1608_AlterAddsAnOperationAndLeavesTheOthersAsStored(t *testing.T) { + exec, out := restProject(t) + run(t, exec, bookingAPIWithQuery) + seedStudioProQueryParameter(t, exec) + _, before := rawRestService(t, exec, "BookingApi") + storedPost := marshalD(t, rawRestOperation(t, before, "PostBooking")) + + out.Reset() + run(t, exec, `alter consumed rest service MyFirstModule.BookingApi add operation GetBooking ( + Method: get, + Path: '/bookings/{id}', + Parameters: ($id: String), + Response: json as $Result +);`) + if got := out.String(); !strings.Contains(got, "Altered consumed rest service: MyFirstModule.BookingApi (2 operations)") { + t.Errorf("unexpected output:\n%s", got) + } + if ops := restOperations(t, exec, "BookingApi"); strings.Join(ops, ",") != "PostBooking,GetBooking" { + t.Fatalf("operations = %v", ops) + } + _, after := rawRestService(t, exec, "BookingApi") + if !bytes.Equal(marshalD(t, rawRestOperation(t, after, "PostBooking")), storedPost) { + t.Error("adding GetBooking rewrote PostBooking") + } + get := restService(t, exec, "BookingApi").Operations[1] + if get.HttpMethod != "GET" || get.Path != "/bookings/{id}" || len(get.Parameters) != 1 || get.ResponseType != "JSON" { + t.Errorf("GetBooking stored as %+v", get) + } + + // Adding it again is refused, naming the spelling that changes it. + prog, errs := visitor.Build(`alter consumed rest service MyFirstModule.BookingApi add operation GetBooking (Method: get, Path: '/x', Response: none);`) + if len(errs) > 0 { + t.Fatal(errs) + } + err := exec.Execute(prog.Statements[0]) + if err == nil || !strings.Contains(err.Error(), "modify operation GetBooking") { + t.Errorf("a second add of GetBooking: %v", err) + } +} + +// Control for the test above: the rebuild CREATE OR MODIFY does — here from +// describe's own output — resets the seeded query parameter, so the seed is +// one a rewrite can be caught losing. +func TestIssue1608_CreateOrModifyRebuildResetsStudioProState(t *testing.T) { + exec, out := restProject(t) + run(t, exec, bookingAPIWithQuery) + seedStudioProQueryParameter(t, exec) + out.Reset() + run(t, exec, "describe consumed rest service MyFirstModule.BookingApi") + described := strings.Replace(out.String(), "Timeout: 300", "Timeout: 299", 1) // force the write + run(t, exec, described) + if got := queryUsage(t, exec); got == "Rest$OptionalQueryParameterUsage" { + t.Errorf("CREATE OR MODIFY kept the seeded query parameter usage, so the seed no longer shows " + + "a rebuild: seed a property it still resets, or the ALTER tests above prove nothing") + } +} + +func TestIssue1608_AlterModifyWritesOnlyTheNamedKeys(t *testing.T) { + exec, out := restProject(t) + run(t, exec, bookingAPIWithQuery) + seedStudioProQueryParameter(t, exec) + _, before := rawRestService(t, exec, "BookingApi") + storedQuery := marshalD(t, bson.D{{Key: "q", Value: bsonnav.DGet(rawRestOperation(t, before, "PostBooking"), "QueryParameters")}}) + + run(t, exec, `alter consumed rest service MyFirstModule.BookingApi modify operation PostBooking (Timeout: 60, Method: put);`) + op := restService(t, exec, "BookingApi").Operations[0] + if op.Timeout != 60 || op.HttpMethod != "PUT" { + t.Errorf("modify did not write Timeout/Method: %+v", op) + } + // The body is in the method element: a method change that keeps it a + // method with a body keeps the stored body. + if op.BodyType != "JSON" || op.BodyVariable != "$Payload" || op.Path != "/bookings" || op.ResponseType != "JSON" { + t.Errorf("modify changed a property it did not name: %+v", op) + } + _, after := rawRestService(t, exec, "BookingApi") + if got := marshalD(t, bson.D{{Key: "q", Value: bsonnav.DGet(rawRestOperation(t, after, "PostBooking"), "QueryParameters")}}); !bytes.Equal(got, storedQuery) { + t.Error("modify rewrote the query parameters it did not name") + } + + // The same statement again changes nothing. + out.Reset() + run(t, exec, `alter consumed rest service MyFirstModule.BookingApi modify operation PostBooking (Timeout: 60, Method: put);`) + if got := out.String(); !strings.Contains(got, "Unchanged") { + t.Errorf("re-running the modify wrote the service:\n%s", got) + } +} + +func TestIssue1608_AlterDropAndSet(t *testing.T) { + exec, out := restProject(t) + run(t, exec, bookingAPIStep1) + run(t, exec, `alter consumed rest service MyFirstModule.BookingApi + add operation GetBooking (Method: get, Path: '/bookings/{id}', Parameters: ($id: String), Response: json), + set (BaseUrl: 'https://api.example.com/v2');`) + if got := restService(t, exec, "BookingApi").BaseUrl; got != "https://api.example.com/v2" { + t.Errorf("BaseUrl = %q", got) + } + + run(t, exec, `alter consumed rest service MyFirstModule.BookingApi drop operation PostBooking;`) + if ops := restOperations(t, exec, "BookingApi"); strings.Join(ops, ",") != "GetBooking" { + t.Errorf("operations after drop = %v", ops) + } + + // A name that is not there is refused, listing the ones that are. + prog, errs := visitor.Build(`alter consumed rest service MyFirstModule.BookingApi drop operation PostBooking;`) + if len(errs) > 0 { + t.Fatal(errs) + } + err := exec.Execute(prog.Statements[0]) + if err == nil || !strings.Contains(err.Error(), "GetBooking") { + t.Errorf("dropping a missing operation: %v", err) + } + + // describe still runs again after the edits. + out.Reset() + run(t, exec, "describe consumed rest service MyFirstModule.BookingApi") + described := out.String() + out.Reset() + run(t, exec, described) + if got := out.String(); !strings.Contains(got, "Unchanged") { + t.Errorf("describe after alter does not re-run unchanged:\n%s\n%s", described, got) + } +} + +// `drop operation X, add operation X (…)` in one statement replaces X. +func TestIssue1608_AlterDropThenAddReplaces(t *testing.T) { + exec, _ := restProject(t) + run(t, exec, bookingAPIStep1) + run(t, exec, `alter consumed rest service MyFirstModule.BookingApi + drop operation PostBooking, + add operation PostBooking (Method: post, Path: '/v2/bookings', Body: json from $Payload, Response: none);`) + svc := restService(t, exec, "BookingApi") + if len(svc.Operations) != 1 || svc.Operations[0].Path != "/v2/bookings" || svc.Operations[0].ResponseType != "NONE" { + t.Errorf("operations = %+v", svc.Operations) + } +} diff --git a/mdl/executor/issue1608_consumed_rest_dropped_ops_test.go b/mdl/executor/issue1608_consumed_rest_dropped_ops_test.go new file mode 100644 index 0000000000..ab1bbad406 --- /dev/null +++ b/mdl/executor/issue1608_consumed_rest_dropped_ops_test.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" +) + +// Step 4: CREATE OR MODIFY naming only new operations removes PostBooking. +// That is what "modify to this shape" means, but it has to say so. +func TestIssue1608_CreateOrModifyNamesTheOperationsItRemoves(t *testing.T) { + exec, out := restProject(t) + run(t, exec, bookingAPIStep1) + out.Reset() + run(t, exec, `create or modify consumed rest service MyFirstModule.BookingApi ( + BaseUrl: 'https://api.example.com', + Authentication: none +) +{ + operation GetBooking (Method: get, Path: '/bookings/{id}', Parameters: ($id: String), Response: json as $Result) + operation PutBooking (Method: put, Path: '/bookings/{id}', Parameters: ($id: String), Body: json from $Payload, Response: json as $Result) +};`) + got := out.String() + for _, want := range []string{"PostBooking", "alter consumed rest service MyFirstModule.BookingApi add operation"} { + if !strings.Contains(got, want) { + t.Errorf("output does not mention %q:\n%s", want, got) + } + } + + // Control: a CREATE OR MODIFY that keeps every operation says nothing of + // the kind. + out.Reset() + run(t, exec, `create or modify consumed rest service MyFirstModule.BookingApi ( + BaseUrl: 'https://api.example.com', + Authentication: none +) +{ + operation GetBooking (Method: get, Path: '/bookings/{id}', Parameters: ($id: String), Response: json) + operation PutBooking (Method: put, Path: '/bookings/{id}', Parameters: ($id: String), Body: json from $Payload, Response: json) +};`) + if got := out.String(); strings.Contains(got, "drops") { + t.Errorf("a statement that keeps every operation reported a drop:\n%s", got) + } +} diff --git a/mdl/executor/issue1608_consumed_rest_ops_test.go b/mdl/executor/issue1608_consumed_rest_ops_test.go new file mode 100644 index 0000000000..0987e4face --- /dev/null +++ b/mdl/executor/issue1608_consumed_rest_ops_test.go @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend" + modelsdkbackend "github.com/mendixlabs/mxcli/mdl/backend/modelsdk" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/model" +) + +// mendixlabs/mxcli#1608 (ako/mxcli#1536), against a real project: the report's +// script, step by step. A consumed REST service with one operation could not +// gain a second from MDL — no ALTER, a DESCRIBE that did not re-parse, and a +// CREATE OR MODIFY that deleted the unlisted operation saying only +// "Modified rest client: MyFirstModule.BookingApi (2 operations)". + +const bookingAPIStep1 = `create consumed rest service MyFirstModule.BookingApi ( + BaseUrl: 'https://api.example.com', + Authentication: none +) +{ + operation PostBooking ( + Method: post, + Path: '/bookings', + Body: json from $Payload, + Response: json as $Result + ) +};` + +// restProject is an executor connected to a copy of the fixture project, +// writing its output to the returned buffer. +func restProject(t *testing.T) (*Executor, *bytes.Buffer) { + t.Helper() + var out bytes.Buffer + exec := New(&out) + exec.SetBackendFactory(func() backend.FullBackend { return modelsdkbackend.New() }) + t.Cleanup(func() { exec.Close() }) + run(t, exec, "CONNECT LOCAL '"+visitor.QuoteString(projectFixture(t))+"'") + out.Reset() + return exec, &out +} + +// restOperations reads the stored service's operation names. +func restOperations(t *testing.T, exec *Executor, name string) []string { + t.Helper() + svcs, err := exec.backend.ListConsumedRestServices() + if err != nil { + t.Fatal(err) + } + for _, s := range svcs { + if s.Name == name { + var names []string + for _, op := range s.Operations { + names = append(names, op.Name) + } + return names + } + } + t.Fatalf("no consumed rest service %s", name) + return nil +} + +func restService(t *testing.T, exec *Executor, name string) *model.ConsumedRestService { + t.Helper() + svcs, err := exec.backend.ListConsumedRestServices() + if err != nil { + t.Fatal(err) + } + for _, s := range svcs { + if s.Name == name { + return s + } + } + t.Fatalf("no consumed rest service %s", name) + return nil +} + +// Step 3: DESCRIBE's output runs again, and changes nothing. +func TestIssue1608_DescribeOutputReExecutes(t *testing.T) { + exec, out := restProject(t) + run(t, exec, bookingAPIStep1) + out.Reset() + run(t, exec, "describe consumed rest service MyFirstModule.BookingApi") + described := out.String() + if !strings.Contains(described, "Response: json") { + t.Fatalf("unexpected describe output:\n%s", described) + } + + prog, errs := visitor.Build(described) + if len(errs) > 0 { + t.Fatalf("describe output does not re-parse: %v\n%s", errs, described) + } + out.Reset() + for _, stmt := range prog.Statements { + if err := exec.Execute(stmt); err != nil { + t.Fatalf("re-executing describe output: %v\n%s", err, described) + } + } + if got := out.String(); !strings.Contains(got, "Unchanged") { + t.Errorf("re-executing describe output wrote the service:\n%s", got) + } + if ops := restOperations(t, exec, "BookingApi"); len(ops) != 1 || ops[0] != "PostBooking" { + t.Errorf("operations after re-exec = %v", ops) + } +} diff --git a/mdl/executor/issue1608_consumed_rest_response_reparse_test.go b/mdl/executor/issue1608_consumed_rest_response_reparse_test.go new file mode 100644 index 0000000000..d0c66356ad --- /dev/null +++ b/mdl/executor/issue1608_consumed_rest_response_reparse_test.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/model" +) + +// mendixlabs/mxcli#1608 (ako/mxcli#1536): `describe consumed rest service` +// printed `Response: json` for an operation created with +// `Response: json as $Result`, and the parser rejected it: +// +// line 14:2 missing VARIABLE at ')' +// +// The variable is not stored — a consumed REST operation has no response +// variable; it is named by the call activity — so there is nothing for +// describe to print, and the bare form has to be the language's. + +func TestIssue1608_BareResponseKindParses(t *testing.T) { + for _, kind := range []string{"json", "string", "file", "status"} { + src := "create consumed rest service M.S (BaseUrl: 'https://x', Authentication: none) {\n" + + " operation Op (Method: get, Path: '/x', Response: " + kind + ")\n};" + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Errorf("Response: %s does not parse: %v", kind, errs) + continue + } + stmt := prog.Statements[0].(*ast.CreateRestClientStmt) + if got := stmt.Operations[0].ResponseType; got != kind { + t.Errorf("Response: %s read as ResponseType %q", kind, got) + } + } +} + +// The response variable is not stored, so describe prints each kind bare and +// the output runs again as the same response. +func TestIssue1608_DescribeResponseKindsReparse(t *testing.T) { + for _, kind := range []string{"JSON", "STRING", "FILE", "STATUS", "NONE"} { + op := &model.RestClientOperation{Name: "Op", HttpMethod: "GET", Path: "/x", ResponseType: kind} + var buf bytes.Buffer + outputRestOperation(&buf, op) + src := "create consumed rest service M.S (BaseUrl: 'https://x', Authentication: none) {\n" + buf.String() + "};" + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Errorf("%s: describe output does not re-parse: %v\n%s", kind, errs, buf.String()) + continue + } + reop, err := buildRestClientOperation(prog.Statements[0].(*ast.CreateRestClientStmt).Operations[0]) + if err != nil { + t.Fatal(err) + } + if reop.ResponseType != kind { + t.Errorf("%s re-parsed as %s:\n%s", kind, reop.ResponseType, buf.String()) + } + } +} diff --git a/mdl/executor/issue1622_page_rewrite_unauthorable_test.go b/mdl/executor/issue1622_page_rewrite_unauthorable_test.go new file mode 100644 index 0000000000..0536d346c7 --- /dev/null +++ b/mdl/executor/issue1622_page_rewrite_unauthorable_test.go @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "strings" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/backend" + modelsdkbackend "github.com/mendixlabs/mxcli/mdl/backend/modelsdk" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/model" +) + +// mendixlabs/mxcli#1622 (ako/mxcli#1554): `describe page M.P` prints a classic +// data grid, a reference selector, a drop-down or a reference set selector as +// a `-- NOT re-executable` comment, and executing that output replaced the +// page with the widget gone — "Replaced page M.P", no warning. MDL cannot +// author those widgets, so a rewrite cannot keep them; it is refused instead, +// naming each one and the ALTER PAGE that edits the page without them. + +func pageFixtureExecutor(t *testing.T) (*Executor, *bytes.Buffer) { + t.Helper() + var out bytes.Buffer + exec := New(&out) + exec.SetBackendFactory(func() backend.FullBackend { return modelsdkbackend.New() }) + t.Cleanup(func() { exec.Close() }) + run(t, exec, "CONNECT LOCAL '"+visitor.QuoteString(projectFixture(t))+"'") + out.Reset() + return exec, &out +} + +// runErr executes mdl and returns the first statement error. +func runErr(t *testing.T, exec *Executor, mdl string) error { + t.Helper() + prog, errs := visitor.Build(mdl) + if len(errs) > 0 { + t.Fatalf("parsing %q: %v", mdl, errs) + } + for _, stmt := range prog.Statements { + if err := exec.Execute(stmt); err != nil { + return err + } + } + return nil +} + +func toBsonD(t *testing.T, m map[string]any) bson.D { + t.Helper() + raw, err := bson.Marshal(m) + if err != nil { + t.Fatal(err) + } + var d bson.D + if err := bson.Unmarshal(raw, &d); err != nil { + t.Fatal(err) + } + return d +} + +// referenceSelectorBSON is a Forms$ReferenceSelector as Studio Pro stores one +// (its kind is enough: describe has no case for it). +func referenceSelectorBSON() map[string]any { + return map[string]any{ + "$Type": "Forms$ReferenceSelector", + "Name": "referenceSelector1", + } +} + +// unitIDByName finds a page or snippet unit. +func unitIDByName(t *testing.T, exec *Executor, kind, name string) string { + t.Helper() + switch kind { + case "page": + ps, err := exec.backend.ListPages() + if err != nil { + t.Fatal(err) + } + for _, p := range ps { + if p.Name == name { + return string(p.ID) + } + } + case "snippet": + ss, err := exec.backend.ListSnippets() + if err != nil { + t.Fatal(err) + } + for _, s := range ss { + if s.Name == name { + return string(s.ID) + } + } + } + t.Fatalf("no %s %s", kind, name) + return "" +} + +// seedIntoContainer appends Studio Pro-authored widgets to the stored +// container named container, the way a page built in Studio Pro holds them. +func seedIntoContainer(t *testing.T, exec *Executor, unitID, container string, widgets ...map[string]any) { + t.Helper() + raw, err := exec.backend.GetRawUnitBytes(model.ID(unitID)) + if err != nil { + t.Fatal(err) + } + var doc bson.D + if err := bson.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + found := false + var walk func(v any) any + walk = func(v any) any { + switch n := v.(type) { + case bson.D: + isTarget := false + for _, e := range n { + if e.Key == "Name" && e.Value == container { + isTarget = true + } + } + for i, e := range n { + if isTarget && e.Key == "Widgets" { + arr := e.Value.(bson.A) + for _, w := range widgets { + arr = append(arr, toBsonD(t, w)) + } + n[i].Value = arr + found = true + continue + } + n[i].Value = walk(e.Value) + } + return n + case bson.A: + for i := range n { + n[i] = walk(n[i]) + } + return n + } + return v + } + doc = walk(doc).(bson.D) + if !found { + t.Fatalf("no container %s with a Widgets list in the stored unit", container) + } + out, err := bson.Marshal(doc) + if err != nil { + t.Fatal(err) + } + if err := exec.backend.UpdateRawUnit(unitID, out); err != nil { + t.Fatal(err) + } +} + +func storedHas(t *testing.T, exec *Executor, unitID, needle string) bool { + t.Helper() + raw, err := exec.backend.GetRawUnitBytes(model.ID(unitID)) + if err != nil { + t.Fatal(err) + } + return bytes.Contains(raw, []byte(needle)) +} + +const issue1622Page = `create page MyFirstModule.Orders (Title: 'Orders', Layout: Atlas_Core.Atlas_Default) { + container ctn1 { + dynamictext txtHeading (Content: 'Orders') + } +};` + +// The report: describe → exec of a page holding a classic data grid and a +// reference selector. +func TestIssue1622_RewriteOfPageWithUnauthorableWidgetsIsRefused(t *testing.T) { + exec, out := pageFixtureExecutor(t) + run(t, exec, issue1622Page) + id := unitIDByName(t, exec, "page", "Orders") + seedIntoContainer(t, exec, id, "ctn1", legacyDataGridBSON(), referenceSelectorBSON()) + + out.Reset() + run(t, exec, "describe page MyFirstModule.Orders") + described := out.String() + for _, want := range []string{"-- Forms$DataGrid (grid1)", "-- Forms$ReferenceSelector (referenceSelector1)"} { + if !strings.Contains(described, want) { + t.Fatalf("describe output lacks %q:\n%s", want, described) + } + } + + out.Reset() + err := runErr(t, exec, described) + if err == nil { + t.Fatalf("executing the describe output replaced the page:\n%s", out.String()) + } + t.Log(err) + for _, want := range []string{"grid1", "Forms$DataGrid", "referenceSelector1", "Forms$ReferenceSelector", "alter page MyFirstModule.Orders"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal does not name %q:\n%v", want, err) + } + } + for _, kept := range []string{"Forms$DataGrid", "Forms$ReferenceSelector"} { + if !storedHas(t, exec, id, kept) { + t.Errorf("the stored page lost its %s", kept) + } + } +} + +// The refusal is about what would be lost, not about the page: once ALTER PAGE +// has removed the widgets on purpose, the rewrite goes through — and a page +// with no such widget was never affected. +func TestIssue1622_RewriteAfterDroppingTheWidgetsGoesThrough(t *testing.T) { + exec, out := pageFixtureExecutor(t) + run(t, exec, issue1622Page) + id := unitIDByName(t, exec, "page", "Orders") + seedIntoContainer(t, exec, id, "ctn1", legacyDataGridBSON(), referenceSelectorBSON()) + + run(t, exec, "alter page MyFirstModule.Orders { drop grid1, referenceSelector1; };") + out.Reset() + run(t, exec, "describe page MyFirstModule.Orders") + described := out.String() + if strings.Contains(described, "NOT re-executable") { + t.Fatalf("the widgets were not dropped:\n%s", described) + } + if err := runErr(t, exec, described); err != nil { + t.Errorf("a page with nothing MDL cannot author was refused: %v", err) + } +} + +// A snippet is rewritten by the same builder and loses the same widgets. +func TestIssue1622_RewriteOfSnippetWithUnauthorableWidgetIsRefused(t *testing.T) { + exec, out := pageFixtureExecutor(t) + run(t, exec, `create snippet MyFirstModule.OrderList { + container ctn1 { + dynamictext txtHeading (Content: 'Orders') + } +};`) + id := unitIDByName(t, exec, "snippet", "OrderList") + seedIntoContainer(t, exec, id, "ctn1", legacyDataGridBSON()) + + out.Reset() + run(t, exec, "describe snippet MyFirstModule.OrderList") + err := runErr(t, exec, out.String()) + if err == nil || !strings.Contains(err.Error(), "grid1") || !strings.Contains(err.Error(), "alter snippet MyFirstModule.OrderList") { + t.Errorf("snippet rewrite: %v", err) + } + if !storedHas(t, exec, id, "Forms$DataGrid") { + t.Error("the stored snippet lost its data grid") + } +} + +// `check -p` says the same before anything is written. +func TestIssue1622_CheckReportsTheWidgetsARewriteWouldDrop(t *testing.T) { + exec, out := pageFixtureExecutor(t) + run(t, exec, issue1622Page) + id := unitIDByName(t, exec, "page", "Orders") + seedIntoContainer(t, exec, id, "ctn1", legacyDataGridBSON(), referenceSelectorBSON()) + out.Reset() + run(t, exec, "describe page MyFirstModule.Orders") + + prog, errs := visitor.Build(out.String()) + if len(errs) > 0 { + t.Fatal(errs) + } + verrs := exec.ValidateProgram(prog) + joined := "" + for _, e := range verrs { + joined += e.Error() + "\n" + } + if !strings.Contains(joined, "grid1") || !strings.Contains(joined, "referenceSelector1") { + t.Errorf("check does not report the widgets a rewrite would drop:\n%s", joined) + } +} + +// A layout is described as `create or modify layout` and rebuilt by the same +// builder, so it loses the same widgets. +func TestIssue1622_RewriteOfLayoutWithUnauthorableWidgetIsRefused(t *testing.T) { + exec, out := pageFixtureExecutor(t) + run(t, exec, `create layout MyFirstModule.Shell (layouttype: 'Responsive') { + scrollcontainer sc1 { + region center { + container ctn1 { + dynamictext txtBrand (Content: 'Shop') + } + placeholder Main + } + } +};`) + ls, err := exec.backend.ListLayouts() + if err != nil { + t.Fatal(err) + } + id := "" + for _, l := range ls { + if l.Name == "Shell" { + id = string(l.ID) + } + } + seedIntoContainer(t, exec, id, "ctn1", referenceSelectorBSON()) + + out.Reset() + run(t, exec, "describe layout MyFirstModule.Shell") + described := out.String() + if !strings.Contains(described, "-- Forms$ReferenceSelector (referenceSelector1)") { + t.Fatalf("describe output lacks the reference selector:\n%s", described) + } + err = runErr(t, exec, described) + if err == nil || !strings.Contains(err.Error(), "referenceSelector1") || !strings.Contains(err.Error(), "alter layout MyFirstModule.Shell") { + t.Errorf("layout rewrite: %v", err) + } + if !storedHas(t, exec, id, "Forms$ReferenceSelector") { + t.Error("the stored layout lost its reference selector") + } +} + +// check predicts the stored state a statement meets: an ALTER earlier in the +// same script that drops the widgets means the rewrite after it loses nothing. +func TestIssue1622_CheckFollowsAnEarlierAlterInTheScript(t *testing.T) { + exec, out := pageFixtureExecutor(t) + run(t, exec, issue1622Page) + id := unitIDByName(t, exec, "page", "Orders") + seedIntoContainer(t, exec, id, "ctn1", legacyDataGridBSON(), referenceSelectorBSON()) + out.Reset() + run(t, exec, "describe page MyFirstModule.Orders") + + described := strings.Replace(out.String(), "mdl 1;\n", "", 1) + prog, errs := visitor.Build("mdl 1;\nalter page MyFirstModule.Orders { drop grid1, referenceSelector1; };\n" + described) + if len(errs) > 0 { + t.Fatal(errs) + } + for _, e := range exec.ValidateProgram(prog) { + if strings.Contains(e.Error(), "cannot author") { + t.Errorf("check refused a rewrite that follows the ALTER dropping the widgets: %v", e) + } + } +} + +// A widget the statement declares by name is the author writing it: a rewrite +// that restates it does not count it as lost. +func TestIssue1622_DeclaredWidgetIsNotCountedAsLost(t *testing.T) { + exec, _ := pageFixtureExecutor(t) + run(t, exec, issue1622Page) + id := unitIDByName(t, exec, "page", "Orders") + seedIntoContainer(t, exec, id, "ctn1", legacyDataGridBSON(), referenceSelectorBSON()) + + err := runErr(t, exec, `create or modify page MyFirstModule.Orders (Title: 'Orders', Layout: Atlas_Core.Atlas_Default) { + container ctn1 { + dynamictext txtHeading (Content: 'Orders') + container grid1 { dynamictext txtGrid (Content: 'replaced on purpose') } + } +};`) + if err == nil || strings.Contains(err.Error(), "grid1 (") || !strings.Contains(err.Error(), "referenceSelector1") { + t.Errorf("the declared grid1 was counted, or the undeclared selector was not: %v", err) + } +} diff --git a/mdl/executor/output_guard.go b/mdl/executor/output_guard.go index f9dc5bd174..b124f79d66 100644 --- a/mdl/executor/output_guard.go +++ b/mdl/executor/output_guard.go @@ -26,11 +26,13 @@ func newOutputGuard(w io.Writer, maxLines int) *outputGuard { return &outputGuard{w: w, maxLines: maxLines} } -// reset clears the line count for the next statement. -func (g *outputGuard) reset() { +// reset clears the line count for the next statement and sets its limit; +// maxLines <= 0 means no limit. +func (g *outputGuard) reset(maxLines int) { g.mu.Lock() g.lines = 0 g.exceeded = false + g.maxLines = maxLines g.mu.Unlock() } @@ -44,7 +46,7 @@ func (g *outputGuard) Write(p []byte) (int, error) { } g.lines += bytes.Count(p, []byte{'\n'}) - if g.lines > g.maxLines { + if g.maxLines > 0 && g.lines > g.maxLines { g.exceeded = true // Write the current chunk first so output isn't abruptly cut mid-line. _, _ = g.w.Write(p) @@ -70,5 +72,5 @@ func (g *outputGuard) exceededError() error { func (g *outputGuard) limitError() error { return mdlerrors.NewValidationf("output line limit exceeded (%d lines); output truncated and statement aborted. "+ - "Describe a large module per document (e.g. describe entity / microflow / page ...) instead of all at once", g.maxLines) + "Raise the limit with MXCLI_OUTPUT_LINE_LIMIT (0 for none), or describe per document (e.g. describe entity / microflow / page ...)", g.maxLines) } diff --git a/mdl/executor/register_stubs.go b/mdl/executor/register_stubs.go index f883442c37..841fd0f313 100644 --- a/mdl/executor/register_stubs.go +++ b/mdl/executor/register_stubs.go @@ -415,6 +415,9 @@ func registerRESTHandlers(r *Registry) { r.Register(&ast.DropRestClientStmt{}, func(ctx *ExecContext, stmt ast.Statement) error { return dropRestClient(ctx, stmt.(*ast.DropRestClientStmt)) }) + r.Register(&ast.AlterRestClientStmt{}, func(ctx *ExecContext, stmt ast.Statement) error { + return alterRestClient(ctx, stmt.(*ast.AlterRestClientStmt)) + }) r.Register(&ast.DescribeContractFromOpenAPIStmt{}, func(ctx *ExecContext, stmt ast.Statement) error { return withDescribeHeader(ctx, func() error { return describeContractFromOpenAPI(ctx, stmt.(*ast.DescribeContractFromOpenAPIStmt)) }) }) diff --git a/mdl/executor/registry_test.go b/mdl/executor/registry_test.go index a2dfca43cd..4ab7e4b321 100644 --- a/mdl/executor/registry_test.go +++ b/mdl/executor/registry_test.go @@ -184,6 +184,7 @@ func allKnownStatements() []ast.Statement { &ast.AlterPagesStylingStmt{}, &ast.AlterProjectSecurityStmt{}, &ast.AlterPublishedRestServiceStmt{}, + &ast.AlterRestClientStmt{}, &ast.AlterSettingsStmt{}, &ast.AlterStylingStmt{}, &ast.AlterUserRoleStmt{}, diff --git a/mdl/executor/strict_parsing_test.go b/mdl/executor/strict_parsing_test.go index 1d821273d6..8af6a052ee 100644 --- a/mdl/executor/strict_parsing_test.go +++ b/mdl/executor/strict_parsing_test.go @@ -24,7 +24,7 @@ var strictParsingCases = []struct { {"missing semicolon", "show modules\nshow entities;", "MDL-V1-SEMI", "no terminating `;`"}, {"slash terminator", "show modules;\n/\nshow entities;", "MDL-V1-SLASH", "`/` is not a statement terminator"}, {"unknown property key", "show modules;\ncreate published rest service M.S (Path: 'rest/s', Verison: '1.0') { };", "MDL-V1-PROP", "did you mean 'Version'"}, - {"mis-shaped property value", "show modules;\ncreate rest client M.Api (BaseUrl: 'https://x', Authentication: none) { operation Post { Method: post, Path: '/u', Response: json from $X } };", "MDL-V1-PROPVALUE", "takes none or json as $var"}, + {"mis-shaped property value", "show modules;\ncreate rest client M.Api (BaseUrl: 'https://x', Authentication: none) { operation Post { Method: post, Path: '/u', Response: json from $X } };", "MDL-V1-PROPVALUE", "takes none or json or json as $var"}, // A change of meaning, not a rejection: see TestStringEscapeUnderEachVersion. // R7 (ako/mxcli#755): a session command belongs at the REPL or on the // command line; under mdl 0 the script still sets the format. diff --git a/mdl/executor/system_enum_flow_type_test.go b/mdl/executor/system_enum_flow_type_test.go new file mode 100644 index 0000000000..1e6639eae2 --- /dev/null +++ b/mdl/executor/system_enum_flow_type_test.go @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// writtenMicroflowFromMDL parses one CREATE MICROFLOW statement and returns the +// microflow handed to the backend. Parsing (rather than building the AST by +// hand) is the point: the visitor types every bare qualified name in a +// parameter or return position as an entity, and the executor is what has to +// tell an enumeration apart. +func writtenMicroflowFromMDL(t *testing.T, src string) *microflows.Microflow { + t.Helper() + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parse error: %v", errs[0]) + } + stmt, ok := prog.Statements[0].(*ast.CreateMicroflowStmt) + if !ok { + t.Fatalf("expected CreateMicroflowStmt, got %T", prog.Statements[0]) + } + ctx, written := microflowWriteProbe(t, nil, model.ID("module-1")) + if err := execCreateMicroflow(ctx, stmt); err != nil { + t.Fatalf("CREATE MICROFLOW failed: %v", err) + } + if *written == nil { + t.Fatal("no microflow was written") + } + return *written +} + +func wantEnumerationType(t *testing.T, what string, dt microflows.DataType, qn string) { + t.Helper() + et, ok := dt.(*microflows.EnumerationType) + if !ok { + t.Fatalf("%s: got %T (%s), want *microflows.EnumerationType %s", what, dt, dt.GetTypeName(), qn) + } + if et.EnumerationQualifiedName != qn { + t.Fatalf("%s: enumeration = %q, want %q", what, et.EnumerationQualifiedName, qn) + } +} + +// TestCreateMicroflow_SystemEnumerationParameter: a System enumeration named +// bare in a parameter used to be written as DataTypes$ObjectType with Entity +// "System.UserType" — on every engine. The System module is not stored in the +// project, so the entity-then-enumeration lookup was skipped for it outright +// and the visitor's entity guess stood. The local engines wrote that without +// complaint (and describe rendered it as a plausible bare `System.UserType`); +// Studio Pro's PED refused it over --mcp: "Reference with qualified name +// System.UserType of type DomainModels$Entity not found." +func TestCreateMicroflow_SystemEnumerationParameter(t *testing.T) { + mf := writtenMicroflowFromMDL(t, `create microflow MyModule.MF ($T: System.UserType) +begin +end;`) + if len(mf.Parameters) != 1 { + t.Fatalf("got %d parameters, want 1", len(mf.Parameters)) + } + wantEnumerationType(t, "parameter $T", mf.Parameters[0].Type, "System.UserType") +} + +func TestCreateMicroflow_SystemEnumerationReturnType(t *testing.T) { + mf := writtenMicroflowFromMDL(t, `create microflow MyModule.MF () +returns System.DeviceType +begin + return System.DeviceType.Desktop; +end;`) + wantEnumerationType(t, "return type", mf.ReturnType, "System.DeviceType") +} + +// The explicit spelling is what DESCRIBE emits for an enumeration parameter, so +// it must round-trip. It used to be refused outright: the System module's +// enumerations are not in ListEnumerations, so validation reported +// "enumeration 'System.UserType' not found". +func TestCreateMicroflow_ExplicitSystemEnumerationParameter(t *testing.T) { + mf := writtenMicroflowFromMDL(t, `create microflow MyModule.MF ($T: enum System.UserType) +begin +end;`) + wantEnumerationType(t, "parameter $T", mf.Parameters[0].Type, "System.UserType") +} + +// Control: a System *entity* must stay an object type, or the fix has merely +// flipped the guess. +func TestCreateMicroflow_SystemEntityParameterStaysObject(t *testing.T) { + mf := writtenMicroflowFromMDL(t, `create microflow MyModule.MF ($U: System.User) +begin +end;`) + ot, ok := mf.Parameters[0].Type.(*microflows.ObjectType) + if !ok { + t.Fatalf("parameter $U: got %T, want *microflows.ObjectType", mf.Parameters[0].Type) + } + if ot.EntityQualifiedName != "System.User" { + t.Fatalf("parameter $U: entity = %q, want System.User", ot.EntityQualifiedName) + } +} + +// The nanoflow builder carries its own copy of the parameter resolution, so it +// needs its own case. +func TestCreateNanoflow_SystemEnumerationParameter(t *testing.T) { + prog, errs := visitor.Build(`create nanoflow MyModule.NF ($D: System.DeviceType) +begin +end;`) + if len(errs) > 0 { + t.Fatalf("parse error: %v", errs[0]) + } + stmt, ok := prog.Statements[0].(*ast.CreateNanoflowStmt) + if !ok { + t.Fatalf("expected CreateNanoflowStmt, got %T", prog.Statements[0]) + } + const moduleID = model.ID("module-1") + var written *microflows.Nanoflow + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { + return []*model.Module{{BaseElement: model.BaseElement{ID: moduleID}, Name: "MyModule"}}, nil + }, + GetModuleByNameFunc: func(name string) (*model.Module, error) { + return &model.Module{BaseElement: model.BaseElement{ID: moduleID}, Name: "MyModule"}, nil + }, + ListNanoflowsFunc: func() ([]*microflows.Nanoflow, error) { return nil, nil }, + CreateNanoflowFunc: func(nf *microflows.Nanoflow) error { + written = nf + return nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb)) + if err := execCreateNanoflow(ctx, stmt); err != nil { + t.Fatalf("CREATE NANOFLOW failed: %v", err) + } + if written == nil { + t.Fatal("no nanoflow was written") + } + wantEnumerationType(t, "parameter $D", written.Parameters[0].Type, "System.DeviceType") +} diff --git a/mdl/executor/validate.go b/mdl/executor/validate.go index 46cdc17273..43a994e59a 100644 --- a/mdl/executor/validate.go +++ b/mdl/executor/validate.go @@ -435,6 +435,9 @@ func validateProgramWithWarnings(ctx *ExecContext, prog *ast.Program) ([]error, } } errors = append(errors, validateForwardPageRefs(ctx, prog)...) + // A rewrite exec would refuse for the widgets MDL cannot author that it + // deletes (mendixlabs/mxcli#1622). + errors = append(errors, validateUnauthorableRewrites(ctx, prog)...) // The same for the other references exec resolves in statement order, where // the later definition is a `create or modify` (mendixlabs/mxcli#1211, #1212). errors = append(errors, validateForwardDefRefs(ctx, prog)...) diff --git a/mdl/executor/validate_rest_mapping.go b/mdl/executor/validate_rest_mapping.go index d4f22a6975..b63f48231d 100644 --- a/mdl/executor/validate_rest_mapping.go +++ b/mdl/executor/validate_rest_mapping.go @@ -24,11 +24,23 @@ import ( func ValidateRestClientMappings(prog *ast.Program) []linter.Violation { var out []linter.Violation for _, stmt := range prog.Statements { - createStmt, ok := stmt.(*ast.CreateRestClientStmt) - if !ok { + var opDefs []*ast.RestOperationDef + switch s := stmt.(type) { + case *ast.CreateRestClientStmt: + opDefs = s.Operations + case *ast.AlterRestClientStmt: + for _, a := range s.Actions { + switch a := a.(type) { + case *ast.RestClientAddOperationAction: + opDefs = append(opDefs, a.Operation) + case *ast.RestClientModifyOperationAction: + opDefs = append(opDefs, a.Operation) + } + } + default: continue } - for _, opDef := range createStmt.Operations { + for _, opDef := range opDefs { if err := checkInlineMappingBody(opDef); err != nil { out = append(out, linter.Violation{ RuleID: "MDL-REST01", diff --git a/mdl/executor/validate_widget_attribute_type.go b/mdl/executor/validate_widget_attribute_type.go index 11d6c31285..97545efa62 100644 --- a/mdl/executor/validate_widget_attribute_type.go +++ b/mdl/executor/validate_widget_attribute_type.go @@ -24,6 +24,9 @@ import ( // - MDL-WIDGET40 / CE0582: the classic drop-down on a project that uses the // React client. // +// MDL-WIDGET39 also judges the pluggable combo box in enumeration mode, the one +// pluggable input whose attribute type mxbuild checks the same way (CE7247). +// // Both are questions about the model (the attribute's type, the project's client // setting), so they live in the --references tier with the pluggable-widget // attribute-scope check, sharing its data-context walk. Without -p neither can @@ -39,7 +42,8 @@ const ( type inputAttributeRule struct { label string // mxbuild's name for the widget, as its messages print it accepts map[string]bool // attribute kinds (see attributeKind) it takes - listed string // mxbuild's own list, verbatim from CE2421 + listed string // mxbuild's own list, verbatim from its message + code string // the error mxbuild reports for a type outside accepts } // builtinInputAttributeRules is the measured matrix, not the Model SDK's: every @@ -52,12 +56,37 @@ type inputAttributeRule struct { // widget's CE2421 list is what it offers, not proof of what it refuses. var builtinInputAttributeRules = map[string]inputAttributeRule{ "textbox": {"text box", kindSet("String", "HashedString", "Integer", "Long", "Decimal", "AutoNumber"), - "Hashed string, Integer, Long, String, Decimal, AutoNumber"}, - "textarea": {"text area", kindSet("String"), "String"}, - "datepicker": {"date picker", kindSet("DateTime"), "Date and time"}, - "checkbox": {"check box", kindSet("Boolean"), "Boolean"}, - "radiobuttons": {"radio buttons", kindSet("Boolean", "Enumeration"), "Boolean, Enumeration"}, - "dropdown": {"drop-down", kindSet("Enumeration"), "Enumeration"}, + "Hashed string, Integer, Long, String, Decimal, AutoNumber", "CE2421"}, + "textarea": {"text area", kindSet("String"), "String", "CE2421"}, + "datepicker": {"date picker", kindSet("DateTime"), "Date and time", "CE2421"}, + "checkbox": {"check box", kindSet("Boolean"), "Boolean", "CE2421"}, + "radiobuttons": {"radio buttons", kindSet("Boolean", "Enumeration"), "Boolean, Enumeration", "CE2421"}, + "dropdown": {"drop-down", kindSet("Enumeration"), "Enumeration", "CE2421"}, +} + +// comboBoxEnumerationRule is the pluggable combo box with no options data +// source. Enumeration mode writes the attribute to `attributeEnumeration`, whose +// AllowedTypes is Enum alone. Measured on fresh 11.14.0 and 11.13.0 projects, +// one combo box per kind: every kind but Enumeration was CE7247 "Only attributes +// of type Enumeration are allowed here." Boolean is among them — the widget has +// a separate Boolean source (`attributeBoolean`), which MDL does not write. +var comboBoxEnumerationRule = inputAttributeRule{"combo box", kindSet("Enumeration"), "Enumeration", "CE7247"} + +// inputAttributeRuleFor is the rule w's attribute is judged by. A combo box +// with an options data source is in association mode and binds an +// association, not an attribute, so it has none. +func inputAttributeRuleFor(w *ast.WidgetV3) (rule inputAttributeRule, isComboBox, ok bool) { + if w.TypeIsGeneric { + return inputAttributeRule{}, false, false + } + if strings.EqualFold(w.Type, "combobox") { + if comboBoxHasOptionsDataSource(w) { + return inputAttributeRule{}, true, false + } + return comboBoxEnumerationRule, true, true + } + rule, ok = builtinInputAttributeRules[strings.ToLower(w.Type)] + return rule, false, ok } // measuredAttributeKinds are the kinds the matrix was measured against. A kind @@ -300,11 +329,12 @@ func overlayScriptMembers(ix memberTypeIndex, sc *scriptContext) memberTypeIndex return ix } -// checkInputBinding judges one built-in input widget's attribute against the -// entity its enclosing data container supplies. +// checkInputBinding judges one input widget's attribute — a built-in input, or +// a combo box in enumeration mode — against the entity its enclosing data +// container supplies. func (v *attributeScopeValidator) checkInputBinding(w *ast.WidgetV3, enclosing dataContext) { - rule, ok := builtinInputAttributeRules[strings.ToLower(w.Type)] - if !ok || w.TypeIsGeneric { + rule, isComboBox, ok := inputAttributeRuleFor(w) + if !ok { return } attr := strings.TrimPrefix(w.GetAttribute(), "$currentObject/") @@ -329,7 +359,9 @@ func (v *attributeScopeValidator) checkInputBinding(w *ast.WidgetV3, enclosing d return } if !found { - if len(segs) == 1 { + // The CE1613 below was measured for the built-in inputs only; a combo + // box over a bare association is left unjudged rather than guessed at. + if len(segs) == 1 && !isComboBox { if assocQN, _, isAssoc := v.types.follow(name, entity); isAssoc { v.errs = append(v.errs, fmt.Sprintf( "widget `%s` (%s): `%s` is the association %s, not an attribute — an input widget binds an "+ @@ -347,8 +379,8 @@ func (v *attributeScopeValidator) checkInputBinding(w *ast.WidgetV3, enclosing d } msg := fmt.Sprintf( "widget `%s` (%s) is bound to %s.%s, a%s %s attribute; the %s widget takes only attributes of type %s — "+ - "mxbuild reports CE2421 \"Only attributes of type %s are allowed here.\"", - w.Name, strings.ToLower(w.Type), declaring, a.name, anPrefix(a.kind), a.kind, rule.label, rule.listed, rule.listed) + "mxbuild reports %s \"Only attributes of type %s are allowed here.\"", + w.Name, strings.ToLower(w.Type), declaring, a.name, anPrefix(a.kind), a.kind, rule.label, rule.listed, rule.code, rule.listed) if alt := widgetsAccepting(a.kind); alt != "" { msg += " For " + a.kind + ", use " + alt + "." } diff --git a/mdl/executor/validate_widget_attribute_type_test.go b/mdl/executor/validate_widget_attribute_type_test.go index 6d190a8f93..0e7fdc539a 100644 --- a/mdl/executor/validate_widget_attribute_type_test.go +++ b/mdl/executor/validate_widget_attribute_type_test.go @@ -293,3 +293,57 @@ func TestCheck_ClassicDropDownOnNativePageIsClean(t *testing.T) { t.Fatalf("native page: MDL-WIDGET39 (CE2421) still applies, got %q", errs) } } + +// A combo box in enumeration mode (no options data source) binds its attribute +// to `attributeEnumeration`, which takes only an Enumeration. Measured on fresh +// 11.14.0 and 11.13.0 projects, one combo box per attribute type: every type but +// Enumeration was CE7247 "Only attributes of type Enumeration are allowed here." +// — Boolean included, though the widget has a separate Boolean source that MDL +// does not write. `check --references` passed all eight. +func TestCheck_ComboBoxEnumerationModeAttributeTypes(t *testing.T) { + ctx := typeCheckCtx(t, "No") + attrs := map[string]string{ + "Name": "String", "Color": "Enumeration", "Flag": "Boolean", "WhenAt": "DateTime", + "Num": "Integer", "LongNum": "Long", "Amount": "Decimal", "Seq": "AutoNumber", "Blob": "Binary", + } + for attrName, kind := range attrs { + body := fmt.Sprintf(`dataview dv (datasource: $Thing) { combobox cb1 (label: 'x', attribute: %s) }`, attrName) + errs := typeErrs(t, ctx, body, nil) + if want := kind != "Enumeration"; (len(errs) > 0) != want { + t.Errorf("combobox on %s (%s): flagged=%v, want %v (%q)", attrName, kind, len(errs) > 0, want, errs) + } + } +} + +// The reported case, and what its message must carry. +func TestCheck_ComboBoxOnBooleanIsCE7247(t *testing.T) { + ctx := typeCheckCtx(t, "Yes") + errs := typeErrs(t, ctx, `dataview dv (datasource: $Thing) { combobox cbFlag (label: 'F', attribute: Flag) }`, nil) + if len(errs) != 1 { + t.Fatalf("want one error, got %q", errs) + } + for _, want := range []string{"MDL-WIDGET39", "CE7247", "cbFlag", "Sales.Thing.Flag", "Boolean", + "Only attributes of type Enumeration are allowed here.", "`checkbox`", "`radiobuttons`"} { + if !strings.Contains(errs[0], want) { + t.Errorf("error should mention %q, got: %s", want, errs[0]) + } + } + if strings.Contains(errs[0], "CE2421") { + t.Errorf("a combo box is CE7247, not the built-in widgets' CE2421: %s", errs[0]) + } +} + +// Association mode binds an association, not `attributeEnumeration`: a data +// source puts the combo box in it whichever key names the association, so the +// attribute-type rule does not apply. +func TestCheck_ComboBoxAssociationModeIsNotTypeChecked(t *testing.T) { + ctx := typeCheckCtx(t, "Yes") + errs := typeErrs(t, ctx, `dataview dv (datasource: $Thing) { + combobox cb1 (label: 'O', attribute: Thing_Other, datasource: database Sales.Other, CaptionAttribute: Txt) + }`, nil) + for _, e := range errs { + if strings.Contains(e, "MDL-WIDGET39") { + t.Fatalf("association mode judged as an enumeration binding: %s", e) + } + } +} diff --git a/mdl/executor/validate_widget_expression_value.go b/mdl/executor/validate_widget_expression_value.go index 4af0d74504..4a08a385fe 100644 --- a/mdl/executor/validate_widget_expression_value.go +++ b/mdl/executor/validate_widget_expression_value.go @@ -77,6 +77,12 @@ func validateWidgetExpressionValuesIn(w *ast.WidgetV3, def *WidgetDefinition, it switch { case op == "expression": continue + case isExpr && isParameterAttributeText(text) && (op == "attribute" || !known): + // `$Param.Attr` reads an attribute of a page or snippet parameter + // (mendixlabs/mxcli#1591); exec resolves the parameter. It is no + // Mendix expression (that is `$Param/Attr`), so a key whose kind is + // out of sight — no project, no schema — is not refused as one. + continue case op == "image" && isImageObjectVariable(text): // `Image: $currentObject` binds the object in context (#1476). continue diff --git a/mdl/executor/validate_widget_kind.go b/mdl/executor/validate_widget_kind.go index 483a4d4a63..652961f342 100644 --- a/mdl/executor/validate_widget_kind.go +++ b/mdl/executor/validate_widget_kind.go @@ -39,7 +39,7 @@ func explicitWidgetID(w *ast.WidgetV3) string { // validateWidgetKind reports a widget whose kind mxcli cannot resolve, and a // container keyword the parent's definition does not declare. func validateWidgetKind(w *ast.WidgetV3, registry *WidgetRegistry, parentDef *WidgetDefinition, - parentObjectLists map[string]*ObjectListMapping, locationPrefix string) []linter.Violation { + parentObjectLists map[string]*ObjectListMapping, parentItem *ObjectListMapping, locationPrefix string) []linter.Violation { if w == nil || registry == nil { return nil } @@ -90,6 +90,13 @@ func validateWidgetKind(w *ast.WidgetV3, registry *WidgetRegistry, parentDef *Wi if _, known := registry.Get(strings.ToUpper(w.Type)); known { return nil } + // A slot of the enclosing object-list item — `headercontent { … }` in an + // Accordion group — is a container the item declares, routed by the + // item builder's slot pass. Refusing it as "not a widget" made the very + // block DESCRIBE prints for that slot impossible to run (#1453). + if itemDeclaresSlot(parentItem, w.Type) { + return nil + } // A container the parent declares is not a widget and never resolves in // the registry — that is what makes it a container. Accept it here and // let the object-list engine validate its properties. @@ -287,3 +294,17 @@ func nearestWidgetNames(registry *WidgetRegistry, name string) string { } return "" } + +// itemDeclaresSlot reports whether an object-list item mapping has a widget +// slot whose container keyword is kw (case-insensitive). +func itemDeclaresSlot(item *ObjectListMapping, kw string) bool { + if item == nil { + return false + } + for _, s := range item.ItemSlots { + if strings.EqualFold(s.MDLContainer, kw) || strings.EqualFold(s.PropertyKey, kw) { + return true + } + } + return false +} diff --git a/mdl/executor/validate_widget_kind_test.go b/mdl/executor/validate_widget_kind_test.go index b8a227df2b..2ef38e3dd2 100644 --- a/mdl/executor/validate_widget_kind_test.go +++ b/mdl/executor/validate_widget_kind_test.go @@ -77,7 +77,7 @@ func TestValidateWidgetKind_ContainerNotDeclaredByTheParentIsReported(t *testing }, } got := validateWidgetKind(&ast.WidgetV3{Type: "group", Name: "g1"}, - registry, parent, objectListMappingSet(parent), "page X") + registry, parent, objectListMappingSet(parent), nil, "page X") if len(got) == 0 || got[0].RuleID != "MDL-WIDGET26" { t.Fatalf("an undeclared container was not reported; got %v", got) @@ -102,7 +102,7 @@ func TestValidateWidgetKind_ContainerDeclaredByTheParentIsSilent(t *testing.T) { }, } if got := validateWidgetKind(&ast.WidgetV3{Type: "group", Name: "g1"}, - registry, parent, objectListMappingSet(parent), "page X"); len(got) != 0 { + registry, parent, objectListMappingSet(parent), nil, "page X"); len(got) != 0 { t.Errorf("the parent declares `group`, so it must not be reported: %v", got) } } @@ -116,7 +116,7 @@ func TestValidateWidgetKind_DeclaredChildSlotIsSilent(t *testing.T) { ChildSlots: []ChildSlotMapping{{MDLContainer: "GROUP", PropertyKey: "groups"}}, } if got := validateWidgetKind(&ast.WidgetV3{Type: "group", Name: "g1"}, - registry, parent, nil, "page X"); len(got) != 0 { + registry, parent, nil, nil, "page X"); len(got) != 0 { t.Errorf("a declared child slot must not be reported: %v", got) } } @@ -127,7 +127,7 @@ func TestValidateWidgetKind_DeclaredChildSlotIsSilent(t *testing.T) { func TestValidateWidgetKind_UnresolvableParentSilencesTheContainerRule(t *testing.T) { registry := LoadWidgetRegistry("") if got := validateWidgetKind(&ast.WidgetV3{Type: "group", Name: "g1"}, - registry, nil, nil, "page X"); len(got) != 0 { + registry, nil, nil, nil, "page X"); len(got) != 0 { t.Errorf("with no resolvable parent the rule must stay silent: %v", got) } } diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index 22d9638236..0693d99bdc 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -195,7 +195,7 @@ func validateWidgetTree(widgets []*ast.WidgetV3, registry *WidgetRegistry, locat // validateDocumentWidgetTree is validateWidgetTree for a document whose // entity-typed parameters are known (a CREATE PAGE / SNIPPET's `Params:`). func validateDocumentWidgetTree(widgets []*ast.WidgetV3, registry *WidgetRegistry, locationPrefix string, params []string) []linter.Violation { - out := validateWidgetTreeIn(widgets, registry, locationPrefix, nil, nil, atDocumentRoot().withDocumentParameters(params)) + out := validateWidgetTreeIn(widgets, registry, locationPrefix, nil, nil, nil, atDocumentRoot().withDocumentParameters(params)) // Only a whole document: an ALTER fragment's parent is out of sight, and // whether a row or column stores a name depends on it (#749). for _, msg := range checkMissingWidgetNames(widgets) { @@ -214,7 +214,7 @@ func validateDocumentWidgetTree(widgets []*ast.WidgetV3, registry *WidgetRegistr // encloses them is unknown, so rules that depend on the enclosing context stand // down rather than guess. func validateWidgetSubtree(widgets []*ast.WidgetV3, registry *WidgetRegistry, locationPrefix string) []linter.Violation { - return validateWidgetTreeIn(widgets, registry, locationPrefix, nil, nil, pageArgContext{}) + return validateWidgetTreeIn(widgets, registry, locationPrefix, nil, nil, nil, pageArgContext{}) } // validateWidgetTreeIn is validateWidgetTree with the *parent* widget's @@ -225,7 +225,11 @@ func validateWidgetSubtree(widgets []*ast.WidgetV3, registry *WidgetRegistry, lo // must be exempt from the MDL-WIDGET07 "unrecognized property, silently dropped" // warning. When the parent mapping is known, the child's enumeration // sub-properties are validated against their member keys (MDL-WIDGET08). (9a) -func validateWidgetTreeIn(widgets []*ast.WidgetV3, registry *WidgetRegistry, locationPrefix string, parentObjectLists map[string]*ObjectListMapping, parent *ast.WidgetV3, argCtx pageArgContext) []linter.Violation { +// +// parentItem is the object-list mapping of the parent when the parent is an +// object-list ITEM (an Accordion group): its widget slots (`headercontent`) +// are containers of that item, not widgets (#1453). +func validateWidgetTreeIn(widgets []*ast.WidgetV3, registry *WidgetRegistry, locationPrefix string, parentObjectLists map[string]*ObjectListMapping, parent *ast.WidgetV3, parentItem *ObjectListMapping, argCtx pageArgContext) []linter.Violation { var out []linter.Violation for _, w := range widgets { if w == nil { @@ -235,7 +239,7 @@ func validateWidgetTreeIn(widgets []*ast.WidgetV3, registry *WidgetRegistry, loc isObjectListItem := mapping != nil || isUniversalObjectListKeyword(w.Type) // Slice 0: is this a widget at all, and does the parent declare this // container? Both were previously left to `exec`. - out = append(out, validateWidgetKind(w, registry, lookupWidgetDef(parent, registry), parentObjectLists, locationPrefix)...) + out = append(out, validateWidgetKind(w, registry, lookupWidgetDef(parent, registry), parentObjectLists, parentItem, locationPrefix)...) // A keyword whose stored $Type Mendix no longer has. Unlike MDL-WIDGET25 // this needs no project: the type is unknown to every Mendix version. out = append(out, validateRetiredWidgetKind(w, locationPrefix)...) @@ -370,7 +374,7 @@ func validateWidgetTreeIn(widgets []*ast.WidgetV3, registry *WidgetRegistry, loc out = append(out, validateWidgetItemVisibility(parent, w, mapping, registry, locationPrefix)...) } if len(w.Children) > 0 { - out = append(out, validateWidgetTreeIn(w.Children, registry, locationPrefix, objectListMappingSet(def), w, argContextForChildren(w, argCtx))...) + out = append(out, validateWidgetTreeIn(w.Children, registry, locationPrefix, objectListMappingSet(def), w, mapping, argContextForChildren(w, argCtx))...) } } out = append(out, validateConsecutiveDynamicText(widgets, locationPrefix)...) diff --git a/mdl/executor/version_gates.go b/mdl/executor/version_gates.go index a4b77ae048..3e8cc9fd80 100644 --- a/mdl/executor/version_gates.go +++ b/mdl/executor/version_gates.go @@ -36,6 +36,9 @@ func statementVersionGates(stmt ast.Statement) []versionGate { case *ast.CreateRestClientStmt: return []versionGate{{"integration", "rest_client_basic", "create rest client", "upgrade your project to 10.1+"}} + case *ast.AlterRestClientStmt: + return []versionGate{{"integration", "rest_client_basic", "alter consumed rest service", + "upgrade your project to 10.1+"}} case *ast.CreateDataTransformerStmt: return []versionGate{{"integration", "data_transformer", "create data transformer", "upgrade your project to 11.9+"}} diff --git a/mdl/executor/widget_attribute_scope.go b/mdl/executor/widget_attribute_scope.go index 418b06d91a..c21b8f964b 100644 --- a/mdl/executor/widget_attribute_scope.go +++ b/mdl/executor/widget_attribute_scope.go @@ -132,13 +132,17 @@ func (ix attributeIndex) declares(entityQN, attr string) (found, known bool) { // wrote Entity.Missing and the first word of it was mx check's CE1613 // (mendixlabs/mxcli#1591). // -// Nothing is refused where the question cannot be answered: a qualified name or -// path, a scope the rule did not decide (scopeShared), or a model the index -// cannot read. +// A qualified name is judged by foreignQualifiedAttributeError. +// +// Nothing is refused where the question cannot be answered: a path, a scope the +// rule did not decide (scopeShared), or a model the index cannot read. func misboundAttributeError(where, attr string, scope bindingScope, outer string, dsEntities map[string]string, ix attributeIndex) error { - if attr == "" || strings.ContainsAny(attr, "./$") || scope.Kind == scopeShared || scope.Entity == "" { + if attr == "" || strings.ContainsAny(attr, "/$") || scope.Kind == scopeShared || scope.Entity == "" { return nil } + if strings.Contains(attr, ".") { + return foreignQualifiedAttributeError(where, attr, scope, ix) + } if found, known := ix.declares(scope.Entity, attr); found || !known { return nil } @@ -174,3 +178,40 @@ func misboundAttributeError(where, attr string, scope bindingScope, outer string "the widget links to that entity", where, attr, scope.Entity, scope.describe(), strings.Join(candidates, " and "), scope.Entity) } + +// foreignQualifiedAttributeError refuses a qualified attribute whose entity is +// neither the one the property binds to nor a generalization of it. The writer +// stores a qualified name as written, with no SourceVariable, and Mendix reads +// it against the bound object: mx check CE7247 "Attribute +// 'MyFirstModule.ChartContext.EventData' is not an attribute of entity +// 'MyFirstModule.ChartEntry'" (mendixlabs/mxcli#1591, 11.12.5). An attribute of +// another object is read through a parameter, `$Param.Attr` — what Studio Pro +// stores for that binding. +// +// An inherited attribute is qualified with the entity that declares it, which is +// on the bound entity's generalization chain, so it passes. A chain the index +// cannot follow, or an entity it does not hold, is not judged. +func foreignQualifiedAttributeError(where, attr string, scope bindingScope, ix attributeIndex) error { + owner := attributeEntity(attr) + if strings.Count(attr, ".") != 2 || ix.owners == nil { + return nil + } + if _, held := ix.owners[owner]; !held { + return nil + } + seen := map[string]bool{} + for cur := scope.Entity; cur != "" && !seen[cur]; cur = ix.parents[cur] { + if strings.EqualFold(cur, owner) { + return nil + } + if _, held := ix.owners[cur]; !held { + return nil + } + seen[cur] = true + } + return mdlerrors.NewValidationf( + "%s: `%s` is an attribute of %s, not of %s, the entity this property binds to (%s), "+ + "and writing it is CE7247 \"… is not an attribute of entity %s\". Bind an attribute of %s here, "+ + "or read the other object through a page or snippet parameter: `$Param.%s`", + where, attr, owner, scope.Entity, scope.describe(), scope.Entity, scope.Entity, shortAttributeName(attr)) +} diff --git a/mdl/executor/widget_engine.go b/mdl/executor/widget_engine.go index 495a5344fb..ecc3d17a88 100644 --- a/mdl/executor/widget_engine.go +++ b/mdl/executor/widget_engine.go @@ -663,9 +663,32 @@ func (e *PluggableWidgetEngine) Build(def *WidgetDefinition, w *ast.WidgetV3) (* // re-applies a property authored by its schema key AFTER every // datasource has run, so the shared context here is always the LAST // datasource's entity, which is what #647 wrote for every name. + if name, rest, named := namedObjectBinding(strVal); named { + // `$Param.Attr`: the attribute of a page or snippet parameter, + // whatever encloses the widget — the AttributeRef plus a + // SourceVariable naming the parameter, as on a mapped key. + // Written as the qualified name alone it is CE7247 + // (mendixlabs/mxcli#1591). + wv, entity, err := e.parameterBinding(w, propName, strVal, name) + if err != nil { + return nil, err + } + where := fmt.Sprintf("widget `%s` property `%s`", w.Name, propName) + if err := e.pageBuilder.rejectAssociationAsAttributeOn(w, name, rest, entity, where); err != nil { + return nil, err + } + builder.SetAttribute(propName, e.pageBuilder.resolveAttributePathForEntity(rest, entity)) + builder.SetSourceVariable(propName, &wv) + break + } attrPath := "" scope := e.bindingScopeFor(propName) if strings.Count(strVal, ".") >= 2 { + // Qualified: stored as written, so an attribute of an entity + // other than the bound one is refused (CE7247). + if err := e.refuseMisboundAttribute(w, propName, strVal, scope); err != nil { + return nil, err + } attrPath = strVal } else if scope.Entity != "" { if err := e.refuseMisboundAttribute(w, propName, strVal, scope); err != nil { @@ -1502,6 +1525,9 @@ func (e *PluggableWidgetEngine) resolveMapping(mapping PropertyMapping, w *ast.W // GenerateDefJSON orders first). Fall back to the generic single // `Attribute:` keyword for single-attribute widgets (ComboBox, filters). attr := namedPropValue(mapping, w) + if raw, _, ok := namedPropRaw(mapping, w); ok && attr == "" { + attr, _ = parameterAttributeValue(raw) + } if attr == "" && e.isPrimaryAttributeMapping(mapping) { // The bare `Attribute:` keyword names ONE attribute — the widget's // primary one. Applying it to every attribute-typed mapping wrote the @@ -2386,15 +2412,30 @@ func applyColumnSortableFallback(spec *backend.ObjectListItemSpec, sortableExpli } func defaultItemSlotKey(mapping *ObjectListMapping) string { - if len(mapping.ItemSlots) == 0 { - return "" - } + keys := make([]string, 0, len(mapping.ItemSlots)) for _, s := range mapping.ItemSlots { - if s.PropertyKey == "content" { - return s.PropertyKey + keys = append(keys, s.PropertyKey) + } + return bodyItemSlotKey(keys) +} + +// bodyItemSlotKey picks an object-list item's BODY among its widget slots +// (property keys, in schema order): `content` when the item has one, else the +// first. CREATE routes an item's loose children there; DESCRIBE prints that +// slot as the item's children and every other slot as a ` { … }` +// block. One function, so the two cannot disagree about which slot loose +// widgets belong to — they did, and an Accordion group's headerContent +// round-tripped into its content (mendixlabs/mxcli#1453). +func bodyItemSlotKey(keys []string) string { + for _, k := range keys { + if k == "content" { + return k } } - return mapping.ItemSlots[0].PropertyKey + if len(keys) == 0 { + return "" + } + return keys[0] } // lookupProperty does a case-insensitive lookup against an AST property map. diff --git a/mdl/executor/widget_expression_value.go b/mdl/executor/widget_expression_value.go index 55f2fb4536..3a547423f6 100644 --- a/mdl/executor/widget_expression_value.go +++ b/mdl/executor/widget_expression_value.go @@ -64,6 +64,9 @@ func scalarPropertyText(w *ast.WidgetV3, key string, raw any, op string) (string if op == "expression" { return v.Text, nil } + if text, ok := parameterAttributeValue(v); ok && op == "attribute" { + return text, nil // `$Param.Attr`, not an expression + } return "", expressionInScalarPropertyError(w, key, op, v.Text) case *ast.DataSourceV3: src, hasSrc := w.ExpressionSource(key) diff --git a/mdl/executor/widget_parameter_attribute_1591_test.go b/mdl/executor/widget_parameter_attribute_1591_test.go new file mode 100644 index 0000000000..ae457353c5 --- /dev/null +++ b/mdl/executor/widget_parameter_attribute_1591_test.go @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/model" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// mendixlabs/mxcli#1591, the half its first fix left open. A Custom chart +// inside a data view over $Entry had its eventDataAttribute bound to an +// attribute of the OTHER page parameter, $Ctx. Studio Pro stores that as the +// AttributeRef plus a SourceVariable naming the parameter, and builds clean. +// MDL had no spelling for it: +// +// - `eventDataAttribute: $Ctx.EventData` was MDL-WIDGET42 "is a attribute +// property and cannot hold the expression $Ctx.EventData"; +// - `eventDataAttribute: MyFirstModule.ChartContext.EventData` passed check +// and exec and wrote the AttributeRef alone — mx check CE7247 "Attribute +// 'MyFirstModule.ChartContext.EventData' is not an attribute of entity +// 'MyFirstModule.ChartEntry'" (measured on 11.12.5); +// - and DESCRIBE printed that stored pair as the qualified name, which +// re-executes into the CE7247 shape. +// +// The property here is the combo box's staticAttribute: linked to no +// datasource and mapped by no .def.json entry, so — like eventDataAttribute — +// it is written by the explicit-key pass. + +// paramDataViewEngine is scopeEngine inside a data view over Sales.Invoice on +// a page that also declares $Ctx: Sales.Customer. +func paramDataViewEngine(t *testing.T) *PluggableWidgetEngine { + t.Helper() + e := scopeEngine(t) + e.pageBuilder.paramScope = map[string]model.ID{"Ctx": "e-customer"} + e.pageBuilder.paramEntityNames["Ctx"] = "Sales.Customer" + return e +} + +func TestBuild_ExplicitAttributeKeyReadsAPageParameter(t *testing.T) { + e := paramDataViewEngine(t) + w := twoSourceWidget(map[string]any{"staticAttribute": &ast.WidgetExpressionV3{Text: "$Ctx.Name"}}) + widget, err := e.Build(multiSourceDef(), w) + if err != nil { + t.Fatalf("Build: %v", err) + } + val := widgetValueFor(widget.RawObject, "Sales.Customer.Name") + if val == nil { + t.Fatal("no WidgetValue binds Sales.Customer.Name — the binding was dropped") + } + sv, ok := dGet(val, "SourceVariable").(bson.D) + if !ok { + t.Fatalf("SourceVariable = %#v, want a Forms$PageVariable naming $Ctx", dGet(val, "SourceVariable")) + } + for k, want := range map[string]any{"$Type": "Forms$PageVariable", "PageParameter": "Ctx", + "SnippetParameter": "", "LocalVariable": "", "Widget": ""} { + if got := dGet(sv, k); got != want { + t.Errorf("SourceVariable.%s = %#v, want %#v", k, got, want) + } + } + + // Control: a `$name` that is no parameter is refused, as on a mapped key. + w = twoSourceWidget(map[string]any{"staticAttribute": &ast.WidgetExpressionV3{Text: "$Nope.Name"}}) + if _, err := paramDataViewEngine(t).Build(multiSourceDef(), w); err == nil { + t.Error("staticAttribute: $Nope.Name built; want a refusal") + } +} + +// The qualified name of an attribute the bound entity does not have is the +// CE7247 shape: refused by exec and check alike, pointing at `$Param.Attr`. +func TestCheck_QualifiedAttributeOfAnotherEntityIsRefused(t *testing.T) { + ctx := scopeCheckCtx(t) + s := scopeCheckPage(t, `staticAttribute: Sales.Customer.Name`) + errs := validatePluggableAttributeScopes(ctx, s.Layout, s.Parameters, allPageWidgets(s), newScriptContext()) + if len(errs) != 1 { + t.Fatalf("want one binding error, got %q", errs) + } + for _, want := range []string{"viewer", "staticAttribute", "Sales.Customer.Name", "Sales.Invoice", "CE7247", "$Param.Name"} { + if !strings.Contains(errs[0], want) { + t.Errorf("error should mention %q, got: %s", want, errs[0]) + } + } + + _, err := scopeEngine(t).Build(multiSourceDef(), twoSourceWidget(map[string]any{"staticAttribute": "Sales.Customer.Name"})) + if err == nil || !strings.Contains(err.Error(), errs[0]) { + t.Errorf("check and exec disagree:\n check: %s\n exec: %v", errs[0], err) + } + + // Controls: the bound entity's own attribute, qualified, passes both. + s = scopeCheckPage(t, `staticAttribute: Sales.Invoice.Reference`) + if errs := validatePluggableAttributeScopes(ctx, s.Layout, s.Parameters, allPageWidgets(s), newScriptContext()); len(errs) > 0 { + t.Errorf("Sales.Invoice.Reference: unexpected refusal %q", errs) + } + if _, err := scopeEngine(t).Build(multiSourceDef(), twoSourceWidget(map[string]any{"staticAttribute": "Sales.Invoice.Reference"})); err != nil { + t.Errorf("Sales.Invoice.Reference: unexpected exec refusal %v", err) + } +} + +// An inherited attribute stays qualified with the entity that declares it, and +// that entity is a generalization of the bound one — not "another entity". +func TestMisbound_QualifiedInheritedAttributePasses(t *testing.T) { + ix := attributeIndex{ + owners: map[string]map[string]bool{"Sales.Invoice": {"reference": true}, "Sales.Document": {"title": true}}, + parents: map[string]string{"Sales.Invoice": "Sales.Document"}, + } + scope := bindingScope{Entity: "Sales.Invoice", Kind: scopeEnclosing} + if err := misboundAttributeError("w", "Sales.Document.Title", scope, "Sales.Invoice", nil, ix); err != nil { + t.Errorf("inherited Sales.Document.Title refused: %v", err) + } + if err := misboundAttributeError("w", "Sales.Order.Number", scope, "Sales.Invoice", nil, ix); err != nil { + t.Errorf("an entity the index does not hold must not be judged: %v", err) + } +} + +// Check takes `$Param.Attr` on an attribute property, and still refuses a real +// expression there. +func TestCheck_ParameterAttributeOnAttributeKeyIsNotAnExpression(t *testing.T) { + widget42 := func(value string) []string { + t.Helper() + src := "create page M.P (Title: 'P', Layout: Atlas_Core.Atlas_Default, Params: ($Ctx: M.A)) {\n" + + " combobox cmb (Attribute: Name, staticAttribute: " + value + ")\n};" + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parsing %q: %v", src, errs) + } + var out []string + for _, v := range ValidateWidgetPropertiesForStatement(prog.Statements[0], LoadWidgetRegistry("")) { + if v.RuleID == "MDL-WIDGET42" { + out = append(out, v.Message) + } + } + return out + } + if got := widget42("$Ctx.Name"); len(got) > 0 { + t.Errorf("staticAttribute: $Ctx.Name refused: %q", got) + } + if got := widget42("$Ctx.Name + 'x'"); len(got) != 1 { + t.Errorf("staticAttribute: $Ctx.Name + 'x': want MDL-WIDGET42, got %q", got) + } +} + +func TestDescribePluggableAttributeOfAPageParameter(t *testing.T) { + ctx, _ := newMockCtx(t) + dv := chartInParameterDataView("MyFirstModule.ChartContext.EventData") + chart := dv["Widgets"].([]any)[0].(map[string]any) + props := chart["Object"].(map[string]any)["Properties"].([]any) + props[1].(map[string]any)["Value"].(map[string]any)["SourceVariable"] = map[string]any{ + "$Type": "Forms$PageVariable", "PageParameter": "Ctx", "SnippetParameter": "", "LocalVariable": "", "Widget": "", + } + dvs := parseRawWidget(ctx, dv) + got := map[string]string{} + for _, p := range dvs[0].Children[0].ExplicitProperties { + got[p.Key] = p.Value + } + if got["eventDataAttribute"] != "$Ctx.EventData" { + t.Errorf("eventDataAttribute = %q, want $Ctx.EventData (qualified, re-exec drops the parameter: CE7247)", + got["eventDataAttribute"]) + } + if got["dataAttribute"] != "JsonData" { + t.Errorf("dataAttribute = %q, want the bare JsonData", got["dataAttribute"]) + } +} + +// With no project the Custom chart has neither a definition nor a schema, so +// eventDataAttribute's kind is unknown. `$Param.Attr` is no Mendix expression +// (that is `$Param/Attr`), so it is not refused as one; a real expression on +// the same key still is. +func TestCheck_ParameterAttributeOnAnUnknownKeyIsNotAnExpression(t *testing.T) { + widget42 := func(value string) []string { + t.Helper() + src := "create page M.P (Title: 'P', Layout: Atlas_Core.Atlas_Default, Params: ($Ctx: M.A, $Entry: M.B)) {\n" + + " dataview dv (DataSource: $Entry) {\n customchart chart1 (eventDataAttribute: " + value + ")\n }\n};" + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parsing %q: %v", src, errs) + } + var out []string + for _, v := range ValidateWidgetPropertiesForStatement(prog.Statements[0], LoadWidgetRegistry("")) { + if v.RuleID == "MDL-WIDGET42" { + out = append(out, v.Message) + } + } + return out + } + if got := widget42("$Ctx.EventData"); len(got) > 0 { + t.Errorf("eventDataAttribute: $Ctx.EventData refused: %q", got) + } + if got := widget42("$Ctx.EventData + 'x'"); len(got) != 1 { + t.Errorf("eventDataAttribute: $Ctx.EventData + 'x': want MDL-WIDGET42, got %q", got) + } +} diff --git a/mdl/grammar/MDLParser.g4 b/mdl/grammar/MDLParser.g4 index b31d8cfd7e..63fda217df 100644 --- a/mdl/grammar/MDLParser.g4 +++ b/mdl/grammar/MDLParser.g4 @@ -284,6 +284,9 @@ alterStatement | alterMessageDefinitionCollectionStatement | alterMessageDefinitionStatement | ALTER PUBLISHED REST SERVICE qualifiedName alterPublishedRestServiceAction (COMMA? alterPublishedRestServiceAction)* + // Edits the stored consumed REST service in place: an operation the statement + // does not name is written back as stored (mendixlabs/mxcli#1608). + | ALTER consumedRestServiceKw qualifiedName alterConsumedRestServiceAction (COMMA? alterConsumedRestServiceAction)* | ALTER aiModelKw qualifiedName SET agentEditorAlterAssignment (COMMA agentEditorAlterAssignment)* | ALTER KNOWLEDGE BASE qualifiedName SET agentEditorAlterAssignment (COMMA agentEditorAlterAssignment)* | ALTER CONSUMED MCP SERVICE qualifiedName SET agentEditorAlterAssignment (COMMA agentEditorAlterAssignment)* diff --git a/mdl/grammar/domains/MDLService.g4 b/mdl/grammar/domains/MDLService.g4 index f8a0eda056..11812ae887 100644 --- a/mdl/grammar/domains/MDLService.g4 +++ b/mdl/grammar/domains/MDLService.g4 @@ -107,6 +107,27 @@ createRestClientStatement (LBRACE restClientOperation* RBRACE)? ; +/** + * ALTER CONSUMED REST SERVICE — the operation body is CREATE's; MODIFY writes + * only the keys it names and DROP removes one operation, so every operation the + * statement does not name stays as stored, including one made in Studio Pro. + * + * @example + * ```mdl + * alter consumed rest service Shop.BookingApi + * add operation GetBooking (Method: get, Path: '/bookings/{id}', Parameters: ($id: String), Response: json), + * modify operation PostBooking (Timeout: 60), + * drop operation OldBooking, + * set (BaseUrl: 'https://api.example.com/v2'); + * ``` + */ +alterConsumedRestServiceAction + : SET LPAREN restClientProperty (COMMA restClientProperty)* COMMA? RPAREN + | ADD restClientOperation + | MODIFY restClientOperation + | DROP OPERATION (identifierOrKeyword | STRING_LITERAL) + ; + restClientProperty : identifierOrKeyword COLON STRING_LITERAL // BaseUrl: '...', Username: '...' | identifierOrKeyword COLON VARIABLE /* @alias MDL-DEPR083 */ // Username: $Constant (a constant of the client's own module) @@ -132,7 +153,10 @@ restClientOpProp | identifierOrKeyword COLON NONE // Response: NONE | identifierOrKeyword COLON LPAREN restClientParamItem (COMMA restClientParamItem)* RPAREN // Parameters/Query | identifierOrKeyword COLON LPAREN restClientHeaderItem (COMMA restClientHeaderItem)* RPAREN // Headers - | identifierOrKeyword COLON (JSON | FILE_KW | STRING_TYPE | STATUS) (FROM | AS) VARIABLE // Body: JSON FROM $v, Response: JSON AS $v + // Body: JSON FROM $v, Response: JSON AS $v. A response's variable is optional: + // an operation stores none (the call activity names it), so describe prints + // `Response: json` and that has to run again (mendixlabs/mxcli#1608). + | identifierOrKeyword COLON (JSON | FILE_KW | STRING_TYPE | STATUS) ((FROM | AS) VARIABLE)? | identifierOrKeyword COLON TEMPLATE STRING_LITERAL // Body: TEMPLATE '...' | identifierOrKeyword COLON MAPPING qualifiedName (LBRACE restClientMappingEntry* RBRACE)? // Body/Response: MAPPING Entity { ... } ; diff --git a/mdl/openapi/parser.go b/mdl/openapi/parser.go index c3062e89f5..f6ae6ef5bc 100644 --- a/mdl/openapi/parser.go +++ b/mdl/openapi/parser.go @@ -19,6 +19,10 @@ import ( // Spec is a minimal OpenAPI 3.0 representation covering fields needed for REST client generation. type Spec struct { + // OpenAPI is the 3.x version string; Swagger is set instead by a 2.0 + // document, which this parser does not read (see ParseSpec). + OpenAPI string `json:"openapi" yaml:"openapi"` + Swagger string `json:"swagger" yaml:"swagger"` Info Info `json:"info" yaml:"info"` Servers []Server `json:"servers" yaml:"servers"` Paths map[string]PathItem `json:"paths" yaml:"paths"` @@ -125,6 +129,16 @@ func ParseSpec(data []byte, ext string) (*Spec, error) { return nil, fmt.Errorf("YAML parse error: %w", err) } } + // A Swagger 2.0 document unmarshals without error into the OpenAPI 3 types, + // but none of the fields that carry its contract (host/basePath, response + // `schema`, `in: body`, `definitions`) exist there. Converting it produced + // an empty contract — BaseUrl '', no request bodies, every response + // `none` — with no warning (mendixlabs/mxcli#280). Refuse it instead. + if spec.Swagger != "" { + return nil, fmt.Errorf("this is a Swagger %s document; only OpenAPI 3 is supported. "+ + "Convert it to OpenAPI 3 first (for example with https://converter.swagger.io "+ + "or `npx swagger2openapi`) and import the converted file", spec.Swagger) + } return &spec, nil } diff --git a/mdl/openapi/parser_test.go b/mdl/openapi/parser_test.go new file mode 100644 index 0000000000..3863c5979e --- /dev/null +++ b/mdl/openapi/parser_test.go @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: Apache-2.0 + +package openapi + +import ( + "strings" + "testing" +) + +// mendixlabs/mxcli#280 (ako/mxcli#1234) used the Swagger 2.0 Petstore. The +// parser reads only OpenAPI 3 fields, so a 2.0 document parsed "successfully" +// into an empty contract — BaseUrl ”, request bodies dropped, every response +// `none` — and nothing said so. Refusing is strictly better than that. +const swagger2Petstore = `{ + "swagger": "2.0", + "info": {"title": "Swagger Petstore", "version": "1.0.7"}, + "host": "petstore.swagger.io", + "basePath": "/v2", + "schemes": ["https"], + "paths": { + "/pet/{petId}": { + "get": { + "operationId": "getPetById", + "produces": ["application/json"], + "parameters": [{"name": "petId", "in": "path", "required": true, "type": "integer", "format": "int64"}], + "responses": {"200": {"description": "ok", "schema": {"$ref": "#/definitions/Pet"}}} + } + } + }, + "definitions": {"Pet": {"type": "object", "properties": {"id": {"type": "integer"}}}} +}` + +func TestParseSpec_RefusesSwagger2(t *testing.T) { + for _, tc := range []struct{ name, ext, doc string }{ + {"json", ".json", swagger2Petstore}, + {"yaml", ".yaml", "swagger: '2.0'\ninfo:\n title: T\n version: '1'\npaths: {}\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + spec, err := ParseSpec([]byte(tc.doc), tc.ext) + if err == nil { + t.Fatalf("a Swagger 2.0 document was accepted (%d paths) — it converts into an empty contract with no warning", len(spec.Paths)) + } + for _, want := range []string{"Swagger 2.0", "OpenAPI 3"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error should name %q so the user knows what to do: %v", want, err) + } + } + }) + } +} + +// CONTROL: OpenAPI 3.0 and 3.1 still parse. +func TestParseSpec_AcceptsOpenAPI3(t *testing.T) { + for _, v := range []string{"3.0.2", "3.1.0"} { + doc := `{"openapi": "` + v + `", "info": {"title": "T", "version": "1"}, "servers": [{"url": "https://x.example/api"}], + "paths": {"/a": {"get": {"operationId": "getA", "responses": {"200": {"description": "ok", "content": {"application/json": {}}}}}}}}` + spec, err := ParseSpec([]byte(doc), ".json") + if err != nil { + t.Fatalf("OpenAPI %s refused: %v", v, err) + } + svc, _, err := ToRestClientModel(spec, "Svc", "") + if err != nil || len(svc.Operations) != 1 || svc.Operations[0].ResponseType != "JSON" { + t.Errorf("OpenAPI %s did not convert to one JSON operation: %+v, %v", v, svc, err) + } + } +} diff --git a/mdl/roundtrip/dangling_selection_1415_test.go b/mdl/roundtrip/dangling_selection_1415_test.go index dd0d5343d4..86d83723d0 100644 --- a/mdl/roundtrip/dangling_selection_1415_test.go +++ b/mdl/roundtrip/dangling_selection_1415_test.go @@ -19,6 +19,9 @@ import ( // `DataSource: selection grid1`, which check -p and exec refused as a // reference to a widget the script does not contain. // +// Since mendixlabs/mxcli#1622 the describe output is still refused, for the +// right reason: executing it would delete the grid. +// // mxcli cannot write a Forms$DataGrid, so the fixture gets one by retyping a // list view in place: "Forms$ListView" and "Forms$DataGrid" are the same // length, so the BSON string is replaced byte for byte and nothing else in the @@ -70,10 +73,24 @@ create page RT1415.P (Title: 'P', Layout: Atlas_Core.Atlas_Default) { t.Errorf("check refuses the describe output: [%s] %s", v.RuleID, v.Message) } } - if errs := h.exe.ValidateProgram(prog); len(errs) > 0 { - t.Errorf("check --references refuses the describe output: %v\n%s", errs, described) + // The #1415 refusal — a selection naming a widget the script lacks — is + // gone. What remains is mendixlabs/mxcli#1622's: the rewrite would delete + // the grid MDL cannot author, so check and exec both refuse it, naming + // the grid, and the stored page keeps it. + errs = h.exe.ValidateProgram(prog) + if len(errs) != 1 || !strings.Contains(errs[0].Error(), "lv1 (Forms$DataGrid)") { + t.Errorf("check --references: want only the refusal to delete lv1, got %v\n%s", errs, described) + } + for _, e := range errs { + if strings.Contains(e.Error(), "no widget with that name exists") { + t.Errorf("the #1415 dangling-selection refusal is back: %v", e) + } + } + err = h.exe.ExecuteProgram(prog) + if err == nil || !strings.Contains(err.Error(), "lv1 (Forms$DataGrid)") { + t.Errorf("exec of the describe output: want the refusal to delete lv1, got %v", err) } - if err := h.exe.ExecuteProgram(prog); err != nil { - t.Errorf("exec the describe output: %v\n%s", err, described) + if again, derr := h.describe("page RT1415.P"); derr != nil || !strings.Contains(again, "Forms$DataGrid (lv1)") { + t.Errorf("the stored page lost its grid (err %v):\n%s", derr, again) } } diff --git a/mdl/roundtrip/testapp_allowlist_test.go b/mdl/roundtrip/testapp_allowlist_test.go index 1f61b39ac5..330ac976d3 100644 --- a/mdl/roundtrip/testapp_allowlist_test.go +++ b/mdl/roundtrip/testapp_allowlist_test.go @@ -151,7 +151,6 @@ var testAppKnownFailures = map[string]knownFailure{ "page WorkflowCommons.WorkflowUserTaskView_View": {laws: []law{lawGetPut}, issue: "#721 #826", why: "executes since describe prints snippet call Params (#826); the page Appearance keeps 1 of 3 DesignProperties (#721 C)"}, "page WorkflowCommons.Workflow_Dashboard": {laws: []law{lawGetPut}, issue: "#721", why: "page: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, "rest client Clients.OrdersRestClient": {laws: []law{lawParse}, issue: "#721", why: "rest client: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "rest client Mappings.RestClient": {laws: []law{lawParse}, issue: "#721", why: "rest client: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, "rule Rules.Rule1": {laws: []law{lawGetPut}, issue: "#721", why: "rule: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, "rule Rules.Rule2": {laws: []law{lawGetPut}, issue: "#721", why: "rule: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, "scheduled event WorkflowCommons.SE_WorkflowAuditTrailRecord_CleanUp": {laws: []law{lawGetPut}, issue: "#721", why: "scheduled event: breaks getput on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, diff --git a/mdl/visitor/visitor_alter.go b/mdl/visitor/visitor_alter.go index 05968da86c..700d994f41 100644 --- a/mdl/visitor/visitor_alter.go +++ b/mdl/visitor/visitor_alter.go @@ -60,6 +60,12 @@ func (b *Builder) ExitAlterStatement(ctx *parser.AlterStatementContext) { return } + // Handle ALTER CONSUMED REST SERVICE + if ctx.ConsumedRestServiceKw() != nil { + b.exitAlterRestClientStatement(ctx) + return + } + // Handle agent-editor ALTER statements if ctx.AiModelKw() != nil || ctx.AGENT() != nil || (ctx.KNOWLEDGE() != nil && ctx.BASE() != nil) || diff --git a/mdl/visitor/visitor_rest.go b/mdl/visitor/visitor_rest.go index 8a90ac38af..bde927f380 100644 --- a/mdl/visitor/visitor_rest.go +++ b/mdl/visitor/visitor_rest.go @@ -59,38 +59,7 @@ func (b *Builder) ExitCreateRestClientStatement(ctx *parser.CreateRestClientStat } case "authentication": if pc.BASIC() != nil { - authDef := &ast.RestAuthDef{Scheme: "BASIC"} - for _, subProp := range pc.AllRestClientProperty() { - sp, spOk := subProp.(*parser.RestClientPropertyContext) - if !spOk || sp == nil || sp.IdentifierOrKeyword() == nil { - continue - } - rawSubKey := identifierOrKeywordText(sp.IdentifierOrKeyword().(*parser.IdentifierOrKeywordContext)) - b.checkProperty(sp, &restClientBasicAuthSchema, rawSubKey, restClientPropertyShape(sp)) - subKey := strings.ToLower(rawSubKey) - var val string - if sl := sp.STRING_LITERAL(); sl != nil { - val = unquoteStringLit(sl) - } else if v := sp.VARIABLE(); v != nil { - // $Constant (MDL-DEPR083): a constant of the service's own - // module, stored qualified as `@Module.Const` stores it. - val = "$" + dollarConstantName(stmt.Name.Module, v.GetText()) - b.recordDollarConstant(v, stmt.Name.Module) - } else if sp.AT() != nil { - // @Module.Constant reference (preferred Mendix convention) - // Store with $ prefix so the writer serializes as Rest$ConstantValue - if qn := sp.QualifiedName(); qn != nil { - val = "$" + getQualifiedNameText(qn) - } - } - switch subKey { - case "username": - authDef.Username = val - case "password": - authDef.Password = val - } - } - stmt.Authentication = authDef + stmt.Authentication = b.restClientBasicAuth(pc, stmt.Name.Module) } // NONE: leave Authentication nil } @@ -98,18 +67,9 @@ func (b *Builder) ExitCreateRestClientStatement(ctx *parser.CreateRestClientStat // Parse operations for _, opCtx := range ctx.AllRestClientOperation() { - oc, ok := opCtx.(*parser.RestClientOperationContext) - if !ok || oc == nil { - continue - } - for _, p := range oc.AllRestClientOpProp() { - if pc, ok := p.(*parser.RestClientOpPropContext); ok && pc != nil && pc.IdentifierOrKeyword() != nil { - key := identifierOrKeywordText(pc.IdentifierOrKeyword().(*parser.IdentifierOrKeywordContext)) - b.checkProperty(pc, &restClientOperationSchema, key, restClientOpPropShape(pc)) - } + if opDef := b.restClientOperation(opCtx); opDef != nil { + stmt.Operations = append(stmt.Operations, opDef) } - opDef := parseRestClientOperation(oc) - stmt.Operations = append(stmt.Operations, opDef) } // Check for CREATE OR MODIFY @@ -124,6 +84,58 @@ func (b *Builder) ExitCreateRestClientStatement(ctx *parser.CreateRestClientStat b.statements = append(b.statements, stmt) } +// restClientBasicAuth reads `Authentication: basic (Username: …, Password: …)` +// for the service of module moduleName. +func (b *Builder) restClientBasicAuth(pc *parser.RestClientPropertyContext, moduleName string) *ast.RestAuthDef { + authDef := &ast.RestAuthDef{Scheme: "BASIC"} + for _, subProp := range pc.AllRestClientProperty() { + sp, spOk := subProp.(*parser.RestClientPropertyContext) + if !spOk || sp == nil || sp.IdentifierOrKeyword() == nil { + continue + } + rawSubKey := identifierOrKeywordText(sp.IdentifierOrKeyword().(*parser.IdentifierOrKeywordContext)) + b.checkProperty(sp, &restClientBasicAuthSchema, rawSubKey, restClientPropertyShape(sp)) + subKey := strings.ToLower(rawSubKey) + var val string + if sl := sp.STRING_LITERAL(); sl != nil { + val = unquoteStringLit(sl) + } else if v := sp.VARIABLE(); v != nil { + // $Constant (MDL-DEPR083): a constant of the service's own + // module, stored qualified as `@Module.Const` stores it. + val = "$" + dollarConstantName(moduleName, v.GetText()) + b.recordDollarConstant(v, moduleName) + } else if sp.AT() != nil { + // @Module.Constant reference (preferred Mendix convention) + // Store with $ prefix so the writer serializes as Rest$ConstantValue + if qn := sp.QualifiedName(); qn != nil { + val = "$" + getQualifiedNameText(qn) + } + } + switch subKey { + case "username": + authDef.Username = val + case "password": + authDef.Password = val + } + } + return authDef +} + +// restClientOperation checks and reads one `operation Name ( … )`. +func (b *Builder) restClientOperation(opCtx parser.IRestClientOperationContext) *ast.RestOperationDef { + oc, ok := opCtx.(*parser.RestClientOperationContext) + if !ok || oc == nil { + return nil + } + for _, p := range oc.AllRestClientOpProp() { + if pc, ok := p.(*parser.RestClientOpPropContext); ok && pc != nil && pc.IdentifierOrKeyword() != nil { + key := identifierOrKeywordText(pc.IdentifierOrKeyword().(*parser.IdentifierOrKeywordContext)) + b.checkProperty(pc, &restClientOperationSchema, key, restClientOpPropShape(pc)) + } + } + return parseRestClientOperation(oc) +} + // parseRestClientOperation parses a single OPERATION { Key: Value, ... } block. func parseRestClientOperation(ctx *parser.RestClientOperationContext) *ast.RestOperationDef { op := &ast.RestOperationDef{} @@ -147,6 +159,11 @@ func parseRestClientOperation(ctx *parser.RestClientOperationContext) *ast.RestO continue } parseRestClientOpProp(pc, op) + if iok := pc.IdentifierOrKeyword(); iok != nil { + if k, ok := restClientOperationSchema.lookup(identifierOrKeywordText(iok.(*parser.IdentifierOrKeywordContext))); ok { + op.Keys = append(op.Keys, k.name) + } + } } return op @@ -225,6 +242,24 @@ func parseRestClientOpProp(ctx *parser.RestClientOpPropContext, op *ast.RestOper return } + // Response: JSON (no variable — an operation stores none; mendixlabs/mxcli#1608) + if key == "response" { + switch { + case ctx.JSON() != nil: + op.ResponseType = "json" + return + case ctx.STRING_TYPE() != nil: + op.ResponseType = "string" + return + case ctx.FILE_KW() != nil: + op.ResponseType = "file" + return + case ctx.STATUS() != nil: + op.ResponseType = "status" + return + } + } + // NONE if ctx.NONE() != nil { switch key { @@ -552,6 +587,65 @@ func buildPublishedRestResourceDef(rc *parser.PublishedRestResourceContext) *ast return resDef } +// exitAlterRestClientStatement handles ALTER CONSUMED REST SERVICE +// (mendixlabs/mxcli#1608). +func (b *Builder) exitAlterRestClientStatement(ctx *parser.AlterStatementContext) { + qn := ctx.QualifiedName() + if qn == nil { + return + } + stmt := &ast.AlterRestClientStmt{Name: buildQualifiedName(qn)} + for _, actCtx := range ctx.AllAlterConsumedRestServiceAction() { + ac, ok := actCtx.(*parser.AlterConsumedRestServiceActionContext) + if !ok || ac == nil { + continue + } + switch { + case ac.SET() != nil: + set := &ast.RestClientSetAction{} + for _, propCtx := range ac.AllRestClientProperty() { + pc, ok := propCtx.(*parser.RestClientPropertyContext) + if !ok || pc == nil || pc.IdentifierOrKeyword() == nil { + continue + } + rawKey := identifierOrKeywordText(pc.IdentifierOrKeyword().(*parser.IdentifierOrKeywordContext)) + b.checkProperty(pc, &restClientAlterSetSchema, rawKey, restClientPropertyShape(pc)) + switch strings.ToLower(rawKey) { + case "baseurl": + if sl := pc.STRING_LITERAL(); sl != nil { + set.BaseUrl, set.BaseUrlSet = unquoteStringLit(sl), true + } + case "authentication": + switch { + case pc.BASIC() != nil: + set.Authentication, set.AuthenticationSet = b.restClientBasicAuth(pc, stmt.Name.Module), true + case pc.NONE() != nil: + set.AuthenticationSet = true + } + } + } + stmt.Actions = append(stmt.Actions, set) + case ac.ADD() != nil: + if op := b.restClientOperation(ac.RestClientOperation()); op != nil { + stmt.Actions = append(stmt.Actions, &ast.RestClientAddOperationAction{Operation: op}) + } + case ac.MODIFY() != nil: + if op := b.restClientOperation(ac.RestClientOperation()); op != nil { + stmt.Actions = append(stmt.Actions, &ast.RestClientModifyOperationAction{Operation: op}) + } + case ac.DROP() != nil: + name := "" + if iok := ac.IdentifierOrKeyword(); iok != nil { + name = identifierOrKeywordText(iok.(*parser.IdentifierOrKeywordContext)) + } else if sl := ac.STRING_LITERAL(); sl != nil { + name = unquoteStringLit(sl) + } + stmt.Actions = append(stmt.Actions, &ast.RestClientDropOperationAction{Name: name}) + } + } + b.statements = append(b.statements, stmt) +} + // exitAlterPublishedRestServiceStatement handles ALTER PUBLISHED REST SERVICE. func (b *Builder) exitAlterPublishedRestServiceStatement(ctx *parser.AlterStatementContext) { qn := ctx.QualifiedName() diff --git a/mdl/visitor/visitor_rest_test.go b/mdl/visitor/visitor_rest_test.go index d73608eddc..93432aa31d 100644 --- a/mdl/visitor/visitor_rest_test.go +++ b/mdl/visitor/visitor_rest_test.go @@ -3,6 +3,7 @@ package visitor import ( + "strings" "testing" "github.com/mendixlabs/mxcli/mdl/ast" @@ -137,3 +138,56 @@ func TestCreatePublishedRestService_NonStringPropertyValue_NoPanic(t *testing.T) }) } } + +// mendixlabs/mxcli#1608: ALTER CONSUMED REST SERVICE, every action, and the +// keys a modify names (only those are rewritten). +func TestAlterRestClient_Actions(t *testing.T) { + prog, errs := Build(`alter consumed rest service M.Api + add operation GetBooking (Method: get, Path: '/b/{id}', Parameters: ($id: String), Response: json), + modify operation PostBooking (Timeout: 60, Method: put), + drop operation OldBooking, + set (BaseUrl: 'https://x/v2', Authentication: basic (Username: 'u', Password: @M.Pw)); +alter rest client M.Api set (Authentication: none);`) + if len(errs) > 0 { + t.Fatalf("parse errors: %v", errs) + } + if len(prog.Statements) != 2 { + t.Fatalf("got %d statements", len(prog.Statements)) + } + stmt, ok := prog.Statements[0].(*ast.AlterRestClientStmt) + if !ok { + t.Fatalf("got %T", prog.Statements[0]) + } + if stmt.Name.String() != "M.Api" || len(stmt.Actions) != 4 { + t.Fatalf("got %s with %d actions", stmt.Name, len(stmt.Actions)) + } + add := stmt.Actions[0].(*ast.RestClientAddOperationAction) + if add.Operation.Name != "GetBooking" || add.Operation.Path != "/b/{id}" { + t.Errorf("add: %+v", add.Operation) + } + mod := stmt.Actions[1].(*ast.RestClientModifyOperationAction) + if got := strings.Join(mod.Operation.Keys, ","); got != "Timeout,Method" || mod.Operation.Timeout != 60 { + t.Errorf("modify keys %q, timeout %d", got, mod.Operation.Timeout) + } + if drop := stmt.Actions[2].(*ast.RestClientDropOperationAction); drop.Name != "OldBooking" { + t.Errorf("drop: %q", drop.Name) + } + set := stmt.Actions[3].(*ast.RestClientSetAction) + if !set.BaseUrlSet || set.BaseUrl != "https://x/v2" || !set.AuthenticationSet || set.Authentication == nil || + set.Authentication.Password != "$M.Pw" { + t.Errorf("set: %+v", set) + } + none := prog.Statements[1].(*ast.AlterRestClientStmt).Actions[0].(*ast.RestClientSetAction) + if !none.AuthenticationSet || none.Authentication != nil || none.BaseUrlSet { + t.Errorf("set (Authentication: none): %+v", none) + } +} + +// A property ALTER cannot set is reported, not dropped. +func TestAlterRestClient_SetRejectsUnknownProperty(t *testing.T) { + _, errs := Build(`mdl 1; +alter consumed rest service M.Api set (OpenApi: 'spec.json');`) + if len(errs) == 0 || !strings.Contains(errs[0].Error(), "OpenApi") { + t.Errorf("errs = %v", errs) + } +} diff --git a/mdl/visitor/visitor_strict_properties.go b/mdl/visitor/visitor_strict_properties.go index 7af31f8d09..ae1533e7a2 100644 --- a/mdl/visitor/visitor_strict_properties.go +++ b/mdl/visitor/visitor_strict_properties.go @@ -67,6 +67,10 @@ const ( shapeStringAs propShape = "string as $var" shapeFileAs propShape = "file as $var" shapeStatusAs propShape = "status as $var" + shapeJSON propShape = "json" + shapeStringKind propShape = "string" + shapeFileKind propShape = "file" + shapeStatusKind propShape = "status" shapeOther propShape = "some other value" ) @@ -164,6 +168,14 @@ var restClientSchema = propSchema{on: "a REST client", keys: []propKey{ {"Folder", []propShape{shapeString}}, }} +// What `alter consumed rest service … set ( … )` can change. The folder is +// not a property of the document's contents, and the OpenAPI source is +// CREATE's input, not something stored to set. +var restClientAlterSetSchema = propSchema{on: "an ALTER of a REST client", keys: []propKey{ + {"BaseUrl", []propShape{shapeString}}, + {"Authentication", []propShape{shapeNone, shapeBasic}}, +}} + var restClientBasicAuthSchema = propSchema{on: "REST client basic authentication", keys: []propKey{ {"Username", []propShape{shapeString, shapeConstant, shapeVariable}}, {"Password", []propShape{shapeString, shapeConstant, shapeVariable}}, @@ -177,7 +189,8 @@ var restClientOperationSchema = propSchema{on: "a REST client operation", keys: {"Headers", []propShape{shapeHeaderList}}, {"Body", []propShape{shapeJSONFrom, shapeFileFrom, shapeTemplate, shapeMapping}}, {"Timeout", []propShape{shapeNumber}}, - {"Response", []propShape{shapeNone, shapeJSONAs, shapeStringAs, shapeFileAs, shapeStatusAs, shapeMapping}}, + {"Response", []propShape{shapeNone, shapeJSON, shapeJSONAs, shapeStringKind, shapeStringAs, + shapeFileKind, shapeFileAs, shapeStatusKind, shapeStatusAs, shapeMapping}}, }} var publishedRestSchema = propSchema{on: "a published REST service", keys: []propKey{ @@ -361,6 +374,14 @@ func restClientOpPropShape(pc *parser.RestClientOpPropContext) propShape { return propShape(kind + " from $var") } return propShape(kind + " as $var") + case pc.JSON() != nil: + return shapeJSON + case pc.STRING_TYPE() != nil: + return shapeStringKind + case pc.FILE_KW() != nil: + return shapeFileKind + case pc.STATUS() != nil: + return shapeStatusKind case pc.NONE() != nil: return shapeNone case len(pc.AllRestClientParamItem()) > 0: diff --git a/modelsdk/mpr/reader.go b/modelsdk/mpr/reader.go index ca369c3e64..6d287fecd3 100644 --- a/modelsdk/mpr/reader.go +++ b/modelsdk/mpr/reader.go @@ -42,6 +42,16 @@ type Reader struct { unitCache []cachedUnit unitCacheValid bool + // v2Types is MPR v2's unit-type index (reader_units_v2.go): ContentsHash + // → $Type, loaded from .mxcli/ on first use. v2Hashes is every hash the + // Unit table held at the last rebuild — what a persisted index keeps. + // v2TypesMu guards these and the unit cache rebuild. + v2TypesMu sync.Mutex + v2Types map[string]string + v2TypesLoaded bool + v2TypesDirty bool + v2Hashes map[string]struct{} + // contentCache stores raw BSON bytes per unit ID (MPR v2 only). // Populated on first read while caching is on (EnableContentCache, or a // CacheUnitReads scope); cleared by InvalidateCache, which every write @@ -210,6 +220,9 @@ func (r *Reader) ListUnitHashes() (map[string]string, error) { // Close closes the reader and releases resources. func (r *Reader) Close() error { + r.v2TypesMu.Lock() + r.persistUnitTypeIndex() + r.v2TypesMu.Unlock() if r.db != nil && r.ownsDB { return r.db.Close() } diff --git a/modelsdk/mpr/reader_units.go b/modelsdk/mpr/reader_units.go index 98c8b5cf5d..931451565b 100644 --- a/modelsdk/mpr/reader_units.go +++ b/modelsdk/mpr/reader_units.go @@ -5,6 +5,8 @@ package mpr import ( "bytes" + "crypto/sha256" + "encoding/base64" "fmt" "os" "path/filepath" @@ -77,6 +79,28 @@ func (r *Reader) ListUnitsByType(typePrefix string) ([]UnitRef, error) { return result, nil } +// ListUnitsByTypeInContainer returns the units of exactly typeName whose +// ContainerID is containerID, reading the contents of those units only: a +// lookup of one module's domain model must not read every module's +// (mendixlabs/mxcli#1616). +func (r *Reader) ListUnitsByTypeInContainer(typeName, containerID string) ([]UnitRef, error) { + var units []rawUnit + var err error + if r.version == MPRVersionV2 { + units, err = r.listUnitsByTypeV2In(typeName, containerID) + } else { + units, err = r.listUnitsByTypeV1In(typeName, containerID) + } + if err != nil { + return nil, err + } + result := make([]UnitRef, len(units)) + for i, u := range units { + result[i] = UnitRef{ID: u.ID, ContainerID: u.ContainerID, Type: u.Type, Contents: u.Contents} + } + return result, nil +} + // listUnitsByType returns all units of exactly the given storage type. An empty // typeName returns every unit. // @@ -91,8 +115,10 @@ func (r *Reader) listUnitsByType(typeName string) ([]rawUnit, error) { } // listUnitTypes returns every unit's metadata and $Type without reading the -// contents of any unit whose type is already indexed (Contents is nil). For -// callers that need the project's shape, not its documents. +// contents of any unit whose type is already indexed (Contents is nil): on v2 +// the index is keyed by ContentsHash and persisted (reader_units_v2.go), on v1 +// by blob length (reader_units_v1.go). For callers that need the project's +// shape, not its documents. func (r *Reader) listUnitTypes() ([]rawUnit, error) { if r.version != MPRVersionV2 { return r.listUnitTypesV1() @@ -112,6 +138,12 @@ func (r *Reader) listUnitTypes() ([]rawUnit, error) { // listUnitsByTypeV2 handles MPR v2 format (contents in mprcontents folder). // Uses caching to avoid reading every file for each query. func (r *Reader) listUnitsByTypeV2(typeName string) ([]rawUnit, error) { + return r.listUnitsByTypeV2In(typeName, "") +} + +// listUnitsByTypeV2In is listUnitsByTypeV2 restricted to one container ("" for +// any), filtered before any contents are read. +func (r *Reader) listUnitsByTypeV2In(typeName, containerID string) ([]rawUnit, error) { if !r.unitCacheValid { if err := r.buildUnitCache(); err != nil { return nil, err @@ -121,6 +153,9 @@ func (r *Reader) listUnitsByTypeV2(typeName string) ([]rawUnit, error) { // Filter by type using cache, only read contents for matching units. var units []rawUnit for _, cu := range r.unitCache { + if containerID != "" && cu.ContainerID != containerID { + continue + } if typeName == "" || cu.Type == typeName { contents, err := r.readMprContents(cu.ID) if err != nil { @@ -138,10 +173,12 @@ func (r *Reader) listUnitsByTypeV2(typeName string) ([]rawUnit, error) { return units, nil } -// buildUnitCache reads all unit metadata once and caches it. +// buildUnitCache reads all unit metadata once and caches it. A unit's $Type +// comes from the type index when its ContentsHash is known there, so only new +// and changed units are opened (reader_units_v2.go, mendixlabs/mxcli#1603). func (r *Reader) buildUnitCache() error { rows, err := r.db.Query(` - SELECT UnitID, ContainerID, ContainmentName + SELECT UnitID, ContainerID, ContainmentName, COALESCE(ContentsHash, '') FROM Unit `) if err != nil { @@ -149,30 +186,56 @@ func (r *Reader) buildUnitCache() error { } defer rows.Close() + r.v2TypesMu.Lock() + defer r.v2TypesMu.Unlock() + r.unitCache = nil + hashes := make(map[string]struct{}) + opened := 0 for rows.Next() { var unitID, containerID []byte - var containmentName string + var containmentName, hash string - if err := rows.Scan(&unitID, &containerID, &containmentName); err != nil { + if err := rows.Scan(&unitID, &containerID, &containmentName, &hash); err != nil { return fmt.Errorf("failed to scan unit row: %w", err) } unitUUID := blobToUUID(unitID) - contents, err := r.readMprContents(unitUUID) - if err != nil { - continue + var typeName string + if held, ok := r.overlaid(unitUUID); ok { + // Bytes held in memory are what the unit is now; its stored hash + // describes the file it will replace. + typeName = getTypeFromContents(held) + } else if known, ok := r.knownUnitType(hash); ok { + typeName = known + } else { + contents, err := r.readMprContents(unitUUID) + if err != nil { + continue + } + opened++ + typeName = getTypeFromContents(contents) + r.learnUnitType(contentsHash(contents), typeName) + } + if hash != "" { + hashes[hash] = struct{}{} } - - typeName := getTypeFromContents(contents) r.unitCache = append(r.unitCache, cachedUnit{ - ID: blobToUUID(unitID), + ID: unitUUID, ContainerID: blobToUUID(containerID), ContainmentName: containmentName, Type: typeName, }) } + if err := rows.Err(); err != nil { + return fmt.Errorf("failed to scan units: %w", err) + } + first := r.v2Hashes == nil + r.v2Hashes = hashes + if first || opened >= unitTypeIndexPersistAfter { + r.persistUnitTypeIndex() + } r.unitCacheValid = true return nil } @@ -291,6 +354,13 @@ func (r *Reader) readMprContents(unitUUID string) ([]byte, error) { return data, nil } +// contentsHash is a unit's ContentsHash as Studio Pro and the writer compute +// it: base64(SHA-256(contents)). +func contentsHash(contents []byte) string { + sum := sha256.Sum256(contents) + return base64.StdEncoding.EncodeToString(sum[:]) +} + // getTypeFromContents extracts the $Type field from BSON contents. // Uses bson.Raw.LookupErr for O(1) field extraction instead of unmarshalling // the entire document into map[string]any. diff --git a/modelsdk/mpr/reader_units_v1.go b/modelsdk/mpr/reader_units_v1.go index 5535d9099b..917bb797fb 100644 --- a/modelsdk/mpr/reader_units_v1.go +++ b/modelsdk/mpr/reader_units_v1.go @@ -65,6 +65,15 @@ func (r *Reader) listUnitsByTypeV1(typeName string) ([]rawUnit, error) { if typeName == "" { return r.listAllUnitsV1() } + return r.listUnitsByTypeV1In(typeName, "") +} + +// listUnitsByTypeV1In is listUnitsByTypeV1 for a non-empty typeName, +// restricted to one container ("" for any) before any blob is read. +func (r *Reader) listUnitsByTypeV1In(typeName, containerID string) ([]rawUnit, error) { + if typeName == "" { + return nil, fmt.Errorf("listUnitsByTypeV1In: empty type name") + } metas, err := r.scanUnitMetaV1() if err != nil { @@ -84,18 +93,23 @@ func (r *Reader) listUnitsByTypeV1(typeName string) ([]rawUnit, error) { } }() var units []rawUnit + // Formatting a UUID costs more than the rest of the row, so the container + // is compared only for units of the wanted type. + inContainer := func(m v1UnitMeta) bool { + return containerID == "" || blobToUUID(m.containerBlob) == containerID + } for _, m := range metas { var contents []byte var unitType string if m.overlaid { contents = m.held unitType = getTypeFromContents(contents) - if unitType != typeName { + if unitType != typeName || !inContainer(m) { continue } } else { unitType = r.v1Types[m.key].typ - if unitType != typeName { + if unitType != typeName || !inContainer(m) { continue } if stmt == nil { diff --git a/modelsdk/mpr/reader_units_v2.go b/modelsdk/mpr/reader_units_v2.go new file mode 100644 index 0000000000..25bc23de37 --- /dev/null +++ b/modelsdk/mpr/reader_units_v2.go @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import ( + "bufio" + "bytes" + "os" + "path/filepath" + "strings" +) + +// The MPR v2 unit-type index. +// +// A v2 project keeps each unit in its own file under mprcontents/, and the +// Unit table has no type column, so a unit's $Type is only known from its +// BSON. buildUnitCache used to open every file to learn it and throw the bytes +// away — 393 of a blank app's 394 files for one `describe microflow`, and all +// of them again after every write (mendixlabs/mxcli#1603). +// +// What the table does carry is ContentsHash, base64(SHA-256) of the file, +// which every v2 writer — Studio Pro's and ours (writer_core.go) — keeps in +// step with the bytes. The index maps that hash to the $Type, so a unit is +// opened only when its hash is one the index has not seen. Keying by content +// rather than by unit means an entry can never be stale: a unit whose bytes +// change — including a re-type in place (#503) — has a new hash and is read +// again; unchanged bytes have the type they had. +// +// The index is persisted to /.mxcli/unit-types.idx so the next +// process starts warm. The file is a pure cache: absent, unreadable, corrupt +// or unwritable all mean "type from the file", never a wrong answer. A unit +// with no ContentsHash cannot be keyed and is read every time, as before. +// +// An entry is learnt under the hash of the bytes actually read, not the hash +// the row claims, so a file caught between a writer's rename and its hash +// update — or one edited behind the table's back — cannot teach the index a +// wrong pair. What the index does trust is that a row's ContentsHash describes +// its file: where it does not, the project is already broken (`mxcli mpr +// verify-hashes` finds it, ako/mxcli#972), and a lookup by that stale hash +// returns the type of the bytes it was last true of. + +const unitTypeIndexName = "unit-types.idx" + +// unitTypeIndexHeader is the first line of the persisted index; a file +// without it is ignored, so a format change only costs one cold start. +const unitTypeIndexHeader = "mxcli-unit-types 1" + +// unitTypeIndexPersistAfter is how many files a later rebuild must have +// opened before it rewrites the persisted index straight away. A reader's +// first rebuild always persists what it learnt; a rebuild after a write opens +// one or two files, and those are saved by Close instead of rewriting the +// whole index after every statement of a script. +const unitTypeIndexPersistAfter = 32 + +// unitTypeIndexPath is where the persisted index lives, or "" when the reader +// has no project path to put it next to. +func (r *Reader) unitTypeIndexPath() string { + if r.path == "" { + return "" + } + return filepath.Join(filepath.Dir(r.path), ".mxcli", unitTypeIndexName) +} + +// knownUnitType returns the indexed $Type for a ContentsHash, loading the +// persisted index on first use. Caller holds v2TypesMu. +func (r *Reader) knownUnitType(hash string) (string, bool) { + if !r.v2TypesLoaded { + r.v2TypesLoaded = true + r.v2Types = loadUnitTypeIndex(r.unitTypeIndexPath()) + } + if hash == "" { + return "", false + } + typ, ok := r.v2Types[hash] + return typ, ok +} + +// learnUnitType records a hash's $Type. Caller holds v2TypesMu. +func (r *Reader) learnUnitType(hash, typ string) { + if hash == "" { + return + } + if r.v2Types == nil { + r.v2Types = make(map[string]string) + } + if old, ok := r.v2Types[hash]; !ok || old != typ { + r.v2Types[hash] = typ + r.v2TypesDirty = true + } +} + +// persistUnitTypeIndex writes the hashes the project holds now, and drops +// those of units that have since changed or gone, so the file stays the size +// of the project however many writes a session made. Best effort: a project +// directory the user cannot write to only means the next process types its +// units from their files. Caller holds v2TypesMu. +func (r *Reader) persistUnitTypeIndex() { + path := r.unitTypeIndexPath() + if !r.v2TypesDirty || path == "" || r.v2Hashes == nil { + return + } + var buf bytes.Buffer + buf.WriteString(unitTypeIndexHeader) + buf.WriteByte('\n') + for hash := range r.v2Hashes { + typ, ok := r.v2Types[hash] + if !ok { + continue + } + buf.WriteString(hash) + buf.WriteByte('\t') + buf.WriteString(typ) + buf.WriteByte('\n') + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return + } + tmp, err := os.CreateTemp(filepath.Dir(path), unitTypeIndexName+".*.tmp") + if err != nil { + return + } + _, werr := tmp.Write(buf.Bytes()) + cerr := tmp.Close() + _ = os.Chmod(tmp.Name(), 0o644) // CreateTemp's 0600 would differ from every other .mxcli/ file + if werr != nil || cerr != nil || os.Rename(tmp.Name(), path) != nil { + _ = os.Remove(tmp.Name()) + return + } + r.v2TypesDirty = false +} + +// loadUnitTypeIndex reads a persisted index; any problem yields an empty one. +func loadUnitTypeIndex(path string) map[string]string { + out := make(map[string]string) + if path == "" { + return out + } + f, err := os.Open(path) + if err != nil { + return out + } + defer f.Close() + sc := bufio.NewScanner(f) + if !sc.Scan() || sc.Text() != unitTypeIndexHeader { + return out + } + for sc.Scan() { + hash, typ, ok := strings.Cut(sc.Text(), "\t") + if !ok || hash == "" || strings.Contains(typ, "\t") { + return make(map[string]string) // corrupt: start cold rather than half-trust it + } + out[hash] = typ + } + if sc.Err() != nil { + return make(map[string]string) + } + return out +} diff --git a/modelsdk/mpr/reader_units_v2_test.go b/modelsdk/mpr/reader_units_v2_test.go new file mode 100644 index 0000000000..cf5755392e --- /dev/null +++ b/modelsdk/mpr/reader_units_v2_test.go @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// mendixlabs/mxcli#1603: on MPR v2 the first listing of a process opened every +// unit file in mprcontents/ to learn its $Type, and threw the bytes away — +// `describe microflow` read 393 of a blank app's 394 files — and every write +// made the next listing do it again. The type index is keyed by the Unit +// table's ContentsHash, so only a unit whose hash it has not seen is opened, +// and it is persisted under .mxcli/ so the next process starts warm. + +// v2Project is a minimal MPR v2 project on disk: a Unit table plus one +// .mxunit file per unit, laid out as Studio Pro lays them out. +type v2Project struct { + root, mprPath, contentsDir string + db *sql.DB +} + +func newV2Project(t *testing.T) *v2Project { + t.Helper() + root := t.TempDir() + p := &v2Project{root: root, mprPath: filepath.Join(root, "app.mpr"), contentsDir: filepath.Join(root, "mprcontents")} + db, err := sql.Open("sqlite", p.mprPath) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if _, err := db.Exec(` + CREATE TABLE Unit ( + UnitID BLOB PRIMARY KEY NOT NULL, + ContainerID BLOB, + ContainmentName TEXT, + TreeConflict LONG, + ContentsHash TEXT, + ContentsConflicts TEXT + ) + `); err != nil { + t.Fatalf("create Unit table: %v", err) + } + p.db = db + return p +} + +// add stores a unit the way the reader finds it: the file under the path +// readMprContents builds from the cache's ID, the row with its ContentsHash. +func (p *v2Project) add(t *testing.T, i int, typ string) string { + t.Helper() + id := fmt.Sprintf("%08x-0000-0000-0000-000000000000", i+1) + blob := uuidToBlob(id) + doc := typedDoc(t, typ, fmt.Sprintf("Unit%d", i)) + path := p.unitPath(id) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, doc, 0o644); err != nil { + t.Fatalf("write unit: %v", err) + } + if _, err := p.db.Exec(`INSERT INTO Unit (UnitID, ContainerID, ContainmentName, ContentsHash) VALUES (?, ?, 'Documents', ?)`, + blob, uuidToBlob("11111111-2222-3333-4444-555555555555"), hashOf(doc)); err != nil { + t.Fatalf("insert unit row: %v", err) + } + return id +} + +// unitPath is where readMprContents looks for the unit with this ID. +func (p *v2Project) unitPath(id string) string { + cacheID := blobToUUID(uuidToBlob(id)) + return filepath.Join(p.contentsDir, cacheID[0:2], cacheID[2:4], cacheID+".mxunit") +} + +// rewrite replaces a unit's bytes and hash, as a Studio Pro save would. +func (p *v2Project) rewrite(t *testing.T, id string, doc []byte) { + t.Helper() + if err := os.WriteFile(p.unitPath(id), doc, 0o644); err != nil { + t.Fatalf("rewrite unit: %v", err) + } + if _, err := p.db.Exec(`UPDATE Unit SET ContentsHash = ? WHERE UnitID = ?`, hashOf(doc), uuidToBlob(id)); err != nil { + t.Fatalf("update hash: %v", err) + } +} + +// reader opens the project as a fresh process would: nothing carried over +// from any earlier reader except what is on disk. +func (p *v2Project) reader() *Reader { + return &Reader{path: p.mprPath, db: p.db, version: MPRVersionV2, contentsDir: p.contentsDir} +} + +func typedDoc(t *testing.T, typ, name string) []byte { + t.Helper() + b, err := bson.Marshal(bson.D{ + {Key: "$ID", Value: bson.Binary{Subtype: 0x00, Data: uuidToBlob("11111111-1111-1111-1111-111111111111")}}, + {Key: "$Type", Value: typ}, + {Key: "Name", Value: name}, + }) + if err != nil { + t.Fatalf("marshal unit: %v", err) + } + return b +} + +// seedProject is a project of n microflows and one module: the shape of the +// report, where listing the modules opened every microflow too. +func seedProject(t *testing.T, n int) (*v2Project, string) { + t.Helper() + p := newV2Project(t) + module := p.add(t, 0, "Projects$ModuleImpl") + for i := 1; i <= n; i++ { + p.add(t, i, "Microflows$Microflow") + } + return p, module +} + +func listModules(t *testing.T, r *Reader) []UnitRef { + t.Helper() + units, err := r.ListUnitsByType("Projects$ModuleImpl") + if err != nil { + t.Fatalf("list modules: %v", err) + } + return units +} + +// The reported symptom: a second process listing the one module opened every +// unit file. It must open only the module's. +func TestListUnitsV2_SecondProcessOpensOnlyTheMatches(t *testing.T) { + const n = 40 + p, _ := seedProject(t, n) + + first := p.reader() + if got := len(listModules(t, first)); got != 1 { + t.Fatalf("first reader listed %d modules, want 1", got) + } + // The cold reader has no choice: it has never seen these hashes. + if got := first.UnitFileReads(); got < n { + t.Fatalf("control: the cold listing read %d files, want >= %d (the fixture does not exercise the index)", got, n) + } + + second := p.reader() + if got := len(listModules(t, second)); got != 1 { + t.Fatalf("second reader listed %d modules, want 1", got) + } + if got := second.UnitFileReads(); got != 1 { + t.Errorf("a listing of one module in a fresh process read %d unit files, want 1 (#1603: every unit was opened to learn its $Type)", got) + } +} + +// After a write the next listing re-types only the unit that changed. +func TestListUnitsV2_RebuildAfterWriteOpensOnlyTheChange(t *testing.T) { + const n = 40 + p, _ := seedProject(t, n) + r := p.reader() + listModules(t, r) + + p.add(t, n+1, "Microflows$Microflow") // a new unit + r.InvalidateCache() + before := r.UnitFileReads() + listModules(t, r) + if got := r.UnitFileReads() - before; got != 2 { + t.Errorf("a listing after one insert read %d unit files, want 2 (the new unit to type it, the module to return it)", got) + } +} + +// The index is keyed by content, not by unit: a unit whose bytes change type +// — mxcli re-types elements in place (#503) — is listed under its new type. +func TestListUnitsV2_RetypedUnitIsSeenUnderItsNewType(t *testing.T) { + p, _ := seedProject(t, 3) + r := p.reader() + listModules(t, r) + + retyped := p.add(t, 10, "Microflows$Microflow") + r.InvalidateCache() + if got := len(listModules(t, r)); got != 1 { + t.Fatalf("listed %d modules before the re-type, want 1", got) + } + p.rewrite(t, retyped, typedDoc(t, "Projects$ModuleImpl", "Second")) + r.InvalidateCache() + if got := len(listModules(t, r)); got != 2 { + t.Errorf("listed %d modules after a unit was re-typed to a module, want 2", got) + } + // And a fresh process, starting from the persisted index, agrees. + if got := len(listModules(t, p.reader())); got != 2 { + t.Errorf("a fresh reader listed %d modules after the re-type, want 2", got) + } +} + +// A unit with no ContentsHash cannot be keyed, so it is read every time — the +// pre-index behaviour, never a wrong type. +func TestListUnitsV2_UnhashedUnitIsAlwaysRead(t *testing.T) { + p, _ := seedProject(t, 3) + if _, err := p.db.Exec(`UPDATE Unit SET ContentsHash = ''`); err != nil { + t.Fatal(err) + } + listModules(t, p.reader()) + r := p.reader() + if got := len(listModules(t, r)); got != 1 { + t.Fatalf("listed %d modules, want 1", got) + } + if got := r.UnitFileReads(); got < 4 { + t.Errorf("unhashed units read %d times, want every unit typed from its file (>= 4)", got) + } +} + +// A corrupt index file is a cold start, not an error and not a wrong answer. +func TestListUnitsV2_CorruptIndexIsIgnored(t *testing.T) { + p, _ := seedProject(t, 3) + listModules(t, p.reader()) + idx := filepath.Join(p.root, ".mxcli", unitTypeIndexName) + if _, err := os.Stat(idx); err != nil { + t.Fatalf("the index was not persisted at %s: %v", idx, err) + } + if err := os.WriteFile(idx, []byte("garbage\nmore\tgarbage\tthan\tfits\n"), 0o644); err != nil { + t.Fatal(err) + } + if got := len(listModules(t, p.reader())); got != 1 { + t.Errorf("listed %d modules with a corrupt index, want 1", got) + } +} + +// A reader that catches a unit between a writer's file rename and its hash +// update sees new bytes under the old hash. Learning that pair would list the +// unit under the wrong type once the old bytes are back — a rolled-back write, +// a checkout — so the index learns the hash of what it read. +func TestListUnitsV2_FileAheadOfItsHashTeachesNothingWrong(t *testing.T) { + p, _ := seedProject(t, 3) + unit := p.add(t, 10, "Microflows$Microflow") + stored, err := os.ReadFile(p.unitPath(unit)) + if err != nil { + t.Fatal(err) + } + // The window: the file is a module now, the row still hashes the microflow. + if err := os.WriteFile(p.unitPath(unit), typedDoc(t, "Projects$ModuleImpl", "Midway"), 0o644); err != nil { + t.Fatal(err) + } + if got := len(listModules(t, p.reader())); got != 2 { + t.Fatalf("listed %d modules mid-write, want 2 (the file is what is read)", got) + } + // The write is undone: file and row agree again on the microflow. + if err := os.WriteFile(p.unitPath(unit), stored, 0o644); err != nil { + t.Fatal(err) + } + if got := len(listModules(t, p.reader())); got != 1 { + t.Errorf("listed %d modules after the write was undone, want 1: the index learnt the stale hash with the new type", got) + } +} diff --git a/modelsdk/mprread/generic.go b/modelsdk/mprread/generic.go index b52121bd47..588022227c 100644 --- a/modelsdk/mprread/generic.go +++ b/modelsdk/mprread/generic.go @@ -73,6 +73,11 @@ func ListUnitsWithContainerWhere[T element.Element](r *mmpr.Reader, keep func(bs if err != nil { return nil, err } + return decodeUnits[T](refs, typeName, keep) +} + +// decodeUnits decodes the refs of typeName that keep accepts (nil keeps all). +func decodeUnits[T element.Element](refs []mmpr.UnitRef, typeName string, keep func(bson.Raw) bool) ([]Unit[T], error) { dec := codec.NewDecoder(codec.DefaultRegistry) out := make([]Unit[T], 0, len(refs)) for _, ref := range refs { @@ -95,6 +100,18 @@ func ListUnitsWithContainerWhere[T element.Element](r *mmpr.Reader, keep func(bs return out, nil } +// ListUnitsInContainer decodes the units of type T whose ContainerID is +// containerID, and reads no other unit: a lookup of one module's domain model +// reads one document, not the project's (mendixlabs/mxcli#1616). +func ListUnitsInContainer[T element.Element](r *mmpr.Reader, containerID model.ID) ([]Unit[T], error) { + typeName := bsonTypeName[T]() + refs, err := r.ListUnitsByTypeInContainer(typeName, string(containerID)) + if err != nil { + return nil, err + } + return decodeUnits[T](refs, typeName, nil) +} + // UnitHeader is what a name listing needs of a unit, read without decoding it. type UnitHeader struct { ID model.ID diff --git a/sdk/microflows/microflows.go b/sdk/microflows/microflows.go index fdda98db46..89f173c526 100644 --- a/sdk/microflows/microflows.go +++ b/sdk/microflows/microflows.go @@ -405,6 +405,16 @@ type AnnotationFlow struct { Sides *ConnectionSides `json:"sides,omitempty"` } +// SidesOrDefault returns Sides, or — when nothing chose them — the drawing for +// a note sitting above its object: out of the note's bottom edge, into the +// object's top. Every writer uses it, so the engines agree on the default. +func (f *AnnotationFlow) SidesOrDefault() ConnectionSides { + if f.Sides != nil { + return *f.Sides + } + return ConnectionSides{Origin: 2, Destination: 0} +} + // ConnectionSides is the pair of connection indexes a flow attaches at, in the // numbering a SequenceFlow uses for an activity — 0 top, 1 right, 2 bottom, // 3 left. Studio Pro numbers a note's sides the same way.