From 306764b55da9dbd1fd5d3cdb98e466c21cdd2829 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 10:25:15 +0000 Subject: [PATCH 01/13] fix: a change next to a find/filter `where` with a bare member is refused (ako/mxcli#856) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `find $L where Code = 'x'` is stored as `$currentObject/Code = 'x'`: the builder qualifies a bare member of a find/filter-by-expression predicate (qualifyIteratorAttributes, #1002), and describe prints the stored form. create or modify matches the declared statements against that description, so the statement as the script spells it never matched its own stored activity. An unchanged re-run still read as Unchanged (the built flow is the stored one), but a re-run with one real change sent the flow to the statement diff, where the find read as changed too: needlessly replaced away from the change, and next to it merged with the change into one replace whose fragment declares $A while the stored find still does — refused under mdl 1 ("the fragment declares $A, which the microflow already has"), the whole flow rebuilt under mdl 0. This is the "other clauses with two parse paths" item of the issue. describedMemberSpellings now rewrites a find/filter-by-expression predicate with the builder's own qualifyIteratorAttributes before the statement diff, keeping it as written where the builder would refuse it. Found by probing: each flow created, then re-run with one inserted statement, flagging any summary but "1 inserted". Over the 118 flows of 02-microflow-examples and 23 hand-picked respellings (sort without asc, qualified members, association spellings, `<>`, aggregates, enumeration values, …) this clause was the only mismatch left. The issue's other two items need no change: `diff` runs a script's earlier statements since 4c1e5c384 (#907: it runs exec on a scratch copy); verified on a blank 11.13 app — a script creating SUB_New and calling it from a create or modify of a stored flow diffs as "1 new, 1 modified". The TestApp parity test's CI time is a scheduling note, not a defect. Verification: - TestCreateOrModify_ListOperationWhereBareMemberIsItsStoredActivity (mdl 1 and mdl 0): before the fix refused / rebuilt; after, "spliced: 1 replaced" with the find and filter keeping their $IDs and places. Control TestCreateOrModify_ListOperationWhereChangedPredicateIsAChange: a changed predicate, in either spelling, is still replaced. - Revert proof: with flow_member_spelling.go restored both tests fail with the reported refusal. - Blank 11.13.0 app, mdl-examples/bug-tests/splice-856-…mdl: baseline refuses statement 4 and rolls back; fixed prints "Modified microflow: BugTest856.F (spliced: 1 replaced)"; mx check 0 errors. Co-Authored-By: Claude Opus 5.5 --- ...ilter-where-bare-member-never-matches.json | 1 + ...e-856-list-operation-where-bare-member.mdl | 58 +++++++++ mdl/executor/flow_member_spelling.go | 29 +++++ ...ow_splice_list_op_where_856_pedapp_test.go | 114 ++++++++++++++++++ 4 files changed, 202 insertions(+) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-25-find-filter-where-bare-member-never-matches.json create mode 100644 mdl-examples/bug-tests/splice-856-list-operation-where-bare-member.mdl create mode 100644 mdl/executor/flow_splice_list_op_where_856_pedapp_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-find-filter-where-bare-member-never-matches.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-find-filter-where-bare-member-never-matches.json new file mode 100644 index 000000000..7bd2fc992 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-find-filter-where-bare-member-never-matches.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-25", "symptom": "`create or modify microflow` of a flow holding `$A = find $L where Code = 'x'` (or `filter … where` with a bare member) and one real change next to it was refused under mdl 1 — 'replace $N: the fragment declares $A, which the microflow already has' — and rebuilt whole (MDL-V1-REBUILD) under mdl 0; away from the change the find was needlessly replaced on every run. An unchanged re-run was still 'Unchanged'", "cause": "The builder stores a bare member of a find/filter-by-expression predicate as `$currentObject/` (qualifyIteratorAttributes, #1002) and describe prints the stored form, so the statement as the script spells it never matched its own stored activity in the statement diff; the unchanged re-run escaped only because builtAsStored compares built graphs before the statement diff runs", "file": "mdl/executor/flow_member_spelling.go (memberSpeller.respell, qualifiedPredicate)", "fix": "describedMemberSpellings rewrites a find/filter-by-expression predicate with the builder's own qualifyIteratorAttributes before the statement diff, keeping the predicate as written when the builder would refuse it", "insight": "A spelling the builder normalises is invisible to the identity check (an unchanged re-run passes) and only shows when a REAL change elsewhere sends the flow to the statement diff, so the probe that finds these is: create each flow, re-run it with one inserted statement, and flag any summary other than '1 inserted'. Over 02-microflow-examples (118 flows) and 23 hand-picked respellings this was the only clause left; sort without asc, member qualification, association spelling in expressions, `<>`, aggregates (stored as written) all matched", "refs": ["ako/mxcli#856", "#1002", "#839"], "test": "TestCreateOrModify_ListOperationWhereBareMemberIsItsStoredActivity, TestCreateOrModify_ListOperationWhereChangedPredicateIsAChange (mdl/executor/flow_splice_list_op_where_856_pedapp_test.go)"} diff --git a/mdl-examples/bug-tests/splice-856-list-operation-where-bare-member.mdl b/mdl-examples/bug-tests/splice-856-list-operation-where-bare-member.mdl new file mode 100644 index 000000000..18ab2f0ec --- /dev/null +++ b/mdl-examples/bug-tests/splice-856-list-operation-where-bare-member.mdl @@ -0,0 +1,58 @@ +-- ============================================================================ +-- ako/mxcli#856 — a find/filter `where` naming a member bare never matched +-- its own stored activity, so a one-line change next to it was refused +-- ============================================================================ +-- +-- `find $L where Code = 'x'` is stored as `$currentObject/Code = 'x'` (Mendix +-- evaluates the predicate once per item, and a bare name is CE0117), and +-- describe prints the stored form. create or modify matches the statements it +-- is given against that description, so the statement as written read as +-- changed on every run: +-- +-- - a re-run that changed nothing was still Unchanged (the built flow is +-- the stored one); +-- - a re-run with a real change elsewhere replaced the find too, and next +-- to the change the two became one replace whose fragment declares $A +-- while the stored find still does. Under `mdl 1;`: +-- +-- this change cannot be spliced into the stored flow: alter microflow +-- BugTest856.F: replace $N: the fragment declares $A, which the +-- microflow already has; choose another name. Nothing was written +-- +-- and under mdl 0 the whole flow was rebuilt (MDL-V1-REBUILD warning). +-- +-- The predicate is now compared as the builder stores it. +-- +-- To reproduce: exec this file against a project. Expected: +-- Created microflow: BugTest856.F +-- Modified microflow: BugTest856.F (spliced: 1 replaced) +-- Before the fix the second statement was refused and exec rolled the whole +-- script back. Running the file again sets the declare back and forth, and +-- each statement again replaces the declare alone (spliced: 1 replaced). + +mdl 1; + +create module BugTest856; + +create or modify persistent entity BugTest856.Row ( + Code: String(100), + Qty: Integer +); + +create or modify microflow BugTest856.F ($L: List of BugTest856.Row) +begin + declare $N Integer = 0; + $A = find $L where Code = 'x'; + $B = filter $L where Code != 'y' and Qty > 0; + log info node 'BugTest856' toString($N); +end; + +-- The same flow with one change, next to the find: only the declare is +-- replaced; the find and the filter keep their $IDs. +create or modify microflow BugTest856.F ($L: List of BugTest856.Row) +begin + declare $N Integer = 1; + $A = find $L where Code = 'x'; + $B = filter $L where Code != 'y' and Qty > 0; + log info node 'BugTest856' toString($N); +end; diff --git a/mdl/executor/flow_member_spelling.go b/mdl/executor/flow_member_spelling.go index b953ba724..449cd1aef 100644 --- a/mdl/executor/flow_member_spelling.go +++ b/mdl/executor/flow_member_spelling.go @@ -129,6 +129,14 @@ func (s *memberSpeller) respell(st ast.MicroflowStatement) { if entity == "" { return } + if (x.Operation == ast.ListOpFind || x.Operation == ast.ListOpFilter) && (x.ByExpression || !ast.IsMemberEquality(x.Condition)) { + // The by-expression operation. The builder writes a bare member + // of the list's entity as `$currentObject/` (#1002), and + // describe prints what is stored, so the bare spelling never + // matched its own stored activity (ako/mxcli#856). + x.Condition = s.qualifiedPredicate(x.Condition, x.InputVariable, entity) + return + } if (x.Operation == ast.ListOpFind || x.Operation == ast.ListOpFilter) && !x.ByExpression && ast.IsMemberEquality(x.Condition) { binary := x.Condition.(*ast.BinaryExpr) name, ok := listOperationFieldName(binary.Left) @@ -170,6 +178,27 @@ func (s *memberSpeller) member(entity, name string, spell func(attr, assoc strin return short } +// qualifiedPredicate returns a find or filter predicate over list (whose +// elements are entity) with its bare members written as the builder stores +// them, by the builder's own rewrite (qualifyIteratorAttributes). A predicate +// the builder would refuse — a bare name that is no member — is returned as +// written: the build reports it. +func (s *memberSpeller) qualifiedPredicate(cond ast.Expression, list, entity string) ast.Expression { + if cond == nil { + return nil + } + saved := s.fb.varTypes + s.fb.varTypes = map[string]string{list: "List of " + entity} + s.fb.errors = nil + q := s.fb.qualifyIteratorAttributes(cond, list, "find") + failed := len(s.fb.errors) > 0 + s.fb.varTypes, s.fb.errors = saved, nil + if failed { + return cond + } + return q +} + func (s *memberSpeller) resolve(entity, name string) (attr, assoc string, ok bool) { s.fb.errors = nil mc := µflows.MemberChange{} diff --git a/mdl/executor/flow_splice_list_op_where_856_pedapp_test.go b/mdl/executor/flow_splice_list_op_where_856_pedapp_test.go new file mode 100644 index 000000000..9bbbd7ac4 --- /dev/null +++ b/mdl/executor/flow_splice_list_op_where_856_pedapp_test.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/model" +) + +// ako/mxcli#856: a find or filter by expression may name a member of the +// list's entity bare — `find $L where Subject = 'x'` — and the builder stores +// it as `$currentObject/Subject = 'x'` (#1002: Mendix evaluates the predicate +// per item, and a bare name is CE0117). describe prints the stored form, so the +// statement as the script spells it never matched its own stored activity. +// +// A re-run that changed nothing was still Unchanged (the built flow is the +// stored one, ako/mxcli#859), but a re-run with a real change elsewhere went +// to the statement diff, where the find read as changed too: replaced with new +// $IDs at best, and next to the real change the two merged into one replace +// whose fragment declares $A while the stored find still does, which mdl 1 +// refused ("the fragment declares $A, which the microflow already has"). + +const listOpWhereFlow = `mdl 1; +create or modify microflow MyFirstModule.ListOpWhere ($L: List of FeedbackModule.Feedback) +begin + declare $N Integer = 0; + $A = find $L where Subject = 'x'; + $B = filter $L where Subject != 'y' and ScreenWidth > 0; + log info node 'T' toString($N); +end;` + +func TestCreateOrModify_ListOperationWhereBareMemberIsItsStoredActivity(t *testing.T) { + for _, header := range []string{"mdl 1;", ""} { + t.Run("header "+header, func(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + if err := agreeExec(t, exec, listOpWhereFlow); err != nil { + t.Fatalf("setup: %v\n%s", err, out.String()) + } + desc := describeFlow(t, exec, "ListOpWhere") + if !strings.Contains(desc, "where $currentObject/Subject = 'x'") { + t.Fatalf("the stored find is not described in its qualified spelling, so the case would not reach the bug:\n%s", desc) + } + before := flowObjectIDs(loopEditFlowOf(t, exec, "ListOpWhere").ObjectCollection) + + // One real change, next to the find; the find and filter are + // restated exactly as the script first wrote them. + edited := strings.Replace(listOpWhereFlow, "declare $N Integer = 0;", "declare $N Integer = 1;", 1) + edited = strings.TrimPrefix(edited, "mdl 1;\n") + if header != "" { + edited = header + "\n" + edited + } + out.Reset() + if err := agreeExec(t, exec, edited); err != nil { + t.Fatalf("a one-line change next to a find by expression is refused: %v", err) + } + if strings.Contains(out.String(), "MDL-V1-REBUILD") { + t.Errorf("the change fell back to rebuilding the flow:\n%s", out.String()) + } + if !strings.Contains(out.String(), "Modified microflow: MyFirstModule.ListOpWhere (spliced: 1 replaced)") { + t.Errorf("want only the changed declare replaced, got:\n%s", out.String()) + } + if got := describeFlow(t, exec, "ListOpWhere"); !strings.Contains(got, "declare $N Integer = 1;") { + t.Errorf("the change was not written:\n%s", got) + } + + // Only the declare is new: the find, the filter and the rest keep + // their $IDs and places. + after := flowObjectIDs(loopEditFlowOf(t, exec, "ListOpWhere").ObjectCollection) + var gone, added []model.ID + for id := range before { + if _, ok := after[id]; !ok { + gone = append(gone, id) + } + } + for id, p := range after { + if q, ok := before[id]; !ok { + added = append(added, id) + } else if p != q { + t.Errorf("object %s moved from %v to %v", id, q, p) + } + } + if len(gone) != 1 || len(added) != 1 { + t.Errorf("want exactly the declare replaced (1 gone, 1 new), got %d gone, %d new", len(gone), len(added)) + } + }) + } +} + +// Control: a predicate that does change is still a change, in either spelling. +func TestCreateOrModify_ListOperationWhereChangedPredicateIsAChange(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + if err := agreeExec(t, exec, listOpWhereFlow); err != nil { + t.Fatalf("setup: %v\n%s", err, out.String()) + } + for _, c := range []struct{ from, to string }{ + {"where Subject = 'x'", "where Subject = 'z'"}, + {"where Subject = 'x'", "where $currentObject/Description = 'x'"}, + {"where Subject != 'y' and ScreenWidth > 0", "where Subject != 'y' and ScreenHeight > 0"}, + } { + out.Reset() + if err := agreeExec(t, exec, strings.Replace(listOpWhereFlow, c.from, c.to, 1)); err != nil { + t.Fatalf("%s → %s: %v", c.from, c.to, err) + } + if !strings.Contains(out.String(), "(spliced: 1 replaced)") { + t.Errorf("%s → %s: want the operation replaced, got:\n%s", c.from, c.to, out.String()) + } + // Back to the stored statement for the next case. + if err := agreeExec(t, exec, listOpWhereFlow); err != nil { + t.Fatalf("restore: %v", err) + } + } +} From 957fe86d8c3d0739e2d3d6d2fb345b6b4405bd8e Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 10:29:16 +0000 Subject: [PATCH 02/13] fix: drop entity leaves the association member in the other entity's access rule (ako/mxcli#1449) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `drop entity` removes every association with the entity at either end — the backend cascades into other modules' domain models too — and reconciled no access rule, so the member entry for each removed association stayed on the entity at the other end: mx check CE1613 "The selected association 'Shop.Order_Customer' no longer exists" (upstream mendixlabs/mxcli#1551). Cause, in two layers: - execDropEntity (mdl/executor/cmd_entities.go) never reconciled, and execDropAssociation reconciled only the declaring module, though under OWNER Both a cross-module association is a member of its TO entity in another module. - ReconcileMemberAccesses (mdl/backend/modelsdk/domainmodel_security_write.go) preserved every association reference qualified by another module — the rule written for associations inherited across modules (#758) — so even a reconcile of the right module could not remove an entry for an association that no longer exists. `update security` reported "All entity access rules are up to date" over it. Fix: drop entity collects, before the delete, every module that can hold a member entry for one of the entity's associations (its own, each cross-module TO end's, each module declaring a cross-module association to it) and reconciles them through reconcileModulesAccess — the helper alter association already used, now also behind drop association, which passes the TO end's module as well. The reconcile now removes a foreign reference whose declaring module exists and no longer declares the association; System (virtual domain model), absent modules and existing associations stay preserved. Verified on a blank 11.13.0 app with security on, mx check before (baseline binary) / after: drop of a same-module TO end 1 -> 0 errors, of a cross-module TO end 1 -> 0, of the FROM end of a cross-module OWNER Both association 1 -> 0, of a FROM end 1 -> 0, `drop association` of a cross-module OWNER Both association 1 -> 0, the TO end being Administration.Account (a System.User specialization) 3 -> 0; `update security` repairs a project broken by the baseline binary (1 -> 0). An association inherited across modules (Shop.Special extends Crm.Lead) keeps its entry. Tests revert-proven: with the fix reverted, the backend and executor tests fail with the reported symptoms. Co-Authored-By: Claude Opus 5.5 --- ...tity-leaves-association-member-access.json | 1 + .../domain-model/drop-association.md | 4 +- .../src/reference/domain-model/drop-entity.md | 14 +- ...ss-1449-drop-entity-association-member.mdl | 62 ++++++ .../modelsdk/domainmodel_security_write.go | 114 ++++++++--- .../reconcile_foreign_stale_assoc_test.go | 180 ++++++++++++++++++ mdl/executor/cmd_associations.go | 47 +++-- mdl/executor/cmd_entities.go | 44 +++++ .../cmd_entities_drop_reconcile_test.go | 129 +++++++++++++ 9 files changed, 546 insertions(+), 49 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-25-drop-entity-leaves-association-member-access.json create mode 100644 mdl-examples/bug-tests/access-1449-drop-entity-association-member.mdl create mode 100644 mdl/backend/modelsdk/reconcile_foreign_stale_assoc_test.go create mode 100644 mdl/executor/cmd_entities_drop_reconcile_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-drop-entity-leaves-association-member-access.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-drop-entity-leaves-association-member-access.json new file mode 100644 index 000000000..99eab4ff0 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-drop-entity-leaves-association-member-access.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-25", "symptom": "`drop entity Shop.Customer` reported \"Dropped entity\" and mx check then failed CE1613 \"The selected association 'Shop.Order_Customer' no longer exists.\" at Access rule of entity 'Shop.Order'; the same across modules (`drop entity Crm.Contact` leaving Shop.Order_Contact in Shop.Order's rule), and for the FROM end of a cross-module OWNER Both association (`drop entity Crm.Lead` leaving Crm.Lead_Order on its TO entity Shop.Order). `drop association Crm.Lead_Order` left the same entry, and `update security` printed \"All entity access rules are up to date\" over it (ako/mxcli#1449, mendixlabs/mxcli#1551).", "cause": "Two gaps. (1) execDropEntity called DeleteEntity, which cascades the associations away in every module, and reconciled nothing; execDropAssociation reconciled only the DECLARING module, though under OWNER Both a cross-module association is a member of its TO entity in another module. (2) ReconcileMemberAccesses kept every association reference qualified by another module (the 'preserve what cannot be checked' branch for associations inherited across modules, #758), so even a reconcile of the right module could not remove an entry for an association that no longer exists anywhere.", "file": "mdl/executor/cmd_entities.go (associationMemberModules, read before the delete), mdl/executor/cmd_associations.go (reconcileModulesAccess, now behind create/alter/drop association and drop entity); mdl/backend/modelsdk/domainmodel_security_write.go (scanProjectAssociations records what every module declares; staleForeignRef)", "insight": "Which modules to reconcile is not 'the module written': a rule naming an association lives on the FROM entity (the declaring module, or ANOTHER module when the dropped entity is the TO end of a cross-module association) and under OWNER Both on the TO entity too — and that set must be read BEFORE the delete, while the associations exist. 'Preserve what cannot be checked' was right for its case and had quietly widened to every foreign reference: existence in the declaring module is checkable from the same scan the reconcile already did for incoming associations, and 'not declared' is a positive finding, not an unknown. Keep the boundary narrow — System (virtual domain model) and absent modules stay preserved, and an existing association on a specialization of another module's entity stays (control test). A single MDL script is a weak repro here: the end-of-program reconcile repairs the same-module and FROM-side cases on the old binary, so only the OWNER Both one shows in a combined script — run the drop as its own script to see all three. Measured on mxbuild 11.13.0: baseline 1 CE1613 per drop, fixed 0; `update security` on a project broken by the old binary now repairs it.", "refs": ["ako/mxcli#1449", "mendixlabs/mxcli#1551", "ako/mxcli#802"], "ce": ["CE1613"], "test": "mdl/executor/cmd_entities_drop_reconcile_test.go (TestExecDropEntity_ReconcilesEveryModuleHoldingAMemberOfItsAssociations, TestExecDropAssociation_CrossModuleReconcilesTheToModule); mdl/backend/modelsdk/reconcile_foreign_stale_assoc_test.go (TestReconcile_DropsAForeignAssociationThatNoLongerExists, control TestReconcile_KeepsAForeignAssociationInheritedAcrossModules); mdl-examples/bug-tests/access-1449-drop-entity-association-member.mdl"} diff --git a/docs-site/src/reference/domain-model/drop-association.md b/docs-site/src/reference/domain-model/drop-association.md index 445895c8a..6383d1cac 100644 --- a/docs-site/src/reference/domain-model/drop-association.md +++ b/docs-site/src/reference/domain-model/drop-association.md @@ -8,7 +8,9 @@ `DROP ASSOCIATION` removes an association (relationship) between two entities from the domain model. The entities themselves are not affected -- only the relationship is removed. -After dropping an association, any microflows, pages, or access rules that traverse it will become invalid. Use `LIST IMPACT OF Module.AssociationName` to check references before dropping. +The access rules that named the association are reconciled: the entry is removed from the FROM entity's rules and, for an `OWNER Both` association, from the TO entity's — which for a cross-module association are in another module. A message definition that still exposes the association blocks the drop, naming the member to remove first. + +Microflows and pages that traverse the association become invalid. Use `LIST IMPACT OF Module.AssociationName` to check references before dropping. ## Parameters diff --git a/docs-site/src/reference/domain-model/drop-entity.md b/docs-site/src/reference/domain-model/drop-entity.md index 811f6aefe..0c36baaa8 100644 --- a/docs-site/src/reference/domain-model/drop-entity.md +++ b/docs-site/src/reference/domain-model/drop-entity.md @@ -8,7 +8,14 @@ `DROP ENTITY` removes an entity from a module's domain model. The entity and all of its attributes, validation rules, indexes, and access rules are deleted. -Associations that reference the dropped entity are **not** automatically removed. Drop those separately with `DROP ASSOCIATION` before or after dropping the entity to avoid dangling references. +Every association with the dropped entity at either end is removed with it — including a cross-module association declared in another module. The access rules of the entities at the other end are reconciled afterwards, in whichever module they live, so no rule is left naming an association that no longer exists (Mendix would reject that with CE1613): + +``` +Reconciled 1 access rule(s) in module Shop after dropping the entity +Dropped entity: Crm.Contact +``` + +Microflows, pages and other documents that use the entity or one of those associations are not changed. Use `LIST IMPACT OF Module.EntityName` to check which microflows, pages, and associations reference the entity before dropping it. @@ -32,10 +39,7 @@ mdl 1; -- Check what references this entity LIST IMPACT OF Sales.OldEntity; --- Drop related association first -DROP ASSOCIATION Sales.OldEntity_Customer; - --- Then drop the entity +-- Drops the entity and its associations, and reconciles the access rules DROP ENTITY Sales.OldEntity; ``` diff --git a/mdl-examples/bug-tests/access-1449-drop-entity-association-member.mdl b/mdl-examples/bug-tests/access-1449-drop-entity-association-member.mdl new file mode 100644 index 000000000..b3dae3236 --- /dev/null +++ b/mdl-examples/bug-tests/access-1449-drop-entity-association-member.mdl @@ -0,0 +1,62 @@ +mdl 1; +-- ============================================================================ +-- ako/mxcli#1449 (upstream mendixlabs/mxcli#1551) — `drop entity` left the +-- association's member entry in the access rule of the entity at the OTHER end. +-- +-- Dropping an entity removes every association with it at either end, in other +-- modules too, but reconciled no access rule. `drop association` did reconcile +-- — only the declaring module, though, so a cross-module OWNER Both +-- association left its TO entity's entry behind as well. And the reconcile +-- itself kept any reference to another module's association on faith, so even +-- `update security` printed "All entity access rules are up to date" over it. +-- +-- mxcli exec this-file -p app.mpr ; mx check app.mpr +-- before: [CE1613] "The selected association 'Access1449Crm.Lead_Order' +-- no longer exists." at Access rule of entity 'Access1449Shop.Order' +-- (with the drops run as their own script, one CE1613 per drop) +-- after: 0 errors; `describe entity Access1449Shop.Order` still grants +-- read *, write * and names none of the three associations +-- +-- Access rules are only checked with security on. +-- ============================================================================ + +alter app security ( SecurityLevel: prototype ); + +create module Access1449Shop; +create module role Access1449Shop.User; +create module Access1449Crm; +create module role Access1449Crm.User; + +@position(100, 100) +create persistent entity Access1449Shop.Customer ( Name: string(200) ); +@position(400, 100) +create persistent entity Access1449Shop.Order ( Number: string(20) ); +@position(100, 100) +create persistent entity Access1449Crm.Contact ( Name: string(200) ); +@position(400, 100) +create persistent entity Access1449Crm.Lead ( Name: string(200) ); + +-- Same module: the entry is on Order, FROM end. +create association Access1449Shop.Order_Customer + from Access1449Shop.Order to Access1449Shop.Customer; +-- Cross-module: the entry is on Order, in a module that is not the one of the +-- entity dropped below. +create association Access1449Shop.Order_Contact + from Access1449Shop.Order to Access1449Crm.Contact; +-- Cross-module, OWNER Both: a member of the TO end too, so Order carries an +-- entry for an association its own module does not declare. +create association Access1449Crm.Lead_Order + from Access1449Crm.Lead to Access1449Shop.Order + owner both; + +grant create, delete, read *, write * on entity Access1449Shop.Customer to Access1449Shop.User; +grant create, delete, read *, write * on entity Access1449Shop.Order to Access1449Shop.User; +grant create, delete, read *, write * on entity Access1449Crm.Contact to Access1449Crm.User; +grant create, delete, read *, write * on entity Access1449Crm.Lead to Access1449Crm.User; + +-- Each drop removes one association that Access1449Shop.Order's rule names. +drop entity Access1449Shop.Customer; +drop entity Access1449Crm.Contact; +drop entity Access1449Crm.Lead; + +describe entity Access1449Shop.Order; diff --git a/mdl/backend/modelsdk/domainmodel_security_write.go b/mdl/backend/modelsdk/domainmodel_security_write.go index 067d66576..2e61471b1 100644 --- a/mdl/backend/modelsdk/domainmodel_security_write.go +++ b/mdl/backend/modelsdk/domainmodel_security_write.go @@ -402,10 +402,13 @@ func (b *Backend) ReconcileMemberAccesses(unitID model.ID, moduleName string) (i // Cross-module associations declared ELSEWHERE whose TO end is an entity of // this module. Under `OWNER Both` such an association is a member of that // entity too, and nothing in this domain model says so (ako/mxcli#802). - incoming, err := b.incomingCrossAssociations(moduleName) + // The same scan records what every other module declares, which is what + // lets a foreign reference be judged rather than kept on faith. + elsewhere, err := b.scanProjectAssociations(moduleName) if err != nil { return 0, err } + incoming := elsewhere.incoming modified := 0 for _, el := range dm.EntitiesItems() { @@ -658,14 +661,23 @@ func (b *Backend) ReconcileMemberAccesses(unitID model.ID, moduleName string) (i // whose owner is no longer Both (ako/mxcli#802). rule.RemoveMemberAccesses(i) changed = true + case !assocRefBelongsTo(assocRef, moduleName) && + elsewhere.staleForeignRef(assocRef): + // A foreign reference its module no longer declares (CE1613). + // Kept before ako/mxcli#1449, so `drop entity` on the FROM end + // of a cross-module OWNER Both association, and `drop + // association` on one, left the entry on the TO end for good — + // and `update security` reported nothing to do. + rule.RemoveMemberAccesses(i) + changed = true case !assocRefBelongsTo(assocRef, moduleName): // An association is qualified by the module that DECLARES it, so // one inherited from a generalization in another module names - // that module. Its domain model is not loaded here, so the - // reference cannot be validated at all — preserve it, as the - // attribute branch above does, rather than delete what cannot be - // checked. Without this, every rule on a specialization of a - // marketplace entity lost its inherited associations. + // that module. Whether this entity inherits it cannot be decided + // from here — preserve it, as the attribute branch above does, + // rather than delete what cannot be checked. Without this, every + // rule on a specialization of a marketplace entity lost its + // inherited associations. covAssoc[assocRef] = true default: // Genuinely stale: an association of this module that no longer @@ -773,40 +785,66 @@ type incomingCross struct { both bool // OWNER Both: a member of the TO entity as well } -// incomingCrossAssociations maps each entity name of moduleName to the -// cross-module associations other modules declare TO it. A CrossAssociation -// names its TO entity by qualified name, so this is a scan of every other -// domain model — the only place the fact is stored. -func (b *Backend) incomingCrossAssociations(moduleName string) (map[string][]incomingCross, error) { +// projectAssociations is what a reconcile of one module needs to know about the +// associations every other module declares. +type projectAssociations struct { + // incoming maps each entity name of the module being reconciled to the + // cross-module associations other modules declare TO it. + incoming map[string][]incomingCross + // declared maps every module with a stored domain model (lower-cased) to the + // lower-cased names of the associations, regular and cross-module, it + // declares. System has no stored domain model and is absent. + declared map[string]map[string]bool +} + +// scanProjectAssociations reads every domain model once. A CrossAssociation +// names its TO entity by qualified name, so the incoming half is a scan of every +// other domain model — the only place the fact is stored. +func (b *Backend) scanProjectAssociations(moduleName string) (*projectAssociations, error) { units, err := mprread.ListUnitsWithContainer[*genDm.DomainModel](b.reader) if err != nil { return nil, fmt.Errorf("list domain models: %w", err) } + mods, err := b.reader.ListModules() + if err != nil { + return nil, fmt.Errorf("list modules: %w", err) + } + moduleNameOf := make(map[model.ID]string, len(mods)) + for _, m := range mods { + moduleNameOf[model.ID(m.ID)] = m.Name + } prefix := moduleName + "." - out := map[string][]incomingCross{} + out := &projectAssociations{incoming: map[string][]incomingCross{}, declared: map[string]map[string]bool{}} for _, u := range units { - var declaring string + declaring := moduleNameOf[u.ContainerID] + if declaring == "" { + continue // an orphaned domain model declares nothing reachable + } + names := out.declared[strings.ToLower(declaring)] + if names == nil { + names = map[string]bool{} + out.declared[strings.ToLower(declaring)] = names + } + for _, ae := range u.Element.AssociationsItems() { + if a, ok := ae.(*genDm.Association); ok && a.Name() != "" { + names[strings.ToLower(a.Name())] = true + } + } for _, ce := range u.Element.CrossAssociationsItems() { ca, ok := ce.(*genDm.CrossAssociation) if !ok || ca.Name() == "" { continue } + names[strings.ToLower(ca.Name())] = true + if strings.EqualFold(declaring, moduleName) { + continue + } child := ca.ChildQualifiedName() if len(child) <= len(prefix) || !strings.EqualFold(child[:len(prefix)], prefix) { continue } - if declaring == "" { - mi, err := b.reader.GetModule(string(u.ContainerID)) - if err != nil || mi == nil { - break // an orphaned domain model declares nothing reachable - } - declaring = mi.Name - } - if strings.EqualFold(declaring, moduleName) { - break - } name := child[len(prefix):] - out[name] = append(out[name], incomingCross{ + out.incoming[name] = append(out.incoming[name], incomingCross{ qn: declaring + "." + ca.Name(), module: declaring, both: ca.Owner() == "Both", }) } @@ -814,6 +852,34 @@ func (b *Backend) incomingCrossAssociations(moduleName string) (map[string][]inc return out, nil } +// staleForeignRef reports whether a MemberAccess naming an association declared +// in ANOTHER module is positively stale on an entity of the module being +// reconciled. +// +// The reconcile used to keep every such reference, because the one it was +// written for — an association inherited from a generalization in another module +// — cannot be validated without walking that chain. One case can be decided +// without it: the declaring module exists and no longer declares the +// association. Then it is a member of nothing, and mx check reports CE1613 "The +// selected association … no longer exists". That is what `drop entity` on the +// FROM end of a cross-module OWNER Both association, or `drop association` on +// one, leaves on the TO end (ako/mxcli#1449). +// +// A reference into a module the project does not store — System, whose domain +// model is virtual, or one that is simply absent — is still preserved, and so is +// any association that still exists. +func (p *projectAssociations) staleForeignRef(ref string) bool { + idx := strings.LastIndex(ref, ".") + if idx < 0 { + return false + } + names, known := p.declared[strings.ToLower(ref[:idx])] + if !known { + return false + } + return !names[strings.ToLower(ref[idx+1:])] +} + // foreignAncestorModule returns the module of the first generalization in ent's // chain that lies outside moduleName, or "" when the chain stays inside it. // ancestors is sameModuleAncestors(ent, …), nearest first. diff --git a/mdl/backend/modelsdk/reconcile_foreign_stale_assoc_test.go b/mdl/backend/modelsdk/reconcile_foreign_stale_assoc_test.go new file mode 100644 index 000000000..f0c972e9e --- /dev/null +++ b/mdl/backend/modelsdk/reconcile_foreign_stale_assoc_test.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 + +// A MemberAccess for an association is qualified by the module that DECLARES the +// association. On the TO entity of a cross-module `OWNER Both` association that +// is another module, and ReconcileMemberAccesses preserved every such foreign +// reference it could not find among the entity's members — the "preserve what +// cannot be checked" rule written for associations inherited from a +// generalization in another module (#758). So once the association was gone, +// the entry stayed for good. +// +// ako/mxcli#1449: `drop entity` on the FROM end removes the association, and the +// TO entity's rule went on naming it — mx check CE1613 "The selected association +// 'Crm.Lead_Order' no longer exists". `drop association` on the same association +// left the same entry, and `update security` printed "All entity access rules +// are up to date" over both. Measured on mxbuild 11.13.0. +// +// That case is checkable after all: an association its declaring module no +// longer declares is a member of nothing. +package modelsdkbackend + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// The #1449 shape: the FROM end of a cross-module OWNER Both association goes +// away (drop entity), or the association itself does (drop association), and the +// TO entity — Administration.Account, a specialization of System.User — must +// lose the entry. +func TestReconcile_DropsAForeignAssociationThatNoLongerExists(t *testing.T) { + const ref = "MyFirstModule.ZzNote_Account" + for _, tc := range []struct { + name string + remove func(t *testing.T, b *Backend, from *model.Module) + }{ + {"drop entity on the FROM end", func(t *testing.T, b *Backend, from *model.Module) { + dm, err := b.GetDomainModel(from.ID) + if err != nil { + t.Fatalf("GetDomainModel: %v", err) + } + if err := b.DeleteEntity(dm.ID, entityIDs(t, b, from.ID)["ZzNote"]); err != nil { + t.Fatalf("DeleteEntity: %v", err) + } + }}, + {"drop association", func(t *testing.T, b *Backend, from *model.Module) { + dm, err := b.GetDomainModel(from.ID) + if err != nil { + t.Fatalf("GetDomainModel: %v", err) + } + for _, ca := range dm.CrossAssociations { + if ca.Name == "ZzNote_Account" { + if err := b.DeleteCrossAssociation(dm.ID, ca.ID); err != nil { + t.Fatalf("DeleteCrossAssociation: %v", err) + } + } + } + }}, + } { + t.Run(tc.name, func(t *testing.T) { + b, from, to := crossBothFixture(t, "Both") + reconcileModule(t, b, to) + populated, naming := accountRulesNaming(t, b, to, ref) + if populated == 0 || naming != populated { + t.Fatalf("precondition: %d of %d populated rules name %s", naming, populated, ref) + } + + tc.remove(t, b, from) + + n := reconcileModule(t, b, to) + if _, naming := accountRulesNaming(t, b, to, ref); naming != 0 { + t.Errorf("%d rule(s) on the TO entity still name %s after it was removed — CE1613", naming, ref) + } + // The count drives `update security`'s "All entity access rules are + // up to date", so it is part of the contract. + if n != populated { + t.Errorf("reconcile reported %d modified rule(s), want %d", n, populated) + } + }) + } +} + +// targetFixture adds, beside crossBothFixture's ZzNote, an entity +// Administration.ZzTarget with no generalization and a populated rule, plus the +// cross-module association MyFirstModule.ZzNote_Target FROM ZzNote TO it, owned +// by both ends. +func targetFixture(t *testing.T) (b *Backend, from, to *model.Module) { + t.Helper() + b, from, to = crossBothFixture(t, "Default") + toDM, err := b.GetDomainModel(to.ID) + if err != nil { + t.Fatalf("GetDomainModel: %v", err) + } + if err := b.CreateEntity(toDM.ID, &domainmodel.Entity{Name: "ZzTarget", Persistable: true, + Attributes: []*domainmodel.Attribute{{Name: "Code", Type: &domainmodel.StringAttributeType{}}}}); err != nil { + t.Fatalf("CreateEntity: %v", err) + } + if err := b.AddEntityAccessRule(backend.EntityAccessRuleParams{ + UnitID: toDM.ID, EntityName: "ZzTarget", + RoleNames: []string{"Administration.Administrator"}, + DefaultMemberAccess: "ReadWrite", + MemberAccesses: []types.EntityMemberAccess{ + {AttributeRef: "Administration.ZzTarget.Code", AccessRights: "ReadWrite"}, + }, + }); err != nil { + t.Fatalf("AddEntityAccessRule: %v", err) + } + fromDM, err := b.GetDomainModel(from.ID) + if err != nil { + t.Fatalf("GetDomainModel: %v", err) + } + if err := b.CreateCrossAssociation(fromDM.ID, &domainmodel.CrossModuleAssociation{ + Name: "ZzNote_Target", ParentID: entityIDs(t, b, from.ID)["ZzNote"], ChildRef: "Administration.ZzTarget", + Type: "Reference", Owner: "Both", + }); err != nil { + t.Fatalf("CreateCrossAssociation: %v", err) + } + return b, from, to +} + +// entityNamesRef reports whether any of an entity's rules carries a +// MemberAccess for the association ref. +func entityNamesRef(t *testing.T, b *Backend, mod *model.Module, entity, ref string) bool { + t.Helper() + dm, err := b.GetDomainModel(mod.ID) + if err != nil { + t.Fatalf("GetDomainModel: %v", err) + } + for _, e := range dm.Entities { + if e.Name != entity { + continue + } + for _, r := range e.AccessRules { + for _, ma := range r.MemberAccesses { + if ma.AssociationName == ref { + return true + } + } + } + return false + } + t.Fatalf("entity %s not found in %s", entity, mod.Name) + return false +} + +// Control: an association declared in another module that this entity INHERITS +// — from a generalization in that module — still exists and is a member, and +// the reconcile cannot see that from here. Judging foreign references must not +// reach it: it stays. +func TestReconcile_KeepsAForeignAssociationInheritedAcrossModules(t *testing.T) { + const ref = "MyFirstModule.ZzNote_Target" + b, from, to := targetFixture(t) + toDM, err := b.GetDomainModel(to.ID) + if err != nil { + t.Fatalf("GetDomainModel: %v", err) + } + if err := b.CreateEntity(toDM.ID, &domainmodel.Entity{Name: "ZzSpecial", Persistable: true, + GeneralizationRef: from.Name + ".ZzNote"}); err != nil { + t.Fatalf("CreateEntity: %v", err) + } + if err := b.AddEntityAccessRule(backend.EntityAccessRuleParams{ + UnitID: toDM.ID, EntityName: "ZzSpecial", + RoleNames: []string{"Administration.Administrator"}, + DefaultMemberAccess: "ReadWrite", + MemberAccesses: []types.EntityMemberAccess{ + {AttributeRef: "MyFirstModule.ZzNote.Title", AccessRights: "ReadWrite"}, + {AssociationRef: ref, AccessRights: "ReadWrite"}, + }, + }); err != nil { + t.Fatalf("AddEntityAccessRule: %v", err) + } + + reconcileModule(t, b, to) + if !entityNamesRef(t, b, to, "ZzSpecial", ref) { + t.Fatalf("an association inherited from a generalization in another module was removed — CE0066") + } +} diff --git a/mdl/executor/cmd_associations.go b/mdl/executor/cmd_associations.go index c42329f65..81ec49f32 100644 --- a/mdl/executor/cmd_associations.go +++ b/mdl/executor/cmd_associations.go @@ -370,11 +370,7 @@ func execAlterAssociation(ctx *ExecContext, s *ast.AlterAssociationStmt) error { if err := verifyAssociationAltered(ctx, module.ID, s, want); err != nil { return err } - toModule := "" - if i := strings.LastIndex(ca.ChildRef, "."); i > 0 { - toModule = ca.ChildRef[:i] - } - if err := reconcileAfterAssociationAlter(ctx, s, module.Name, toModule); err != nil { + if err := reconcileAfterAssociationAlter(ctx, s, module.Name, qualifiedNameModule(ca.ChildRef)); err != nil { return err } fmt.Fprintf(ctx.Output, "Altered association: %s\n", s.Name) @@ -400,15 +396,29 @@ func reconcileAfterAssociationAlter(ctx *ExecContext, s *ast.AlterAssociationStm if s.Operation != ast.AlterAssociationSetOwner { return nil } + return reconcileModulesAccess(ctx, "after changing the owner", modules...) +} + +// reconcileModulesAccess reconciles the entity access rules of each named module +// once, after a write that changed which entities an association is a member of +// — or removed associations outright. It is the one reconcile behind create, +// alter and drop association and drop entity, so that every statement that can +// leave a member entry behind reconciles the same set of modules the same way. +// +// The modules are named by the caller because the rule that needs fixing is not +// always in the module written: an association is a member of its FROM entity, +// and under OWNER Both of its TO entity, which for a cross-module association +// lives in another module. +func reconcileModulesAccess(ctx *ExecContext, why string, modules ...string) error { invalidateHierarchy(ctx) invalidateDomainModelsCache(ctx) seen := map[string]bool{} for _, name := range modules { - if name == "" || seen[name] { + if name == "" || seen[strings.ToLower(name)] { continue } - seen[name] = true - if err := reconcileModuleAccess(ctx, name, "after changing the owner"); err != nil { + seen[strings.ToLower(name)] = true + if err := reconcileModuleAccess(ctx, name, why); err != nil { return err } } @@ -555,15 +565,10 @@ func execDropAssociation(ctx *ExecContext, s *ast.DropAssociationStmt) error { // rule that named it, which Mendix rejects with CE1613 "The selected // association … no longer exists". Creating one already reconciles; dropping // one has to as well, or the drop produces a model that will not build. - afterDrop := func() { - invalidateHierarchy(ctx) - invalidateDomainModelsCache(ctx) - if freshDM, err := ctx.Backend.GetDomainModel(module.ID); err == nil { - if count, err := ctx.Backend.ReconcileMemberAccesses(freshDM.ID, module.Name); err == nil && count > 0 { - fmt.Fprintf(ctx.Output, "Reconciled %d access rule(s) after dropping the association\n", count) - } - } - ctx.trackModifiedDomainModel(module.ID, module.Name) + // toModule is a cross-module association's TO end: under OWNER Both its + // entity holds an entry too, in a module this one is not (ako/mxcli#1449). + afterDrop := func(toModule string) error { + return reconcileModulesAccess(ctx, "after dropping the association", module.Name, toModule) } // A message definition holds the association by qualified name, and nothing @@ -585,7 +590,9 @@ func execDropAssociation(ctx *ExecContext, s *ast.DropAssociationStmt) error { if err := ctx.Backend.DeleteAssociation(dm.ID, assoc.ID); err != nil { return mdlerrors.NewBackend("delete association", err) } - afterDrop() + if err := afterDrop(""); err != nil { + return err + } fmt.Fprintf(ctx.Output, "Dropped association: %s\n", s.Name) return nil } @@ -595,7 +602,9 @@ func execDropAssociation(ctx *ExecContext, s *ast.DropAssociationStmt) error { if err := ctx.Backend.DeleteCrossAssociation(dm.ID, ca.ID); err != nil { return mdlerrors.NewBackend("delete cross-module association", err) } - afterDrop() + if err := afterDrop(qualifiedNameModule(ca.ChildRef)); err != nil { + return err + } fmt.Fprintf(ctx.Output, "Dropped cross-module association: %s\n", s.Name) return nil } diff --git a/mdl/executor/cmd_entities.go b/mdl/executor/cmd_entities.go index e486a0eb0..78b198097 100644 --- a/mdl/executor/cmd_entities.go +++ b/mdl/executor/cmd_entities.go @@ -2057,6 +2057,16 @@ func execDropEntity(ctx *ExecContext, s *ast.DropEntityStmt) error { // Warn about references before deleting (best-effort) warnEntityReferences(ctx, s.Name.String()) + // DeleteEntity removes every association with this entity at either + // end, in other modules too, and their member entries stay behind on + // the entity at the other end: CE1613 "The selected association … + // no longer exists" (ako/mxcli#1449). Which modules hold those rules + // has to be read before the delete, while the associations exist. + reconcile, err := associationMemberModules(ctx, module.Name, entity) + if err != nil { + return err + } + // If this is a view entity, also delete the associated ViewEntitySourceDocument if entity.Source == "DomainModels$OqlViewEntitySource" { if err := ctx.Backend.DeleteViewEntitySourceDocumentByName(s.Name.Module, s.Name.Name); err != nil { @@ -2067,6 +2077,9 @@ func execDropEntity(ctx *ExecContext, s *ast.DropEntityStmt) error { return mdlerrors.NewBackend("delete entity", err) } invalidateDomainModelsCache(ctx) + if err := reconcileModulesAccess(ctx, "after dropping the entity", reconcile...); err != nil { + return err + } fmt.Fprintf(ctx.Output, "Dropped entity: %s\n", s.Name) return nil } @@ -2075,6 +2088,37 @@ func execDropEntity(ctx *ExecContext, s *ast.DropEntityStmt) error { return mdlerrors.NewNotFoundMsg("entity", fmt.Sprint(s.Name), fmt.Sprintf("entity not found: %s", s.Name)) } +// associationMemberModules names every module whose access rules can hold a +// member entry for an association with entity (of module moduleName) at one end: +// the entity's own module, which declares the associations it is the FROM end +// of; the module of each TO end of those that are cross-module, whose entity is +// a member under OWNER Both; and every module declaring a cross-module +// association TO the entity, whose FROM entity holds the entry. The last two are +// why reconciling the dropped entity's module alone is not enough. +func associationMemberModules(ctx *ExecContext, moduleName string, entity *domainmodel.Entity) ([]string, error) { + modules := []string{moduleName} + dms, err := ctx.Backend.ListDomainModels() + if err != nil { + return nil, mdlerrors.NewBackend("list domain models", err) + } + h, err := getHierarchy(ctx) + if err != nil { + return nil, mdlerrors.NewBackend("build hierarchy", err) + } + qn := moduleName + "." + entity.Name + for _, dm := range dms { + for _, ca := range dm.CrossAssociations { + switch { + case ca.ParentID == entity.ID: + modules = append(modules, qualifiedNameModule(ca.ChildRef)) + case strings.EqualFold(ca.ChildRef, qn): + modules = append(modules, h.GetModuleName(dm.ContainerID)) + } + } + } + return modules, nil +} + // warnEntityReferences prints a warning if the entity is referenced by other elements. // Uses the catalog if available; silently skips if catalog is not built. func warnEntityReferences(ctx *ExecContext, entityName string) { diff --git a/mdl/executor/cmd_entities_drop_reconcile_test.go b/mdl/executor/cmd_entities_drop_reconcile_test.go new file mode 100644 index 000000000..dd0f2c96a --- /dev/null +++ b/mdl/executor/cmd_entities_drop_reconcile_test.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "sort" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// ako/mxcli#1449 (mendixlabs/mxcli#1551): `drop entity` removes every +// association with the entity at either end — the backend cascades into other +// modules' domain models too — but reconciled no access rule. The member entry +// for each removed association stayed on the entity at the OTHER end, and mx +// check failed with CE1613 "The selected association 'Shop.Order_Customer' no +// longer exists" at the access rule of that entity. `drop association` on the +// same association reconciles; `drop entity` skipped it. +// +// The rule that keeps the stale entry can live in a module other than the +// dropped entity's: the FROM entity of a cross-module association TO it, or the +// TO entity of an OWNER Both cross-module association FROM it. Each of those +// modules has to be reconciled, not only the dropped entity's own. + +// dropEntityFixture: module Shop holds Order, Customer and Lonely, with +// Shop.Order_Customer (Order → Customer) and the cross-module Shop.Order_Contact +// (Order → Crm.Contact). Module Crm holds Contact and Lead, with the cross-module +// Crm.Lead_Order (Lead → Shop.Order) under OWNER Both. Hr is unrelated. +func dropEntityFixture(t *testing.T) (*ExecContext, *[]string, func() string) { + t.Helper() + shop, crm, hr := mkModule("Shop"), mkModule("Crm"), mkModule("Hr") + order, customer, lonely := mkEntity(shop.ID, "Order"), mkEntity(shop.ID, "Customer"), mkEntity(shop.ID, "Lonely") + contact, lead := mkEntity(crm.ID, "Contact"), mkEntity(crm.ID, "Lead") + employee := mkEntity(hr.ID, "Employee") + + shopDM := mkDomainModel(shop.ID, order, customer, lonely) + shopDM.Associations = []*domainmodel.Association{mkAssociation(shop.ID, "Order_Customer", order.ID, customer.ID)} + orderContact := &domainmodel.CrossModuleAssociation{Name: "Order_Contact", ParentID: order.ID, ChildRef: "Crm.Contact", + Type: "Reference", Owner: "Default"} + orderContact.ID = nextID("xassoc") + shopDM.CrossAssociations = []*domainmodel.CrossModuleAssociation{orderContact} + + crmDM := mkDomainModel(crm.ID, contact, lead) + leadOrder := &domainmodel.CrossModuleAssociation{Name: "Lead_Order", ParentID: lead.ID, ChildRef: "Shop.Order", + Type: "Reference", Owner: "Both"} + leadOrder.ID = nextID("xassoc") + crmDM.CrossAssociations = []*domainmodel.CrossModuleAssociation{leadOrder} + + hrDM := mkDomainModel(hr.ID, employee) + + dms := map[model.ID]*domainmodel.DomainModel{shop.ID: shopDM, crm.ID: crmDM, hr.ID: hrDM} + mods := []*model.Module{shop, crm, hr} + + var reconciled []string + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return mods, nil }, + GetDomainModelFunc: func(id model.ID) (*domainmodel.DomainModel, error) { + return dms[id], nil + }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { + return []*domainmodel.DomainModel{shopDM, crmDM, hrDM}, nil + }, + DeleteEntityFunc: func(model.ID, model.ID) error { return nil }, + ReconcileMemberAccessesFunc: func(_ model.ID, moduleName string) (int, error) { + reconciled = append(reconciled, moduleName) + return 1, nil + }, + } + ctx, buf := newMockCtx(t, withBackend(mb), withHierarchy(mkHierarchy(mods...))) + return ctx, &reconciled, buf.String +} + +func TestExecDropEntity_ReconcilesEveryModuleHoldingAMemberOfItsAssociations(t *testing.T) { + for _, tc := range []struct { + name string + entity ast.QualifiedName + want []string // modules whose access rules are reconciled + }{ + // The reported case: Order's rule names Shop.Order_Customer. + {"same-module TO end", ast.QualifiedName{Module: "Shop", Name: "Customer"}, []string{"Shop"}}, + // Reported too: the rule naming Shop.Order_Contact is Shop.Order's, in the + // module that is NOT the dropped entity's. + {"cross-module TO end", ast.QualifiedName{Module: "Crm", Name: "Contact"}, []string{"Crm", "Shop"}}, + // OWNER Both: Shop.Order's rule names Crm.Lead_Order as its TO end. + {"cross-module FROM end, owner Both", ast.QualifiedName{Module: "Crm", Name: "Lead"}, []string{"Crm", "Shop"}}, + {"FROM end of both kinds", ast.QualifiedName{Module: "Shop", Name: "Order"}, []string{"Crm", "Shop"}}, + // Control: no association, nothing in another module to reconcile. + {"no associations", ast.QualifiedName{Module: "Shop", Name: "Lonely"}, []string{"Shop"}}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, reconciled, out := dropEntityFixture(t) + assertNoError(t, execDropEntity(ctx, &ast.DropEntityStmt{Name: tc.entity})) + got := append([]string(nil), *reconciled...) + sort.Strings(got) + if strings.Join(got, ",") != strings.Join(tc.want, ",") { + t.Fatalf("reconciled modules %v, want %v — a module left out keeps a member for an association that no longer exists (CE1613)", got, tc.want) + } + if !strings.Contains(out(), "Reconciled 1 access rule(s) in module "+tc.want[0]) { + t.Errorf("output does not report the reconcile: %q", out()) + } + if !strings.Contains(out(), "Dropped entity: "+tc.entity.String()) { + t.Errorf("output does not report the drop: %q", out()) + } + }) + } +} + +// The `drop association` half of the same defect: a cross-module association +// under OWNER Both is a member of its TO entity, whose rules live in the other +// module, and only the declaring module was reconciled. +func TestExecDropAssociation_CrossModuleReconcilesTheToModule(t *testing.T) { + ctx, reconciled, _ := dropEntityFixture(t) + mb := ctx.Backend.(*mock.MockBackend) + mb.DeleteCrossAssociationFunc = func(model.ID, model.ID) error { return nil } + + assertNoError(t, execDropAssociation(ctx, &ast.DropAssociationStmt{ + Name: ast.QualifiedName{Module: "Crm", Name: "Lead_Order"}, + })) + got := append([]string(nil), *reconciled...) + sort.Strings(got) + if strings.Join(got, ",") != "Crm,Shop" { + t.Fatalf("reconciled modules %v, want [Crm Shop] — Shop.Order keeps Crm.Lead_Order (CE1613)", got) + } +} From 36fe5a91a271df2a09e9775dbfaec46834885f7b Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 09:21:40 +0000 Subject: [PATCH 03/13] fix: multi-line @expect silently truncated to its first line (ako/mxcli#1514) Mirror of mendixlabs/mxcli#1604. Cause: parseAnnotations (cmd/mxcli/testrunner/parser.go) read the doc comment line by line, and the @expect/@verify patterns captured only the rest of the tag's own line. A continuation line carries no tag, so it matched nothing and was dropped without a message. When the first line was a complete expression the test asserted something else: with `and` it passed on less, with `or` it failed although the whole assertion held. A bare `@expect` with the expression below it was dropped whole, because the pattern required text on the tag line. Fix: @expect and @verify open an annotation whose text runs, as a javadoc block tag's does, to the next @tag or the end of the comment; the joined text is what ParseExpect/ParseVerify compile. Text after the tag that does not continue the expression is therefore an ERROR (MDL-TEST01 in check) whose message says the annotation spans N lines and that a description goes before the first tag; it is never skipped. @test/@setup/@cleanup/@throws stay one line. Case B: `check -p` on a lone test file reported "microflow not found" for a target a setup script creates. The script-set form already resolves it; the not-found report now names it, and check/test --help say a test file can be one of the files. Verification: parser_multiline_annotation_test.go and TestCheck_TestFileMissingTargetPointsAtTheScriptSet fail with the fix reverted (Raw = "$r = 'zzz'", no hint). Runtime A/B on 11.13.0 with `mxcli test --local --db-type hsqldb`: the baseline PASSED `$r = 'a;b'` + `and length($r) = 4` and FAILED the true `or` assertion ("expected $r = 'zzz', actual: a;b"); the fix inverts both. The bug-test fixtures discriminate in `make check-mdl` (baseline fails the positive one and passes the .fail one). Co-Authored-By: Claude Opus 5.5 --- ...tiline-expect-truncated-to-first-line.json | 1 + .../skills/mendix/test-microflows/SKILL.md | 26 +- .../test-microflows/reference/annotations.md | 20 ++ cmd/mxcli/check_test_file_setup_hint_test.go | 79 ++++++ cmd/mxcli/cmd_check.go | 35 ++- cmd/mxcli/cmd_test_run.go | 11 + cmd/mxcli/testrunner/parser.go | 98 ++++++-- .../parser_multiline_annotation_test.go | 226 ++++++++++++++++++ ...test-runner-1514-multiline-expect.test.mdl | 47 ++++ .../bug-tests/test-runner-1514-setup.mdl | 16 ++ ...unner-1514-text-after-expect.fail.test.mdl | 20 ++ 11 files changed, 557 insertions(+), 22 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/cmd-mxcli/2026-10-25-multiline-expect-truncated-to-first-line.json create mode 100644 cmd/mxcli/check_test_file_setup_hint_test.go create mode 100644 cmd/mxcli/testrunner/parser_multiline_annotation_test.go create mode 100644 mdl-examples/bug-tests/test-runner-1514-multiline-expect.test.mdl create mode 100644 mdl-examples/bug-tests/test-runner-1514-setup.mdl create mode 100644 mdl-examples/bug-tests/test-runner-1514-text-after-expect.fail.test.mdl diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-25-multiline-expect-truncated-to-first-line.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-25-multiline-expect-truncated-to-first-line.json new file mode 100644 index 000000000..4a27f3f7b --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-25-multiline-expect-truncated-to-first-line.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-25","symptom":"`mxcli test`: an @expect written over two lines (` * @expect $r = 'a;b'` then ` * and length($r) = 3`) kept only its first line with no message; --list, check and the run all asserted `$r = 'a;b'`, so with `and` the test passed on less than it said and with `or` it failed although the whole assertion held. Only a first line that did not parse on its own was reported, as a confusing \"expected a value at column 16\". Separately, `check -p` on a lone test file reported \"microflow not found\" for a target a setup script creates, with no sign that the script-set form (`check setup.mdl x.test.mdl -p`) resolves it","cause":"parseAnnotations read the doc comment one line at a time and expectPattern/verifyPattern captured the rest of the tag's own line; a following ` * …` line carries no tag, so it matched nothing and fell through every branch. The patterns also required text on the tag line (`\\s+(.+)`), so a bare `@expect` with the expression below it was dropped whole","file":"cmd/mxcli/testrunner/parser.go (parseAnnotations, openAssertion, continuationHint), cmd/mxcli/cmd_check.go (testFileMissingTargetHint, check --help)","fix":"@expect and @verify open an annotation whose text runs, as a javadoc block tag's does, to the next `@tag` line or the end of the comment; the joined text (single spaces) is what ParseExpect/ParseVerify compile. Text after the tag that is not part of the expression therefore makes the assertion an ERROR whose message says it spans N lines and that a description goes before the first tag. @test/@setup/@cleanup/@throws stay one line, so prose under a test's name is still description. check prints a hint naming the script-set form under a test file's \"not found\" reference errors, and check/test --help say a test file can be one of the set","insight":"The same silent-absence shape as the original @expect fix (#46), one level up: that fix made a line the parser could not compile an error, but a line the parser never LOOKED at was still dropped. `--list` printing the truncated text back is what made it look wired. Javadoc's rule (a block tag's text continues to the next tag) is the fail-closed choice; stopping at a blank line was considered and rejected because it re-opens the silent drop for an expression with a blank line in it, at the cost of turning prose written after the tags (seen once, in upgrade_source_test's fixture) into an ERROR with a message naming the fix. Runtime A/B on 11.13.0 (`mxcli test --local --db-type hsqldb`): baseline PASSED `$r = 'a;b'` + `and length($r) = 4` and FAILED the true `or` assertion with \"expected $r = 'zzz', actual: a;b\"; fixed inverts both. The paren-on-line-1 fixture is what makes `make check-mdl` fail if the continuation is lost again. Sibling not fixed: an untagged line after @throws is still ignored, so `@throws` + a message on the next line becomes \"any exception\"","test":"cmd/mxcli/testrunner/parser_multiline_annotation_test.go (8 tests; fail on revert with Raw = \"$r = 'zzz'\" etc.), cmd/mxcli/check_test_file_setup_hint_test.go TestCheck_TestFileMissingTargetPointsAtTheScriptSet (with the set-form and no-hint controls); mdl-examples/bug-tests/test-runner-1514-*.mdl","refs":["ako/mxcli#1514","mendixlabs/mxcli#1604"]} diff --git a/.claude/skills/mendix/test-microflows/SKILL.md b/.claude/skills/mendix/test-microflows/SKILL.md index fed89b612..ff1b9e6e7 100644 --- a/.claude/skills/mendix/test-microflows/SKILL.md +++ b/.claude/skills/mendix/test-microflows/SKILL.md @@ -17,13 +17,13 @@ description: "Write and run MDL-based microflow tests with `mxcli test` — anno | 223-238 | string Operations | | 239-262 | Running Tests | | 263-450 | How It Works | -| 451-477 | Check a test file before you run it | -| 478-584 | Writing Good Tests | -| 585-599 | Test File Organization | -| 600-612 | Interpreting Failures | -| 613-634 | CI Integration | -| 635-641 | Related Skills | -| 642-644 | Warm test loop (`mxcli test --local [--watch]`, `--attach`, `run --local --test-endpoint`) | +| 451-489 | Check a test file before you run it | +| 490-596 | Writing Good Tests | +| 597-611 | Test File Organization | +| 612-624 | Interpreting Failures | +| 625-646 | CI Integration | +| 647-653 | Related Skills | +| 654-656 | Warm test loop (`mxcli test --local [--watch]`, `--attach`, `run --local --test-endpoint`) | # Test Microflows Skill @@ -459,6 +459,18 @@ whose body would not compile is reported here instead of failing the injection with nothing but "the project cannot be deployed". An `@expect` or `@verify` that cannot be evaluated is reported here too, as `MDL-TEST01`. +A test file **creates nothing itself**, so the microflows its tests call must be +in the project already. When a setup script creates them, check the two in one +call, the setup script first — each file is resolved against a scratch copy +holding what the files before it write, and the project is not changed: + +```bash +mxcli check setup.mdl tests/orders.test.mdl -p app.mpr +``` + +Checked alone, the test file reports the microflow as not found, with a hint +naming this form. + One rule to know about, because its symptom is confusing and its shape is common in tests: `retrieve $x … first` — and `retrieve $x … limit 1` in a script without the `mdl 1;` header — binds a **single object**, not a one-element list, so diff --git a/.claude/skills/mendix/test-microflows/reference/annotations.md b/.claude/skills/mendix/test-microflows/reference/annotations.md index 932d0a5cb..8ac83034f 100644 --- a/.claude/skills/mendix/test-microflows/reference/annotations.md +++ b/.claude/skills/mendix/test-microflows/reference/annotations.md @@ -21,6 +21,26 @@ indentation). Quoting one inside a sentence — ``a test with `@expect $x = 1` asserts …`` — is documentation, not an annotation, so a doc comment can explain itself without giving the test assertions nobody wrote. +An `@expect` or `@verify` **may span several lines**. Its text runs on, as a +javadoc block tag's does, to the next tag or the end of the comment, and all of +it is the one assertion: + +```mdl +/** + * @test the separator survives + * @expect $r = 'a;b' + * and length($r) = 3 + */ +``` + +So a test's description goes **before** its first tag. Text under an `@expect` +that does not continue the expression is not skipped — it makes the assertion +an ERROR that says the annotation spans several lines. Reading only the tag's +own line used to drop the continuation without a word, and the test then +asserted `$r = 'a;b'` alone: less than it said (ako/mxcli#1514). `@test`, +`@setup`, `@cleanup` and `@throws` are one line each, so prose under a test's +name is still description. + ### A test run leaves the project byte-identical `mxcli test` injects an `MxTest` module, builds, runs, and takes the injection diff --git a/cmd/mxcli/check_test_file_setup_hint_test.go b/cmd/mxcli/check_test_file_setup_hint_test.go new file mode 100644 index 000000000..04592279f --- /dev/null +++ b/cmd/mxcli/check_test_file_setup_hint_test.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// ako/mxcli#1514 case B (upstream mendixlabs/mxcli#1604): `check -p` on a test +// file whose microflow a setup script has not created yet reported only +// "microflow not found". The way to check such a file exists — pass the setup +// script first, as a script set, and each file resolves against what the files +// before it write (mendixlabs/mxcli#1355) — but neither the message nor +// `mxcli help check` said it applies to test files, which create nothing of +// their own and so always depend on something else for their targets. +// +// Controls: the set form passes (so the hint points at something that works), +// and a test file whose references resolve gets no hint. +func TestCheck_TestFileMissingTargetPointsAtTheScriptSet(t *testing.T) { + src := filepath.Join("..", "..", "testdata", "pedapp") + if _, err := os.Stat(filepath.Join(src, "PedApp.mpr")); err != nil { + t.Skipf("PedApp fixture not found: %v", err) + } + dir := t.TempDir() + if err := copyTree(src, dir); err != nil { + t.Fatal(err) + } + mpr := filepath.Join(dir, "PedApp.mpr") + _ = checkCmd.InheritedFlags() + _ = rootCmd.PersistentFlags().Set("project", mpr) + defer func() { + _ = rootCmd.PersistentFlags().Set("project", "") + rootCmd.PersistentFlags().Lookup("project").Changed = false + }() + if f := checkCmd.Flags().Lookup("format"); f != nil { + _ = f.Value.Set(f.DefValue) + } + + scripts := t.TempDir() + setup := writeScript(t, scripts, "setup.mdl", `mdl 1; +create module Probe; +create microflow Probe.SUB_Make () returns String as $r begin return 'a;b'; end; +`) + testFile := writeScript(t, scripts, "missing.test.mdl", `mdl 1; +/** + * @test later + * @expect $r = 'a;b' + */ +$r = call microflow Probe.SUB_Make(); +/ +`) + + var code int + out := captureStd(t, func() { code = runCheckFiles(checkCmd, []string{testFile}) }) + if code == 0 || !strings.Contains(out, "microflow not found: Probe.SUB_Make") { + t.Fatalf("the missing microflow was not reported (exit %d):\n%s", code, out) + } + for _, want := range []string{"a test file creates nothing", "mxcli check " + testFile} { + if !strings.Contains(out, want) { + t.Errorf("the not-found report does not say %q, so the script-set form stays undiscovered:\n%s", want, out) + } + } + + // Control: the form the hint names works. + out = captureStd(t, func() { code = runCheckFiles(checkCmd, []string{setup, testFile}) }) + if code != 0 { + t.Errorf("control: the setup script and the test file as one set failed (exit %d):\n%s", code, out) + } + + // Control: nothing missing, nothing to hint at. + ok := writeScript(t, scripts, "ok.test.mdl", "mdl 1;\n/**\n * @test fine\n * @expect $a = 1\n */\ndeclare $a Integer = 1;\n/\n") + out = captureStd(t, func() { code = runCheckFiles(checkCmd, []string{ok}) }) + if code != 0 || strings.Contains(out, "a test file creates nothing") { + t.Errorf("control: a test file whose references resolve got the hint (exit %d):\n%s", code, out) + } +} diff --git a/cmd/mxcli/cmd_check.go b/cmd/mxcli/cmd_check.go index 2eca2946a..3cf51ee41 100644 --- a/cmd/mxcli/cmd_check.go +++ b/cmd/mxcli/cmd_check.go @@ -87,7 +87,11 @@ Several files are checked as one script set, run in the order given, as resolved against a scratch copy of it that the files before it have been applied to, so a name an earlier file creates resolves in a later one; a file that does not pass is left out of the copy, as exec's pre-flight would refuse -it and write nothing. The project itself is not changed. The set is also first +it and write nothing. The project itself is not changed. A test file +(.test.mdl, .test.md) can be one of the files: it creates nothing itself, so +when its tests call a microflow a setup script creates, check the two together, +the setup script first — checked alone, the test file reports the microflow as +not found. The set is also first read as a whole for two "create or modify" statements declaring the same flow — a placeholder ("stub") followed by the real flow. That is reported as an MDL-STUB01 warning naming both statements: run in order, the stub replaces the real flow before @@ -138,6 +142,9 @@ Examples: # Check a script set, run in this order mxcli check 10-domain.mdl 20-flows.mdl 30-pages.mdl -p app.mpr + # Check a test file against what its setup script creates + mxcli check setup.mdl tests/orders.test.mdl -p app.mpr + # Read the script from stdin cat script.mdl | mxcli check - `, @@ -422,6 +429,9 @@ func runCheckFileAgainst(cmd *cobra.Command, filePath, against string) int { fmt.Fprintf(os.Stderr, " %v\n", err) } fmt.Fprintf(os.Stderr, "\n✗ %d reference error(s) found\n", len(validationErrors)) + if hint := testFileMissingTargetHint(filePath, validationErrors); hint != "" { + fmt.Fprint(os.Stderr, hint) + } } return finish(1) } @@ -672,6 +682,29 @@ func applyToScratch(scratch *scriptdiff.Scratch, file string, checkCode int) str return "" } +// testFileMissingTargetHint is the line under a test file's "not found" +// reference errors that names the way to check it (ako/mxcli#1514). +// +// A test file creates nothing of its own — applyToScratch does not even run +// it — so whatever its tests call has to be in the project already or come +// from a script. Check resolves a later file against what the files before it +// write, but a lone "microflow not found" gave no sign that this applies to +// test files; the user had to know the script-set form existed. +func testFileMissingTargetHint(filePath string, errs []error) string { + if !testrunner.IsTestFile(filePath) { + return "" + } + for _, err := range errs { + if strings.Contains(err.Error(), "not found") { + return fmt.Sprintf(" Hint: a test file creates nothing itself, so what its tests use must already be in\n"+ + " the project. If a script creates it, check the two together, that script first —\n"+ + " each file is resolved against what the files before it write:\n"+ + " mxcli check %s -p \n", mdlSourceLabel(filePath)) + } + } + return "" +} + // parseScriptSet reads and parses each file for the set-level checks. A file // that cannot be read or parsed is left out (Prog nil): its own check reports // that. diff --git a/cmd/mxcli/cmd_test_run.go b/cmd/mxcli/cmd_test_run.go index 6fde43122..eebd11894 100644 --- a/cmd/mxcli/cmd_test_run.go +++ b/cmd/mxcli/cmd_test_run.go @@ -28,6 +28,17 @@ Tests use MDL syntax with javadoc-style annotations for expectations: ); / +An @expect or @verify may run over several lines: its text continues, as a +javadoc tag's does, to the next @tag or the end of the comment, and all of it +is the assertion. Put a test's description before its first tag — text after an +@expect that is not part of the expression makes the assertion an ERROR rather +than being ignored. + +Check a test file without running it with 'mxcli check file.test.mdl -p app.mpr'. +A test file creates nothing itself: when its tests call what a setup script +creates, check the two together, setup first ('mxcli check setup.mdl +file.test.mdl -p app.mpr'). + With --local the app runs on mxcli's own runtime instead of a container — the same boot as 'mxcli run --local', so no Docker daemon is needed. It uses its own ports (8081/8091) and its own '_test' database, so a warm 'run --local' diff --git a/cmd/mxcli/testrunner/parser.go b/cmd/mxcli/testrunner/parser.go index 06673490b..13a251513 100644 --- a/cmd/mxcli/testrunner/parser.go +++ b/cmd/mxcli/testrunner/parser.go @@ -761,8 +761,11 @@ var ( // in a sentence" gave the test an assertion nobody wrote, and "@cleanup none // would apply here" changed the cleanup strategy. The leading `*` and its // indentation are stripped before these run. - expectPattern = regexp.MustCompile(`^@expect\s+(.+)`) - verifyPattern = regexp.MustCompile(`^@verify\s+(.+)`) + // + // @expect and @verify match on their own as well: their text may start on + // the next line, and a bare tag that matched nothing was dropped. + expectPattern = regexp.MustCompile(`^@expect\b(.*)$`) + verifyPattern = regexp.MustCompile(`^@verify\b(.*)$`) testPattern = regexp.MustCompile(`^@test\s+(.+)`) setupPattern = regexp.MustCompile(`^@setup\s+(\S+)`) cleanupPattern = regexp.MustCompile(`^@cleanup\s+(\S+)`) @@ -770,9 +773,30 @@ var ( // message is optional — the old pattern required it, so a bare @throws // matched nothing at all and was dropped without a word. throwsPattern = regexp.MustCompile(`^@throws\b(.*)$`) + // tagPattern is any javadoc block tag, known or not: it ends the text of the + // annotation before it. + tagPattern = regexp.MustCompile(`^@[A-Za-z]`) ) +// openAssertion is an @expect or @verify whose text may still continue on the +// lines below it. +type openAssertion struct { + tag string // "@expect" or "@verify" + parts []string +} + // parseAnnotations extracts test annotations from a javadoc comment. +// +// The text of @expect and @verify runs on, as a javadoc block tag's does, to +// the next tag or the end of the comment, and the whole of it is parsed as one +// assertion (ako/mxcli#1514). Reading only the tag's own line truncated an +// assertion written over two lines to its first, without a word — and when +// that first line was complete, the test asserted less than it said: `$r = +// 'a;b'` instead of `$r = 'a;b' and length($r) = 3`. Text after one of these +// tags that does not continue the expression is therefore an error, not +// ignored; a description belongs before the first tag. @test, @setup, +// @cleanup and @throws keep to their own line, so prose under a test's name +// is still description. func parseAnnotations(doc string) annotations { var a annotations a.Cleanup = "rollback" // default @@ -781,6 +805,31 @@ func parseAnnotations(doc string) annotations { doc = strings.TrimPrefix(doc, "/**") doc = strings.TrimSuffix(doc, "*/") + var open *openAssertion + closeOpen := func() { + if open == nil { + return + } + text := strings.Join(open.parts, " ") + switch open.tag { + case "@expect": + exp, err := ParseExpect(text) + if err != nil { + a.AssertionErrors = append(a.AssertionErrors, continuationHint(err, open)) + } else { + a.Expects = append(a.Expects, exp) + } + case "@verify": + v, err := ParseVerify(text) + if err != nil { + a.AssertionErrors = append(a.AssertionErrors, continuationHint(err, open)) + } else { + a.Verify = append(a.Verify, v) + } + } + open = nil + } + // Process line by line scanner := bufio.NewScanner(strings.NewReader(doc)) for scanner.Scan() { @@ -790,24 +839,22 @@ func parseAnnotations(doc string) annotations { line = strings.TrimPrefix(line, "*") line = strings.TrimSpace(line) + if !tagPattern.MatchString(line) { + if open != nil && line != "" { + open.parts = append(open.parts, line) + } + continue + } + closeOpen() + if m := testPattern.FindStringSubmatch(line); m != nil { a.Test = strings.TrimSpace(m[1]) } if m := expectPattern.FindStringSubmatch(line); m != nil { - exp, err := ParseExpect(m[1]) - if err != nil { - a.AssertionErrors = append(a.AssertionErrors, err.Error()) - } else { - a.Expects = append(a.Expects, exp) - } + open = &openAssertion{tag: "@expect", parts: nonEmpty(m[1])} } if m := verifyPattern.FindStringSubmatch(line); m != nil { - v, err := ParseVerify(m[1]) - if err != nil { - a.AssertionErrors = append(a.AssertionErrors, err.Error()) - } else { - a.Verify = append(a.Verify, v) - } + open = &openAssertion{tag: "@verify", parts: nonEmpty(m[1])} } if m := setupPattern.FindStringSubmatch(line); m != nil { a.Setups = append(a.Setups, strings.TrimSpace(m[1])) @@ -820,12 +867,35 @@ func parseAnnotations(doc string) annotations { parseThrows(&a, m[1]) } } + closeOpen() checkVerifyCleanup(&a) checkThrowsExpect(&a) return a } +// nonEmpty is the tag line's own text as the first part of an annotation, or +// nothing when the tag stands alone and its text starts on the next line. +func nonEmpty(s string) []string { + if s = strings.TrimSpace(s); s == "" { + return nil + } + return []string{s} +} + +// continuationHint is the parse error of an annotation written over several +// lines, saying so. The error quotes the joined text, which reads as though the +// author wrote it on one line; without the hint, a sentence of description +// under an @expect looks like a mangled expression. +func continuationHint(err error, open *openAssertion) string { + if len(open.parts) < 2 { + return err.Error() + } + return fmt.Sprintf("%v (this %s spans %d lines: an annotation's text runs on to the next "+ + "@tag or the end of the comment, and all of it is the assertion; put a "+ + "description before the first tag)", err, open.tag, len(open.parts)) +} + // parseThrows records an @throws annotation and its optional message. // // Three forms, and the third is why this is a function rather than a regexp diff --git a/cmd/mxcli/testrunner/parser_multiline_annotation_test.go b/cmd/mxcli/testrunner/parser_multiline_annotation_test.go new file mode 100644 index 000000000..e74310961 --- /dev/null +++ b/cmd/mxcli/testrunner/parser_multiline_annotation_test.go @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testrunner + +import ( + "strings" + "testing" +) + +// ako/mxcli#1514 (upstream mendixlabs/mxcli#1604): an @expect written over two +// lines kept only its first. parseAnnotations read the doc comment one line at a +// time and the tag's pattern captured the rest of ITS line, so the continuation +// line — having no tag of its own — matched nothing and was dropped without a +// word. When the first line was a complete expression the test then ran a +// weaker assertion than the one written: `$r = 'a;b'` instead of +// `$r = 'a;b' and length($r) = 3`, which can pass on less. +// +// An annotation's text now runs on, as a javadoc block tag's does, to the next +// tag or the end of the comment, and the whole of it is what gets parsed. + +func TestMultiLineExpectIsOneAssertion(t *testing.T) { + cases := []struct { + name, doc, raw string + }{ + { + name: "or on the next line", + doc: `/** + * @test multiline3 + * @expect $r = 'zzz' + * or $r = 'qqq' + */`, + raw: `$r = 'zzz' or $r = 'qqq'`, + }, + { + name: "and on the next line", + doc: `/** + * @test multiline + * @expect $r = 'a;b' + * and length($r) = 3 + */`, + raw: `$r = 'a;b' and length($r) = 3`, + }, + { + // The first line alone does not parse, which used to be the one + // shape that was reported — as "expected a value at column 16". + name: "open parenthesis on the first line", + doc: `/** + * @test multiline2 + * @expect ($r = 'a;b' and + * length($r) = 3) + */`, + raw: `($r = 'a;b' and length($r) = 3)`, + }, + { + name: "three lines, with a blank line inside", + doc: `/** + * @test three + * @expect $r = 'a' + * + * or $r = 'b' + * or $r = 'c' + */`, + raw: `$r = 'a' or $r = 'b' or $r = 'c'`, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + a := parseAnnotations(c.doc) + if len(a.AssertionErrors) != 0 { + t.Fatalf("AssertionErrors = %v, want none", a.AssertionErrors) + } + if len(a.Expects) != 1 { + t.Fatalf("Expects = %+v, want exactly one assertion", a.Expects) + } + if got := a.Expects[0].Raw; got != c.raw { + t.Errorf("Raw = %q, want %q: the continuation line was not part of the assertion", got, c.raw) + } + // The condition evaluated is the joined one, not the first line's. + want, err := ParseExpect(c.raw) + if err != nil { + t.Fatalf("ParseExpect(%q): %v", c.raw, err) + } + if a.Expects[0].Condition != want.Condition { + t.Errorf("Condition = %q, want %q", a.Expects[0].Condition, want.Condition) + } + }) + } +} + +// The continuation ends at the next tag: two @expect lines are still two +// assertions, and a tag after a multi-line one is still read as itself. +func TestMultiLineExpectEndsAtTheNextTag(t *testing.T) { + a := parseAnnotations(`/** + * @test boundaries + * @expect $a = 1 + * and $b = 2 + * @expect $c = 3 + * @cleanup none + */`) + if len(a.AssertionErrors) != 0 { + t.Fatalf("AssertionErrors = %v", a.AssertionErrors) + } + var raws []string + for _, e := range a.Expects { + raws = append(raws, e.Raw) + } + if strings.Join(raws, " | ") != "$a = 1 and $b = 2 | $c = 3" { + t.Errorf("Expects = %q, want [$a = 1 and $b = 2, $c = 3]", raws) + } + if a.Cleanup != CleanupNone { + t.Errorf("Cleanup = %q: the tag after the continuation was swallowed", a.Cleanup) + } +} + +// Fail closed: text after an @expect that does not continue the expression is +// not quietly ignored either — ignoring it is how the truncation went unseen. +// It makes the assertion uncompilable, which is an ERROR, and the message says +// the following lines were read as part of the annotation and where prose goes. +func TestTextAfterExpectIsNotSilentlyDropped(t *testing.T) { + a := parseAnnotations(`/** + * @test trailing prose + * @expect $r = 'a;b' + * Checks that the separator survives. + */`) + if len(a.Expects) != 0 { + t.Errorf("Expects = %+v, want none: a line that is not part of the expression was dropped", a.Expects) + } + if len(a.AssertionErrors) != 1 { + t.Fatalf("AssertionErrors = %v, want one", a.AssertionErrors) + } + msg := a.AssertionErrors[0] + for _, want := range []string{"Checks that the separator survives", "2 lines", "before the first tag"} { + if !strings.Contains(msg, want) { + t.Errorf("error %q does not mention %q", msg, want) + } + } +} + +// @verify has the same shape — an OQL query is the annotation most likely to be +// wrapped — and was truncated the same way. +func TestMultiLineVerifyIsOneQuery(t *testing.T) { + a := parseAnnotations(`/** + * @test verify wrapped + * @cleanup none + * @verify select count(*) as n + * from Probe.Item + * where Status = 'Open' = 2 + */`) + if len(a.AssertionErrors) != 0 { + t.Fatalf("AssertionErrors = %v", a.AssertionErrors) + } + if len(a.Verify) != 1 { + t.Fatalf("Verify = %+v, want one", a.Verify) + } + v := a.Verify[0] + if v.Query != "select count(*) as n from Probe.Item where Status = 'Open'" || v.Expected != "2" { + t.Errorf("Verify = {Query: %q, Expected: %q}, want the whole query compared with 2", v.Query, v.Expected) + } +} + +// Controls: prose BEFORE the tags is the description and stays ignored, and so +// do the lines after a tag that takes no expression (@test's name is one line). +func TestDescriptionLinesAreNotAnnotationText(t *testing.T) { + a := parseAnnotations(`/** + * Orders over the limit need approval. + * @test approval needed + * The limit is configured per customer. + * @expect $r = true + */`) + if a.Test != "approval needed" { + t.Errorf("Test = %q", a.Test) + } + if len(a.AssertionErrors) != 0 { + t.Errorf("AssertionErrors = %v, want none", a.AssertionErrors) + } + if len(a.Expects) != 1 || a.Expects[0].Raw != "$r = true" { + t.Errorf("Expects = %+v, want [$r = true]", a.Expects) + } +} + +// End to end through the file parser, as `mxcli test --list` reads it: the test +// case carries the joined assertion, so --list, check and the generated +// microflow all see the expression that was written. +func TestParseMDLTestsKeepsAMultiLineExpect(t *testing.T) { + tests, err := parseMDLTests(`mdl 1; +/** + * @test multiline + * @expect $r = 'a;b' + * and length($r) = 3 + */ +$r = call microflow Probe.SUB_Make(); +/ +`, "multi.test.mdl") + if err != nil { + t.Fatal(err) + } + if len(tests) != 1 || len(tests[0].Expects) != 1 { + t.Fatalf("tests = %+v", tests) + } + if got := tests[0].Expects[0].Raw; got != "$r = 'a;b' and length($r) = 3" { + t.Errorf("Raw = %q: --list and the run would assert less than the file says", got) + } +} + +// A tag on a line of its own, with its text starting on the next, is the same +// annotation. The old pattern required text on the tag's line, so a bare +// @expect matched nothing and the assertion below it was dropped; one that +// stays bare is an error, like any @expect that cannot be compiled. +func TestBareExpectTagTakesTheNextLines(t *testing.T) { + a := parseAnnotations(`/** + * @test bare tag + * @expect + * $r = 'a;b' + */`) + if len(a.AssertionErrors) != 0 || len(a.Expects) != 1 || a.Expects[0].Raw != "$r = 'a;b'" { + t.Errorf("Expects = %+v, AssertionErrors = %v, want [$r = 'a;b']", a.Expects, a.AssertionErrors) + } + + a = parseAnnotations(`/** + * @test bare tag, nothing after it + * @expect + */`) + if len(a.Expects) != 0 || len(a.AssertionErrors) != 1 { + t.Errorf("Expects = %+v, AssertionErrors = %v, want one error", a.Expects, a.AssertionErrors) + } +} diff --git a/mdl-examples/bug-tests/test-runner-1514-multiline-expect.test.mdl b/mdl-examples/bug-tests/test-runner-1514-multiline-expect.test.mdl new file mode 100644 index 000000000..d98d9251a --- /dev/null +++ b/mdl-examples/bug-tests/test-runner-1514-multiline-expect.test.mdl @@ -0,0 +1,47 @@ +-- ako/mxcli#1514 (upstream mendixlabs/mxcli#1604): an @expect written over two +-- lines kept only its first, with no message. When the first line was a +-- complete expression the test ran a different assertion from the one written: +-- with `and` it could pass on less, with `or` it could fail although the whole +-- assertion held. An @expect (and an @verify) now runs on, as a javadoc tag's +-- text does, to the next @tag or the end of the comment. +-- +-- Setup: test-runner-1514-setup.mdl. Then: +-- +-- mxcli test mdl-examples/bug-tests/test-runner-1514-multiline-expect.test.mdl --list +-- +-- lists the whole assertion for each test (before the fix: only the first line, +-- and an "expected a value at column 16" ERROR for test_3), and +-- +-- mxcli test mdl-examples/bug-tests/test-runner-1514-multiline-expect.test.mdl -p app.mpr --local +-- +-- reports test_1 FAIL ("expected $r = 'a;b' and length($r) = 4") and test_2 and +-- test_3 PASS. Before the fix test_1 PASSED — it asserted only $r = 'a;b' — and +-- test_2 FAILED with "expected $r = 'zzz', actual: a;b". Measured on 11.13.0. +-- +-- `make check-mdl` sweeps this file without a project: test_3 is the shape that +-- did not compile before, so the sweep fails if the continuation is lost again. +-- The negative twin is test-runner-1514-text-after-expect.fail.test.mdl. +mdl 1; +/** + * @test and on the next line: the whole assertion is false + * @expect $r = 'a;b' + * and length($r) = 4 + */ +$r = call microflow Probe.SUB_Make(); +/ + +/** + * @test or on the next line: the whole assertion is true + * @expect $r = 'zzz' + * or $r = 'a;b' + */ +$r = call microflow Probe.SUB_Make(); +/ + +/** + * @test open parenthesis on the first line + * @expect ($r = 'a;b' and + * length($r) = 3) + */ +$r = call microflow Probe.SUB_Make(); +/ diff --git a/mdl-examples/bug-tests/test-runner-1514-setup.mdl b/mdl-examples/bug-tests/test-runner-1514-setup.mdl new file mode 100644 index 000000000..f9f09f09b --- /dev/null +++ b/mdl-examples/bug-tests/test-runner-1514-setup.mdl @@ -0,0 +1,16 @@ +-- ako/mxcli#1514 (upstream mendixlabs/mxcli#1604): the setup script for +-- test-runner-1514-multiline-expect.test.mdl. Apply it before running the tests: +-- +-- mxcli exec mdl-examples/bug-tests/test-runner-1514-setup.mdl -p app.mpr +-- +-- Case B of the issue is about checking a test file before this has run. A test +-- file creates nothing itself, so checked alone it reports Probe.SUB_Make as +-- "microflow not found", and the report now names the way to check it — this +-- script first, as one set, each file resolved against what the ones before it +-- write (the project is not changed): +-- +-- mxcli check mdl-examples/bug-tests/test-runner-1514-setup.mdl \ +-- mdl-examples/bug-tests/test-runner-1514-multiline-expect.test.mdl -p app.mpr +mdl 1; +create module Probe; +create microflow Probe.SUB_Make () returns String as $r begin return 'a;b'; end; diff --git a/mdl-examples/bug-tests/test-runner-1514-text-after-expect.fail.test.mdl b/mdl-examples/bug-tests/test-runner-1514-text-after-expect.fail.test.mdl new file mode 100644 index 000000000..b0bbff721 --- /dev/null +++ b/mdl-examples/bug-tests/test-runner-1514-text-after-expect.fail.test.mdl @@ -0,0 +1,20 @@ +-- ako/mxcli#1514 (upstream mendixlabs/mxcli#1604), the negative side: text +-- under an @expect that does not continue the expression is not skipped. Its +-- text runs on to the next @tag or the end of the comment, so the sentence +-- below is part of the assertion, which then cannot be compiled — an ERROR +-- (MDL-TEST01 in check), never a pass on the first line alone. Skipping +-- untagged lines is exactly how the truncation went unseen. +-- +-- mxcli check mdl-examples/bug-tests/test-runner-1514-text-after-expect.fail.test.mdl +-- +-- fails with: @expect $r = 'a;b' Checks that the separator survives.: unexpected +-- trailing input ... (this @expect spans 2 lines ... put a description before +-- the first tag). Before the fix it passed, asserting $r = 'a;b'. +mdl 1; +/** + * @test trailing prose + * @expect $r = 'a;b' + * Checks that the separator survives. + */ +$r = call microflow Probe.SUB_Make(); +/ From ef3b0ea1ddc6370dfd8b5cdb5864c9b3e11cc5f2 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 10:29:14 +0000 Subject: [PATCH 04/13] fix: page check gave its verdict before the data source rendered (ako/mxcli#845) Cause: pageProbeJS (cmd/mxcli/docker/pagecheck.go) counted a page as settled once its body text held still for three 250ms polls with no progress element, capped at 4s, and left the cap silently. Measured on 11.13.0 with a list view whose microflow data source takes 2s and 8s: while the data is out, .mx-listview sits empty with no progress element, no aria-busy and no "no items" placeholder, and the text holds still. The only sign is the open POST /xas/ runtimeOperation. So both pages were measured empty ("text=37", no rows= field) and `playwright check` exited 0. A data grid 2 does show an aria-busy spinner, which is why grids were measured correctly. Fix: the probe tracks the page's same-origin requests and settles only when none has been open for 500ms, nothing is busy and the text is unchanged. A page not settled by the cap is reported STILL LOADING with a WAIT line naming the open requests, and EvaluatePage fails it. The cap (a cap, not a sleep) is now 15s for run --page-check and the playwright check --wait default; the run timeout scales with it. A page with a list view, data grid or gallery always prints rows=, rows=0 included, so an empty list no longer reads like a page without one. Verification: pagecheck_settle_test.go drives the real probe against a fake page whose /xas/ answers late; with the old probe JS it fails with "rows = 0, want 2" and "not reported as still loading", and the verdict/EvaluatePage tests fail with the Go side reverted (the browser tests skip without Node/Playwright). Runtime A/B on 11.13.0 (mdl-examples/bug-tests/pagecheck-845-slow-datasource.mdl): baseline `run --local --page-check` gave slow2/slow8 "text=37" with no rows and `playwright check` "OK 4 page(s)"; fixed gives rows=2 everywhere, and `--wait 3s` on the 8s page gives STILL LOADING and exit 1 where the baseline exited 0. Co-Authored-By: Claude Opus 5.5 --- ...ck-verdict-before-datasource-rendered.json | 1 + .claude/skills/mendix/test-app/SKILL.md | 32 ++-- .claude/skills/verify-in-runtime.md | 5 +- cmd/mxcli/cmd_playwright.go | 17 +- cmd/mxcli/docker/pagecheck.go | 101 ++++++++++-- cmd/mxcli/docker/pagecheck_assert.go | 10 +- cmd/mxcli/docker/pagecheck_settle_test.go | 154 ++++++++++++++++++ cmd/mxcli/docker/runlocal.go | 4 +- docs-site/src/tools/playwright.md | 19 ++- .../pagecheck-845-slow-datasource.mdl | 65 ++++++++ 10 files changed, 376 insertions(+), 32 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/cmd-mxcli/2026-10-25-page-check-verdict-before-datasource-rendered.json create mode 100644 cmd/mxcli/docker/pagecheck_settle_test.go create mode 100644 mdl-examples/bug-tests/pagecheck-845-slow-datasource.mdl diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-25-page-check-verdict-before-datasource-rendered.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-25-page-check-verdict-before-datasource-rendered.json new file mode 100644 index 000000000..16d9a25f0 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-25-page-check-verdict-before-datasource-rendered.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-25","symptom":"`mxcli run --local --page-check` and `mxcli playwright check` reported a page whose list view had not yet received its data as `text=37` with no `rows=` field, and the check exited 0 (\"OK 4 page(s)\"); the same URL checked again later read `rows=2 text=49`. A data source slower than ~1s, or than the 4s cap, was never waited for and nothing flagged the verdict as incomplete","cause":"pageProbeJS settled a page when its body text held still for three 250ms polls with no `.mx-progress/.mx-underlay-loading/[aria-busy]` element, capped at 4s, and broke out of the cap silently. Measured on 11.13.0: while a list view's microflow data source runs, `.mx-listview` sits EMPTY with no progress element, no aria-busy and no `.mx-listview-empty` placeholder, and the text holds still for the whole 2s/8s — the only sign is the open POST /xas/ runtimeOperation. Data grid 2 does render `widget-datagrid-spinner` with aria-busy=true, which is why grids were measured correctly and list views were not","file":"cmd/mxcli/docker/pagecheck.go (pageProbeJS settle loop, PageSignals.Unsettled/PendingRequests/WaitedMs/DataWidgets, formatPageVerdict, DefaultPageWaitMs), cmd/mxcli/docker/pagecheck_assert.go (EvaluatePage), cmd/mxcli/docker/runlocal.go (reportPageChecks), cmd/mxcli/cmd_playwright.go (--wait default + help)","fix":"the probe tracks same-origin requests (request/requestfinished/requestfailed, redirects collapsed) and counts a page settled only when none has been open for 500ms AND the text is unchanged AND nothing is busy; a page not settled by the cap is reported `STILL LOADING` with a `WAIT` line naming the open requests, and EvaluatePage fails it. The cap is now 15s (a cap, not a sleep: a fast page is still measured in ~1s). A page with a list view/data grid/gallery always prints `rows=`, rows=0 included","insight":"Decide what 'rendered' means by sampling the DOM and the network every 50ms during a deliberately slow data source (a Java action calling Thread.sleep behind a microflow data source), not by reasoning about Mendix: the list view gives NO DOM signal at all, so no selector-based wait could have been right, and the grid's spinner made the bug look intermittent. Also from the timeline: on a fast page the data request starts ~3ms AFTER `.mx-page` mounts, so 'no request open' must hold for a window (500ms), not an instant. Repro of the reported shape: the ledger's cold-boot first check is the same thing with the JIT as the slow part. A fake page in an httptest server reproduces it in a unit test; its /xas/ handler must read the request body before blocking, or r.Context() never sees the browser hang up and srv.Close() waits out the full simulated delay. Sibling not fixed: `run --local --screenshot` (screenshot.go) still shoots after a fixed 4s wait, so a PNG can show the same empty list","test":"cmd/mxcli/docker/pagecheck_settle_test.go: TestProbeWaitsForTheDataSourceToRender / TestProbeReportsAPageThatOutlastsTheWait drive the real probe against a fake page (skip without Node/Playwright; run with NODE_PATH set) and fail with the old probe JS (rows = 0, want 2; not reported as still loading); verdict/EvaluatePage tests fail with the Go side reverted. Runtime A/B on 11.13.0 (mdl-examples/bug-tests/pagecheck-845-slow-datasource.mdl): baseline `run --page-check` slow2/slow8 `text=37` no rows, `playwright check` OK exit 0; fixed rows=2 on all, and `--wait 3s` on the 8s page gives STILL LOADING, exit 1","refs":["ako/mxcli#845","ako/mxcli#714"]} diff --git a/.claude/skills/mendix/test-app/SKILL.md b/.claude/skills/mendix/test-app/SKILL.md index d299ce6bb..d996b98c6 100644 --- a/.claude/skills/mendix/test-app/SKILL.md +++ b/.claude/skills/mendix/test-app/SKILL.md @@ -11,18 +11,18 @@ description: "Verify a running Mendix app in a browser: `mxcli playwright check` | lines | section | |---|---| | 32-42 | When to Use This Skill | -| 43-84 | Start Here: `mxcli playwright check` (one call, text, no screenshot) | -| 85-135 | Prerequisites | -| 136-160 | Quick Start | -| 161-182 | Widget Name Selectors | -| 183-298 | Verification Patterns | -| 299-372 | CI/CD: Test Scripts | -| 373-416 | Session Management | -| 417-444 | Debugging | -| 445-477 | Selector Rules | -| 478-494 | Known Gotchas | -| 495-528 | Feedback Loop Workflow | -| 529-536 | Related Skills | +| 43-90 | Start Here: `mxcli playwright check` (one call, text, no screenshot) | +| 91-141 | Prerequisites | +| 142-166 | Quick Start | +| 167-188 | Widget Name Selectors | +| 189-304 | Verification Patterns | +| 305-378 | CI/CD: Test Scripts | +| 379-422 | Session Management | +| 423-450 | Debugging | +| 451-483 | Selector Rules | +| 484-500 | Known Gotchas | +| 501-534 | Feedback Loop Workflow | +| 535-542 | Related Skills | # Test App Skill @@ -62,7 +62,13 @@ FAIL 1 of 2 page(s) - **Exit status:** 0 all pages passed; 1 a page failed (error banner/dialog, console error, failed request, HTTP error, sign-in form instead of the page, failed - assertion); 2 the check could not run. + assertion, or `STILL LOADING`); 2 the check could not run. +- **It waits for the data.** A page is measured once no request to the app has been + open for half a second and its text holds still — a list view shows no spinner + while its data source runs, so the text alone would hold still on an empty list. + A page still loading after `--wait` (default 15s) is reported `STILL LOADING`, + with a `WAIT` line naming the open request, and fails: do not read its `rows=` as + the answer. A page with a list or grid always prints `rows=`, `rows=0` included. - **Login is handled:** `--user U --password P`, or `--role R` (the project's demo user holding user role R), or nothing but `-p` (a demo user, an administrator first). The session is saved under `.mxcli/playwright-check/` and reused by the next check; diff --git a/.claude/skills/verify-in-runtime.md b/.claude/skills/verify-in-runtime.md index 1b777c346..899e8dd93 100644 --- a/.claude/skills/verify-in-runtime.md +++ b/.claude/skills/verify-in-runtime.md @@ -78,7 +78,10 @@ So: `--user/--password`, `--role R` (the project's demo user for that role) or, with only `-p`, a demo user; it saves the session under `.mxcli/playwright-check/` and reuses it, renewing it when the runtime restarted; it waits for the page to - settle instead of sleeping. Hand-written `playwright-cli` is for **interaction** + settle instead of sleeping — including for its data sources, whose requests it + watches, since a list view shows no spinner while one runs. A page that does not + settle within `--wait` is reported `STILL LOADING` and fails, rather than passing + on whatever had rendered. Hand-written `playwright-cli` is for **interaction** (click, fill, submit) — and then put it in a `playwright verify` script. - **Take a screenshot when the question is genuinely about appearance** — layout, spacing, colour, "does this look right" — and then take it **once**, at the end: diff --git a/cmd/mxcli/cmd_playwright.go b/cmd/mxcli/cmd_playwright.go index 33ef67610..80873fd1f 100644 --- a/cmd/mxcli/cmd_playwright.go +++ b/cmd/mxcli/cmd_playwright.go @@ -181,9 +181,18 @@ the project's demo users (an administrator first). The session is saved under /.mxcli/playwright-check/ and reused by later checks; it is renewed automatically when the runtime has forgotten it (after a restart). -Exit status: 0 every page passed; 1 a page failed (HTTP error, sign-in form -instead of the page, error banner or dialog, console error, failed same-origin -request, or a failed assertion); 2 the check could not run. +A page is measured once it has settled: no request to the app open for half a +second, no progress indicator, and its text no longer changing. Waiting for the +requests matters because a list view shows nothing while its data source runs — +no spinner, no placeholder — so a page caught then looks finished and empty. A +page that has not settled within --wait is reported STILL LOADING, naming the +requests it was waiting for, and fails. A page with a list view or grid always +reports its rows, rows=0 included. + +Exit status: 0 every page passed; 1 a page failed (still loading when measured, +HTTP error, sign-in form instead of the page, error banner or dialog, console +error, failed same-origin request, or a failed assertion); 2 the check could +not run. Examples: mxcli playwright check /p/customers /p/orders -p app.mpr @@ -282,7 +291,7 @@ func init() { playwrightCheckCmd.Flags().StringArray("assert-text", nil, "Fail unless the page text contains this (repeatable)") playwrightCheckCmd.Flags().StringArray("assert-count", nil, "Fail unless SELECTOR OP N holds, e.g. '.mx-listview li>=1' (ops >= <= == != > <; bare selector = >=1; repeatable)") playwrightCheckCmd.Flags().String("screenshot", "", "Also write a full-page PNG here (per-page suffix for several pages); only the path is printed") - playwrightCheckCmd.Flags().Duration("wait", 4*time.Second, "Longest a page may keep changing after it mounts before it is measured") + playwrightCheckCmd.Flags().Duration("wait", time.Duration(docker.DefaultPageWaitMs)*time.Millisecond, "Longest a page may keep loading after it mounts (open requests, changing text) before it is measured; a page still loading then fails as STILL LOADING") playwrightCheckCmd.Flags().Bool("fresh-login", false, "Ignore the saved session and sign in again") playwrightCheckCmd.Flags().Bool("allow-console-errors", false, "Report console errors but do not fail on them") diff --git a/cmd/mxcli/docker/pagecheck.go b/cmd/mxcli/docker/pagecheck.go index 4901d23ba..628027f6d 100644 --- a/cmd/mxcli/docker/pagecheck.go +++ b/cmd/mxcli/docker/pagecheck.go @@ -58,6 +58,22 @@ type PageSignals struct { Counts []int `json:"counts"` // Screenshot is the PNG written for this page, when one was asked for. Screenshot string `json:"screenshot"` + + // Unsettled reports that the page was still loading when the wait ran out: + // a same-origin request was still open, or the text was still changing. + // Everything else then describes a partly rendered page, so a check fails + // on it rather than vouching for what it happened to catch (ako/mxcli#845). + Unsettled bool `json:"unsettled"` + // PendingRequests are the same-origin requests still open at that point, + // as "METHOD /path" (at most three). + PendingRequests []string `json:"pendingRequests"` + // WaitedMs is how long the probe waited for the page to settle once it had + // mounted. + WaitedMs int `json:"waitedMs"` + // DataWidgets counts the list views, data grids and galleries on the page. + // With one present the verdict always reports rows — rows=0 included — so + // an empty list does not read the same as a page that has none. + DataWidgets int `json:"dataWidgets"` } // pageProbeJS runs in node with Playwright. It reports signals and never @@ -102,27 +118,58 @@ const req = JSON.parse(require('fs').readFileSync(process.argv[2], 'utf8')); } } catch (e) {} }); + // Same-origin requests still open. A data source is fetched after the page + // mounts, and while it is out a list view shows nothing that says so: + // measured on 11.13, .mx-listview sits empty — no progress element, no + // aria-busy, no "no items" placeholder — for as long as its microflow runs, + // and the text holds still. The open POST /xas/ is the only sign the page + // is not finished (ako/mxcli#845). + const inflight = new Map(); + let lastNet = Date.now(); + const pathOf = u => { try { return new URL(u).pathname; } catch (e) { return u; } }; + const sameOrigin = u => { try { return new URL(u).origin === origin; } catch (e) { return false; } }; + p.on('request', r => { + // A redirect is one request as far as the page is concerned. + if (r.redirectedFrom()) inflight.delete(r.redirectedFrom()); + if (!sameOrigin(r.url())) return; + inflight.set(r, r.method() + ' ' + pathOf(r.url())); + lastNet = Date.now(); + }); + const finished = r => { if (inflight.delete(r)) lastNet = Date.now(); }; + p.on('requestfinished', finished); + p.on('requestfailed', finished); const out = { title: '', headings: [], alerts: [], errorAlerts: [], textLen: 0, rows: 0, consoleErrors, finalUrl: '', status: 0, httpErrors, loginForm: false, - textFound: [], counts: [], screenshot: '' }; + textFound: [], counts: [], screenshot: '', + unsettled: false, pendingRequests: [], waitedMs: 0, dataWidgets: 0 }; try { const resp = await p.goto(target.url, { waitUntil: 'domcontentloaded', timeout: 30000 }); out.status = resp ? resp.status() : 0; // Wait for the Mendix client to mount a page (or the login form), then - // for the body text to stop changing — data sources load after the page - // shell, so a fixed sleep is either too short or wasted. + // for the page to settle: no same-origin request open for 500ms, no + // progress indicator, and the body text unchanged over three polls. Data + // sources load after the page shell, so a fixed sleep is either too short + // or wasted — and the text alone holds still while a list view's data is + // out, which is how a verdict used to be taken before the rows arrived. await p.waitForSelector('.mx-page, #usernameInput', { timeout: 20000 }).catch(() => {}); - const deadline = Date.now() + (req.waitMs || 4000); - let last = -1, stable = 0; + const mounted = Date.now(); + const deadline = mounted + (req.waitMs || 15000); + let last = -1, stable = 0, settled = false; while (Date.now() < deadline) { const st = await p.evaluate(() => ({ len: (document.body && document.body.innerText || '').length, busy: document.querySelectorAll('.mx-progress, .mx-underlay-loading, [aria-busy="true"]').length, })).catch(() => ({ len: -2, busy: 0 })); - if (st.len === last && st.busy === 0) { if (++stable >= 3) break; } else { stable = 0; } + const quiet = inflight.size === 0 && Date.now() - lastNet >= 500; + if (st.len === last && st.busy === 0 && quiet) { + if (++stable >= 3) { settled = true; break; } + } else { stable = 0; } last = st.len; await p.waitForTimeout(250); } + out.unsettled = !settled; + out.waitedMs = Date.now() - mounted; + out.pendingRequests = [...inflight.values()].slice(0, 3); out.finalUrl = p.url(); out.title = await p.title(); out.loginForm = (await p.locator('#usernameInput').count()) > 0; @@ -139,6 +186,7 @@ const req = JSON.parse(require('fs').readFileSync(process.argv[2], 'utf8')); // Data rows only: a list view's "No items found" placeholder is an
  • // too, and a data grid 2 header row is a role=row without grid cells. out.rows = await p.$$eval('.mx-datagrid tbody tr, .mx-listview > ul > li:not(.mx-listview-empty), [role="row"]:has([role="gridcell"])', ns => ns.length); + out.dataWidgets = await p.$$eval('.mx-listview, .mx-datagrid, .widget-datagrid, .widget-gallery', ns => ns.length); out.textFound = (req.texts || []).map(t => body.includes(t)); for (const sel of (req.countSelectors || [])) { out.counts.push(await p.locator(sel).count().catch(() => -1)); @@ -168,16 +216,26 @@ type PageTarget struct { type ProbeOptions struct { Pages []PageTarget Storage string // Playwright storage-state file, for pages behind a login - // WaitMs caps how long a page may keep settling after it mounts (default 4000). + // WaitMs caps how long a page may keep loading and changing after it mounts + // (default DefaultPageWaitMs). A page that settles sooner is measured then; + // one that has not settled by the cap is reported as still loading. WaitMs int // Texts and CountSelectors are evaluated on every page; the answers come // back index-aligned in PageSignals.TextFound / Counts. Texts []string CountSelectors []string - Timeout time.Duration // whole run; default 60s + 30s per page + Timeout time.Duration // whole run; default 60s + (30s + WaitMs) per page MxBuildPath string // node fallback when none is on PATH } +// DefaultPageWaitMs is how long a page may take to settle before the probe +// gives up and reports it as still loading. It is a cap, not a sleep: a page +// is measured as soon as it settles. It was 4s while the probe waited only for +// the text to stop changing; now that it also waits for the page's requests, a +// slow first data source (a cold runtime) is worth waiting for rather than +// reporting as unfinished. +const DefaultPageWaitMs = 15000 + // probeRequest is the JSON handed to pageProbeJS. type probeRequest struct { Pages []PageTarget `json:"pages"` @@ -217,7 +275,7 @@ func ProbePages(opts ProbeOptions) ([]PageSignals, error) { waitMs := opts.WaitMs if waitMs == 0 { - waitMs = 4000 + waitMs = DefaultPageWaitMs } req := probeRequest{ Pages: opts.Pages, Storage: opts.Storage, WaitMs: waitMs, @@ -237,7 +295,11 @@ func ProbePages(opts ProbeOptions) ([]PageSignals, error) { timeout := opts.Timeout if timeout == 0 { - timeout = 60*time.Second + time.Duration(len(opts.Pages))*30*time.Second + // Each page may spend its whole settle wait on top of loading, and a + // page that does is reported STILL LOADING — which the run must live + // long enough to say, rather than timing out on all of them. + perPage := 30*time.Second + time.Duration(waitMs)*time.Millisecond + timeout = 60*time.Second + time.Duration(len(opts.Pages))*perPage } cmd := exec.Command(node, scriptPath, reqPath) cmd.Env = append(os.Environ(), "NODE_PATH="+playwrightNodePath(node)) @@ -353,7 +415,12 @@ func formatPageVerdict(label string, s PageSignals) string { if len(s.Headings) > 0 { fmt.Fprintf(&b, " h=%q", clip(s.Headings[0], 60)) } - if s.Rows > 0 { + if s.Unsettled { + b.WriteString(" STILL LOADING") + } + // rows=0 is said when there is a list to fill: leaving the field out made an + // empty list look like a page without one (ako/mxcli#845). + if s.Rows > 0 || s.DataWidgets > 0 { fmt.Fprintf(&b, " rows=%d", s.Rows) } if s.TextLen == 0 { @@ -374,9 +441,21 @@ func formatPageVerdict(label string, s PageSignals) string { for _, h := range firstN(s.HTTPErrors, 2) { fmt.Fprintf(&b, " HTTP %s\n", clip(h, 140)) } + if s.Unsettled { + fmt.Fprintf(&b, " WAIT %s\n", clip(s.unsettledReason(), 140)) + } return b.String() } +// unsettledReason says why a page counted as still loading. +func (s PageSignals) unsettledReason() string { + waited := fmt.Sprintf("%.1fs", float64(s.WaitedMs)/1000) + if len(s.PendingRequests) > 0 { + return fmt.Sprintf("not settled after %s: still waiting for %s", waited, strings.Join(s.PendingRequests, ", ")) + } + return fmt.Sprintf("not settled after %s: the page was still changing", waited) +} + func dedup(ss []string) []string { seen := map[string]bool{} var out []string diff --git a/cmd/mxcli/docker/pagecheck_assert.go b/cmd/mxcli/docker/pagecheck_assert.go index f0995b70f..373ff967a 100644 --- a/cmd/mxcli/docker/pagecheck_assert.go +++ b/cmd/mxcli/docker/pagecheck_assert.go @@ -108,13 +108,21 @@ func (s CheckSpec) CountSelectors() []string { // EvaluatePage returns the reasons a page fails, empty when it passes. // -// A page fails when the browser was refused it (HTTP >= 400), when it shows +// A page fails when it was still loading when the wait ran out, when the +// browser was refused it (HTTP >= 400), when it shows // the sign-in form instead of itself, when it shows an error banner or error // dialog, when the console reports an error, when a same-origin request it // made failed, or when an assertion does not hold. Warnings and validation // messages are reported in the verdict but do not fail it. func EvaluatePage(sig PageSignals, spec CheckSpec) []string { var fails []string + // A page measured before it finished loading may be missing exactly what + // the check is for — the rows of a list whose data had not arrived + // (ako/mxcli#845) — so it cannot pass, whatever else it shows. + if sig.Unsettled { + fails = append(fails, "still loading when measured, so the verdict may describe a partly "+ + "rendered page (raise --wait if it is only slow)") + } if sig.Status >= 400 { fails = append(fails, fmt.Sprintf("HTTP %d", sig.Status)) } diff --git a/cmd/mxcli/docker/pagecheck_settle_test.go b/cmd/mxcli/docker/pagecheck_settle_test.go new file mode 100644 index 000000000..b5fea1ad8 --- /dev/null +++ b/cmd/mxcli/docker/pagecheck_settle_test.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// ako/mxcli#845: `--page-check` could give its verdict before the page's data +// source had rendered, and nothing in the verdict said so. Measured on 11.13.0 +// against a list view whose microflow data source takes 2s and 8s: while the +// data is out, the list view shows NOTHING that says it is loading — `.mx-listview` +// sits empty, with no progress element, no aria-busy, no "no items" placeholder, +// and the body text holds still. The only sign is the request itself, a +// POST /xas/ runtimeOperation that stays open. The old probe waited for the text +// to stop changing (750ms) or 4s, whichever came first, so both pages were +// reported `text=37` with no rows and `playwright check` exited 0 — "OK 4 page(s)". +// (A data grid 2 does show a spinner with aria-busy, which is why grids were +// measured correctly and list views were not.) +// +// The probe now also waits for the page's same-origin requests to finish, and a +// page still loading when the wait runs out is reported as such and fails. + +func TestVerdictSaysWhenThePageWasStillLoading(t *testing.T) { + got := formatPageVerdict("/p/items", PageSignals{ + Title: "Items", TextLen: 37, Unsettled: true, WaitedMs: 4000, + PendingRequests: []string{"POST /xas/"}, DataWidgets: 1, + }) + for _, want := range []string{"STILL LOADING", "POST /xas/", "4.0s", "rows=0"} { + if !strings.Contains(got, want) { + t.Errorf("verdict omits %q; a half-rendered page must not read as a finished one:\n%s", want, got) + } + } +} + +func TestEvaluatePage_StillLoadingFails(t *testing.T) { + sig := healthy() + sig.Unsettled, sig.WaitedMs, sig.PendingRequests = true, 4000, []string{"POST /xas/"} + fails := EvaluatePage(sig, CheckSpec{}) + if len(fails) != 1 || !strings.Contains(fails[0], "still loading") || !strings.Contains(fails[0], "--wait") { + t.Errorf("a page measured before it finished loading must fail, naming --wait: %v", fails) + } + // Control: the same page, settled, passes. + if f := EvaluatePage(healthy(), CheckSpec{}); len(f) != 0 { + t.Errorf("control: a settled healthy page failed: %v", f) + } +} + +// The rows field used to be left out when it was 0, so "no rows" and "no grid" +// read the same — the missing key was the only trace of #845's symptom. A page +// that has a list view or grid now always says how many rows it rendered; a +// page without one still says nothing about rows. +func TestVerdictShowsRowsZeroOnlyWhenThereIsAListToFill(t *testing.T) { + if got := formatPageVerdict("/p/items", PageSignals{Title: "Items", TextLen: 37, DataWidgets: 1}); !strings.Contains(got, "rows=0") { + t.Errorf("an empty list view is not reported as rows=0:\n%s", got) + } + if got := formatPageVerdict("/p/about", PageSignals{Title: "About", TextLen: 300}); strings.Contains(got, "rows=") { + t.Errorf("a page with no list or grid reports rows:\n%s", got) + } +} + +// fakeMendixPage serves the shape measured on 11.13: the page shell (.mx-page and +// an empty .mx-listview) renders at once, then the list asks POST /xas/ for its +// rows and draws them when the answer comes. delay is how long the "data source" +// takes — simulated latency, the thing under test, not a synchronization. +func fakeMendixPage(delay time.Duration) *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/p/items", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Items +

    Item overview

    +
      +`) + }) + mux.HandleFunc("/xas/", func(w http.ResponseWriter, r *http.Request) { + // Reading the body to the end is what lets the server notice the + // browser hanging up (r.Context() is cancelled by the background read + // that starts after it), so a test that gives up early ends early. + _, _ = io.Copy(io.Discard, r.Body) + select { + case <-time.After(delay): + case <-r.Context().Done(): // the probe gave up and closed the page + return + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `["Row A","Row B"]`) + }) + return httptest.NewServer(mux) +} + +// probeOrSkip runs the real probe, skipping when this machine has no Node or no +// Playwright package — which reads like a pass, so the PR that touches the +// probe states the run it was verified with. +func probeOrSkip(t *testing.T, opts ProbeOptions) PageSignals { + t.Helper() + opts.MxBuildPath = anyMxBuildPath() + sigs, err := ProbePages(opts) + if err != nil { + if strings.Contains(err.Error(), "node not found") || strings.Contains(err.Error(), "Cannot find module") || + strings.Contains(err.Error(), "Executable doesn't exist") { + t.Skipf("no browser tooling here (set NODE_PATH to a Playwright install): %v", err) + } + t.Fatalf("probe failed: %v", err) + } + return sigs[0] +} + +// The reported symptom: a data source slower than the old text-stable window. +// Before the fix this came back rows=0 with nothing flagged. +func TestProbeWaitsForTheDataSourceToRender(t *testing.T) { + srv := fakeMendixPage(2 * time.Second) + defer srv.Close() + sig := probeOrSkip(t, ProbeOptions{Pages: []PageTarget{{URL: srv.URL + "/p/items"}}}) + if sig.Rows != 2 { + t.Errorf("rows = %d, want 2: the verdict was taken before the data source answered (text=%d)", sig.Rows, sig.TextLen) + } + if sig.Unsettled { + t.Errorf("a page that finished loading inside the wait is reported as still loading: %+v", sig) + } +} + +// A data source slower than the whole wait cannot be waited for, and must not +// be passed off as a finished page either: the verdict says the page was still +// loading and names the open request. +func TestProbeReportsAPageThatOutlastsTheWait(t *testing.T) { + srv := fakeMendixPage(20 * time.Second) + defer srv.Close() + sig := probeOrSkip(t, ProbeOptions{Pages: []PageTarget{{URL: srv.URL + "/p/items"}}, WaitMs: 2500}) + if !sig.Unsettled { + t.Fatalf("a page whose data never arrived was not reported as still loading: rows=%d text=%d", sig.Rows, sig.TextLen) + } + if len(sig.PendingRequests) == 0 || sig.PendingRequests[0] != "POST /xas/" { + t.Errorf("pending = %v, want the open POST /xas/", sig.PendingRequests) + } + if fails := EvaluatePage(sig, CheckSpec{}); len(fails) == 0 { + t.Errorf("a page measured mid-load passed the check") + } + if sig.DataWidgets != 1 { + t.Errorf("dataWidgets = %d, want 1 (the list view)", sig.DataWidgets) + } +} diff --git a/cmd/mxcli/docker/runlocal.go b/cmd/mxcli/docker/runlocal.go index 51de11feb..6738483eb 100644 --- a/cmd/mxcli/docker/runlocal.go +++ b/cmd/mxcli/docker/runlocal.go @@ -1634,7 +1634,9 @@ func reportPageChecks(opts LocalRunOptions, rt *LocalRuntime) { for i, t := range targets { pages[i] = PageTarget{URL: resolveScreenshotURL(rt.AppURL(), t)} } - sigs, err := ProbePages(ProbeOptions{Pages: pages, Storage: opts.screenshotStorage, WaitMs: 4000}) + // WaitMs is left to the probe's default: a cap that a page settling sooner + // does not pay, and long enough for the first data source of a cold runtime. + sigs, err := ProbePages(ProbeOptions{Pages: pages, Storage: opts.screenshotStorage}) if err != nil { fmt.Fprintf(opts.Stderr, " page check skipped: %v\n", err) return diff --git a/docs-site/src/tools/playwright.md b/docs-site/src/tools/playwright.md index aa3992da1..f51270550 100644 --- a/docs-site/src/tools/playwright.md +++ b/docs-site/src/tools/playwright.md @@ -66,10 +66,27 @@ FAIL 1 of 2 page(s) The verdict line is the same one `mxcli run --local --page-check` prints; both use one probe, and every page of a call is loaded in one headless browser. +A page is measured once it has **settled**: no request to the app open for half a +second, no progress indicator, and its text unchanged. The requests are the part +that matters. While a list view's data source runs, the list view shows nothing +that says so — no spinner, no "no items" placeholder — so a page caught then looks +finished and simply empty (ako/mxcli#845). A page that has not settled within +`--wait` (default 15s; a page that settles sooner is measured then) is reported as +still loading and fails, naming what it was waiting for: + +``` +page /p/Order_Overview title="Orders" STILL LOADING rows=0 text=37 console-errors=0 + WAIT not settled after 3.1s: still waiting for POST /xas/ + FAIL still loading when measured, so the verdict may describe a partly rendered page (raise --wait if it is only slow) +``` + +A page with a list view, data grid or gallery always reports `rows=`, `rows=0` +included, so an empty list does not read the same as a page without one. + | Exit status | Meaning | |---|---| | 0 | every page passed | -| 1 | a page failed: HTTP error, sign-in form instead of the page (or a 401 from the runtime), error banner or error dialog, console error, failed same-origin request, or a failed `--assert-*` | +| 1 | a page failed: still loading when measured, HTTP error, sign-in form instead of the page (or a 401 from the runtime), error banner or error dialog, console error, failed same-origin request, or a failed `--assert-*` | | 2 | the check could not run (no Node/Playwright, sign-in rejected, bad flag) | A warning banner or validation message is shown but does not fail the page. diff --git a/mdl-examples/bug-tests/pagecheck-845-slow-datasource.mdl b/mdl-examples/bug-tests/pagecheck-845-slow-datasource.mdl new file mode 100644 index 000000000..872e701c3 --- /dev/null +++ b/mdl-examples/bug-tests/pagecheck-845-slow-datasource.mdl @@ -0,0 +1,65 @@ +-- ako/mxcli#845: `--page-check` (and `mxcli playwright check`) could give its +-- verdict before the page's data source had rendered, and passed the page. +-- +-- Measured on 11.13.0: while a list view's microflow data source runs, the list +-- view shows nothing that says it is loading — `.mx-listview` sits empty, with +-- no progress element, no aria-busy and no "no items" placeholder, and the body +-- text holds still. The probe waited for the text to stop changing (750ms) or +-- 4s, so it measured the empty list. The only sign of the unfinished page is the +-- open POST /xas/ runtimeOperation, which the probe now waits for. (A data grid 2 +-- shows a spinner with aria-busy, so grids were measured correctly all along — +-- Probe.Grid2 is the control for that.) +-- +-- Reproduce against a blank app (security off): +-- +-- mxcli exec mdl-examples/bug-tests/pagecheck-845-slow-datasource.mdl -p app.mpr +-- mxcli run --local --detach -p app.mpr --db-type hsqldb \ +-- --page-check --screenshot-url /p/slow2 --screenshot-url /p/slow8 --screenshot-url /p/fast +-- mxcli playwright check /p/fast /p/slow2 /p/slow8 /p/grid2 -p app.mpr +-- mxcli playwright check /p/slow8 -p app.mpr --wait 3s +-- +-- Before the fix both slow pages read `text=37` with no rows= field and the check +-- exited 0 ("OK 4 page(s)"); with --wait 3s too. After it, every page reports +-- rows=2, and with --wait 3s /p/slow8 is reported STILL LOADING ("still waiting +-- for POST /xas/") and the check exits 1. +mdl 1; +create module Probe; +create non-persistent entity Probe.Row (Code: String(50)); + +/** Holds the data source back, standing in for a slow query or a cold runtime. */ +create java action Probe.SleepMs(Ms: integer) returns boolean +as $$ +try { Thread.sleep(Ms); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } +return true; +$$; + +create microflow Probe.DS_Rows (Delay: Integer) returns list of Probe.Row as $Rows +begin + $Slept = call java action Probe.SleepMs(Ms = $Delay); + $Rows = create list of Probe.Row; + $A = create Probe.Row (Code = 'Row A'); + add $A to $Rows; + $B = create Probe.Row (Code = 'Row B'); + add $B to $Rows; + return $Rows; +end; +create microflow Probe.DS_Fast () returns list of Probe.Row as $Rows begin $Rows = call microflow Probe.DS_Rows(Delay = 0); return $Rows; end; +create microflow Probe.DS_Slow2 () returns list of Probe.Row as $Rows begin $Rows = call microflow Probe.DS_Rows(Delay = 2000); return $Rows; end; +create microflow Probe.DS_Slow8 () returns list of Probe.Row as $Rows begin $Rows = call microflow Probe.DS_Rows(Delay = 8000); return $Rows; end; + +create page Probe.Fast (title: 'Fast', layout: Atlas_Core.Atlas_Default, url: 'fast') { + dynamictext hdr (content: 'Fast list') + listview lvRows (datasource: microflow Probe.DS_Fast) { dynamictext code (content: '{1}', contentparams: ({1} = Code)) } +}; +create page Probe.Slow2 (title: 'Slow2', layout: Atlas_Core.Atlas_Default, url: 'slow2') { + dynamictext hdr (content: 'Slow list') + listview lvRows (datasource: microflow Probe.DS_Slow2) { dynamictext code (content: '{1}', contentparams: ({1} = Code)) } +}; +create page Probe.Slow8 (title: 'Slow8', layout: Atlas_Core.Atlas_Default, url: 'slow8') { + dynamictext hdr (content: 'Slow list') + listview lvRows (datasource: microflow Probe.DS_Slow8) { dynamictext code (content: '{1}', contentparams: ({1} = Code)) } +}; +create page Probe.Grid2 (title: 'Grid2', layout: Atlas_Core.Atlas_Default, url: 'grid2') { + dynamictext hdr (content: 'Slow grid') + datagrid dgRows (datasource: microflow Probe.DS_Slow2) { column (attribute: Code, caption: 'Code') } +}; From dccdfe656707440510f7d143837669fc9c288b89 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 10:33:30 +0000 Subject: [PATCH 05/13] fix: STRING_AGG in a view entity is refused as not aggregated (ako/mxcli#1650) A view using STRING_AGG(expr, 'sep') next to GROUP BY was reported as MDL036 "not aggregated and is not a GROUP BY expression", and check, exec and exec --no-check all refused it (upstream mendixlabs/mxcli#1641). Cause: the GROUP BY rules (MDL034/035/036) and MDL037 recognised an aggregate with a hand-written count|sum|avg|min|max regex. mxbuild's OQL grammar has six aggregate tokens ('SUM' 'COUNT' 'AVG' 'MIN' 'MAX' 'STRING_AGG', DeprecatedOqlLexer in Mendix.Modeler.Core.dll). --no-check did not help because execCreateViewEntity runs ValidateOQLSyntax itself as a write barrier, and MDL036 is one of its rules. That barrier stays: MDL036 also fires on shapes mxbuild rejects (`length(s.Name)` next to `group by b.Code` is CE0174 on 11.13.0), so --no-check must not write it. Fix: - oqlAggregateFunctions is the one list both regexes are built from, now with string_agg. A STRING_AGG over a grouped column is MDL034, as mxbuild reports (CE0174 "cannot both be aggregated and appear in the GROUP BY"). - Version gate oql_functions.string_agg: 11.2.0, and 10.24.22 on the 10.24 line (max_version 10.99.99). Below it every such view is CE0174 "Unknown function 'string_agg'". Found by reading the lexer tokens out of each mxbuild's Mendix.Modeler.Core.dll and bisecting CDN tarballs, then confirmed with mx check at both ends of each boundary. - The registry's display paths (show features, added since, the gate's "requires" text) took the lowest min_version, which for a two-line feature reported STRING_AGG unavailable on every 11.x and told an 11.1 project it needed 10.24.22. entryAt picks the entry covering the version, else the next floor above it. - The STRING_AGG column type changed in 11.7.0: the argument's own type on 11.2-11.6.12 (String(100) over a String(100), exactly), a derived String(200) from 11.7.0 and on 10.24.22+. It is judged where the project version is known (check -p / exec's reference pass) and left unjudged without one; on 11.2-11.6 only when the aggregated attribute already exists. The issue's own `Names: String` is CE6770 on every version, and check -p now says so with the fix for the project. Verification: - Unit tests for the grouping rules, the exec handler, the version gate, the type rule per era and the registry; with the production change reverted they fail with the reported MDL036 / "invalid OQL in view entity" refusal. - mx check, probe views over String(100)/String(500)/unlimited sources: 10.24.21, 11.0.0, 11.1.0 CE0174 "Unknown function 'string_agg'"; 10.24.22, 10.24.24, 11.7.0, 11.8.0, 11.9.1, 11.10.0, 11.12.2, 11.13.0, 11.15.0 String(200); 11.2.0, 11.6.6, 11.6.8, 11.6.12 the argument's type. - mdl-examples/bug-tests/view-entity-1650-string-agg.mdl: check -p passes, exec creates both views, mx check 0 errors and describe -> exec reports Unchanged on 10.24.22, 11.7.0, 11.12.2, 11.13.0, 11.15.0; its String(100) variant likewise on 11.2.0 and 11.6.6. On 11.1.0 and 10.24.21, check -p, exec and exec --no-check refuse it naming 11.2.0+ / 10.24.22+. The issue's step2 (`Names: String`) on 11.12.2: check -p and exec refuse with "Fix: change to 'Names: String(200)'". Co-Authored-By: Claude Opus 5.5 --- ...ng-agg-view-refused-as-not-aggregated.json | 1 + .../skills/mendix/write-oql-queries/SKILL.md | 33 ++- .claude/skills/version-awareness.md | 1 + cmd/mxcli/syntax/features_domain_model.go | 8 + .../src/appendixes/version-compatibility.md | 1 + .../bug-tests/view-entity-1650-string-agg.mdl | 59 ++++ mdl/executor/cmd_features.go | 6 +- mdl/executor/oql_type_inference.go | 82 +++++- mdl/executor/oql_view_select_checks.go | 17 +- mdl/executor/version_gates.go | 19 ++ mdl/executor/version_gates_test.go | 3 + mdl/executor/view_entity_string_agg_test.go | 261 ++++++++++++++++++ sdk/versions/mendix-10.yaml | 12 + sdk/versions/mendix-11.yaml | 9 + sdk/versions/registry.go | 82 +++--- sdk/versions/registry_test.go | 64 +++++ 16 files changed, 609 insertions(+), 49 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-25-string-agg-view-refused-as-not-aggregated.json create mode 100644 mdl-examples/bug-tests/view-entity-1650-string-agg.mdl create mode 100644 mdl/executor/view_entity_string_agg_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-string-agg-view-refused-as-not-aggregated.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-string-agg-view-refused-as-not-aggregated.json new file mode 100644 index 000000000..53b52afde --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-string-agg-view-refused-as-not-aggregated.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-25", "symptom": "ako/mxcli#1650 (upstream mendixlabs/mxcli#1641): a view entity using STRING_AGG(s.Name, '; ') next to GROUP BY was MDL036 \"not aggregated and is not a GROUP BY expression\" — refused by check, by exec, and by exec --no-check (\"invalid OQL in view entity\"), while the same view with COUNT passed", "cause": "The GROUP BY rules (MDL034/035/036) and MDL037 recognised an aggregate by a hand-written regex, count|sum|avg|min|max. mxbuild's OQL grammar has a sixth aggregate token, STRING_AGG. --no-check did not help because execCreateViewEntity runs ValidateOQLSyntax itself as a write barrier, and MDL036 lives there", "file": "mdl/executor/oql_view_select_checks.go (oqlAggregateFunctions), oql_type_inference.go (stringAggType, validateViewEntityTypes), version_gates.go (oqlUsesStringAgg), sdk/versions/registry.go (entryAt), sdk/versions/mendix-10.yaml + mendix-11.yaml", "insight": "Read the platform's own list instead of guessing: `strings -a -el Mendix.Modeler.Core.dll` shows the OQL lexer's literal tokens ('SUM' 'COUNT' 'AVG' 'MIN' 'MAX' 'STRING_AGG'), and the same grep over every cached mxbuild — then streaming `tar -xz --wildcards '*Mendix.Modeler.Core.dll'` out of CDN tarballs — bisected the floor without a single full download: 11.2.0 (11.0/11.1 lack it) and, separately, 10.24.22 on the 10.24 line. mx check confirmed both ends (CE0174 \"Unknown function 'string_agg'\" below). The trap was the column TYPE: it changed in 11.7.0 — the argument's own type on 11.2–11.6.12 (String(100) over a String(100), exactly), a derived String(200) from 11.7 and on 10.24.22+ — so a version-blind type rule is a false refusal on one side or the other; it is judged only where the project version is known. Measure a grid (String(100)/String(500)/unlimited sources × declarations) on BOTH sides of a boundary before encoding a type. A feature on two release lines needs two registry entries (min/max on the older line), and every registry view has to pick the entry covering the version — 'lowest min_version' listed it unavailable on every 11.x. Do not 'fix' --no-check by taking MDL036 off the handler's barrier: MDL036 covers two shapes, and only one is runtime-only — `datepart(MONTH, e.D)` next to `group by datepart(YEAR, e.D)` builds at 0 errors (mx check 11.13.0), but `length(s.Name)` next to `group by b.Code` is CE0174 \"Every expression in the SELECT clause must be included in the GROUP BY clause\", despite MDL036's \"MxBuild accepts the view\" message.", "refs": ["ako/mxcli#1650", "mendixlabs/mxcli#1641"], "ce": ["CE0174", "CE6770"], "rules": ["MDL034", "MDL035", "MDL036", "MDL031", "MDL-VERSION01"]} diff --git a/.claude/skills/mendix/write-oql-queries/SKILL.md b/.claude/skills/mendix/write-oql-queries/SKILL.md index 2dc15ea21..1aa47778d 100644 --- a/.claude/skills/mendix/write-oql-queries/SKILL.md +++ b/.claude/skills/mendix/write-oql-queries/SKILL.md @@ -13,13 +13,13 @@ description: "Write OQL for Mendix VIEW entities — joins, aggregates, calculat | 27-32 | Reference files | | 33-35 | Purpose | | 36-42 | When to Use This Skill | -| 43-387 | Critical OQL Syntax Rules | -| 388-524 | Step-by-Step Process | -| 525-620 | Common Mistakes to Avoid | -| 621-652 | Testing OQL Queries | -| 653-671 | Integration with MDL Linter | -| 672-679 | References | -| 680-704 | Summary Checklist | +| 43-402 | Critical OQL Syntax Rules | +| 403-539 | Step-by-Step Process | +| 540-635 | Common Mistakes to Avoid | +| 636-667 | Testing OQL Queries | +| 668-686 | Integration with MDL Linter | +| 687-694 | References | +| 695-719 | Summary Checklist | # Skill: Write OQL Queries for Mendix VIEW Entities @@ -204,8 +204,23 @@ fine as long as nothing aggregates it. | `max(expr)` / `min(expr)` | DateTime | DateTime | `attr: datetime` | | `datepart(part, expr)` | DateTime | Integer | `attr: integer` | | `length(expr)` | String | Integer | `attr: integer` | - -**Key rule**: `count()` and `avg()` have fixed return types. `sum()`, `min()`, `max()` preserve the input type. +| `string_agg(expr, 'sep')` | String | String(200) — 11.7+ and 10.24.22+ | `attr: string(200)` | +| `string_agg(expr, 'sep')` | String(n) | String(n), the argument's own length — 11.2 to 11.6 | `attr: string(n)` | + +`string_agg(expr, 'separator')` joins the values of a group into one string — +`string_agg(s.Name, ', ')` next to `group by b.Code` lists each booking's names. +The separator must be a string literal (CE0174 otherwise). It needs **Mendix +11.2+, or 10.24.22+** on the 10.24 line; below that every such view is CE0174 +"Unknown function 'string_agg'", and `mxcli check -p` refuses it (MDL-VERSION01). + +Its column type **changed in 11.7**: from 11.7 (and on 10.24.22+) it is a derived +String(200) whatever it aggregates; on 11.2–11.6 it is the argument's own type, so +over a `string(100)` attribute it must be declared `string(100)`. Anything else is +CE6770, and `mxcli check -p` reports it with the fix for the project's version. +There is no `group_concat`, `listagg` or `array_agg` in OQL — the six aggregates +are count, sum, avg, min, max and string_agg. + +**Key rule**: `count()` and `avg()` have fixed return types (and `string_agg()` from 11.7). `sum()`, `min()`, `max()` preserve the input type. ### 3. DATEPART Function (Comma Syntax) ```sql diff --git a/.claude/skills/version-awareness.md b/.claude/skills/version-awareness.md index ba45f7de6..134ce27fb 100644 --- a/.claude/skills/version-awareness.md +++ b/.claude/skills/version-awareness.md @@ -27,6 +27,7 @@ Common version gates: | Feature | Requires | Workaround for older versions | |---------|----------|-------------------------------| | VIEW ENTITY | 10.18+ | Regular entity with microflow data source | +| `STRING_AGG(expr, 'sep')` in a view entity's OQL | 11.2+, or 10.24.22+ on the 10.24 line | Build the joined string in a microflow. Its column is String(200) from 11.7 (and on 10.24), but the argument's own type on 11.2–11.6 | | REST query params | 11.0+ | Build query string manually in microflow | | DB runtime connection | 11.0+ | Hardcode connection in Database Connector config | | Design properties v3 | 11.0+ | Use Atlas v2 design properties | diff --git a/cmd/mxcli/syntax/features_domain_model.go b/cmd/mxcli/syntax/features_domain_model.go index 1dcc9795d..57a78f7a3 100644 --- a/cmd/mxcli/syntax/features_domain_model.go +++ b/cmd/mxcli/syntax/features_domain_model.go @@ -183,6 +183,7 @@ func init() { "oql", "view entity oql", "select", "from", "join", "group by", "order by", "limit", "union", "alias", "as alias", "cast", "MDL030", "MDL031", "MDL072", "CE0174", "CE6770", + "string_agg", "aggregate functions", }, MinVersion: "10.18.0", Syntax: "BOTH CLAUSE ORDERS ARE ACCEPTED — `select … from …` and Mendix's own\n" + @@ -205,6 +206,13 @@ func init() { " (MDL072). So a view attribute can never be called `Month` or `Year` —\n" + " that one is renamed, not quoted. An MDL keyword that is NOT an OQL one\n" + " (Role, Status, Value) is a fine FROM / JOIN alias after `as`.\n\n" + + "AGGREGATES: count, sum, avg, min, max and string_agg(expr, 'separator'),\n" + + "which joins a group's values into one string; the separator must be a string\n" + + "literal. string_agg needs Mendix 11.2+, or 10.24.22+ on the 10.24 line (below\n" + + "that: CE0174 \"Unknown function\"). Its column is String(200) from 11.7 and on\n" + + "10.24, but its argument's own type on 11.2-11.6 (CE6770 otherwise; check -p\n" + + "gives the fix for the project). An aggregate may not read a GROUP BY column\n" + + "(CE0174, MDL034); every other column must be grouped (MDL035/036).\n\n" + "UNION / UNION ALL are supported and round-trip; column count and types must\n" + "line up across branches, and an ORDER BY applies to the whole result.", Example: "mdl 1;\n" + diff --git a/docs-site/src/appendixes/version-compatibility.md b/docs-site/src/appendixes/version-compatibility.md index f37bdb081..e8f16a921 100644 --- a/docs-site/src/appendixes/version-compatibility.md +++ b/docs-site/src/appendixes/version-compatibility.md @@ -135,6 +135,7 @@ body on Mendix 10 too, and Studio Pro could not open the project. |---------|-----------|--------|-------| | Basic SELECT | `SELECT, FROM, WHERE` | Yes | Yes | | Aggregate functions | `COUNT, SUM, AVG, MIN, MAX, GROUP BY` | Yes | Yes | +| String aggregate | `STRING_AGG(expr, 'separator')` | 10.24.22+ only | 11.2+ | | Subqueries | Inline subqueries in SELECT/WHERE | Yes | Yes | | JOIN types | `INNER/LEFT/RIGHT/FULL JOIN` | Yes | Yes | diff --git a/mdl-examples/bug-tests/view-entity-1650-string-agg.mdl b/mdl-examples/bug-tests/view-entity-1650-string-agg.mdl new file mode 100644 index 000000000..20845cd69 --- /dev/null +++ b/mdl-examples/bug-tests/view-entity-1650-string-agg.mdl @@ -0,0 +1,59 @@ +mdl 1; +-- ============================================================================ +-- ako/mxcli#1650 (upstream mendixlabs/mxcli#1641) — STRING_AGG in a view +-- entity was reported as "not aggregated" (MDL036), so check, exec and even +-- exec --no-check refused every view that used it. +-- +-- The GROUP BY rules knew five aggregates; mxbuild's OQL grammar has six +-- (SUM COUNT AVG MIN MAX STRING_AGG). They now read one list. +-- +-- Two version facts came with it, both measured with mx check: +-- * STRING_AGG exists from 11.2.0, and from 10.24.22 on the 10.24 line. +-- Below that every such view is CE0174 "Unknown function 'string_agg'", +-- and `check -p` / exec refuse it (MDL-VERSION01). +-- * Its column type changed in 11.7.0: from 11.7 (and on 10.24.22+) it is a +-- derived String(200) whatever it aggregates; on 11.2–11.6 it is the +-- argument's own type (String(100) below). Anything else is CE6770. +-- +-- Run against an 11.7+ (or 10.24.22+) project. Measured on 11.13.0: check +-- passes, exec creates both views, mx check reports 0 errors, and +-- `describe entity` → exec reports them Unchanged. +-- ============================================================================ + +create module Sagg; + +create persistent entity Sagg.Booking ( + Code: String(20) +); + +create persistent entity Sagg.Sale ( + Name: String(100), + Amount: Decimal +); + +create association Sagg.Sale_Booking from Sagg.Sale to Sagg.Booking; + +-- The issue's view: the names of each booking's sales. +create view entity Sagg.BookingNames ( + Code: String(20), + Names: String(200) +) as ( + select b.Code as Code, STRING_AGG(s.Name, '; ') as Names + from Sagg.Sale as s + join s/Sagg.Sale_Booking/Sagg.Booking as b + group by b.Code +); + +-- A CASE inside the aggregate, beside a COUNT. +create view entity Sagg.BookingSummary ( + Code: String(20), + Paid: String(200), + Sales: Integer +) as ( + select b.Code as Code, + string_agg(case when s.Amount > 0 then s.Name else '-' end, ', ') as Paid, + count(s.Name) as Sales + from Sagg.Sale as s + join s/Sagg.Sale_Booking/Sagg.Booking as b + group by b.Code +); diff --git a/mdl/executor/cmd_features.go b/mdl/executor/cmd_features.go index ab57832ca..8cc30c8c3 100644 --- a/mdl/executor/cmd_features.go +++ b/mdl/executor/cmd_features.go @@ -38,8 +38,10 @@ func checkFeature(ctx *ExecContext, area, name, statement, hint string) error { return nil } - // Find the min_version for the error message. - features := reg.FeaturesForVersion(versions.SemVer{Major: 99, Minor: 0, Patch: 0}) + // Find the min_version for the error message: the floor an upgrade from + // this project reaches first. A feature on two release lines has two + // (STRING_AGG: 10.24.22 and 11.2.0), and an 11.1 project needs the second. + features := reg.FeaturesForVersion(pv) minV := "a newer version" for _, f := range features { if f.Area == area && f.Name == name { diff --git a/mdl/executor/oql_type_inference.go b/mdl/executor/oql_type_inference.go index d002e29f5..ca13090b1 100644 --- a/mdl/executor/oql_type_inference.go +++ b/mdl/executor/oql_type_inference.go @@ -11,6 +11,7 @@ import ( mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" "github.com/mendixlabs/mxcli/mdl/langver" "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/sdk/domainmodel" ) @@ -313,6 +314,13 @@ func inferTypeStatic(expr string) ast.DataType { return ast.DataType{Kind: ast.TypeUnknown} } + // string_agg(...) → depends on the project's Mendix version, which this + // path does not know: String(200) on one, its argument's type on another + // (see stringAggType). Unknown, so the column is not judged here. + if strings.HasPrefix(upper, "STRING_AGG(") { + return ast.DataType{Kind: ast.TypeUnknown} + } + // datepart(...) → Integer if strings.HasPrefix(upper, "DATEPART(") { return ast.DataType{Kind: ast.TypeInteger} @@ -361,6 +369,54 @@ func inferTypeStatic(expr string) ast.DataType { // such a column makes mxbuild report CE6770 "View Entity is out of sync". const derivedStringLength = 200 +// firstOQLArgument returns the first argument of a function's argument list: +// the text before the first comma outside parentheses and string literals, so +// the separator of string_agg(case when … then 'a,b' end, ', ') is not mistaken +// for part of the expression. +func firstOQLArgument(args string) string { + depth := 0 + for i := 0; i < len(args); i++ { + switch args[i] { + case '\'': + for i++; i < len(args) && args[i] != '\''; i++ { + } + case '(': + depth++ + case ')': + depth-- + case ',': + if depth == 0 { + return strings.TrimSpace(args[:i]) + } + } + } + return strings.TrimSpace(args) +} + +// stringAggType is the type of a STRING_AGG(expression, 'separator') column. +// Mendix changed it in 11.7 (ako/mxcli#1650). Measured with mx check over +// String(100), String(500) and unlimited attributes: +// +// 11.2.0 – 11.6.12 the type of its argument: String(100) over a String(100) +// attribute, unlimited over an unlimited one — exactly, as +// for a pass-through column; String(200) is CE6770 +// 11.7.0 – 11.15.0, a derived String(200), whatever it aggregates; unlimited, +// 10.24.22 – 10.24.24 String(100) or String(500) are CE6770 +// +// Below 11.2 (and 10.24.22) there is no STRING_AGG at all — the version gate. +// argType infers the argument; the caller's column takes its pass-through +// source from it, which holds the 11.2–11.6 era to the exact-length rule. +// Unknown without a project version: neither answer is right for both eras. +func stringAggType(pv *types.ProjectVersion, argType func() ast.DataType) ast.DataType { + if pv == nil || pv.MajorVersion == 0 { + return ast.DataType{Kind: ast.TypeUnknown} + } + if pv.MajorVersion == 10 || pv.IsAtLeastFull(11, 7, 0) { + return ast.DataType{Kind: ast.TypeString, Length: derivedStringLength} + } + return argType() +} + // castTargetType maps a CAST(expr AS type) to the target Mendix data type. Only // the target matters for view-entity column typing; the source expression is // ignored. Unknown targets return TypeUnknown (the column is then not validated). @@ -521,8 +577,12 @@ func passthroughStringLengthMismatch(declared, inferred ast.DataType, sourceAttr // inherits that branch's length (inferCaseType), and is named as what it is. func passthroughLengthError(attrName string, declared, inferred ast.DataType, expression, sourceEntity, sourceAttr string) string { column, via := "pass-through column", "" - if strings.HasPrefix(strings.ToUpper(expression), "CASE") { + switch upper := strings.ToUpper(expression); { + case strings.HasPrefix(upper, "CASE"): column, via = "CASE column", " through its first branch" + case strings.HasPrefix(upper, "STRING_AGG("): + // Before 11.7 only — see stringAggType. + column, via = "STRING_AGG column", " through its argument" } return fmt.Sprintf( "attribute '%s': declared as %s but %s '%s' inherits length %d%s from source attribute %s.%s — Mendix requires an exact length match (CE6770 \"View Entity out of sync\"). Fix: change to '%s: %s'", @@ -605,8 +665,16 @@ func validateViewEntityTypes(ctx *ExecContext, stmt *ast.CreateViewEntityStmt) ( continue } - // Compare types - if !typesCompatible(attr.Type, col.InferredType) { + // Compare types. A derived STRING_AGG column (11.7+, 10.24.22+) is + // String(200) exactly, like every derived string — the static rule + // cannot apply it, because only here is the version known + // (stringAggType). Unlimited and String(500) are CE6770 too, which the + // "long enough" rule below would let through. + compatible := typesCompatible(attr.Type, col.InferredType) + if col.AggregateFunc == "string_agg" && col.SourceAttr == "" { + compatible = typesStrictlyCompatible(attr.Type, col.InferredType) + } + if !compatible { errors = append(errors, fmt.Sprintf( "attribute '%s': declared as %s but OQL expression '%s' returns %s. Fix: change to '%s: %s'", attr.Name, @@ -1026,6 +1094,14 @@ func inferAggregateType(ctx *ExecContext, expr string, col *OQLColumnInfo, alias return ast.DataType{Kind: ast.TypeUnknown} } + if strings.HasPrefix(upperExpr, "STRING_AGG(") { + col.IsAggregate = true + col.AggregateFunc = "string_agg" + return stringAggType(ctx.Backend.ProjectVersion(), func() ast.DataType { + return inferTypeFromExpression(ctx, firstOQLArgument(extractFunctionArg(expr)), col, aliasMap) + }) + } + // LENGTH returns Integer if strings.HasPrefix(upperExpr, "LENGTH(") { return ast.DataType{Kind: ast.TypeInteger} diff --git a/mdl/executor/oql_view_select_checks.go b/mdl/executor/oql_view_select_checks.go index 5636c7127..61e2c4338 100644 --- a/mdl/executor/oql_view_select_checks.go +++ b/mdl/executor/oql_view_select_checks.go @@ -159,11 +159,24 @@ var ( oqlPlainColumnRe = regexp.MustCompile(`^[a-z_][a-z0-9_]*\.[a-z_][a-z0-9_]*$`) oqlBareIdentRe = regexp.MustCompile(`^[a-z_][a-z0-9_]*$`) // oqlAggregateCallRe matches an aggregate call in a normalized expression. - oqlAggregateCallRe = regexp.MustCompile(`(^|[^a-z0-9_.])(count|sum|avg|min|max)\(`) + oqlAggregateCallRe = regexp.MustCompile(`(^|[^a-z0-9_.])(` + strings.Join(oqlAggregateFunctions, "|") + `)\(`) // oqlRawAggregateCallRe is oqlAggregateCallRe over query text as written. - oqlRawAggregateCallRe = regexp.MustCompile(`(?i)(^|[^A-Za-z0-9_.])(count|sum|avg|min|max)\s*\(`) + oqlRawAggregateCallRe = regexp.MustCompile(`(?i)(^|[^A-Za-z0-9_.])(` + strings.Join(oqlAggregateFunctions, "|") + `)\s*\(`) ) +// oqlAggregateFunctions are the aggregate functions of Mendix OQL — the +// aggregate tokens of mxbuild's own OQL grammar, 'SUM' 'COUNT' 'AVG' 'MIN' +// 'MAX' 'STRING_AGG' (DeprecatedOqlLexer in Mendix.Modeler.Core.dll, the only +// OQL grammar it carries). Every rule that asks "is this column aggregated?" +// reads this one list. It held the first five only, so a STRING_AGG column +// read as an ungrouped expression and every view using it was refused as +// MDL036 — by check, by exec, and by exec --no-check (ako/mxcli#1650). A +// name Mendix does not have is CE0174 "Unknown function '…'": group_concat, +// listagg, array_agg and stddev, measured on 11.13.0. +// +// STRING_AGG has a version floor of its own; see oqlUsesStringAgg. +var oqlAggregateFunctions = []string{"count", "sum", "avg", "min", "max", "string_agg"} + // oqlColumnRefs lists the column references in a normalized expression, // ignoring anything inside a string literal. func oqlColumnRefs(n string) []string { diff --git a/mdl/executor/version_gates.go b/mdl/executor/version_gates.go index ae2224d89..7ec112375 100644 --- a/mdl/executor/version_gates.go +++ b/mdl/executor/version_gates.go @@ -5,6 +5,7 @@ package executor import ( "context" "fmt" + "regexp" "strings" "github.com/mendixlabs/mxcli/mdl/ast" @@ -79,6 +80,15 @@ func statementVersionGates(stmt ast.Statement) []versionGate { "a comment in a view entity's OQL query", "remove the comments from the query — Mendix rejects them as CE0174 before 11.7 — or upgrade the project"}) } + // STRING_AGG is an OQL aggregate from 11.2, and from 10.24.22 on the + // 10.24 line (ako/mxcli#1650). Below that floor mxbuild has no such + // function, and the view fails the build. + if oqlUsesStringAgg(s.Query.RawQuery) { + gates = append(gates, versionGate{"oql_functions", "string_agg", + "STRING_AGG in a view entity's OQL query", + "Mendix added STRING_AGG to OQL in 11.2 and in 10.24.22 — upgrade the project, " + + "or build the list in a microflow"}) + } return gates // A domain-model EXPORT LEVEL is 9.3+ (mendixlabs/mxcli#1498). Gated on the // clause being STATED, as the deep-link URL is: a statement without one @@ -189,6 +199,15 @@ func statementVersionGates(stmt ast.Statement) []versionGate { return nil } +// oqlStringAggCallRe is a STRING_AGG call in query text as written. +var oqlStringAggCallRe = regexp.MustCompile(`(?i)(^|[^A-Za-z0-9_."` + "`" + `])string_agg\s*\(`) + +// oqlUsesStringAgg reports whether a view's query calls STRING_AGG — outside +// comments and string literals, so `'string_agg('` as a label is not a call. +func oqlUsesStringAgg(oql string) bool { + return oqlStringAggCallRe.MatchString(oqlStringLiteralRe.ReplaceAllString(stripOQLComments(oql), "''")) +} + func exportLevelGate(statement string) versionGate { return versionGate{"domain_model", "export_levels", statement, "Mendix stores no export level on domain-model elements before 9.3 — remove the clause, or upgrade the project"} diff --git a/mdl/executor/version_gates_test.go b/mdl/executor/version_gates_test.go index 59bc77916..b26b8d99a 100644 --- a/mdl/executor/version_gates_test.go +++ b/mdl/executor/version_gates_test.go @@ -40,6 +40,9 @@ func TestCheckVersionGates_ReportsEveryHandlerGate(t *testing.T) { // refuses the statement instead of writing an unbuildable model. {"view query comment", &ast.CreateViewEntityStmt{Query: ast.OQLQuery{ RawQuery: "-- totals\nselect sum(s.Amount) as Total\nfrom M.Sale as s"}}, [3]int{11, 6, 6}, "a comment in a view entity's OQL query"}, + // ako/mxcli#1650: STRING_AGG is 11.2+ (and 10.24.22+ on the 10.24 line). + {"view query string_agg", &ast.CreateViewEntityStmt{Query: ast.OQLQuery{ + RawQuery: "select string_agg(s.Name, ', ') as Names\nfrom M.Sale as s"}}, [3]int{11, 1, 0}, "STRING_AGG in a view entity's OQL query"}, {"agent model", &ast.CreateModelStmt{}, [3]int{11, 8, 0}, "create model"}, {"consumed mcp", &ast.CreateConsumedMCPServiceStmt{}, [3]int{11, 8, 0}, "create consumed mcp service"}, {"knowledge base", &ast.CreateKnowledgeBaseStmt{}, [3]int{11, 8, 0}, "create knowledge base"}, diff --git a/mdl/executor/view_entity_string_agg_test.go b/mdl/executor/view_entity_string_agg_test.go new file mode 100644 index 000000000..49083bb82 --- /dev/null +++ b/mdl/executor/view_entity_string_agg_test.go @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 + +// ako/mxcli#1650 (upstream mendixlabs/mxcli#1641): STRING_AGG is an OQL +// aggregate, and the GROUP BY rules did not know it — every view using it was +// MDL036 "not aggregated", refused by check, exec and exec --no-check alike. +package executor + +import ( + "fmt" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// stringAggView is the issue's own view, over +// +// MyFirstModule.Sale (Name: String(100), Amount: Decimal) +// MyFirstModule.Booking (Code: String(20)) +// MyFirstModule.Sale_Booking from Sale to Booking +func stringAggView(column string) string { + return `select b.Code as Code, ` + column + ` as Names + from MyFirstModule.Sale as s + join s/MyFirstModule.Sale_Booking/MyFirstModule.Booking as b + group by b.Code` +} + +// The grouping rules count every OQL aggregate as one. Each "fires: false" +// case below builds at 0 errors on mxbuild (see the version notes on +// oqlAggregateFunctions); each "fires: true" one is the rule still doing its +// job next to a STRING_AGG. +func TestGroupByRules_StringAggIsAnAggregate(t *testing.T) { + for _, rule := range []string{"MDL035", "MDL036"} { + runOQLRuleCases(t, rule, []oqlRuleCase{ + {"the issue's view", stringAggView(`STRING_AGG(s.Name, '; ')`), false}, + {"lower case", stringAggView(`string_agg(s.Name, ', ')`), false}, + {"a space before the parenthesis", stringAggView(`STRING_AGG (s.Name, ', ')`), false}, + {"a CASE as the argument", stringAggView(`STRING_AGG(CASE WHEN s.Amount > 0 THEN s.Name ELSE '-' END, '; ')`), false}, + {"from-first clause order", + `from MyFirstModule.Sale as s join s/MyFirstModule.Sale_Booking/MyFirstModule.Booking as b group by b.Code select b.Code as Code, string_agg(s.Name, ';') as Names`, false}, + }) + } + // Controls: a STRING_AGG elsewhere in the select list does not excuse a + // column that is not aggregated. + runOQLRuleCases(t, "MDL036", []oqlRuleCase{ + {"an ungrouped expression beside a STRING_AGG", + `select b.Code as Code, length(s.Name) as L, string_agg(s.Name, ';') as Names + from MyFirstModule.Sale as s join s/MyFirstModule.Sale_Booking/MyFirstModule.Booking as b group by b.Code`, true}, + }) + runOQLRuleCases(t, "MDL035", []oqlRuleCase{ + {"an ungrouped column beside a STRING_AGG", + `select b.Code as Code, s.Name as Nm, string_agg(s.Name, ';') as Names + from MyFirstModule.Sale as s join s/MyFirstModule.Sale_Booking/MyFirstModule.Booking as b group by b.Code`, true}, + }) +} + +// A STRING_AGG over a grouped column is the same CE0174 as count(r.Season) +// by r.Season (MDL034). +func TestMDL034_StringAggOverAGroupedColumn(t *testing.T) { + runOQLRuleCases(t, "MDL034", []oqlRuleCase{ + {"string_agg(b.Code) by b.Code", stringAggView(`string_agg(b.Code, ';')`), true}, + {"string_agg(s.Name) by b.Code (control)", stringAggView(`string_agg(s.Name, ';')`), false}, + }) +} + +// The separator is a string literal, and Mendix requires one there: it is not +// the untyped aggregate ARGUMENT MDL037 is about. +func TestMDL037_StringAggSeparatorIsNotALiteralArgument(t *testing.T) { + if got := selectRuleViolations(stringAggView(`string_agg(s.Name, '; ')`), "MDL037"); len(got) != 0 { + t.Errorf("the separator literal drew MDL037: %v", got) + } +} + +func stringAggFixture(t *testing.T, version ...[3]int) (*ExecContext, *[]*domainmodel.Entity) { + t.Helper() + mod := mkModule("MyFirstModule") + sale := mkEntity(mod.ID, "Sale") + sale.Attributes = []*domainmodel.Attribute{ + {Name: "Name", Type: &domainmodel.StringAttributeType{Length: 100}}, + {Name: "Amount", Type: &domainmodel.DecimalAttributeType{}}, + } + booking := mkEntity(mod.ID, "Booking") + booking.Attributes = []*domainmodel.Attribute{ + {Name: "Code", Type: &domainmodel.StringAttributeType{Length: 20}}, + } + memo := mkEntity(mod.ID, "Memo") + memo.Attributes = []*domainmodel.Attribute{ + {Name: "Body", Type: &domainmodel.StringAttributeType{}}, + {Name: "Brief", Type: &domainmodel.StringAttributeType{Length: 500}}, + } + dm := mkDomainModel(mod.ID, sale, booking, memo) + var created []*domainmodel.Entity + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { return []*domainmodel.DomainModel{dm}, nil }, + GetDomainModelFunc: func(model.ID) (*domainmodel.DomainModel, error) { return dm, nil }, + CreateEntityFunc: func(_ model.ID, e *domainmodel.Entity) error { + created = append(created, e) + return nil + }, + } + if len(version) == 1 { + v := version[0] + mb.ProjectVersionFunc = func() *types.ProjectVersion { + return &types.ProjectVersion{MajorVersion: v[0], MinorVersion: v[1], PatchVersion: v[2], + ProductVersion: fmt.Sprintf("%d.%d.%d", v[0], v[1], v[2])} + } + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(mkHierarchy(mod))) + return ctx, &created +} + +// The issue's step2.mdl reached the handler under `exec --no-check` and was +// refused there with "invalid OQL in view entity" — the handler runs +// ValidateOQLSyntax itself, so the pre-flight was not the only barrier. +func TestExecCreateViewEntity_WritesAStringAggView(t *testing.T) { + ctx, created := stringAggFixture(t) + s := parseViewStmt(t, `create view entity MyFirstModule.BookingNames (Code: String(20), Names: String(200)) as ( + SELECT b.Code AS Code, STRING_AGG(s.Name, '; ') AS Names + FROM MyFirstModule.Sale AS s + JOIN s/MyFirstModule.Sale_Booking/MyFirstModule.Booking AS b + GROUP BY b.Code +);`) + if err := execCreateViewEntity(ctx, s); err != nil { + t.Fatalf("exec refused the view: %v", err) + } + if len(*created) != 1 || (*created)[0].Name != "BookingNames" { + t.Fatalf("want BookingNames written, got %d entities", len(*created)) + } + if err := strings.TrimSpace((*created)[0].OqlQuery); err == "" { + t.Error("the view was written without its query") + } +} + +// The type of a STRING_AGG column is version-dependent (stringAggType), and +// every row below is an mx check result over the attribute shown: +// +// from 11.7.0 and on 10.24.22+ String(200), whatever it aggregates +// 11.2.0 – 11.6.12 the argument's own type, exactly +// +// So the issue's `Names: String` (unlimited) over Sale.Name, a String(100), is +// CE6770 "View Entity is out of sync with the OQL Query" on every version that +// has STRING_AGG, and check says so before the write — with the fix for that +// project's version. +func TestViewTypes_StringAggColumnTypeFollowsTheVersion(t *testing.T) { + cases := []struct { + v [3]int + column string // over Sale.Name: String(100), Memo.Body: String(unlimited), Memo.Brief: String(500) + declared string + fix string // "" = builds + }{ + // 11.7.0+: a derived String(200) (measured 11.7.0, 11.10.0, 11.12.2, 11.13.0, 11.15.0). + {[3]int{11, 13, 0}, "STRING_AGG(s.Name, '; ')", "String", "Names: String(200)"}, + {[3]int{11, 13, 0}, "STRING_AGG(s.Name, '; ')", "String(100)", "Names: String(200)"}, + {[3]int{11, 13, 0}, "STRING_AGG(s.Name, '; ')", "String(200)", ""}, + {[3]int{11, 7, 0}, "string_agg(m.Body, ',')", "String", "Names: String(200)"}, + {[3]int{11, 7, 0}, "string_agg(m.Brief, ',')", "String(500)", "Names: String(200)"}, + {[3]int{11, 7, 0}, "string_agg(m.Brief, ',')", "String(200)", ""}, + {[3]int{11, 13, 0}, "STRING_AGG(CASE WHEN s.Amount > 0 THEN s.Name ELSE '-' END, '; ')", "String", "Names: String(200)"}, + {[3]int{11, 13, 0}, "string_agg(s.Name, ',')", "Integer", "Names: String(200)"}, + // The 10.24 line got the 11.7 behaviour with the function (measured 10.24.22, 10.24.24). + {[3]int{10, 24, 22}, "STRING_AGG(s.Name, '; ')", "String(100)", "Names: String(200)"}, + {[3]int{10, 24, 24}, "STRING_AGG(s.Name, '; ')", "String(200)", ""}, + // 11.2–11.6: the argument's type (measured 11.2.0, 11.6.6, 11.6.8, 11.6.12). + {[3]int{11, 6, 6}, "STRING_AGG(s.Name, '; ')", "String(200)", "Names: String(100)"}, + {[3]int{11, 6, 6}, "STRING_AGG(s.Name, '; ')", "String", "Names: String(100)"}, + {[3]int{11, 6, 6}, "STRING_AGG(s.Name, '; ')", "String(100)", ""}, + {[3]int{11, 2, 0}, "string_agg(m.Brief, ',')", "String(200)", "Names: String(500)"}, + {[3]int{11, 2, 0}, "string_agg(m.Brief, ',')", "String(500)", ""}, + {[3]int{11, 6, 12}, "STRING_AGG(CASE WHEN s.Amount > 0 THEN s.Name ELSE '-' END, '; ')", "String(100)", ""}, + } + for _, c := range cases { + t.Run(fmt.Sprintf("%d.%d.%d %s as %s", c.v[0], c.v[1], c.v[2], c.column, c.declared), func(t *testing.T) { + ctx, _ := stringAggFixture(t, c.v) + from := "MyFirstModule.Sale as s" + if strings.Contains(c.column, "m.") { + from = "MyFirstModule.Memo as m" + } + s := parseViewStmt(t, `create view entity MyFirstModule.V (Names: `+c.declared+`) as ( + select `+c.column+` as Names from `+from+` +);`) + errs, _ := validateViewEntityTypes(ctx, s) + if c.fix == "" { + if len(errs) != 0 { + t.Fatalf("refused a declaration that builds: %v", errs) + } + return + } + if len(errs) != 1 || !strings.Contains(errs[0], c.fix) { + t.Fatalf("want one error with the fix %q, got %v", c.fix, errs) + } + }) + } + + // Without a project version neither answer is safe, so the column is not + // judged — `check` without -p cannot tell the eras apart. + if vs := ValidateOQLTypes(`select string_agg(s.Name, ',') as Names from MyFirstModule.Sale as s`, + []ast.ViewAttribute{{Name: "Names", Type: ast.DataType{Kind: ast.TypeString, Length: 200}}}); len(vs) != 0 { + t.Errorf("judged a STRING_AGG column with no version: %v", vs) + } +} + +// Mendix added STRING_AGG to OQL twice: in 11.2.0, and in 10.24.22 on the 10.24 +// line. Each floor was found in mxbuild's own OQL lexer (the 'STRING_AGG' +// token) and confirmed with mx check — below it every STRING_AGG view is +// CE0174 "Unknown function 'string_agg'". The refusal names the floor that +// project's upgrade reaches first. +func TestCheckVersionGates_StringAggIsOnTwoReleaseLines(t *testing.T) { + view := &ast.CreateViewEntityStmt{Query: ast.OQLQuery{ + RawQuery: "select b.Code as Code, STRING_AGG(s.Name, '; ') as Names\nfrom M.Sale as s\njoin s/M.Sale_Booking/M.Booking as b\ngroup by b.Code"}} + prog := &ast.Program{Statements: []ast.Statement{view}} + cases := []struct { + v [3]int + floor string // "" = available + }{ + {[3]int{10, 24, 21}, "10.24.22"}, + {[3]int{10, 24, 22}, ""}, + {[3]int{10, 24, 28}, ""}, + {[3]int{11, 0, 0}, "11.2.0"}, + {[3]int{11, 1, 0}, "11.2.0"}, + {[3]int{11, 2, 0}, ""}, + {[3]int{11, 12, 5}, ""}, + } + for _, c := range cases { + vs := versionGateViolations(versionCtx(t, c.v[0], c.v[1], c.v[2]), prog) + switch { + case c.floor == "" && len(vs) != 0: + t.Errorf("%v: STRING_AGG refused where Mendix has it: %+v", c.v, vs) + case c.floor != "" && (len(vs) != 1 || !strings.Contains(vs[0].Message, "STRING_AGG") || + !strings.Contains(vs[0].Message, "Mendix "+c.floor+"+")): + t.Errorf("%v: want one refusal naming STRING_AGG and Mendix %s+, got %+v", c.v, c.floor, vs) + } + } +} + +// The gate fires on a call, not on the letters. +func TestOQLUsesStringAgg(t *testing.T) { + cases := []struct { + oql string + want bool + }{ + {`select string_agg(s.Name, ',') as N from M.Sale as s`, true}, + {`select STRING_AGG (s.Name, ',') as N from M.Sale as s`, true}, + {`from M.Sale as s select String_Agg(s.Name, ',') as N`, true}, + {`select 'string_agg(' as L from M.Sale as s`, false}, + {"select s.Name as N -- string_agg(s.Name, ',')\nfrom M.Sale as s", false}, + {`select s.my_string_agg as N from M.Sale as s`, false}, + {`select s."string_agg" as N from M.Sale as s`, false}, + {`select count(s.Name) as N from M.Sale as s`, false}, + } + for _, c := range cases { + if got := oqlUsesStringAgg(c.oql); got != c.want { + t.Errorf("oqlUsesStringAgg(%q) = %v, want %v", c.oql, got, c.want) + } + } +} diff --git a/sdk/versions/mendix-10.yaml b/sdk/versions/mendix-10.yaml index 9e10a00ac..adebf0d35 100644 --- a/sdk/versions/mendix-10.yaml +++ b/sdk/versions/mendix-10.yaml @@ -42,6 +42,18 @@ features: join_types: min_version: "10.18.0" mdl: "INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN" + string_agg: + # STRING_AGG(expression, 'separator'), the OQL aggregate that joins a + # group's values into one string. On the 10.24 line it arrived in a + # patch: mxbuild 10.24.21's OQL lexer has no 'STRING_AGG' token and mx + # check reports every such view as CE0174 "Unknown function + # 'string_agg'"; 10.24.22 has the token and builds them at 0 errors. + # Never in 10.18-10.23. 11.x has its own floor (mendix-11.yaml), hence + # the max_version (ako/mxcli#1650). + min_version: "10.24.22" + max_version: "10.99.99" + mdl: "STRING_AGG(expression, 'separator') in a view entity's OQL query" + notes: "Measured with mx check: 10.24.21 CE0174 \"Unknown function 'string_agg'\", 10.24.22 and 10.24.24 0 errors. The column is a String(200) whatever it aggregates. ako/mxcli#1650" microflows: basic: diff --git a/sdk/versions/mendix-11.yaml b/sdk/versions/mendix-11.yaml index d5346903b..d3cb1be1e 100644 --- a/sdk/versions/mendix-11.yaml +++ b/sdk/versions/mendix-11.yaml @@ -39,6 +39,15 @@ features: join_types: min_version: "10.18.0" mdl: "INNER JOIN, LEFT JOIN, RIGHT JOIN, FULL JOIN" + string_agg: + # STRING_AGG(expression, 'separator'), the OQL aggregate that joins a + # group's values into one string. mxbuild 11.0.0 and 11.1.0 have no + # 'STRING_AGG' token in their OQL lexer and mx check reports every such + # view as CE0174 "Unknown function 'string_agg'"; 11.2.0 has it. The + # 10.24 line got it separately, in 10.24.22 (ako/mxcli#1650). + min_version: "11.2.0" + mdl: "STRING_AGG(expression, 'separator') in a view entity's OQL query" + notes: "Measured with mx check: 11.0.0 and 11.1.0 CE0174 \"Unknown function 'string_agg'\"; 11.2.0, 11.6.6, 11.6.8, 11.6.12, 11.7.0, 11.8.0, 11.9.1, 11.10.0, 11.12.2, 11.13.0 and 11.15.0 build it. The column's type changed in 11.7.0: before, the type of its argument (String(100) over a String(100)); from 11.7.0, String(200) whatever it aggregates. ako/mxcli#1650" comments: min_version: "11.7.0" mdl: "-- line comment / /* block comment */ in a view entity's OQL query" diff --git a/sdk/versions/registry.go b/sdk/versions/registry.go index b91ef6083..c668e4b59 100644 --- a/sdk/versions/registry.go +++ b/sdk/versions/registry.go @@ -242,19 +242,7 @@ func (r *Registry) FeaturesForVersion(projectVersion SemVer) []FeatureStatus { } seen[key] = true - // Use the entry with the lowest min_version as the canonical one. - best := entries[0] - for _, e := range entries[1:] { - if best.MinVersion.AtLeast(e.MinVersion) && e.MinVersion.String() != best.MinVersion.String() { - best = e - } - } - - available := projectVersion.AtLeast(best.MinVersion) - if best.MaxVersion != nil && !best.MaxVersion.AtLeast(projectVersion) { - available = false - } - + best, available := entryAt(entries, projectVersion) result = append(result, FeatureStatus{ FeatureEntry: best, Available: available, @@ -268,6 +256,51 @@ func (r *Registry) FeaturesForVersion(projectVersion SemVer) []FeatureStatus { return result } +// covers reports whether v lies in the entry's [min_version, max_version]. +func (e FeatureEntry) covers(v SemVer) bool { + return v.AtLeast(e.MinVersion) && (e.MaxVersion == nil || e.MaxVersion.AtLeast(v)) +} + +// entryAt picks the entry that describes a feature at version v, and reports +// whether the feature is available there. +// +// Most features have one floor, so every entry agrees and the lowest +// min_version is the one to show. A feature Mendix shipped on two release +// lines has two that do not: STRING_AGG in OQL is 10.24.22 on the 10.24 line +// (closed by a max_version) and 11.2.0 on 11 (ako/mxcli#1650). Taking the +// lowest floor there reports it unavailable on 11.6 — 10.24.22's range does +// not reach it — and tells an 11.1 project it needs "10.24.22+". So: the +// lowest entry whose range holds v; failing that, the next floor above v, the +// one an upgrade reaches first; failing that (a feature Mendix removed), the +// lowest floor. +func entryAt(entries []FeatureEntry, v SemVer) (FeatureEntry, bool) { + var covering, next *FeatureEntry + lowest := &entries[0] + for i := range entries { + e := &entries[i] + if !e.MinVersion.AtLeast(lowest.MinVersion) { + lowest = e + } + switch { + case e.covers(v): + if covering == nil || !e.MinVersion.AtLeast(covering.MinVersion) { + covering = e + } + case !v.AtLeast(e.MinVersion): + if next == nil || !e.MinVersion.AtLeast(next.MinVersion) { + next = e + } + } + } + if covering != nil { + return *covering, true + } + if next != nil { + return *next, false + } + return *lowest, false +} + // FeaturesAddedSince returns features that become available when upgrading // from sinceVersion. Only features whose min_version > sinceVersion are included. func (r *Registry) FeaturesAddedSince(sinceVersion SemVer) []FeatureEntry { @@ -280,14 +313,8 @@ func (r *Registry) FeaturesAddedSince(sinceVersion SemVer) []FeatureEntry { } seen[key] = true - best := entries[0] - for _, e := range entries[1:] { - if best.MinVersion.AtLeast(e.MinVersion) && e.MinVersion.String() != best.MinVersion.String() { - best = e - } - } - - if !sinceVersion.AtLeast(best.MinVersion) { + // Added since: not available at sinceVersion, and a floor above it. + if best, available := entryAt(entries, sinceVersion); !available && !sinceVersion.AtLeast(best.MinVersion) { result = append(result, best) } } @@ -308,23 +335,12 @@ func (r *Registry) FeaturesInArea(area string, projectVersion SemVer) []FeatureS if seen[key] { continue } - best := entries[0] - for _, e := range entries[1:] { - if best.MinVersion.AtLeast(e.MinVersion) && e.MinVersion.String() != best.MinVersion.String() { - best = e - } - } - + best, available := entryAt(entries, projectVersion) if best.Area != area { continue } seen[key] = true - available := projectVersion.AtLeast(best.MinVersion) - if best.MaxVersion != nil && !best.MaxVersion.AtLeast(projectVersion) { - available = false - } - result = append(result, FeatureStatus{ FeatureEntry: best, Available: available, diff --git a/sdk/versions/registry_test.go b/sdk/versions/registry_test.go index 3d782700f..655326b15 100644 --- a/sdk/versions/registry_test.go +++ b/sdk/versions/registry_test.go @@ -304,3 +304,67 @@ func TestWorkflowGroupsFloorIs11_2(t *testing.T) { } } } + +// TestStringAggIsOnTwoReleaseLines: Mendix added STRING_AGG to OQL in 11.2.0 +// and, separately, in 10.24.22 on the 10.24 line (ako/mxcli#1650). Two entries +// with a max_version on the first; every view of the registry has to agree on +// what that means — IsAvailable alone did, while `show features` took the +// lowest floor and reported the feature missing on every 11.x. +func TestStringAggIsOnTwoReleaseLines(t *testing.T) { + reg, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + cases := []struct { + v SemVer + available bool + since string // the floor shown for v + }{ + {SemVer{10, 24, 21}, false, "10.24.22"}, + {SemVer{10, 24, 22}, true, "10.24.22"}, + {SemVer{11, 0, 0}, false, "11.2.0"}, + {SemVer{11, 1, 0}, false, "11.2.0"}, + {SemVer{11, 2, 0}, true, "11.2.0"}, + {SemVer{11, 6, 6}, true, "11.2.0"}, + } + for _, c := range cases { + if got := reg.IsAvailable("oql_functions", "string_agg", c.v); got != c.available { + t.Errorf("IsAvailable(%s) = %v, want %v", c.v, got, c.available) + } + for _, list := range [][]FeatureStatus{reg.FeaturesForVersion(c.v), reg.FeaturesInArea("oql_functions", c.v)} { + found := false + for _, f := range list { + if f.Area != "oql_functions" || f.Name != "string_agg" { + continue + } + found = true + if f.Available != c.available || f.MinVersion.String() != c.since { + t.Errorf("at %s: listed available=%v since %s, want %v since %s", + c.v, f.Available, f.MinVersion, c.available, c.since) + } + } + if !found { + t.Errorf("at %s: string_agg not listed", c.v) + } + } + } + + // Upgrading from 11.0 adds it at 11.2.0; from 10.24.22 there is nothing to add. + added := func(since SemVer) string { + for _, f := range reg.FeaturesAddedSince(since) { + if f.Area == "oql_functions" && f.Name == "string_agg" { + return f.MinVersion.String() + } + } + return "" + } + if got := added(SemVer{11, 0, 0}); got != "11.2.0" { + t.Errorf("added since 11.0.0 = %q, want 11.2.0", got) + } + if got := added(SemVer{10, 24, 22}); got != "" { + t.Errorf("added since 10.24.22 = %q, want nothing: it is there already", got) + } + if got := added(SemVer{10, 24, 0}); got != "10.24.22" { + t.Errorf("added since 10.24.0 = %q, want 10.24.22", got) + } +} From 7f10d0b651ddfbea86f00b6d383262e05c47692f Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 10:47:40 +0000 Subject: [PATCH 06/13] docs(skills): fence the multi-line @expect fragment as text A doc comment on its own is not an MDL statement, so the conformance gate counted the new ```mdl block as a syntax finding (8 against an allowlist of 7). The gate's own guidance is to fence a non-MDL fragment as ```text. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/mendix/test-microflows/reference/annotations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/mendix/test-microflows/reference/annotations.md b/.claude/skills/mendix/test-microflows/reference/annotations.md index 8ac83034f..3b636fad8 100644 --- a/.claude/skills/mendix/test-microflows/reference/annotations.md +++ b/.claude/skills/mendix/test-microflows/reference/annotations.md @@ -25,7 +25,7 @@ An `@expect` or `@verify` **may span several lines**. Its text runs on, as a javadoc block tag's does, to the next tag or the end of the comment, and all of it is the one assertion: -```mdl +```text /** * @test the separator survives * @expect $r = 'a;b' From f200828254f0f26661f10b9b5f7b3f521b47d652 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 10:47:35 +0000 Subject: [PATCH 07/13] fix: splice refuses a moved producer and a variable an earlier alter operation declares (ako/mxcli#1399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two items of the issue (mendixlabs/mxcli#1517) were still open on main, both because an alter block's variable verdicts — and those of the patch `create or modify` splices with, which is one — were taken against the flow as stored rather than the flow the block writes: - Repro B (variable scope in one alter block): the fragment builder was seeded only with the stored flow's variables, so after `insert after $X begin declare $Late Integer = 0; end;` a replaced loop doing `set $Late = $Late + 1` failed at exec with "variable 'Late' is not declared", while check gave up on it ("not predicted") and passed. 2f42df658 had fixed the scope check for a fragment that only READS such a variable; building one that sets it, or names a member of an entity an inserted retrieve returns, still failed. - Used return: a producer moved upstream — `create or modify` restating `$Code = call …` above the declare it followed, which the diff turns into insert + drop — was refused "the fragment declares $Code, which the microflow already has" by the stored producer the same patch drops, and in the other order "$Code is still used by: log …; return $Code" by readers the inserted producer now feeds. Fix (mdl/executor/cmd_alter_flow.go): - each applied fragment's declared variables, with the types its builder resolved, seed the builders of later fragments (fragmentBuilder, buildFragmentVars, noteFragment); check's pre-build in alterFlowViolations does the same, so the two agree. In scope is still checkFragmentScope's to say, so a declaration downstream of the user is refused as before; - the clash with a stored producer is judged once every operation is applied (checkDeclaredUnclashed), skipping one the block takes away or whose loop it replaces; - checkOutputUnused no longer counts a reader — stored, or in a fragment — that another operation's fragment declares the variable for upstream of it (declaredOnPath), nor one inside a replaced loop. A body that builds declares each variable once, so a create-or-modify clash refusal from the splice was always false: the #1361 test that pinned "replace $Equal: the fragment declares $Different, which the microflow already has" as the expected verdict now asserts that check predicts the splice, exec writes the declared flow, and the start event, the log and the end event keep their $IDs. Kept as is: a declaration made by a LATER operation of the block is not known to an earlier fragment (check: "not predicted"; exec refuses) — the skill now says "declare first, use after". Verification: - New tests (cmd_alter_flow_block_vars_1399_pedapp_test.go): Repro B for a primitive and an entity variable, with check required to PREDICT the statement (no "not predicted" note); a producer moved earlier through create or modify (only the producer re-created: 1 gone, 1 new $ID) and through alter in both orders and with its reader; controls (declared after the loop; producer moved past its reader; stored producer kept; a reader ahead of the new producer) still refused by check and exec. - Revert proof: with cmd_alter_flow.go and validate_flow_verdicts.go restored, all new tests and the updated #1361 test fail with the reported messages. - Blank 11.13.0 app, mdl-examples/bug-tests/splice-1399-…mdl: baseline stops at the alter ("variable 'Late' is not declared") and rolls back; without the alter it stops at "the fragment declares $Code". Fixed: "Altered microflow BugTest1399.LoopFlow", "Modified microflow: BugTest1399.MoveFlow (spliced: 1 inserted, 1 dropped)"; re-runnable; mx check 0 errors. Co-Authored-By: Claude Opus 5.5 --- ...-variables-judged-against-stored-flow.json | 1 + .../write-microflows/reference/pitfalls.md | 12 +- .../splice-1399-alter-block-variables.mdl | 82 +++++++ mdl/executor/cmd_alter_flow.go | 200 ++++++++++++++--- ..._alter_flow_block_vars_1399_pedapp_test.go | 211 ++++++++++++++++++ mdl/executor/validate_flow_verdicts.go | 10 +- ...validate_flow_verdicts_1361_pedapp_test.go | 41 +++- 7 files changed, 521 insertions(+), 36 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-25-alter-block-variables-judged-against-stored-flow.json create mode 100644 mdl-examples/bug-tests/splice-1399-alter-block-variables.mdl create mode 100644 mdl/executor/cmd_alter_flow_block_vars_1399_pedapp_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-alter-block-variables-judged-against-stored-flow.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-alter-block-variables-judged-against-stored-flow.json new file mode 100644 index 000000000..aa59e0cb7 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-alter-block-variables-judged-against-stored-flow.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-25", "symptom": "(1) `alter microflow` with `insert after $X begin declare $Late Integer = 0; end;` and then `replace loop … with begin loop … set $Late = $Late + 1; … end;` passed check with a 'not predicted' note and failed at exec: 'the fragment has errors: variable 'Late' is not declared'. (2) A producer moved upstream — `create or modify` restating `$Code = call …` above the declare it followed, or the same as an alter insert+drop in either order — was refused 'the fragment declares $Code, which the microflow already has' (and, once past that, '$Code is still used by: log …; return $Code')", "cause": "Two variable verdicts of an alter block (and of the patch create or modify splices with, which is one) were still taken against the flow as stored: the fragment builder was seeded only with the stored flow's variables (storedVariables + the full build's entity types), so a later fragment could read but not `set` or member-access what an earlier operation declares; and checkFragmentScope refused a fragment declaring any variable a stored activity declares, at the moment it ran, before a later operation dropped that activity — while checkOutputUnused counted readers the inserted producer now feeds", "file": "mdl/executor/cmd_alter_flow.go (applyTo, fragmentBuilder, buildFragmentVars, noteFragment, checkOutputUnused, checkDeclaredUnclashed); mdl/executor/validate_flow_verdicts.go (alterFlowViolations)", "fix": "Each applied fragment's declared variables (with the types its builder resolved) seed later fragments' builders, in check's pre-build as in exec; the clash with a stored producer is judged once every operation is applied and skips one the block takes away (or whose loop it replaces); a stored or fragment reader of a removed output is no 'user' when another operation's fragment declares the variable upstream of it (declaredOnPath)", "insight": "2f42df658 moved checkOutputUnused to the end of the block but left its two siblings per-operation; any check that answers 'does this variable exist' has to be judged against the flow the block writes, or create or modify's own diff (which turns a reorder into insert+drop) trips it. A useful corollary: a create-or-modify body that builds declares each variable once, so a 'declares $X, which the microflow already has' refusal from the splice is always false — the 1361 test pinned one as expected and now asserts the splice. Known order rule kept: a declaration by a LATER operation is not known to an earlier fragment (check reports it unpredicted, exec refuses it)", "refs": ["ako/mxcli#1399", "mendixlabs/mxcli#1517", "mendixlabs/mxcli#1361"], "test": "TestAlterMicroflow_FragmentBuiltWithAnEarlierOperationsDeclaration, TestCreateOrModify_ProducerMovedEarlierKeepsItsReaders, TestAlterMicroflow_ProducerMovedEarlier (mdl/executor/cmd_alter_flow_block_vars_1399_pedapp_test.go); TestCheckFlowVerdicts_Issue1361_VariableClash"} diff --git a/.claude/skills/mendix/write-microflows/reference/pitfalls.md b/.claude/skills/mendix/write-microflows/reference/pitfalls.md index 184834960..9c34b1408 100644 --- a/.claude/skills/mendix/write-microflows/reference/pitfalls.md +++ b/.claude/skills/mendix/write-microflows/reference/pitfalls.md @@ -362,10 +362,16 @@ the scope checks judge the flow the **whole block** writes: - a fragment may read a variable an earlier operation of the block declares, when that declaration lands upstream of it (an `insert after`/`replace` of a node before it, or - an `insert before` the node it replaces or inserts after); + an `insert before` the node it replaces or inserts after) — and may `set` it or name + its members, since the fragment is built knowing what the earlier operations declare + (ako/mxcli#1399). Declare first, use after: a declaration made by a *later* + operation of the block is not known to an earlier one's fragment; - an activity whose output is taken away is still "used" only by readers the block - leaves in place — so a rename (the producer, every reader and the `return`) goes in - **one** block: + leaves in place, and not by a reader that another operation's fragment declares the + variable for upstream of it — so a producer can be moved up the flow (insert it + where it goes, drop it where it was; `create or modify` does this for a reordered + body), and a rename (the producer, every reader and the `return`) goes in **one** + block: ```mdl mdl 1; diff --git a/mdl-examples/bug-tests/splice-1399-alter-block-variables.mdl b/mdl-examples/bug-tests/splice-1399-alter-block-variables.mdl new file mode 100644 index 000000000..b4a5f220a --- /dev/null +++ b/mdl-examples/bug-tests/splice-1399-alter-block-variables.mdl @@ -0,0 +1,82 @@ +-- ============================================================================ +-- ako/mxcli#1399 (mendixlabs/mxcli#1517) — variables of one alter block, +-- and a producer moved upstream of where it was +-- ============================================================================ +-- +-- The operations of one `alter microflow` block — and the patch a +-- `create or modify` splices with, which is one — write ONE flow, but two of +-- their variable verdicts were still taken against the flow as stored: +-- +-- Repro B a fragment was built knowing only the stored flow's variables, +-- so `set $Late = …` in a replaced loop failed at exec with +-- "variable 'Late' is not declared" although the insert before it +-- in the same block declares $Late upstream — while check passed +-- with a "not predicted" note. +-- +-- Used return a producer moved earlier (`$Code = call …` above the +-- declare it followed) was refused "the fragment declares $Code, +-- which the microflow already has" by the stored $Code the same +-- patch drops, and in the other order "$Code is still used by: +-- log …; return $Code" by readers the moved producer now feeds. +-- +-- To reproduce: exec this file against any project. Expected: +-- Altered microflow BugTest1399.LoopFlow +-- Modified microflow: BugTest1399.MoveFlow (spliced: 1 inserted, 1 dropped) +-- Before the fix exec stopped at the alter ("the fragment has errors: variable +-- 'Late' is not declared") and rolled the whole script back; without the +-- alter it stopped at the second MoveFlow statement. mx check: 0 errors. +-- The file can be run again: LoopFlow is dropped and made afresh, and +-- MoveFlow moves its producer back and forth. + +mdl 1; + +create module BugTest1399; + +create or modify persistent entity BugTest1399.Item ( + Name: String(100) +); + +create or modify microflow BugTest1399.Helper () returns String +begin + return 'x'; +end; + +-- Repro B: the replaced loop sets the variable the insert declares. +drop microflow if exists BugTest1399.LoopFlow; + +create microflow BugTest1399.LoopFlow ($Items: List of BugTest1399.Item) +begin + declare $X Integer = 0; + loop $It in $Items + begin + log info node 'BugTest1399' $It/Name; + end loop; + log info node 'BugTest1399' 'done'; +end; + +alter microflow BugTest1399.LoopFlow { + insert after declare $X Integer = 0 begin declare $Late Integer = 0; end; + replace loop $It in $Items with begin + loop $It in $Items + begin + set $Late = $Late + 1; + end loop; + end; +}; + +-- Used return: the producer of the returned $Code moves above the declare. +create or modify microflow BugTest1399.MoveFlow () returns String +begin + declare $N Integer = 1; + $Code = call microflow BugTest1399.Helper(); + log info node 'BugTest1399' $Code; + return $Code; +end; + +create or modify microflow BugTest1399.MoveFlow () returns String +begin + $Code = call microflow BugTest1399.Helper(); + declare $N Integer = 1; + log info node 'BugTest1399' $Code; + return $Code; +end; diff --git a/mdl/executor/cmd_alter_flow.go b/mdl/executor/cmd_alter_flow.go index 19cebfeb9..da0986766 100644 --- a/mdl/executor/cmd_alter_flow.go +++ b/mdl/executor/cmd_alter_flow.go @@ -163,7 +163,7 @@ func (a *alterFlowContext) applyTo(ctx *ExecContext, mut backend.MicroflowMutato a.noteRewritten(target, "return "+value) continue } - frag, err := a.buildFragment(ctx, op.Body) + frag, vars, err := a.buildFragmentVars(ctx, op.Body) if err != nil { return fail(err) } @@ -186,6 +186,18 @@ func (a *alterFlowContext) applyTo(ctx *ExecContext, mut backend.MicroflowMutato if err := a.checkFragmentScope(ctx, op, target, frag); err != nil { return fail(err) } + // A variable the fragment declares that a stored activity declares + // too is a clash — unless the block takes that activity away, as a + // producer moved upstream is inserted there and dropped here + // (ako/mxcli#1399). Whether it does is known once every operation + // is applied, as for the outputs. + declaring := frag + outputChecks = append(outputChecks, func() error { + if err := a.checkDeclaredUnclashed(declaring); err != nil { + return fail(err) + } + return nil + }) switch op.Op { case ast.AlterFlowInsertAfter: err = mut.InsertAfter(target.ID, frag) @@ -209,7 +221,7 @@ func (a *alterFlowContext) applyTo(ctx *ExecContext, mut backend.MicroflowMutato if op.Op == ast.AlterFlowReplace { a.noteRemoved(target, frag) } - a.noteFragment(ctx, op, target, frag) + a.noteFragment(ctx, op, target, frag, vars) } for _, check := range outputChecks { if err := check(); err != nil { @@ -236,8 +248,15 @@ type alterFlowContext struct { declaredByOps map[string]bool // declaredAt says where each of those fragments went, so a later // operation counts a variable as declared only where it is on its path. - declaredAt map[string][]declSite - readByOps map[string][]string + declaredAt map[string][]declSite + // opVars is what the variables declared by those fragments hold, as each + // fragment's builder resolved them, so a later fragment is built knowing + // them (ako/mxcli#1399): `set $Late = …` after an insert that declares + // $Late, a member of an entity an inserted retrieve returns. + opVars fragmentVars + readByOps map[string][]opRead + // removedByOps are the stored outputs the statement's operations took + // away without declaring them again. removedByOps map[string]bool // removedIDs are the stored activities the statement's operations take // out or rewrite (a return given a new value, a decision a new @@ -346,7 +365,7 @@ func (a *alterFlowContext) noteRemoved(target mfmutator.Candidate, replacement * func (a *alterFlowContext) noteRewritten(target mfmutator.Candidate, text string) { a.removedIDs[target.ID] = true for _, m := range variableRef.FindAllStringSubmatch(withoutStringLiterals(text), -1) { - a.readByOps[m[1]] = append(a.readByOps[m[1]], text) + a.readByOps[m[1]] = append(a.readByOps[m[1]], opRead{text: text, site: declSite{op: ast.AlterFlowReplace, target: target.ID}}) } } @@ -357,9 +376,26 @@ type declSite struct { target model.ID } +// opRead is a read of a variable by what an operation of the statement puts +// in: the activity's text, where the operation puts it, and whether the +// fragment declares the variable itself. +type opRead struct { + text string + site declSite + own bool +} + +// fragmentVars are the variables a fragment declares, as its builder keeps +// them: entity-typed ones with their entity ("M.E" / "List of M.E"), the rest +// with their type name. +type fragmentVars struct { + varTypes, declared map[string]string +} + // noteFragment records what an inserted or replacing fragment declares and -// reads, and where it goes. -func (a *alterFlowContext) noteFragment(ctx *ExecContext, op *ast.AlterFlowOperation, target mfmutator.Candidate, frag *backend.MicroflowFragment) { +// reads, and where it goes. vars is what its builder resolved the variables +// it declares to hold. +func (a *alterFlowContext) noteFragment(ctx *ExecContext, op *ast.AlterFlowOperation, target mfmutator.Candidate, frag *backend.MicroflowFragment, vars fragmentVars) { for _, obj := range frag.Objects { if act, ok := obj.(*microflows.ActionActivity); ok { if v := mfmutator.OutputVariable(act.Action); v != "" { @@ -368,11 +404,46 @@ func (a *alterFlowContext) noteFragment(ctx *ExecContext, op *ast.AlterFlowOpera } } } + a.opVars = a.opVars.with(vars) + site := declSite{op: op.Op, target: target.ID} a.fragmentReads(ctx, frag.Objects, func(v, text string) { - a.readByOps[v] = append(a.readByOps[v], text) + a.readByOps[v] = append(a.readByOps[v], opRead{text: text, site: site, own: fragmentDeclares(frag, v)}) }) } +// with returns fv with the variables of more added, a variable already there +// keeping what it holds. +func (fv fragmentVars) with(more fragmentVars) fragmentVars { + add := func(into, from map[string]string) map[string]string { + for name, t := range from { + if into == nil { + into = map[string]string{} + } + if _, ok := into[name]; !ok { + into[name] = t + } + } + return into + } + fv.varTypes = add(fv.varTypes, more.varTypes) + fv.declared = add(fv.declared, more.declared) + return fv +} + +// takenAway reports whether the statement's operations take the stored object +// id out or rewrite it, itself or with a loop that holds it. +func (a *alterFlowContext) takenAway(id model.ID) bool { + if a.removedIDs[id] { + return true + } + for _, l := range a.loopsAround(id) { + if a.removedIDs[l.ID] { + return true + } + } + return false +} + // fragmentReads calls read with each variable the objects read, and the // activity text that reads it — inside a loop's body too, which the text of // the loop itself (its header) does not show (mendixlabs/mxcli#1517: a @@ -431,7 +502,7 @@ func loadAlterFlow(ctx *ExecContext, s *ast.AlterFlowStmt) (*alterFlowContext, e return nil, mdlerrors.NewBackend("build hierarchy", err) } a := &alterFlowContext{stmt: s, entityNames: getEntityNames(ctx, h), - declaredByOps: map[string]bool{}, declaredAt: map[string][]declSite{}, readByOps: map[string][]string{}, removedByOps: map[string]bool{}, + declaredByOps: map[string]bool{}, declaredAt: map[string][]declSite{}, readByOps: map[string][]opRead{}, removedByOps: map[string]bool{}, removedIDs: map[model.ID]bool{}} // A copy: nanoflow names are added below, and the cached map is shared. a.microflowNames = map[model.ID]string{} @@ -562,9 +633,21 @@ func (a *alterFlowContext) splitCondition(ctx *ExecContext, op *ast.AlterFlowOpe } // fragmentBuilder is the builder `create microflow` uses, seeded with the -// variables the stored flow declares. +// variables the stored flow declares and those the statement's earlier +// operations declare (ako/mxcli#1399). Seeding a variable only lets the +// fragment build; whether it is in scope where the fragment goes is +// checkFragmentScope's to say. func (a *alterFlowContext) fragmentBuilder(ctx *ExecContext) *flowBuilder { varTypes, declared := a.storedVariables(ctx) + for name, t := range a.opVars.varTypes { + varTypes[name] = t + delete(declared, name) + } + for name, t := range a.opVars.declared { + if _, ok := varTypes[name]; !ok { + declared[name] = t + } + } for name, t := range a.declaredVarTypes { varTypes[name] = t delete(declared, name) @@ -596,24 +679,55 @@ func (a *alterFlowContext) fragmentBuilder(ctx *ExecContext) *flowBuilder { // microflow` uses, seeded with the variables the stored flow declares, and // cuts it out of the start and end events the builder wraps it in. func (a *alterFlowContext) buildFragment(ctx *ExecContext, body []ast.MicroflowStatement) (*backend.MicroflowFragment, error) { + frag, _, err := a.buildFragmentVars(ctx, body) + return frag, err +} + +// buildFragmentVars is buildFragment that also returns the variables the +// fragment declares — the outputs of its own activities, not of a loop's body — +// as its builder resolved them. +func (a *alterFlowContext) buildFragmentVars(ctx *ExecContext, body []ast.MicroflowStatement) (*backend.MicroflowFragment, fragmentVars, error) { if len(body) == 0 { - return nil, fmt.Errorf("the fragment is empty; use drop to remove an activity") + return nil, fragmentVars{}, fmt.Errorf("the fragment is empty; use drop to remove an activity") } fb := a.fragmentBuilder(ctx) oc := fb.buildFlowGraph(body, nil) if errs := fb.GetErrors(); len(errs) > 0 { - return nil, fmt.Errorf("the fragment has errors:\n - %s", strings.Join(errs, "\n - ")) + return nil, fragmentVars{}, fmt.Errorf("the fragment has errors:\n - %s", strings.Join(errs, "\n - ")) } if fb.endsWithReturn { - return nil, fmt.Errorf("every path through the fragment ends the flow with a return, so nothing would lead on " + + return nil, fragmentVars{}, fmt.Errorf("every path through the fragment ends the flow with a return, so nothing would lead on " + "to the rest of it; a return can be inserted only on a path that branches off the rest (a guard clause)") } frag, err := cutFragment(oc, fb.fallThroughEndID, fb.returnEndIDs) if err != nil { - return nil, err + return nil, fragmentVars{}, err } frag.Pinned = fb.statedNotes - return frag, nil + return frag, declaredVarsOf(fb, frag.Objects), nil +} + +// declaredVarsOf returns the variables objs declare — the outputs of their +// own activities, not of a loop's body — as fb, which built them, resolved +// them. +func declaredVarsOf(fb *flowBuilder, objs []microflows.MicroflowObject) fragmentVars { + vars := fragmentVars{varTypes: map[string]string{}, declared: map[string]string{}} + for _, obj := range objs { + act, ok := obj.(*microflows.ActionActivity) + if !ok { + continue + } + v := mfmutator.OutputVariable(act.Action) + if v == "" { + continue + } + if t, ok := fb.varTypes[v]; ok { + vars.varTypes[v] = t + } else if t, ok := fb.declaredVars[v]; ok { + vars.declared[v] = t + } + } + return vars } // cutFragment removes the builder's start event and the end event the body @@ -809,20 +923,19 @@ func withoutStringLiterals(text string) string { // checkFragmentScope is plan item 4.2d: the fragment is checked in the scope // of its insertion point. A variable it declares that the flow already has is -// an error (it would shadow or clash with the stored one); a variable it uses -// that is not declared upstream of where it goes, nor by the fragment itself, -// is an error too, since the fragment would read something that does not exist -// yet on that path. +// an error (it would shadow or clash with the stored one; for a stored +// activity's output, see checkDeclaredUnclashed); a variable it uses that is +// not declared upstream of where it goes, nor by the fragment itself, is an +// error too, since the fragment would read something that does not exist yet +// on that path. func (a *alterFlowContext) checkFragmentScope(ctx *ExecContext, op *ast.AlterFlowOperation, target mfmutator.Candidate, frag *backend.MicroflowFragment) error { + // A stored activity declaring the same variable is a clash only if the + // statement keeps it, which is known once every operation is applied + // (checkDeclaredUnclashed); a parameter is never taken away. existing := map[string]bool{} for _, p := range a.mf.Parameters { existing[p.Name] = true } - for _, c := range a.cands { - if c.OutputVariable != "" { - existing[c.OutputVariable] = true - } - } own := map[string]bool{} for _, obj := range frag.Objects { // A loop's iterator exists only inside the loop, which the fragment @@ -992,18 +1105,29 @@ func (a *alterFlowContext) upstreamOf(id model.ID, including bool) map[model.ID] // It runs once every operation of the statement is applied, so the readers it // counts are those of the flow the statement writes: a stored activity that an // operation takes out or rewrites is none, a fragment that reads it is one. +// +// Nor is a reader that another operation's fragment declares the variable +// for, upstream of it (ako/mxcli#1399): a producer moved upstream is +// inserted there and dropped here, and its readers read the inserted one. func (a *alterFlowContext) checkOutputUnused(target mfmutator.Candidate, replacement *backend.MicroflowFragment) error { v := target.OutputVariable if v == "" || fragmentDeclares(replacement, v) { return nil } - if readers := a.readByOps[v]; len(readers) > 0 { + var readers []string + for _, r := range a.readByOps[v] { + if r.own || a.declaredOnPath(v, r.site.target, r.site.op) { + continue + } + readers = append(readers, r.text) + } + if len(readers) > 0 { return fmt.Errorf("$%s is read by what an operation of this alter adds: %s", v, strings.Join(readers, "; ")) } ref := regexp.MustCompile(`\$` + regexp.QuoteMeta(v) + `\b`) var users []string for _, c := range a.cands { - if c.ID == target.ID || a.removedIDs[c.ID] { + if c.ID == target.ID || a.takenAway(c.ID) || a.declaredOnPath(v, c.ID, ast.AlterFlowReplace) { continue } for _, text := range append([]string{c.Statement}, c.Alternates...) { @@ -1018,3 +1142,27 @@ func (a *alterFlowContext) checkOutputUnused(target mfmutator.Candidate, replace } return nil } + +// checkDeclaredUnclashed refuses a fragment that declares a variable a stored +// activity also declares, unless the statement takes that activity away. It +// runs once every operation is applied, like checkOutputUnused: a producer +// moved upstream is inserted by one operation and dropped by another, in +// either order (ako/mxcli#1399). +func (a *alterFlowContext) checkDeclaredUnclashed(frag *backend.MicroflowFragment) error { + for _, obj := range frag.Objects { + act, ok := obj.(*microflows.ActionActivity) + if !ok { + continue + } + v := mfmutator.OutputVariable(act.Action) + if v == "" { + continue + } + for _, c := range a.cands { + if c.OutputVariable == v && !a.takenAway(c.ID) { + return fmt.Errorf("the fragment declares $%s, which the %s already has; choose another name", v, a.stmt.Kind()) + } + } + } + return nil +} diff --git a/mdl/executor/cmd_alter_flow_block_vars_1399_pedapp_test.go b/mdl/executor/cmd_alter_flow_block_vars_1399_pedapp_test.go new file mode 100644 index 000000000..9779233a7 --- /dev/null +++ b/mdl/executor/cmd_alter_flow_block_vars_1399_pedapp_test.go @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/model" +) + +// ako/mxcli#1399 (mendixlabs/mxcli#1517): the operations of one alter block — +// and the patch `create or modify` splices with, which is one — write one +// flow, but two of their variable verdicts were still taken against the flow +// as stored. +// +// - Repro B: a fragment was BUILT knowing only the stored flow's variables, +// so `set $Late = …` in a replaced loop failed "variable 'Late' is not +// declared" although an earlier operation of the block declares $Late +// upstream of it — at exec only: check gave up on the fragment ("not +// predicted") and passed. +// - Used return: a producer moved earlier — inserted upstream, the stored +// one dropped — was refused "the fragment declares $Code, which the +// microflow already has" by the stored producer the same block drops, +// and its readers were "still used by" readers the inserted producer +// now feeds. + +const scopeMoveFlow = `mdl 1; +create microflow MyFirstModule.ScopeMove () returns String +begin + declare $N Integer = 1; + $Code = call microflow MyFirstModule.ScopeHelper(); + log info node 'T' $Code; + return $Code; +end;` + +func openBlockVarsFixture(t *testing.T) *Executor { + t.Helper() + exec := openScopeFixture(t) + if err := agreeExec(t, exec, scopeMoveFlow); err != nil { + t.Fatalf("setup: %v", err) + } + return exec +} + +// predictedAgree is scopeAgree that also requires check to have predicted the +// statement rather than given up on it: a "not predicted" note let check pass +// a statement exec then refused. +func predictedAgree(t *testing.T, exec *Executor, src, want string) { + t.Helper() + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parse: %v\n%s", errs[0], src) + } + if r := exec.CheckFlowVerdictsReport(prog); len(r.Unpredicted) > 0 { + t.Errorf("check must predict the statement, it reported it unpredicted: %+v", r.Unpredicted) + } + scopeAgree(t, exec, src, want) +} + +// Repro B: a declaration an earlier operation of the block makes is known to +// the builder of a later operation's fragment, not only to the scope check. +func TestAlterMicroflow_FragmentBuiltWithAnEarlierOperationsDeclaration(t *testing.T) { + t.Run("a replaced loop sets a variable an earlier insert declares", func(t *testing.T) { + exec := openScopeFixture(t) + predictedAgree(t, exec, `mdl 1; +alter microflow MyFirstModule.ScopeLoop { + insert after log node 'T' 'start' begin declare $Late Integer = 0; end; + replace loop $It in $Items with begin + loop $It in $Items + begin + set $Late = $Late + 1; + commit $It; + end loop; + end; +};`, "") + got := describeFlow(t, exec, "ScopeLoop") + if d, s := strings.Index(got, "declare $Late Integer = 0;"), strings.Index(got, "set $Late = $Late + 1;"); d < 0 || s < d { + t.Errorf("the declaration and the loop that sets it were not written in order:\n%s", got) + } + }) + t.Run("an entity variable an earlier insert retrieves is changed by a later fragment", func(t *testing.T) { + exec := openScopeFixture(t) + predictedAgree(t, exec, `mdl 1; +alter microflow MyFirstModule.ScopeLoop { + insert after log node 'T' 'start' begin retrieve $Me from System.User first; end; + replace log node 'T' 'done' with begin change $Me (Name = 'x'); end; +};`, "") + if got := describeFlow(t, exec, "ScopeLoop"); !strings.Contains(got, "change $Me (Name = 'x')") { + t.Errorf("the change was not written:\n%s", got) + } + }) + // Control: known to the builder is not in scope. A declaration the block + // makes downstream of the loop is still refused, by check and exec alike. + t.Run("control: declared after the loop", func(t *testing.T) { + exec := openScopeFixture(t) + predictedAgree(t, exec, `mdl 1; +alter microflow MyFirstModule.ScopeLoop { + insert after log node 'T' 'done' begin declare $Late Integer = 0; end; + replace loop $It in $Items with begin + loop $It in $Items + begin + set $Late = $Late + 1; + commit $It; + end loop; + end; +};`, "not declared on the path") + }) +} + +// Used return: a producer moved upstream of where it was, its readers left +// where they are. +func TestCreateOrModify_ProducerMovedEarlierKeepsItsReaders(t *testing.T) { + exec := openBlockVarsFixture(t) + before := flowObjectIDs(loopEditFlowOf(t, exec, "ScopeMove").ObjectCollection) + predictedAgree(t, exec, `mdl 1; +create or modify microflow MyFirstModule.ScopeMove () returns String +begin + $Code = call microflow MyFirstModule.ScopeHelper(); + declare $N Integer = 1; + log info node 'T' $Code; + return $Code; +end;`, "") + got := describeFlow(t, exec, "ScopeMove") + if c, n := strings.Index(got, "$Code = call microflow"), strings.Index(got, "declare $N"); c < 0 || n < c { + t.Fatalf("the producer was not moved ahead of the declare:\n%s", got) + } + // The move is a new producer and the stored one gone; the declare, the + // log and the end event keep their $IDs. + after := flowObjectIDs(loopEditFlowOf(t, exec, "ScopeMove").ObjectCollection) + var gone, added []model.ID + for id := range before { + if _, ok := after[id]; !ok { + gone = append(gone, id) + } + } + for id := range after { + if _, ok := before[id]; !ok { + added = append(added, id) + } + } + if len(gone) != 1 || len(added) != 1 { + t.Errorf("want the producer alone re-created (1 gone, 1 new), got %d gone, %d new", len(gone), len(added)) + } + + // Control: moved past its reader, the producer is still refused. + exec = openBlockVarsFixture(t) + predictedAgree(t, exec, `mdl 1; +create or modify microflow MyFirstModule.ScopeMove () returns String +begin + declare $N Integer = 1; + log info node 'T' $Code; + $Code = call microflow MyFirstModule.ScopeHelper(); + return $Code; +end;`, "$Code") +} + +// The same move written as an alter, in either order, and with the moved +// producer's reader in the same fragment. +func TestAlterMicroflow_ProducerMovedEarlier(t *testing.T) { + for name, src := range map[string]string{ + "insert, then drop": `mdl 1; +alter microflow MyFirstModule.ScopeMove { + insert before declare $N Integer = 1 begin $Code = call microflow MyFirstModule.ScopeHelper(); end; + drop $Code = call microflow MyFirstModule.ScopeHelper(); +};`, + "drop, then insert": `mdl 1; +alter microflow MyFirstModule.ScopeMove { + drop $Code = call microflow MyFirstModule.ScopeHelper(); + insert before declare $N Integer = 1 begin $Code = call microflow MyFirstModule.ScopeHelper(); end; +};`, + "the producer and its reader moved together": `mdl 1; +alter microflow MyFirstModule.ScopeMove { + insert before declare $N Integer = 1 begin + $Code = call microflow MyFirstModule.ScopeHelper(); + log info node 'T' $Code; + end; + drop $Code = call microflow MyFirstModule.ScopeHelper(); + drop log node 'T' *; +};`, + } { + t.Run(name, func(t *testing.T) { + exec := openBlockVarsFixture(t) + predictedAgree(t, exec, src, "") + got := describeFlow(t, exec, "ScopeMove") + if c, n := strings.Index(got, "$Code = call microflow"), strings.Index(got, "declare $N"); c < 0 || n < c || strings.Count(got, "$Code = call microflow") != 1 { + t.Errorf("want the one producer ahead of the declare:\n%s", got) + } + }) + } + // Controls: the stored producer stays, so the inserted one clashes with + // it; and a reader the block puts ahead of the new producer reads nothing. + for name, c := range map[string]struct{ src, want string }{ + "the stored producer is kept": {`mdl 1; +alter microflow MyFirstModule.ScopeMove { + insert before declare $N Integer = 1 begin $Code = call microflow MyFirstModule.ScopeHelper(); end; +};`, "which the microflow already has"}, + "a reader ahead of the new producer": {`mdl 1; +alter microflow MyFirstModule.ScopeMove { + insert before declare $N Integer = 1 begin log info node 'T' $Code; end; + insert after declare $N Integer = 1 begin $Code = call microflow MyFirstModule.ScopeHelper(); end; + drop $Code = call microflow MyFirstModule.ScopeHelper(); +};`, "$Code"}, + } { + t.Run("control: "+name, func(t *testing.T) { + exec := openBlockVarsFixture(t) + predictedAgree(t, exec, c.src, c.want) + }) + } +} diff --git a/mdl/executor/validate_flow_verdicts.go b/mdl/executor/validate_flow_verdicts.go index 3feb8b36f..7e106d519 100644 --- a/mdl/executor/validate_flow_verdicts.go +++ b/mdl/executor/validate_flow_verdicts.go @@ -345,17 +345,23 @@ func alterFlowViolations(ctx *ExecContext, s *ast.AlterFlowStmt) (_ []linter.Vio } // A fragment that does not build against the stored project is not // predictable (see CheckFlowVerdicts). Only the builder's own errors say - // so; a fragment that builds and cannot be spliced is a refusal. + // so; a fragment that builds and cannot be spliced is a refusal. Each is + // built knowing what the operations before it declare, as exec builds it + // (ako/mxcli#1399). for _, op := range s.Operations { if len(op.Body) == 0 { continue } fb := a.fragmentBuilder(ctx) - fb.buildFlowGraph(op.Body, nil) + oc := fb.buildFlowGraph(op.Body, nil) if errs := fb.GetErrors(); len(errs) > 0 { return nil, buildDependency(strings.Join(errs, "\n")), nil } + if oc != nil { + a.opVars = a.opVars.with(declaredVarsOf(fb, oc.Objects)) + } } + a.opVars = fragmentVars{} // the plan below starts from the stored flow, as exec's does mut, err := a.plan(ctx) if err == nil { return nil, "", func() bool { return mut.Save() == nil } diff --git a/mdl/executor/validate_flow_verdicts_1361_pedapp_test.go b/mdl/executor/validate_flow_verdicts_1361_pedapp_test.go index d54de9fb2..8dc567062 100644 --- a/mdl/executor/validate_flow_verdicts_1361_pedapp_test.go +++ b/mdl/executor/validate_flow_verdicts_1361_pedapp_test.go @@ -65,13 +65,44 @@ begin log info node 'X' 'done'; end;` +// The first shape is no longer a refusal (ako/mxcli#1399): the patch replaces +// $Equal with the new $Different and drops the stored one, and the clash was +// judged against the stored $Different that the same patch drops. A body that +// builds declares each variable once, so `create or modify` cannot clash with +// a stored activity it keeps. Both variants are now predicted as spliced, and +// exec writes the declared flow; the start event, the log and the end event +// keep their $IDs. func TestCheckFlowVerdicts_Issue1361_VariableClash(t *testing.T) { - exec, out, _ := openPedAppCopy(t) - if err := agreeExec(t, exec, issue1361VarStored); err != nil { - t.Fatalf("setup: %v\n%s", err, out.String()) + for _, tc := range issue1361Variants(issue1361VarModify, "Issue1361VarSub") { + t.Run(tc.name, func(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + if err := agreeExec(t, exec, issue1361VarStored); err != nil { + t.Fatalf("setup: %v\n%s", err, out.String()) + } + before := flowObjectIDs(loopEditFlowOf(t, exec, "Issue1361Var").ObjectCollection) + r := verdictReport(t, exec, tc.src) + if got := verdictRefusals(r.Violations); got != "" || len(r.Unpredicted) != 0 { + t.Fatalf("check must predict the splice, reported refusals %q, unpredicted %+v", got, r.Unpredicted) + } + if err := agreeExec(t, exec, tc.src); err != nil { + t.Fatalf("exec must splice the change: %v", err) + } + got := describeFlow(t, exec, "Issue1361Var") + if strings.Contains(got, "$Equal") || strings.Count(got, "declare $Different") != 1 || + !strings.Contains(got, "declare $Different Boolean = true;") { + t.Errorf("the declared flow was not written:\n%s", got) + } + kept := 0 + for id := range flowObjectIDs(loopEditFlowOf(t, exec, "Issue1361Var").ObjectCollection) { + if _, ok := before[id]; ok { + kept++ + } + } + if kept < 3 { + t.Errorf("want the start event, the log and the end event to keep their $IDs, %d did", kept) + } + }) } - issue1361Expect(t, exec, issue1361VarModify, "Issue1361VarSub", - "replace $Equal: the fragment declares $Different, which the microflow already has") } const issue1361LoopStored = `mdl 1; From 1b343fa05d5f834d6dd0cb686725fd751a754537 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 10:48:22 +0000 Subject: [PATCH 08/13] fix: moving an OWNER Both association's TO end leaves stale member access, and CE0001 associations are written (ako/mxcli#819) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-ups from #802 (PR #809); two were still open on main. 1. `create or modify association` moving a cross-module OWNER Both association's TO end to another module (or entity) printed "Modified association" and mx check reported CE0066 "Entity access is out of date" at the old TO module and the new one. Changing only the owner through create or modify missed the TO module the same way. Cause: the cross-module OR MODIFY branch of execCreateAssociation (mdl/executor/cmd_associations.go) overwrote ChildRef/owner and reconciled nothing; the end-of-program reconcile covers the declaring module only. And ReconcileMemberAccesses kept the old TO entity's entry even when its module was reconciled, because a reference to another module's existing association was preserved on faith (#758's rule for inheritance across modules). Fix: the branch reads the old TO module before overwriting and reconciles declaring, old TO and new TO modules (reconcileModulesAccess). The reconcile treats an existing foreign association as stale on an entity whose generalization chain never leaves its module — its members are then fully known (own + incoming OWNER Both). Inheritance across modules stays preserved (control test). 2. Between a persistent and a non-persistent entity, Mendix accepts only FROM the non-persistent end with owner Default; `owner both`, a persistent FROM end, and `alter association … set owner both` all passed check and exec and failed CE0001. Now refused as MDL-ASSOC01 in `check` (ValidateProgram, script declarations first, then the project) and in `exec` (create, create or modify, alter set owner both — also under --no-check), through one predicate (associationPersistenceViolation) and one stored-ends lookup (storedAssociationEnds). 3. GRANT accepting the TO-side member of a cross-module OWNER Both association already works on main (74bc820b7): `grant read (Number, Lead_Order) on entity Shop.Order …` writes the member, mx check 0 errors. No change. Verified on a blank 11.13.0 app with security on, mx check before (baseline binary) / after: TO end moved to another module 2 -> 0 (CE0066 at both TO modules), within the module 1 -> 0, owner Default->Both via create or modify 1 -> 0; `update security` repairs the move broken by the baseline binary. CE0001 matrix: NP->P both, P->NP default, P->NP both, cross-module NP->P both, ReferenceSet NP->P both and alter set owner both were each 1 CE0001 before and are refused now with the project untouched (0 errors); NP->P default is still written (0 errors). No skill or mdl-examples script trips the new check. Tests revert-proven (fix reverted, predicate stubbed): every new test fails, the controls pass. Co-Authored-By: Claude Opus 5.5 --- ...ation-owner-both-move-and-persistence.json | 1 + .../generate-domain-model/reference/syntax.md | 6 + cmd/mxcli/syntax/features_domain_model.go | 2 +- .../domain-model/create-association.md | 4 + .../access-rule-reconciliation.md | 20 +- ...cess-819-association-owner-both-member.mdl | 81 ++++++ .../modelsdk/domainmodel_security_write.go | 44 ++-- .../reconcile_foreign_stale_assoc_test.go | 44 +++- .../association_persistence_819_test.go | 234 ++++++++++++++++++ mdl/executor/cmd_associations.go | 31 ++- .../validate_association_persistence.go | 232 +++++++++++++++++ mdl/executor/validate_program.go | 5 + 12 files changed, 683 insertions(+), 21 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-25-association-owner-both-move-and-persistence.json create mode 100644 mdl-examples/bug-tests/access-819-association-owner-both-member.mdl create mode 100644 mdl/executor/association_persistence_819_test.go create mode 100644 mdl/executor/validate_association_persistence.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-association-owner-both-move-and-persistence.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-association-owner-both-move-and-persistence.json new file mode 100644 index 000000000..869873ea3 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-association-owner-both-move-and-persistence.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-25", "symptom": "ako/mxcli#819 (follow-ups of #802), measured on a blank 11.13.0 app with security on. (1) `create or modify association Crm.Lead_Order from Crm.Lead to Hr.Employee owner both` — moving a cross-module OWNER Both association's TO end from Shop.Order — printed \"Modified association\" and mx check reported CE0066 \"Entity access is out of date\" at the domain models of both Shop (old TO) and Hr (new TO); moving it to another entity of the same module, or only changing the owner through create or modify, left CE0066 at the TO module. `update security` repaired Hr but not Shop. (2) `create association … from to owner both` passed check and exec and failed CE0001 \"An association between a persistable entity and a non-persistable entity must start in the non-persistable entity and have owner 'Default'.\"; so did a persistent FROM end under owner Default, and `alter association … set owner both`. (3) GRANT naming the TO-side member of a cross-module OWNER Both association (`read (Number, Lead_Order)`) already worked on main — fixed by 74bc820b7, verified 0 errors.", "cause": "(1) The cross-module OR MODIFY branch of execCreateAssociation overwrote ChildRef and owner and reconciled nothing itself; the end-of-program reconcile covered only the declaring module. And ReconcileMemberAccesses kept the old TO entity's entry even when its module was reconciled: a reference to another module's association was preserved on faith (the #758 rule for associations inherited across modules), and this one still exists, just not as a member. (2) No check or exec guard modelled CE0001.", "file": "mdl/executor/cmd_associations.go (OR MODIFY cross branch reconciles declaring, old TO and new TO modules via reconcileModulesAccess; refusals on create and alter set owner both); mdl/executor/validate_association_persistence.go (associationPersistenceViolation, projectEntityPersistence, storedAssociationEnds, ValidateAssociationPersistence — wired into ValidateProgram); mdl/backend/modelsdk/domainmodel_security_write.go (staleForeignRef: an existing foreign association on an entity whose generalization chain never leaves its module is stale unless collected as an incoming OWNER Both)", "insight": "The issue named one half of each defect. The move left a stale entry on the OLD TO module and a missing one on the NEW TO module — read the old ChildRef before overwriting it — and the CE0001 rule is two-sided: mxbuild's message (and a 4-cell measurement: NP->P default ok, NP->P both, P->NP default, P->NP both all CE0001) says FROM must be the non-persistable end AND owner Default, so refusing only `owner both` would have left the persistent-FROM case writing a broken model. Measure the whole matrix before scoping a refusal to the reported cell. For the reconcile, 'cannot be checked' was only true when the entity inherits from another module: with a module-local generalization chain the full member set is known (own + incoming OWNER Both), so anything else foreign is stale; the cross-module-inheritance control stays green. Persistability for the refusal comes from the backend (specializations resolved from their root, mendixlabs/mxcli#1600) in exec and from the script's own declarations first in check, through one predicate. A single bug-test script under-reports the move on the old binary: grants track Hr, so the end-of-program reconcile repairs the new TO module and only the old one shows.", "refs": ["ako/mxcli#819", "ako/mxcli#802", "ako/mxcli#1449"], "ce": ["CE0066", "CE0001"], "rules": ["MDL-ASSOC01"], "test": "mdl/executor/association_persistence_819_test.go (TestAssociationPersistenceViolation_MatchesMxbuild, TestValidateProgram_RefusesOwnerBothAcrossPersistenceFromTheScript, TestExecAssociation_RefusesOwnerBothAcrossPersistence, TestValidateAssociationPersistence_ResolvesTheProject, TestCreateOrModifyAssociation_MovingTheToEndReconcilesBothToModules); mdl/backend/modelsdk/reconcile_foreign_stale_assoc_test.go (TestReconcile_DropsAForeignAssociationWhoseToEndMoved, control TestReconcile_KeepsAForeignAssociationInheritedAcrossModules); mdl-examples/bug-tests/access-819-association-owner-both-member.mdl"} diff --git a/.claude/skills/mendix/generate-domain-model/reference/syntax.md b/.claude/skills/mendix/generate-domain-model/reference/syntax.md index 0d29d1440..4568638a6 100644 --- a/.claude/skills/mendix/generate-domain-model/reference/syntax.md +++ b/.claude/skills/mendix/generate-domain-model/reference/syntax.md @@ -535,6 +535,12 @@ both is refused as **MDL085**. - `Parent` - Only parent (TO) entity owns - `Child` - Only child (FROM) entity owns +> **Persistent ↔ non-persistent: FROM the non-persistent entity, owner `default`.** +> That is the only shape Mendix accepts between a persistable and a +> non-persistable entity; a persistent FROM end or `owner both` is **CE0001**, and +> `mxcli check` refuses it as **MDL-ASSOC01** (`alter association … set owner +> both` too). + > **Use `default` ownership for a normal to-one reference.** Reserve `owner both` > for a `ReferenceSet` (many-to-many). On a plain `type reference`, `owner both` > makes the association navigable **to-one from *both* sides** — so the reverse diff --git a/cmd/mxcli/syntax/features_domain_model.go b/cmd/mxcli/syntax/features_domain_model.go index 1dcc9795d..fa7216f02 100644 --- a/cmd/mxcli/syntax/features_domain_model.go +++ b/cmd/mxcli/syntax/features_domain_model.go @@ -298,7 +298,7 @@ func init() { "many-to-one", "many-to-many", "foreign key", "owner", "delete behavior", }, - Syntax: "[@anchor(from: (x, y), to: (x, y))]\nCREATE [OR MODIFY] ASSOCIATION Module.Name\n FROM Module.FromEntity TO Module.ToEntity\n [TYPE Reference|ReferenceSet]\n [OWNER Default|Both]\n [STORAGE COLUMN|TABLE]\n [ON DELETE CASCADE|RESTRICT|SET NULL [ERROR MESSAGE 'text']];\n-- Documentation: the /** … */ doc comment before the statement\n-- (COMMENT 'text' is its deprecated alias, MDL-DEPR100).\nALTER ASSOCIATION Module.Name SET ANCHOR FROM (x, y) TO (x, y);\nDROP ASSOCIATION Module.Name;\n\nOR MODIFY: updates type/owner/delete behavior in-place, preserves UUID.\n\n@anchor sets the LINE ANCHORS — where the connector attaches to each entity box\nin the domain model editor — as a PERCENTAGE of the box (0..100, whole numbers).\n`from` is the FROM entity's box, `to` the TO entity's: (0, 50) is the middle of\nthe left edge, (100, 50) the right, (50, 100) the bottom centre. Omitting an end\nPRESERVES what is stored, so a CREATE OR MODIFY about something else never\nflattens a hand-tuned line. Cross-module associations have no anchors.\n\nDROP reconciles the entity access rules that named the association, and is\nREFUSED while a message definition still exposes it (that one cannot be\nreconciled -- removing the member would change a published contract). The\nrefusal prints the `alter message definition ... drop member` statement for\neach definition, ready to run.", + Syntax: "[@anchor(from: (x, y), to: (x, y))]\nCREATE [OR MODIFY] ASSOCIATION Module.Name\n FROM Module.FromEntity TO Module.ToEntity\n [TYPE Reference|ReferenceSet]\n [OWNER Default|Both]\n [STORAGE COLUMN|TABLE]\n [ON DELETE CASCADE|RESTRICT|SET NULL [ERROR MESSAGE 'text']];\n-- Documentation: the /** … */ doc comment before the statement\n-- (COMMENT 'text' is its deprecated alias, MDL-DEPR100).\nALTER ASSOCIATION Module.Name SET ANCHOR FROM (x, y) TO (x, y);\nDROP ASSOCIATION Module.Name;\n\nOR MODIFY: updates type/owner/delete behavior in-place, preserves UUID.\n\n@anchor sets the LINE ANCHORS — where the connector attaches to each entity box\nin the domain model editor — as a PERCENTAGE of the box (0..100, whole numbers).\n`from` is the FROM entity's box, `to` the TO entity's: (0, 50) is the middle of\nthe left edge, (100, 50) the right, (50, 100) the bottom centre. Omitting an end\nPRESERVES what is stored, so a CREATE OR MODIFY about something else never\nflattens a hand-tuned line. Cross-module associations have no anchors.\n\nPersistent <-> non-persistent: FROM the non-persistent entity, OWNER Default.\nA persistent FROM end or OWNER Both there is CE0001 and is refused\n(MDL-ASSOC01), on CREATE and on ALTER ... SET OWNER Both.\n\nDROP reconciles the entity access rules that named the association, and is\nREFUSED while a message definition still exposes it (that one cannot be\nreconciled -- removing the member would change a published contract). The\nrefusal prints the `alter message definition ... drop member` statement for\neach definition, ready to run.", Example: "mdl 1;\n-- Many-to-one\nCREATE ASSOCIATION Shop.Order_Customer\n FROM Shop.Order TO Shop.Customer\n TYPE Reference\n OWNER Default\n ON DELETE SET NULL;\n\n-- Many-to-many\nCREATE ASSOCIATION Shop.Product_Tag\n FROM Shop.Product TO Shop.Tag\n TYPE ReferenceSet\n OWNER Both;\n\n-- Line leaving the bottom of Order and entering the top of Customer\n@anchor(from: (50, 100), to: (50, 0))\nCREATE ASSOCIATION Shop.Order_Customer\n FROM Shop.Order TO Shop.Customer;\n\n-- Retune the line without restating the association\nALTER ASSOCIATION Shop.Order_Customer SET ANCHOR FROM (0, 54) TO (100, 54);", SeeAlso: []string{"domain-model.association.create", "domain-model.association.anchor", "domain-model.association.delete-behavior"}, }) diff --git a/docs-site/src/reference/domain-model/create-association.md b/docs-site/src/reference/domain-model/create-association.md index 73f203f85..610bd9a25 100644 --- a/docs-site/src/reference/domain-model/create-association.md +++ b/docs-site/src/reference/domain-model/create-association.md @@ -63,6 +63,10 @@ A documentation comment (`/** ... */`) placed before the statement is preserved **OWNER** : Which side can modify the association. Defaults to `Default` if omitted. +: Between a persistent and a non-persistent entity, Mendix allows one shape only: `FROM` the non-persistent entity, with owner `Default`. A persistent `FROM` end or `OWNER Both` there fails the build with CE0001, so `mxcli check` and `exec` refuse it (MDL-ASSOC01). `ALTER ASSOCIATION … SET OWNER Both` is refused the same way. + +: Under `OWNER Both` the association is a member of the TO entity too, so its access rules get an entry for it — in the TO entity's module, for a cross-module association. `CREATE OR MODIFY` that moves the TO end, or changes the owner, reconciles the old and the new TO module's rules. + **ON DELETE** : What happens to associated objects when a FROM-side object is deleted. If omitted, references are kept (the default Mendix behavior). diff --git a/docs-wiki/bug-patterns/access-rule-reconciliation.md b/docs-wiki/bug-patterns/access-rule-reconciliation.md index 6840916f7..dad9db3a1 100644 --- a/docs-wiki/bug-patterns/access-rule-reconciliation.md +++ b/docs-wiki/bug-patterns/access-rule-reconciliation.md @@ -1,7 +1,7 @@ --- title: Access Rules Are Reconciled, Not Appended category: bug-pattern -last-synced: ced830e0 +last-synced: 1ccc9383e covers: - mdl/backend sources: @@ -50,6 +50,24 @@ local walk cannot see. Dropping what the walk cannot confirm produced it no longer lists. The safe default is to carry an unconfirmable member through rather than to prune it. +**"Cannot be checked" is a claim, and it widens if nobody re-tests it.** The +preserve rule was written for one case — an association inherited from a +generalization in another module — and was implemented as "keep every +reference qualified by another module". That also kept the entry for a +cross-module `OWNER Both` association after its FROM entity was dropped, after +the association was dropped, and after its TO end moved: CE1613 or CE0066, with +`update security` reporting nothing to do. Whether the declaring module still +declares the association was answerable all along, from the same scan the +reconcile already made for incoming associations. **Before preserving, ask +which part is genuinely unknowable, and preserve only that** — and keep a +control for the case the rule was written for. + +**The rule to fix is often not in the module written.** A member entry lives on +the FROM entity and, under `OWNER Both`, on the TO entity — another module for a +cross-module association. A statement that removes or moves an association has +to name every module holding one of those rules, and must collect them *before* +the write, while the associations still exist to be read. + **Under-reporting access is the read-side twin.** A restricted page reported as having "no roles" is the same class seen from the query side, and it is the shape most likely to be believed, because "no roles" reads like a finding rather than a diff --git a/mdl-examples/bug-tests/access-819-association-owner-both-member.mdl b/mdl-examples/bug-tests/access-819-association-owner-both-member.mdl new file mode 100644 index 000000000..425b10a6e --- /dev/null +++ b/mdl-examples/bug-tests/access-819-association-owner-both-member.mdl @@ -0,0 +1,81 @@ +mdl 1; +-- ============================================================================ +-- ako/mxcli#819 — association member access under OWNER Both. +-- +-- 1. `create or modify association` moving the TO end of a cross-module OWNER +-- Both association to an entity of another module reconciled only the +-- declaring module. The old TO entity kept its member entry and the new one +-- never got one: +-- before: [CE0066] "Entity access is out of date…" at Domain model of +-- module 'Access819Shop'; `describe entity Access819Shop.Order` +-- still shows `read (Number, Lead_Order)` for the Viewer. Run as +-- its own script after the setup, also CE0066 at 'Access819Hr' +-- (here the end-of-program reconcile covers Hr, which the grants +-- touched). +-- after: 0 errors; "Reconciled … in module Access819Shop after modifying +-- the association" (and Access819Hr), and the Viewer's rule is +-- `read (Number)`. +-- The old TO entity's entry used to survive even `update security`: the +-- reconcile kept any reference to another module's association on faith. +-- +-- 2. Between a persistent and a non-persistent entity Mendix allows only FROM +-- the non-persistent one with owner Default; anything else is CE0001 +-- "An association between a persistable entity and a non-persistable entity +-- must start in the non-persistable entity and have owner 'Default'." +-- `mxcli check` and `exec` now refuse it (MDL-ASSOC01). Kept out of this +-- script so it checks clean — each of these is refused: +-- +-- create association Access819Shop.Cart_Order +-- from Access819Shop.Cart to Access819Shop.Order owner both; +-- create association Access819Shop.Order_Cart +-- from Access819Shop.Order to Access819Shop.Cart; +-- alter association Access819Shop.Cart_Customer set owner both; +-- +-- 3. GRANT already accepts the TO-side member of a cross-module OWNER Both +-- association by its bare name — `read (Number, Lead_Order)` below. +-- +-- Access rules are only checked with security on. +-- ============================================================================ + +alter app security ( SecurityLevel: prototype ); + +create module Access819Crm; +create module role Access819Crm.User; +create module Access819Shop; +create module role Access819Shop.User; +create module role Access819Shop.Viewer; +create module Access819Hr; +create module role Access819Hr.User; + +@position(100, 100) +create persistent entity Access819Crm.Lead ( Name: string(200) ); +@position(100, 100) +create persistent entity Access819Shop.Order ( Number: string(20) ); +@position(400, 100) +create persistent entity Access819Shop.Customer ( Name: string(200) ); +@position(100, 300) +create non-persistent entity Access819Shop.Cart ( Label: string(50) ); +@position(100, 100) +create persistent entity Access819Hr.Employee ( Name: string(200) ); + +create association Access819Crm.Lead_Order + from Access819Crm.Lead to Access819Shop.Order + owner both; +-- The shape Mendix allows across the persistence boundary. +create association Access819Shop.Cart_Customer + from Access819Shop.Cart to Access819Shop.Customer; + +grant create, delete, read *, write * on entity Access819Crm.Lead to Access819Crm.User; +grant create, delete, read *, write * on entity Access819Shop.Order to Access819Shop.User; +grant read (Number, Lead_Order) on entity Access819Shop.Order to Access819Shop.Viewer; +grant create, delete, read *, write * on entity Access819Shop.Customer to Access819Shop.User; +grant create, delete, read *, write * on entity Access819Shop.Cart to Access819Shop.User; +grant create, delete, read *, write * on entity Access819Hr.Employee to Access819Hr.User; + +-- Move the TO end from Access819Shop.Order to Access819Hr.Employee. +create or modify association Access819Crm.Lead_Order + from Access819Crm.Lead to Access819Hr.Employee + owner both; + +describe entity Access819Shop.Order; +describe entity Access819Hr.Employee; diff --git a/mdl/backend/modelsdk/domainmodel_security_write.go b/mdl/backend/modelsdk/domainmodel_security_write.go index 2e61471b1..2a4cbf96a 100644 --- a/mdl/backend/modelsdk/domainmodel_security_write.go +++ b/mdl/backend/modelsdk/domainmodel_security_write.go @@ -662,12 +662,14 @@ func (b *Backend) ReconcileMemberAccesses(unitID model.ID, moduleName string) (i rule.RemoveMemberAccesses(i) changed = true case !assocRefBelongsTo(assocRef, moduleName) && - elsewhere.staleForeignRef(assocRef): - // A foreign reference its module no longer declares (CE1613). - // Kept before ako/mxcli#1449, so `drop entity` on the FROM end - // of a cross-module OWNER Both association, and `drop - // association` on one, left the entry on the TO end for good — - // and `update security` reported nothing to do. + elsewhere.staleForeignRef(assocRef, foreignAnc): + // A foreign reference that is positively stale: its module no + // longer declares it (CE1613), or this entity cannot be one of + // its members (CE0066). See staleForeignRef. Kept before + // ako/mxcli#1449 / #819, so `drop entity` on the FROM end of a + // cross-module OWNER Both association, `drop association` on + // one, and moving its TO end all left the entry for good — and + // `update security` reported nothing to do. rule.RemoveMemberAccesses(i) changed = true case !assocRefBelongsTo(assocRef, moduleName): @@ -854,21 +856,28 @@ func (b *Backend) scanProjectAssociations(moduleName string) (*projectAssociatio // staleForeignRef reports whether a MemberAccess naming an association declared // in ANOTHER module is positively stale on an entity of the module being -// reconciled. +// reconciled. foreignAnc is foreignAncestorModule's answer for that entity: "" +// when its whole generalization chain lies in its own module. // // The reconcile used to keep every such reference, because the one it was // written for — an association inherited from a generalization in another module -// — cannot be validated without walking that chain. One case can be decided -// without it: the declaring module exists and no longer declares the -// association. Then it is a member of nothing, and mx check reports CE1613 "The -// selected association … no longer exists". That is what `drop entity` on the -// FROM end of a cross-module OWNER Both association, or `drop association` on -// one, leaves on the TO end (ako/mxcli#1449). +// — cannot be validated without walking that chain. Two cases can be decided +// without it, and both were reported (ako/mxcli#1449, ako/mxcli#819): +// +// - The declaring module exists and no longer declares the association. It is a +// member of nothing; mx check reports CE1613 "The selected association … no +// longer exists". This is what `drop entity` on the FROM end of a cross-module +// OWNER Both association, or `drop association` on one, leaves on the TO end. +// - The association still exists, but the entity's generalization chain never +// leaves its module. Then the only way a foreign association is a member of it +// is as the TO end under OWNER Both, which `incoming` enumerates — so one the +// reconcile did not collect is not a member: CE0066 "Entity access is out of +// date". This is what moving the TO end of such an association leaves behind. // // A reference into a module the project does not store — System, whose domain // model is virtual, or one that is simply absent — is still preserved, and so is -// any association that still exists. -func (p *projectAssociations) staleForeignRef(ref string) bool { +// an existing association on an entity that inherits from another module. +func (p *projectAssociations) staleForeignRef(ref, foreignAnc string) bool { idx := strings.LastIndex(ref, ".") if idx < 0 { return false @@ -877,7 +886,10 @@ func (p *projectAssociations) staleForeignRef(ref string) bool { if !known { return false } - return !names[strings.ToLower(ref[idx+1:])] + if !names[strings.ToLower(ref[idx+1:])] { + return true + } + return foreignAnc == "" } // foreignAncestorModule returns the module of the first generalization in ent's diff --git a/mdl/backend/modelsdk/reconcile_foreign_stale_assoc_test.go b/mdl/backend/modelsdk/reconcile_foreign_stale_assoc_test.go index f0c972e9e..907b1ccb5 100644 --- a/mdl/backend/modelsdk/reconcile_foreign_stale_assoc_test.go +++ b/mdl/backend/modelsdk/reconcile_foreign_stale_assoc_test.go @@ -14,8 +14,14 @@ // left the same entry, and `update security` printed "All entity access rules // are up to date" over both. Measured on mxbuild 11.13.0. // -// That case is checkable after all: an association its declaring module no -// longer declares is a member of nothing. +// ako/mxcli#819: `create or modify association` moving the TO end to another +// entity left the old TO entity's entry — mx check CE0066 "Entity access is out +// of date". +// +// Both are checkable after all. An association its declaring module no longer +// declares is a member of nothing; and an entity whose whole generalization +// chain lies in its own module can only have a foreign association as the TO +// end of an `OWNER Both` one, which the reconcile already enumerates. package modelsdkbackend import ( @@ -146,6 +152,40 @@ func entityNamesRef(t *testing.T, b *Backend, mod *model.Module, entity, ref str return false } +// The #819 shape: `create or modify association` points the TO end of an +// OWNER Both cross-module association at another entity. The old TO entity has +// no generalization, so every member it can have is known here, and the foreign +// entry it kept is stale (CE0066). +func TestReconcile_DropsAForeignAssociationWhoseToEndMoved(t *testing.T) { + const ref = "MyFirstModule.ZzNote_Target" + b, from, to := targetFixture(t) + reconcileModule(t, b, to) + if !entityNamesRef(t, b, to, "ZzTarget", ref) { + t.Fatalf("precondition: OWNER Both did not give the TO entity the entry") + } + + dm, err := b.GetDomainModel(from.ID) + if err != nil { + t.Fatalf("GetDomainModel: %v", err) + } + for _, ca := range dm.CrossAssociations { + if ca.Name == "ZzNote_Target" { + ca.ChildRef = "Administration.Account" + } + } + if err := b.UpdateDomainModel(dm); err != nil { + t.Fatalf("UpdateDomainModel: %v", err) + } + + reconcileModule(t, b, to) + if entityNamesRef(t, b, to, "ZzTarget", ref) { + t.Errorf("the old TO entity still names %s after the TO end moved away — CE0066", ref) + } + if !entityNamesRef(t, b, to, "Account", ref) { + t.Errorf("the new TO entity did not get %s", ref) + } +} + // Control: an association declared in another module that this entity INHERITS // — from a generalization in that module — still exists and is a member, and // the reconcile cannot see that from here. Judging foreign references must not diff --git a/mdl/executor/association_persistence_819_test.go b/mdl/executor/association_persistence_819_test.go new file mode 100644 index 000000000..86b5c7e5e --- /dev/null +++ b/mdl/executor/association_persistence_819_test.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "sort" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// ako/mxcli#819, second item: `owner both` between a persistent and a +// non-persistent entity passed check and exec, and mxbuild then failed with +// CE0001 "An association between a persistable entity and a non-persistable +// entity must start in the non-persistable entity and have owner 'Default'." +// The table is what mxbuild 11.13.0 says, one association per blank app. +func TestAssociationPersistenceViolation_MatchesMxbuild(t *testing.T) { + known := map[string]bool{"M.P": true, "M.P2": true, "M.N": false, "M.N2": false} + persistable := func(qn ast.QualifiedName) (bool, bool) { + p, ok := known[qn.String()] + return p, ok + } + q := func(s string) ast.QualifiedName { + i := strings.Index(s, ".") + return ast.QualifiedName{Module: s[:i], Name: s[i+1:]} + } + for _, tc := range []struct { + from, to string + ownerBoth bool + ce0001 bool + }{ + {"M.N", "M.P", false, false}, // the one shape Mendix allows + {"M.N", "M.P", true, true}, // the reported case + {"M.P", "M.N", false, true}, + {"M.P", "M.N", true, true}, + // Controls: no persistence boundary, so owner Both is fine. + {"M.P", "M.P2", true, false}, + {"M.N", "M.N2", true, false}, + // An end nothing resolves is not judged. + {"M.N", "M.Unknown", true, false}, + } { + v := associationPersistenceViolation(q("M.A"), q(tc.from), q(tc.to), tc.ownerBoth, persistable) + if (v != nil) != tc.ce0001 { + t.Errorf("from %s to %s, owner both=%v: violation=%v, want %v", tc.from, tc.to, tc.ownerBoth, v != nil, tc.ce0001) + } + if v != nil && (v.RuleID != associationPersistenceRule || !strings.Contains(v.Message, "CE0001")) { + t.Errorf("violation does not name the rule and CE0001: %+v", v) + } + } +} + +// `check` without a project: the script's own declarations decide, including a +// specialization, which is as persistable as its generalization. +func TestValidateProgram_RefusesOwnerBothAcrossPersistenceFromTheScript(t *testing.T) { + for _, tc := range []struct { + name string + src string + want int + }{ + {"owner both", `create non-persistent entity M.Cart (Label: string(50)); +create persistent entity M.Customer (Name: string(50)); +create association M.Cart_Customer from M.Cart to M.Customer owner both;`, 1}, + {"persistent FROM", `create non-persistent entity M.Cart (Label: string(50)); +create persistent entity M.Customer (Name: string(50)); +create association M.Customer_Cart from M.Customer to M.Cart;`, 1}, + {"specialization of a non-persistent entity", `create non-persistent entity M.Cart (Label: string(50)); +create entity M.BigCart extends M.Cart (Size: integer); +create persistent entity M.Customer (Name: string(50)); +create association M.BigCart_Customer from M.BigCart to M.Customer owner both;`, 1}, + {"alter to owner both", `create non-persistent entity M.Cart (Label: string(50)); +create persistent entity M.Customer (Name: string(50)); +create association M.Cart_Customer from M.Cart to M.Customer; +alter association M.Cart_Customer set owner both;`, 1}, + // Controls. + {"owner default", `create non-persistent entity M.Cart (Label: string(50)); +create persistent entity M.Customer (Name: string(50)); +create association M.Cart_Customer from M.Cart to M.Customer;`, 0}, + {"both persistent", `create persistent entity M.Order (Number: string(50)); +create persistent entity M.Customer (Name: string(50)); +create association M.Order_Customer from M.Order to M.Customer owner both;`, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + n := 0 + for _, v := range ValidateProgram(parseMDL(t, tc.src), "") { + if v.RuleID == associationPersistenceRule { + n++ + } + } + if n != tc.want { + t.Fatalf("%d %s violation(s), want %d", n, associationPersistenceRule, tc.want) + } + }) + } +} + +// persistenceFixture: module M with the non-persistent Cart and the persistent +// Customer, and the stored association M.Cart_Customer (Cart → Customer, owner +// Default) — behind a mock that records what was written. +func persistenceFixture(t *testing.T) (*ExecContext, *mock.MockBackend, *int) { + t.Helper() + mod := mkModule("M") + cart, customer := mkEntity(mod.ID, "Cart"), mkEntity(mod.ID, "Customer") + cart.Persistable = false + dm := mkDomainModel(mod.ID, cart, customer) + dm.Associations = []*domainmodel.Association{mkAssociation(mod.ID, "Cart_Customer", cart.ID, customer.ID)} + writes := 0 + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + GetDomainModelFunc: func(model.ID) (*domainmodel.DomainModel, error) { return dm, nil }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { return []*domainmodel.DomainModel{dm}, nil }, + CreateAssociationFunc: func(model.ID, *domainmodel.Association) error { + writes++ + return nil + }, + UpdateDomainModelFunc: func(*domainmodel.DomainModel) error { + writes++ + return nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(mkHierarchy(mod))) + return ctx, mb, &writes +} + +// `exec` refuses before writing — `check` is skipped under --no-check. +func TestExecAssociation_RefusesOwnerBothAcrossPersistence(t *testing.T) { + q := func(name string) ast.QualifiedName { return ast.QualifiedName{Module: "M", Name: name} } + + t.Run("create", func(t *testing.T) { + ctx, _, writes := persistenceFixture(t) + err := execCreateAssociation(ctx, &ast.CreateAssociationStmt{ + Name: q("Cart_Customer2"), Parent: q("Cart"), Child: q("Customer"), Owner: ast.OwnerBoth, + }) + if err == nil || !strings.Contains(err.Error(), "CE0001") { + t.Fatalf("err = %v, want the CE0001 refusal", err) + } + if *writes != 0 { + t.Errorf("%d write(s) before the refusal", *writes) + } + }) + t.Run("create or modify", func(t *testing.T) { + ctx, _, writes := persistenceFixture(t) + s := &ast.CreateAssociationStmt{ + Name: q("Cart_Customer"), Parent: q("Cart"), Child: q("Customer"), Owner: ast.OwnerBoth, + } + s.CreateOrModify = true + if err := execCreateAssociation(ctx, s); err == nil || !strings.Contains(err.Error(), "CE0001") { + t.Fatalf("err = %v, want the CE0001 refusal", err) + } + if *writes != 0 { + t.Errorf("%d write(s) before the refusal", *writes) + } + }) + t.Run("alter set owner both", func(t *testing.T) { + ctx, _, writes := persistenceFixture(t) + err := execAlterAssociation(ctx, &ast.AlterAssociationStmt{ + Name: q("Cart_Customer"), Operation: ast.AlterAssociationSetOwner, Owner: ast.OwnerBoth, + }) + if err == nil || !strings.Contains(err.Error(), "CE0001") { + t.Fatalf("err = %v, want the CE0001 refusal", err) + } + if *writes != 0 { + t.Errorf("%d write(s) before the refusal", *writes) + } + }) + // Control: the shape Mendix allows is written. + t.Run("owner default", func(t *testing.T) { + ctx, _, writes := persistenceFixture(t) + assertNoError(t, execCreateAssociation(ctx, &ast.CreateAssociationStmt{ + Name: q("Cart_Customer2"), Parent: q("Cart"), Child: q("Customer"), + })) + if *writes != 1 { + t.Errorf("%d write(s), want 1", *writes) + } + }) +} + +// The project half of `check`: an entity the script does not declare is judged +// by what the project stores. +func TestValidateAssociationPersistence_ResolvesTheProject(t *testing.T) { + _, mb, _ := persistenceFixture(t) + project := func() backend.FullBackend { return mb } + for _, tc := range []struct { + src string + want int + }{ + {"create association M.Cart_Customer2 from M.Cart to M.Customer owner both;", 1}, + {"alter association M.Cart_Customer set owner both;", 1}, + {"create association M.Cart_Customer2 from M.Cart to M.Customer;", 0}, + } { + if got := len(ValidateAssociationPersistence(parseMDL(t, tc.src), project)); got != tc.want { + t.Errorf("%q: %d violation(s), want %d", tc.src, got, tc.want) + } + } +} + +// ako/mxcli#819, first item: `create or modify association` moving a +// cross-module association's TO end from one module to another reconciled no +// module but the declaring one. The old TO entity kept its OWNER Both entry and +// the new one never got it — CE0066 at both modules. +func TestCreateOrModifyAssociation_MovingTheToEndReconcilesBothToModules(t *testing.T) { + for _, tc := range []struct { + name string + to ast.QualifiedName + want []string + }{ + {"to another module", ast.QualifiedName{Module: "Hr", Name: "Employee"}, []string{"Crm", "Hr", "Shop"}}, + {"to another entity of the same module", ast.QualifiedName{Module: "Shop", Name: "Customer"}, []string{"Crm", "Shop"}}, + // Owner Default → Both without a move: the TO module still needs it. + {"no move", ast.QualifiedName{Module: "Shop", Name: "Order"}, []string{"Crm", "Shop"}}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, reconciled, _ := dropEntityFixture(t) + s := &ast.CreateAssociationStmt{ + Name: ast.QualifiedName{Module: "Crm", Name: "Lead_Order"}, + Parent: ast.QualifiedName{Module: "Crm", Name: "Lead"}, + Child: tc.to, + Owner: ast.OwnerBoth, + } + s.CreateOrModify = true + assertNoError(t, execCreateAssociation(ctx, s)) + got := append([]string(nil), *reconciled...) + sort.Strings(got) + if strings.Join(got, ",") != strings.Join(tc.want, ",") { + t.Fatalf("reconciled modules %v, want %v", got, tc.want) + } + }) + } +} diff --git a/mdl/executor/cmd_associations.go b/mdl/executor/cmd_associations.go index 81ec49f32..4ce82a91c 100644 --- a/mdl/executor/cmd_associations.go +++ b/mdl/executor/cmd_associations.go @@ -75,6 +75,16 @@ func execCreateAssociation(ctx *ExecContext, s *ast.CreateAssociationStmt) error } } + // A persistable and a non-persistable entity may only be associated FROM the + // non-persistable one under owner Default — anything else is CE0001 + // (ako/mxcli#819). The same function judges it in `check`. + if v := associationPersistenceViolation(s.Name, + ast.QualifiedName{Module: parentModule, Name: s.Parent.Name}, + ast.QualifiedName{Module: childModule, Name: s.Child.Name}, + s.Owner == ast.OwnerBoth, projectEntityPersistence(ctx)); v != nil { + return mdlerrors.NewValidationf("%s\n %s", v.Message, v.Suggestion) + } + // Convert types assocType := domainmodel.AssociationTypeReference if s.Type == ast.AssocReferenceSet { @@ -145,6 +155,8 @@ func execCreateAssociation(ctx *ExecContext, s *ast.CreateAssociationStmt) error childRef := childModule + "." + s.Child.Name for _, ca := range dm.CrossAssociations { if ca.Name == s.Name.Name { + // Read before it is overwritten: the TO end may be moving. + oldToModule := qualifiedNameModule(ca.ChildRef) ca.Type = assocType ca.Owner = owner if s.Storage != ast.StorageDefault { // as above (#704) @@ -164,7 +176,14 @@ func execCreateAssociation(ctx *ExecContext, s *ast.CreateAssociationStmt) error invalidateDomainModelsCache(ctx) ctx.trackModifiedDomainModel(module.ID, module.Name) ctx.ReportMutation("Modified", "association: %s", s.Name) - return nil + // Under OWNER Both the TO entity holds a member entry, in its own + // module. Only the declaring module was reconciled (at the end of + // the program), so moving the TO end left the old TO entity's + // entry behind and never gave the new one its entry — CE0066 at + // both modules; changing the owner alone missed the TO module the + // same way (ako/mxcli#819). + return reconcileModulesAccess(ctx, "after modifying the association", + module.Name, oldToModule, childModule) } } } @@ -294,6 +313,16 @@ func execAlterAssociation(ctx *ExecContext, s *ast.AlterAssociationStmt) error { return mdlerrors.NewBackend("get domain model", err) } + // `set owner both` across the persistence boundary is CE0001, as on create + // (ako/mxcli#819). + if s.Operation == ast.AlterAssociationSetOwner && s.Owner == ast.OwnerBoth { + if from, to, ok := storedAssociationEnds(ctx, s.Name); ok { + if v := associationPersistenceViolation(s.Name, from, to, true, projectEntityPersistence(ctx)); v != nil { + return mdlerrors.NewValidationf("%s\n %s", v.Message, v.Suggestion) + } + } + } + // Try intra-module associations first for _, assoc := range dm.Associations { if assoc.Name == s.Name.Name { diff --git a/mdl/executor/validate_association_persistence.go b/mdl/executor/validate_association_persistence.go new file mode 100644 index 000000000..3c1a424dc --- /dev/null +++ b/mdl/executor/validate_association_persistence.go @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// Mendix lets a persistable and a non-persistable entity be associated in one +// shape only. mxbuild's own words, CE0001: +// +// An association between a persistable entity and a non-persistable entity +// must start in the non-persistable entity and have owner 'Default'. +// +// So two things are refused, and they are one rule: a FROM end that is the +// persistable one (a database row cannot point at an object that has no row), +// and `owner both` (which would make the persistable end own a reference too). +// Measured on mxbuild 11.13.0, a blank app with security on, one association +// each: non-persistable FROM / persistable TO is 0 errors under owner Default +// and CE0001 under owner Both (Reference and ReferenceSet, same-module and +// cross-module alike); persistable FROM / non-persistable TO is CE0001 under +// both owners. +// +// mxcli checked neither, so `create association … owner both` between a +// non-persistent and a persistent entity — the 1-1 shape people reach for, seen +// on PedApp's APD_Account — passed check and exec and failed the build +// (ako/mxcli#819). Refused in `check` and in `exec`, through one function each +// for the rule (associationPersistenceViolation) and for finding the ends of an +// association that is already stored (storedAssociationEnds), so the two cannot +// disagree. +const associationPersistenceRule = "MDL-ASSOC01" + +// associationPersistenceViolation applies the CE0001 rule to an association +// whose two ends are known. persistable answers for one end, and says when it +// cannot tell — an entity neither the script nor the project resolves, a view or +// an external entity — and then nothing is reported: the rule is judged only on +// what is known. +func associationPersistenceViolation(assoc, from, to ast.QualifiedName, ownerBoth bool, + persistable func(ast.QualifiedName) (bool, bool)) *linter.Violation { + fromP, okFrom := persistable(from) + toP, okTo := persistable(to) + if !okFrom || !okTo || fromP == toP { + return nil + } + var why, fix string + switch { + case fromP && ownerBoth: + why = fmt.Sprintf("it starts in the persistable entity %s and has owner Both", from) + fix = fmt.Sprintf("Reverse it with owner Default: `from %s to %s`.", to, from) + case fromP: + why = fmt.Sprintf("it starts in the persistable entity %s", from) + fix = fmt.Sprintf("Reverse it: `from %s to %s`.", to, from) + case ownerBoth: + why = fmt.Sprintf("it has owner Both (%s is persistable, %s is not)", to, from) + fix = "Drop `owner both` — owner Default is the only owner Mendix allows here." + default: + return nil // FROM the non-persistable end, owner Default: the allowed shape + } + return &linter.Violation{ + RuleID: associationPersistenceRule, + Severity: linter.SeverityError, + Message: fmt.Sprintf("association %s links a persistable and a non-persistable entity, and %s — "+ + "Mendix requires such an association to start in the non-persistable entity and have "+ + "owner Default (CE0001 at build time)", assoc, why), + Location: linter.Location{DocumentType: "association", DocumentName: assoc.String()}, + Suggestion: fix, + } +} + +// projectEntityPersistence answers associationPersistenceViolation's question +// from the project: the backend resolves a specialization's persistability from +// its root generalization, System entities included. +func projectEntityPersistence(ctx *ExecContext) func(ast.QualifiedName) (bool, bool) { + return func(qn ast.QualifiedName) (bool, bool) { + if ctx == nil || ctx.Backend == nil || qn.Module == "" { + return false, false + } + ent, err := findEntity(ctx, qn.Module, qn.Name) + if err != nil || ent == nil || !judgedForPersistence(ent) { + return false, false + } + return ent.Persistable, true + } +} + +// judgedForPersistence reports whether an entity's Persistable flag decides the +// CE0001 rule. A view entity cannot be associated at all (CE6771, refused on its +// own), and an external entity's storage is the remote service's — neither has +// been measured against this rule, so neither is judged by it. +func judgedForPersistence(ent *domainmodel.Entity) bool { + return !isViewEntity(ent) && !strings.HasPrefix(ent.Source, "Rest$OData") +} + +// storedAssociationEnds returns the FROM and TO entities of an association that +// is in the project, qualified, as `alter association … set owner both` needs +// them to judge the rule. +func storedAssociationEnds(ctx *ExecContext, name ast.QualifiedName) (from, to ast.QualifiedName, ok bool) { + if ctx == nil || ctx.Backend == nil || name.Module == "" { + return from, to, false + } + mod, err := findModule(ctx, name.Module) + if err != nil || mod == nil { + return from, to, false + } + dm, err := ctx.Backend.GetDomainModel(mod.ID) + if err != nil || dm == nil { + return from, to, false + } + local := func(id string) (ast.QualifiedName, bool) { + for _, e := range dm.Entities { + if string(e.ID) == id { + return ast.QualifiedName{Module: mod.Name, Name: e.Name}, true + } + } + return ast.QualifiedName{}, false + } + for _, a := range dm.Associations { + if a.Name == name.Name { + f, okF := local(string(a.ParentID)) + t, okT := local(string(a.ChildID)) + return f, t, okF && okT + } + } + for _, ca := range dm.CrossAssociations { + if ca.Name == name.Name { + f, okF := local(string(ca.ParentID)) + i := strings.LastIndex(ca.ChildRef, ".") + if i <= 0 { + return from, to, false + } + return f, ast.QualifiedName{Module: ca.ChildRef[:i], Name: ca.ChildRef[i+1:]}, okF + } + } + return from, to, false +} + +// ValidateAssociationPersistence is the `check` side: every CREATE [OR MODIFY] +// ASSOCIATION, and every ALTER ASSOCIATION … SET OWNER BOTH, against the CE0001 +// rule. An entity the script declares answers from its declaration (an +// `extends` from its generalization, as Mendix inherits it); anything else from +// the project, when there is one. +func ValidateAssociationPersistence(prog *ast.Program, project func() backend.FullBackend) []linter.Violation { + if prog == nil || !programAssociatesEntities(prog) { + return nil + } + var ctx *ExecContext + if project != nil { + if b := project(); b != nil { + ctx = &ExecContext{Backend: b} + } + } + fromProject := projectEntityPersistence(ctx) + decls := map[string]*ast.CreateEntityStmt{} + created := map[string][2]ast.QualifiedName{} + + var persistable func(qn ast.QualifiedName, depth int) (bool, bool) + persistable = func(qn ast.QualifiedName, depth int) (bool, bool) { + d, ok := decls[strings.ToLower(qn.String())] + if !ok { + return fromProject(qn) + } + switch { + case d.Kind == ast.EntityView || d.Kind == ast.EntityExternal: + return false, false + case d.Generalization != nil: + if depth > 16 { + return false, false // a cycle is not this rule's concern + } + return persistable(*d.Generalization, depth+1) + } + return d.Kind != ast.EntityNonPersistent, true + } + answer := func(qn ast.QualifiedName) (bool, bool) { return persistable(qn, 0) } + + var out []linter.Violation + for _, stmt := range prog.Statements { + switch s := stmt.(type) { + case *ast.CreateEntityStmt: + decls[strings.ToLower(s.Name.String())] = s + case *ast.CreateAssociationStmt: + from, to := s.Parent, s.Child + if from.Module == "" { + from.Module = s.Name.Module + } + if to.Module == "" { + to.Module = s.Name.Module + } + created[strings.ToLower(s.Name.String())] = [2]ast.QualifiedName{from, to} + if v := associationPersistenceViolation(s.Name, from, to, s.Owner == ast.OwnerBoth, answer); v != nil { + out = append(out, *v) + } + case *ast.AlterAssociationStmt: + if s.Operation != ast.AlterAssociationSetOwner || s.Owner != ast.OwnerBoth { + continue + } + ends, ok := created[strings.ToLower(s.Name.String())] + if !ok { + f, t, found := storedAssociationEnds(ctx, s.Name) + if !found { + continue + } + ends = [2]ast.QualifiedName{f, t} + } + if v := associationPersistenceViolation(s.Name, ends[0], ends[1], true, answer); v != nil { + out = append(out, *v) + } + } + } + return out +} + +// programAssociatesEntities is the cheap pre-test: without a statement the rule +// applies to, the project is never opened. +func programAssociatesEntities(prog *ast.Program) bool { + for _, stmt := range prog.Statements { + switch s := stmt.(type) { + case *ast.CreateAssociationStmt: + return true + case *ast.AlterAssociationStmt: + if s.Operation == ast.AlterAssociationSetOwner && s.Owner == ast.OwnerBoth { + return true + } + } + } + return false +} diff --git a/mdl/executor/validate_program.go b/mdl/executor/validate_program.go index 320917b0d..304dee02a 100644 --- a/mdl/executor/validate_program.go +++ b/mdl/executor/validate_program.go @@ -212,6 +212,11 @@ func ValidateProgram(prog *ast.Program, projectPath string) []linter.Violation { // answer first; the project, when there is one, for the rest. violations = append(violations, ValidateDatasourceXPathComparands(prog, project)...) + // An association between a persistable and a non-persistable entity must + // start in the non-persistable one and have owner Default — CE0001 + // (MDL-ASSOC01, ako/mxcli#819). The script's declarations answer first. + violations = append(violations, ValidateAssociationPersistence(prog, project)...) + // Warn (MDL-OFFLINE01) when a page binds an attribute across more than one // association in a project that has an offline navigation profile. Mendix // rejects those with CE6206 on any page an offline profile can reach, and From ab0b5e0a815d454c8f1be1b326e0c6d451e599a6 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 10:24:11 +0000 Subject: [PATCH 09/13] fix: create java action stores a bare enumeration type as an entity (ako/mxcli#935) `create java action M.JA (C: M.Color) returns Boolean`, with M.Color an enumeration, passed check and exec and was written as an entity parameter; mxbuild reported CE1613 "The selected entity 'M.Color' no longer exists". The same held for the return type, for JavaScript actions, for a System enumeration, and for a name that is neither an entity nor an enumeration. Cause: a bare `Module.Name` in a code-action signature parses as TypeEnumeration without ExplicitEnum (the TypeEnumeration/TypeEntity ambiguity), and astDataTypeToJavaActionParamType / ...ReturnType made every such name a CodeActions$EntityType. Their comment said callers resolve it "via resolveCodeActionParamType", a function that never existed. Fix: settleCodeActionSignature decides each bare name against the project before anything is written - an entity stays an entity, an enumeration (project, System, or created earlier in the script) becomes an explicit enumeration, and a name that is neither is refused. An explicit `enum` is held to enumerations. check --references reports the same verdict through codeActionSignatureErrors (declaredTypeResolver). The stale comment now points at the real function. Verification: tests in cmd_javaactions_bare_enum_935_test.go (Java and JavaScript, parameter and return, refusal, check side) fail with the fix reverted and pass with it. Blank 11.13.0 app with mdl-examples/bug-tests/java-action-935-bare-enumeration-type.mdl: mx check CE1613 x5 with the pre-fix binary, 0 errors fixed; describe -> exec reports "already in sync". Flow parameters already resolved the ambiguity (control microflow in the script). go vet and go test ./mdl/executor/ pass. Co-Authored-By: Claude Opus 5.5 --- ...re-enumeration-type-written-as-entity.json | 1 + .claude/skills/mendix/java-actions/SKILL.md | 2 +- cmd/mxcli/syntax/features_integration.go | 2 +- .../java-action-935-bare-enumeration-type.mdl | 66 ++++++ mdl/executor/cmd_javaactions.go | 25 ++- .../cmd_javaactions_bare_enum_935_test.go | 194 ++++++++++++++++++ mdl/executor/cmd_javascript_actions_write.go | 17 +- mdl/executor/code_action_signature_types.go | 142 +++++++++++++ mdl/executor/validate.go | 10 + 9 files changed, 444 insertions(+), 15 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-25-java-action-bare-enumeration-type-written-as-entity.json create mode 100644 mdl-examples/bug-tests/java-action-935-bare-enumeration-type.mdl create mode 100644 mdl/executor/cmd_javaactions_bare_enum_935_test.go create mode 100644 mdl/executor/code_action_signature_types.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-java-action-bare-enumeration-type-written-as-entity.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-java-action-bare-enumeration-type-written-as-entity.json new file mode 100644 index 000000000..f2a5eecac --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-java-action-bare-enumeration-type-written-as-entity.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-25", "symptom": "`create java action M.JA (C: M.Color) returns Boolean` with M.Color an enumeration passes `check --references` and `exec`; mxbuild then reports CE1613 \"The selected entity 'M.Color' no longer exists\" for the parameter (and for `returns M.Color`). Same for JavaScript actions and for a System enumeration (`System.DeviceType`). A bare name that is neither an entity nor an enumeration (`M.NoSuchThing`) also passed both and failed the same way. Only `C: enum M.Color` worked; describe of a correct action prints `Enum M.Color`, so a bare name in describe output is the symptom", "cause": "In a code-action signature a bare `Module.Name` parses as TypeEnumeration with ExplicitEnum=false (the TypeEnumeration/TypeEntity ambiguity), and `astDataTypeToJavaActionParamType` / `…ReturnType` turned every such name into a CodeActions$EntityType. A comment there said callers 'resolve it via resolveCodeActionParamType' — a function that never existed in the repo. Flows had the entity-then-enumeration fallback (buildMicroflowFromStmt, settleSystemEnumeration); Java and JavaScript actions never got it, and check had no signature pass for them at all", "file": "`mdl/executor/code_action_signature_types.go` (`settleCodeActionSignature`, called from `execCreateJavaAction` and `execCreateJavaScriptAction` before anything is written; `codeActionSignatureErrors`, wired into `validateWithContext` for CreateJavaActionStmt / CreateJavaScriptActionStmt); stale comment fixed in `mdl/executor/cmd_javaactions.go`; tests `mdl/executor/cmd_javaactions_bare_enum_935_test.go`; fixture `mdl-examples/bug-tests/java-action-935-bare-enumeration-type.mdl`", "insight": "**A comment naming a function is a claim — grep for the function.** The converter's comment made the bug look handled; `grep -rn resolveCodeActionParamType` returns only the comment. Settle the name BEFORE the converter (retype an enumeration as an explicit one) rather than teaching the converter to look things up: the converter has no context, and the existing explicit-enum path then does the write. Decide System names from `meta.SystemEnumerations` (systemEnumerations) — System entities are not in the project, so 'not an entity' is not evidence there. Keep check and exec on the same verdict: exec settles with the backend, check uses declaredTypeResolver (which also sees names the script creates), and neither resolves a `list of` element, because exec never refused one. Measured on a blank 11.13.0 app: the bug-test script gives CE1613 x5 with a pre-fix build (Java param, Java return, System enum param, JS param, JS return) and 0 errors fixed; describe -> exec reports 'already in sync'. The generated .java takes the enum as `String _c` and converts with `Color.valueOf`, i.e. the source generator already handled EnumerationType once the model said so", "refs": ["ako/mxcli#935", "ako/mxcli#680"], "ce": ["CE1613"], "test": "TestCreateJavaAction_BareEnumerationNameIsAnEnumeration, TestCreateJavaAction_BareNameThatIsNeitherIsRefused, TestCreateJavaScriptAction_BareEnumerationNameIsAnEnumeration, TestCreateJavaScriptAction_BareNameThatIsNeitherIsRefused, TestValidate_CodeActionSignatureTypes"} diff --git a/.claude/skills/mendix/java-actions/SKILL.md b/.claude/skills/mendix/java-actions/SKILL.md index c800a453b..44874a3d7 100644 --- a/.claude/skills/mendix/java-actions/SKILL.md +++ b/.claude/skills/mendix/java-actions/SKILL.md @@ -254,7 +254,7 @@ from describe output has changed an action's export level to Public. Check it ag | `decimal` | Decimal number | | `boolean` | True/false | | `datetime` | Date and time | -| `Module.Entity` | Entity reference | +| `Module.Entity` | Entity reference — or an enumeration, when `Module.Entity` names one (the project decides; a name that is neither is refused) | | `list of Module.Entity` | List of entities | | `stringtemplate(sql)` | SQL/OQL query template with parameters | | `stringtemplate(text)` | Text template with parameters | diff --git a/cmd/mxcli/syntax/features_integration.go b/cmd/mxcli/syntax/features_integration.go index 12577d7b5..d1b7cae13 100644 --- a/cmd/mxcli/syntax/features_integration.go +++ b/cmd/mxcli/syntax/features_integration.go @@ -657,7 +657,7 @@ DESCRIBE DATABASE CONNECTION Ops.Erp;`, "create java action", "or modify java action", "type parameter", "entity parameter", "exposed as", "returns", "generics", "drop java action", }, - Syntax: "CREATE [OR MODIFY] JAVA ACTION Module.Name[] [FOLDER 'path'](\n [/** description */] Param: Type [NOT NULL],\n EntityType: ENTITY NOT NULL,\n Obj: pEntity\n) RETURNS ReturnType [AS $ReturnName]\n[EXPOSED AS 'Label' IN 'Category'\n [ICON 'icon.png'] [ICON DARK 'icon-dark.png']\n [IMAGE 'image.png'] [IMAGE DARK 'image-dark.png']]\n[NOT EXPOSED]\nAS $$\n// Java code — AS $$ ... $$ is mandatory, cannot be omitted\n$$;\n\nOR MODIFY: updates signature/body in-place, preserves UUID.\n\nEXPOSED AS puts the action in Studio Pro's toolbox. The icon is a 64x64 PNG\nand the image a 256x192 PNG, read from disk relative to the .mdl file's own directory.\nAn OMITTED clause preserves what is stored — including bitmaps set in Studio\nPro — so removing an entry is NOT EXPOSED, and clearing one bitmap is\nDROP ICON / DROP IMAGE [DARK].\n\nA parameter's description is a /** ... */ doc comment before it, as on an\nentity attribute; AS $ReturnName names the return value (the default name of\nthe variable a microflow call stores it in), as on a microflow. Leaving either\nout keeps what is stored; a new action gets no description and ReturnValueName.\n\nType parameters: an ENTITY parameter declares T. A type parameter no\nENTITY parameter names is declared in the header, Module.Name; an\nunqualified type that is neither is refused (MDL-JA01), since it would be\nwritten as an entity with no module (CE1613).", + Syntax: "CREATE [OR MODIFY] JAVA ACTION Module.Name[] [FOLDER 'path'](\n [/** description */] Param: Type [NOT NULL],\n EntityType: ENTITY NOT NULL,\n Obj: pEntity\n) RETURNS ReturnType [AS $ReturnName]\n[EXPOSED AS 'Label' IN 'Category'\n [ICON 'icon.png'] [ICON DARK 'icon-dark.png']\n [IMAGE 'image.png'] [IMAGE DARK 'image-dark.png']]\n[NOT EXPOSED]\nAS $$\n// Java code — AS $$ ... $$ is mandatory, cannot be omitted\n$$;\n\nOR MODIFY: updates signature/body in-place, preserves UUID.\n\nEXPOSED AS puts the action in Studio Pro's toolbox. The icon is a 64x64 PNG\nand the image a 256x192 PNG, read from disk relative to the .mdl file's own directory.\nAn OMITTED clause preserves what is stored — including bitmaps set in Studio\nPro — so removing an entry is NOT EXPOSED, and clearing one bitmap is\nDROP ICON / DROP IMAGE [DARK].\n\nA parameter's description is a /** ... */ doc comment before it, as on an\nentity attribute; AS $ReturnName names the return value (the default name of\nthe variable a microflow call stores it in), as on a microflow. Leaving either\nout keeps what is stored; a new action gets no description and ReturnValueName.\n\nType parameters: an ENTITY parameter declares T. A type parameter no\nENTITY parameter names is declared in the header, Module.Name; an\nunqualified type that is neither is refused (MDL-JA01), since it would be\nwritten as an entity with no module (CE1613).\n\nA bare Module.Name type is an entity or an enumeration, whichever the project\n(or an earlier statement of the script) has; ENUM Module.Name says enumeration\nexplicitly. A name that is neither is refused by check --references and exec,\nsince it would be written as an entity that does not exist (CE1613).", Example: "mdl 1;\nCREATE JAVA ACTION Utils.FormatCurrency(\n /** The amount to format. */\n Amount: Decimal NOT NULL\n) RETURNS String AS $FormattedAmount\nEXPOSED AS 'Format Currency' IN 'Formatting'\n ICON 'assets/currency-64.png'\n IMAGE 'assets/currency-256.png'\nAS $$\nreturn String.format(\"%.2f\", Amount);\n$$;\n\n-- Generic entity validator with type parameter\nCREATE JAVA ACTION Utils.IsValid(\n EntityType: ENTITY NOT NULL,\n Obj: pEntity NOT NULL\n) RETURNS Boolean\nAS $$\nreturn Obj != null;\n$$;\n\n-- Type parameter with no entity selector: the argument's entity binds it\nCREATE JAVA ACTION Utils.Describe(\n Obj: T NOT NULL\n) RETURNS String\nAS $$\nreturn Obj.toString();\n$$;\n\n-- Idempotent update (preserves UUID)\nCREATE OR MODIFY JAVA ACTION Utils.FormatCurrency(\n Amount: Decimal NOT NULL,\n Decimals: Integer NOT NULL\n) RETURNS String\nAS $$\nreturn String.format(\"%.\" + Decimals + \"f\", Amount);\n$$;", SeeAlso: []string{"java-action", "javascript-action"}, }) diff --git a/mdl-examples/bug-tests/java-action-935-bare-enumeration-type.mdl b/mdl-examples/bug-tests/java-action-935-bare-enumeration-type.mdl new file mode 100644 index 000000000..07654d51c --- /dev/null +++ b/mdl-examples/bug-tests/java-action-935-bare-enumeration-type.mdl @@ -0,0 +1,66 @@ +-- ako/mxcli#935: `create java action` stored a bare enumeration parameter type +-- as an ENTITY. +-- +-- The parser cannot tell an entity from an enumeration for a bare +-- `Module.Name`, and the Java/JavaScript action converters called every such +-- name an entity. `check` and `exec` were clean; mxbuild then reported +-- +-- CE1613 "The selected entity 'BugTest935.Color' no longer exists." +-- +-- for the parameter and for the return type. Only the explicit `enum` keyword +-- worked. A name that was neither an entity nor an enumeration was written the +-- same way and failed the same way. +-- +-- Fixed: exec settles each bare name against the project — an entity stays an +-- entity, an enumeration (of the project, of the System module, or created +-- earlier in the script) becomes an enumeration type — and refuses a name that +-- is neither. `check --references` reports the same refusal. Describe prints +-- the explicit `Enum M.Color`, which re-executes to the same document. +-- +-- Verify: exec, `mx check` (0 errors on 11.13.0; a pre-fix build reports +-- CE1613 five times), then describe the actions and re-exec: "already in sync". +-- The misspelled form is refused by check and exec: +-- +-- create java action BugTest935.JA_Ghost (G: BugTest935.NoSuchThing) returns Boolean … +-- → parameter G of java action BugTest935.JA_Ghost: entity or enumeration +-- BugTest935.NoSuchThing does not exist + +mdl 1; + +create module BugTest935; + +create enumeration BugTest935.Color ( + Red 'Red', + Green 'Green' +); + +create persistent entity BugTest935.Canvas ( + Name: String(50) +); + +-- A bare enumeration, a bare entity and a System enumeration in one signature. +create java action BugTest935.JA_Paint ( + C: BugTest935.Color, + Target: BugTest935.Canvas, + Device: System.DeviceType +) returns Boolean +as $$ +return true; +$$; + +create java action BugTest935.JA_PickColor () returns BugTest935.Color +as $$ +return null; +$$; + +create javascript action BugTest935.JSA_PickColor (C: BugTest935.Color) returns BugTest935.Color +as $$ +return null; +$$; + +-- Flows have resolved the same ambiguity since the entity-then-enumeration +-- fallback; this is the control that they still do. +create microflow BugTest935.MF_Paint ($C: BugTest935.Color) returns BugTest935.Color +begin + return $C; +end; diff --git a/mdl/executor/cmd_javaactions.go b/mdl/executor/cmd_javaactions.go index 8938518b4..581d48c9b 100644 --- a/mdl/executor/cmd_javaactions.go +++ b/mdl/executor/cmd_javaactions.go @@ -484,6 +484,13 @@ func execCreateJavaAction(ctx *ExecContext, s *ast.CreateJavaActionStmt) error { return mdlerrors.NewNotFound("module", s.Name.Module) } + // Decide what each bare Module.Name in the signature is — an entity or an + // enumeration — before anything is written (ako/mxcli#935). + params, returnType, err := settleCodeActionSignature(ctx, "java action "+s.Name.String(), s.Parameters, s.ReturnType) + if err != nil { + return err + } + // Check if Java action already exists jas, err := ctx.Backend.ListJavaActions() if err != nil { @@ -598,7 +605,7 @@ func execCreateJavaAction(ctx *ExecContext, s *ast.CreateJavaActionStmt) error { // - TypeEntityTypeParam → EntityTypeParameterType (entity type selector) // - Bare name matching a type parameter → TypeParameter (ParameterizedEntityType) // - Other types → convert normally - for _, param := range s.Parameters { + for _, param := range params { jaParam := &javaactions.JavaActionParameter{ BaseElement: model.BaseElement{ ID: model.ID(types.GenerateID()), @@ -640,17 +647,17 @@ func execCreateJavaAction(ctx *ExecContext, s *ast.CreateJavaActionStmt) error { } // Convert return type — check if it references a type parameter - if isTypeParamRef(s.ReturnType, typeParamNames) { - tpName := getTypeParamRefName(s.ReturnType) + if isTypeParamRef(returnType, typeParamNames) { + tpName := getTypeParamRefName(returnType) ja.ReturnType = &javaactions.TypeParameter{ BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, TypeParameterID: typeParamNameToID[tpName], TypeParameter: tpName, } - } else if l := listOfTypeParameter(s.ReturnType, typeParamNameToID); l != nil { + } else if l := listOfTypeParameter(returnType, typeParamNameToID); l != nil { ja.ReturnType = l } else { - ja.ReturnType = astDataTypeToJavaActionReturnType(s.ReturnType) + ja.ReturnType = astDataTypeToJavaActionReturnType(returnType) } // Fold the EXPOSED AS clause onto the stored toolbox entry rather than @@ -746,8 +753,9 @@ func astDataTypeToJavaActionParamType(dt ast.DataType) javaactions.CodeActionPar // An unambiguous `ENUM`/`Enumeration(...)` type must serialize as an // enumeration parameter, not an entity reference (#680). A bare // qualified name (ExplicitEnum=false) can't be told apart from an entity - // by the parser, so it stays an entity type here — callers that have the - // project loaded resolve it via resolveCodeActionParamType. + // by the parser, so it stays an entity type here: the create handlers + // settle it against the project first (settleCodeActionSignature, + // ako/mxcli#935), which retypes an enumeration as an explicit one. if dt.Kind == ast.TypeEnumeration && dt.ExplicitEnum && dt.EnumRef != nil { return &javaactions.EnumerationType{ BaseElement: model.BaseElement{ @@ -873,7 +881,8 @@ func astDataTypeToJavaActionReturnType(dt ast.DataType) javaactions.CodeActionRe case ast.TypeEntity, ast.TypeEnumeration: // An unambiguous `ENUM`/`Enumeration(...)` return type serializes as an // enumeration, not an entity reference (#680). A bare qualified name - // can't be told apart from an entity, so it stays an entity type. + // can't be told apart from an entity, so it stays an entity type; the + // create handlers settle it first (settleCodeActionSignature). // "void" parses as a bare qualified name; treat it as VoidType. if dt.Kind == ast.TypeEnumeration && dt.ExplicitEnum && dt.EnumRef != nil { return &javaactions.EnumerationType{ diff --git a/mdl/executor/cmd_javaactions_bare_enum_935_test.go b/mdl/executor/cmd_javaactions_bare_enum_935_test.go new file mode 100644 index 000000000..97ecbef4d --- /dev/null +++ b/mdl/executor/cmd_javaactions_bare_enum_935_test.go @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" + "github.com/mendixlabs/mxcli/sdk/javaactions" +) + +// ako/mxcli#935: `create java action M.JA_Paint (C: M.Color) returns Boolean` +// with M.Color an enumeration passed check and exec and was written as an +// ENTITY parameter — mxbuild: CE1613 "The selected entity 'M.Color' no longer +// exists". The parser cannot tell an entity from an enumeration for a bare +// `Module.Name` (both arrive as TypeEnumeration without ExplicitEnum), and the +// converter called that an entity unconditionally. A comment promised a +// resolveCodeActionParamType that would settle it against the project; no such +// function existed. + +// codeActionProject is a mock project with one module holding an entity +// Customer and an enumeration Color. created collects the Java action and +// JavaScript action writes. +type codeActionProject struct { + mb *mock.MockBackend + h *ContainerHierarchy + ja *javaactions.JavaAction + jsa *types.JavaScriptAction + mod *model.Module + enum *model.Enumeration +} + +func newCodeActionProject() *codeActionProject { + p := &codeActionProject{} + p.mod = mkModule("MyFirstModule") + p.h = mkHierarchy(p.mod) + p.enum = mkEnumeration(p.mod.ID, "Color", "Red", "Green") + dm := mkDomainModel(p.mod.ID, mkEntity(p.mod.ID, "Customer")) + p.mb = &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{p.mod}, nil }, + ListEnumerationsFunc: func() ([]*model.Enumeration, error) { return []*model.Enumeration{p.enum}, nil }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { return []*domainmodel.DomainModel{dm}, nil }, + ListJavaActionsFunc: func() ([]*types.JavaAction, error) { return nil, nil }, + ListJavaScriptActionsFunc: func() ([]*types.JavaScriptAction, error) { return nil, nil }, + CreateJavaActionFunc: func(ja *javaactions.JavaAction) error { + p.ja = ja + return nil + }, + CreateJavaScriptActionFunc: func(jsa *types.JavaScriptAction) error { + p.jsa = jsa + return nil + }, + WriteJavaSourceFileFunc: func(string, string, string, []*javaactions.JavaActionParameter, + javaactions.CodeActionReturnType, []string, string) error { + return nil + }, + WriteJavaScriptSourceFileFunc: func(string, string, string, []*types.JavaActionParameter, types.CodeActionReturnType) error { + return nil + }, + } + return p +} + +func TestCreateJavaAction_BareEnumerationNameIsAnEnumeration(t *testing.T) { + p := newCodeActionProject() + ctx, _ := newMockCtx(t, withBackend(p.mb), withHierarchy(p.h)) + + s := parseOne[*ast.CreateJavaActionStmt](t, `create java action MyFirstModule.JA_Paint ( + C: MyFirstModule.Color, + Who: MyFirstModule.Customer +) returns MyFirstModule.Color +as $$ +return null; +$$;`) + assertNoError(t, execCreateJavaAction(ctx, s)) + if p.ja == nil || len(p.ja.Parameters) != 2 { + t.Fatalf("CreateJavaAction not called with two parameters: %+v", p.ja) + } + + et, ok := p.ja.Parameters[0].ParameterType.(*javaactions.EnumerationType) + if !ok { + t.Fatalf("parameter C is %T, want *EnumerationType — an enumeration written as an entity is CE1613", + p.ja.Parameters[0].ParameterType) + } + if et.Enumeration != "MyFirstModule.Color" { + t.Errorf("parameter C enumeration = %q", et.Enumeration) + } + // The other half of the ambiguity: a bare name that IS an entity stays one. + if en, ok := p.ja.Parameters[1].ParameterType.(*javaactions.EntityType); !ok || en.Entity != "MyFirstModule.Customer" { + t.Errorf("parameter Who = %#v, want EntityType MyFirstModule.Customer", p.ja.Parameters[1].ParameterType) + } + rt, ok := p.ja.ReturnType.(*javaactions.EnumerationType) + if !ok || rt.Enumeration != "MyFirstModule.Color" { + t.Errorf("return type = %#v, want EnumerationType MyFirstModule.Color", p.ja.ReturnType) + } +} + +func TestCreateJavaAction_BareNameThatIsNeitherIsRefused(t *testing.T) { + p := newCodeActionProject() + ctx, _ := newMockCtx(t, withBackend(p.mb), withHierarchy(p.h)) + + s := parseOne[*ast.CreateJavaActionStmt](t, `create java action MyFirstModule.JA_Ghost ( + G: MyFirstModule.NoSuchThing +) returns Boolean +as $$ +return true; +$$;`) + err := execCreateJavaAction(ctx, s) + if err == nil { + t.Fatal("expected a refusal for a type that is neither an entity nor an enumeration") + } + if !strings.Contains(err.Error(), "MyFirstModule.NoSuchThing") { + t.Errorf("refusal does not name the type: %v", err) + } + if p.ja != nil { + t.Error("the action was written despite the refusal") + } +} + +func TestCreateJavaScriptAction_BareEnumerationNameIsAnEnumeration(t *testing.T) { + p := newCodeActionProject() + ctx, _ := newMockCtx(t, withBackend(p.mb), withHierarchy(p.h)) + + s := parseOne[*ast.CreateJavaScriptActionStmt](t, `create javascript action MyFirstModule.JSA_Paint ( + C: MyFirstModule.Color +) returns MyFirstModule.Color +as $$ +return null; +$$;`) + assertNoError(t, execCreateJavaScriptAction(ctx, s)) + if p.jsa == nil || len(p.jsa.Parameters) != 1 { + t.Fatalf("CreateJavaScriptAction not called with one parameter: %+v", p.jsa) + } + if et, ok := p.jsa.Parameters[0].ParameterType.(*types.EnumerationType); !ok || et.Enumeration != "MyFirstModule.Color" { + t.Errorf("parameter C = %#v, want EnumerationType MyFirstModule.Color", p.jsa.Parameters[0].ParameterType) + } + if et, ok := p.jsa.ReturnType.(*types.EnumerationType); !ok || et.Enumeration != "MyFirstModule.Color" { + t.Errorf("return type = %#v, want EnumerationType MyFirstModule.Color", p.jsa.ReturnType) + } +} + +func TestCreateJavaScriptAction_BareNameThatIsNeitherIsRefused(t *testing.T) { + p := newCodeActionProject() + ctx, _ := newMockCtx(t, withBackend(p.mb), withHierarchy(p.h)) + + s := parseOne[*ast.CreateJavaScriptActionStmt](t, `create javascript action MyFirstModule.JSA_Ghost () +returns MyFirstModule.NoSuchThing +as $$ +return null; +$$;`) + if err := execCreateJavaScriptAction(ctx, s); err == nil || !strings.Contains(err.Error(), "MyFirstModule.NoSuchThing") { + t.Fatalf("expected a refusal naming MyFirstModule.NoSuchThing, got %v", err) + } + if p.jsa != nil { + t.Error("the action was written despite the refusal") + } +} + +// check --references gives exec's verdict: the name resolves when it is an +// entity or an enumeration — of the project or created earlier in the script — +// and is refused otherwise. +func TestValidate_CodeActionSignatureTypes(t *testing.T) { + p := newCodeActionProject() + ctx, _ := newMockCtx(t, withBackend(p.mb), withHierarchy(p.h)) + + prog, errs := visitor.Build(`create enumeration MyFirstModule.Shade (Light 'Light'); +create java action MyFirstModule.JA_Ok ( + C: MyFirstModule.Color, S: MyFirstModule.Shade, Who: MyFirstModule.Customer +) returns MyFirstModule.Color as $$ return null; $$; +create java action MyFirstModule.JA_Ghost (G: MyFirstModule.NoSuchThing) returns Boolean as $$ return true; $$; +create javascript action MyFirstModule.JSA_Ghost () returns MyFirstModule.Nope as $$ return null; $$;`) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + sc := newScriptContext() + sc.collectDefinitions(prog) + + if err := validateWithContext(ctx, prog.Statements[1], sc); err != nil { + t.Errorf("JA_Ok: unexpected reference error: %v", err) + } + if err := validateWithContext(ctx, prog.Statements[2], sc); err == nil || !strings.Contains(err.Error(), "MyFirstModule.NoSuchThing") { + t.Errorf("JA_Ghost: want a reference error naming MyFirstModule.NoSuchThing, got %v", err) + } + if err := validateWithContext(ctx, prog.Statements[3], sc); err == nil || !strings.Contains(err.Error(), "MyFirstModule.Nope") { + t.Errorf("JSA_Ghost: want a reference error naming MyFirstModule.Nope, got %v", err) + } +} diff --git a/mdl/executor/cmd_javascript_actions_write.go b/mdl/executor/cmd_javascript_actions_write.go index a59ce88b6..029098553 100644 --- a/mdl/executor/cmd_javascript_actions_write.go +++ b/mdl/executor/cmd_javascript_actions_write.go @@ -43,6 +43,13 @@ func execCreateJavaScriptAction(ctx *ExecContext, s *ast.CreateJavaScriptActionS return mdlerrors.NewNotFound("module", s.Name.Module) } + // A bare Module.Name in the signature is an entity or an enumeration, which + // only the project can say (ako/mxcli#935). + params, returnType, err := settleCodeActionSignature(ctx, "javascript action "+s.Name.String(), s.Parameters, s.ReturnType) + if err != nil { + return err + } + // Detect an existing action of the same name (for CREATE OR MODIFY). existing, err := ctx.Backend.ListJavaScriptActions() if err != nil { @@ -144,7 +151,7 @@ func execCreateJavaScriptAction(ctx *ExecContext, s *ast.CreateJavaScriptActionS typeParamNames[tpName] = true } - for _, param := range s.Parameters { + for _, param := range params { p := &types.JavaActionParameter{ BaseElement: model.BaseElement{ ID: model.ID(types.GenerateID()), @@ -185,19 +192,19 @@ func execCreateJavaScriptAction(ctx *ExecContext, s *ast.CreateJavaScriptActionS jsa.Parameters = append(jsa.Parameters, p) } - if isTypeParamRef(s.ReturnType, typeParamNames) { - tpName := getTypeParamRefName(s.ReturnType) + if isTypeParamRef(returnType, typeParamNames) { + tpName := getTypeParamRefName(returnType) jsa.ReturnType = &types.TypeParameter{ BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, TypeParameterID: typeParamNameToID[tpName], TypeParameter: tpName, } - } else if l := listOfTypeParameter(s.ReturnType, typeParamNameToID); l != nil { + } else if l := listOfTypeParameter(returnType, typeParamNameToID); l != nil { // `returns list of T` — without this it became a list of the entity `.T`, // the JavaScript twin of the Java action's #1183. jsa.ReturnType = l } else { - jsa.ReturnType = astDataTypeToJavaActionReturnType(s.ReturnType) + jsa.ReturnType = astDataTypeToJavaActionReturnType(returnType) } if jsa.MicroflowActionInfo, err = mergeMicroflowActionInfo(ctx, diff --git a/mdl/executor/code_action_signature_types.go b/mdl/executor/code_action_signature_types.go new file mode 100644 index 000000000..e26a403cc --- /dev/null +++ b/mdl/executor/code_action_signature_types.go @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" +) + +// A Java or JavaScript action's signature names entities and enumerations by +// a bare `Module.Name`, and the parser cannot tell the two apart: both arrive +// as TypeEnumeration with EnumRef set and ExplicitEnum false (CLAUDE.md, +// "TypeEnumeration vs TypeEntity Ambiguity"). The converters +// (astDataTypeToJavaActionParamType / …ReturnType) called every such name an +// entity, so `create java action M.JA (C: M.Color)` with M.Color an +// enumeration was written as an entity parameter: check and exec were clean +// and mxbuild reported CE1613 "The selected entity 'M.Color' no longer exists" +// (ako/mxcli#935). A comment there promised a resolveCodeActionParamType that +// would settle the name against the project; it never existed. +// +// settleCodeActionSignature is that resolution, run by exec before anything is +// written; codeActionSignatureErrors is check --references' copy of the +// verdict. Flow signatures have done the same since the entity-then- +// enumeration fallback of buildMicroflowFromStmt. + +// settleCodeActionSignature returns params and ret with every bare qualified +// name decided against the project: an entity stays an entity, an enumeration +// becomes an explicit enumeration, and a name that is neither is refused. An +// explicit `enum M.E` is held to enumerations. Unqualified names — type +// parameters, `Microflow`, `Nanoflow` — are left to the converters, and a +// `list of` element is not resolved (exec never refused one). The returned +// slice is a copy; the statement is not changed. +// +// A System name is decided by the System module's enumeration list: its +// entities are not stored in the project, so one that is not an enumeration is +// written by name as before (isBuiltinModuleEntity). +func settleCodeActionSignature(ctx *ExecContext, what string, params []ast.JavaActionParam, ret ast.DataType) ([]ast.JavaActionParam, ast.DataType, error) { + var entities map[string]bool // built on first use: most signatures are primitive + settle := func(dt ast.DataType, where string) (ast.DataType, error) { + qn := codeActionTypeName(dt) + if qn == nil { + return dt, nil + } + name := qn.String() + asEnumeration := ast.DataType{Kind: ast.TypeEnumeration, EnumRef: qn, ExplicitEnum: true} + if dt.Kind == ast.TypeEnumeration && dt.ExplicitEnum { + if isKnownEnumeration(ctx, qn.Module, qn.Name) { + return dt, nil + } + return dt, mdlerrors.NewNotFoundMsg("enumeration", name, + fmt.Sprintf("%s: %s is typed enum %s, and there is no such enumeration", what, where, name)) + } + asEntity := ast.DataType{Kind: ast.TypeEntity, EntityRef: qn} + if isBuiltinModuleEntity(qn.Module) { + if systemEnumerations[name] { + return asEnumeration, nil + } + return asEntity, nil + } + if entities == nil { + entities = buildEntityQualifiedNames(ctx) + } + if entities[name] { + return asEntity, nil + } + if findEnumeration(ctx, qn.Module, qn.Name) != nil { + return asEnumeration, nil + } + return dt, mdlerrors.NewNotFoundMsg("entity or enumeration", name, + fmt.Sprintf("%s: %s is typed %s, which is neither an entity nor an enumeration "+ + "(written as an entity, mxbuild reports CE1613)", what, where, name)) + } + + out := make([]ast.JavaActionParam, len(params)) + copy(out, params) + for i := range out { + dt, err := settle(out[i].Type, "parameter "+out[i].Name) + if err != nil { + return nil, ret, err + } + out[i].Type = dt + } + settledRet, err := settle(ret, "the return type") + if err != nil { + return nil, ret, err + } + return out, settledRet, nil +} + +// codeActionTypeName returns the qualified entity-or-enumeration name a +// code-action signature type names, or nil for any other type — a primitive, +// a list, an `entity ` selector, or an unqualified name (a type parameter, +// `Microflow`, `Nanoflow`, `void`). +func codeActionTypeName(dt ast.DataType) *ast.QualifiedName { + var qn *ast.QualifiedName + switch dt.Kind { + case ast.TypeEntity: + qn = dt.EntityRef + case ast.TypeEnumeration: + qn = dt.EnumRef + } + if qn == nil || qn.Module == "" || qn.Name == "" { + return nil + } + return qn +} + +// codeActionSignatureErrors is check --references' copy of +// settleCodeActionSignature's refusals: a bare name resolves when it is an +// entity or an enumeration of the project or of the script, and an explicit +// `enum` when it is an enumeration. kind is "java action" or "javascript +// action". +func codeActionSignatureErrors(ctx *ExecContext, sc *scriptContext, kind string, name ast.QualifiedName, + params []ast.JavaActionParam, ret ast.DataType) []string { + if !ctx.Connected() { + return nil + } + r := newDeclaredTypeResolver(ctx, sc) + checkType := func(dt ast.DataType, where string) string { + qn := codeActionTypeName(dt) + if qn == nil { + return "" + } + if !(dt.Kind == ast.TypeEnumeration && dt.ExplicitEnum) { + // A bare name: either kind resolves it, and the message says so. + dt = ast.DataType{Kind: ast.TypeEntity, EntityRef: qn} + } + return r.check(dt, where) + } + var errs []string + for _, p := range params { + if msg := checkType(p.Type, fmt.Sprintf("parameter %s of %s %s", p.Name, kind, name.String())); msg != "" { + errs = append(errs, msg) + } + } + if msg := checkType(ret, fmt.Sprintf("return type of %s %s", kind, name.String())); msg != "" { + errs = append(errs, msg) + } + return errs +} diff --git a/mdl/executor/validate.go b/mdl/executor/validate.go index 88bdff7b0..d297bb4d0 100644 --- a/mdl/executor/validate.go +++ b/mdl/executor/validate.go @@ -1023,6 +1023,16 @@ func validateWithContext(ctx *ExecContext, stmt ast.Statement, sc *scriptContext return mdlerrors.NewValidationf("workflow '%s' has reference errors:\n - %s", s.Name.String(), strings.Join(refErrors, "\n - ")) } + case *ast.CreateJavaActionStmt: + // The signature's bare Module.Name types, which exec resolves and + // refuses (ako/mxcli#935). + if errs := codeActionSignatureErrors(ctx, sc, "java action", s.Name, s.Parameters, s.ReturnType); len(errs) > 0 { + return flowValidationError("java action", s.Name.String(), nil, errs) + } + case *ast.CreateJavaScriptActionStmt: + if errs := codeActionSignatureErrors(ctx, sc, "javascript action", s.Name, s.Parameters, s.ReturnType); len(errs) > 0 { + return flowValidationError("javascript action", s.Name.String(), nil, errs) + } case *ast.CreatePublishedRestServiceStmt: return validatePublishedRestAuthMicroflow(ctx, s.Authentication, sc) case *ast.AlterPublishedRestServiceStmt: From 7a6d5c3f98e07d444686eb8aa58ac73459f7b507 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 11:24:20 +0000 Subject: [PATCH 10/13] fix: create java action writes a StringTemplate parameter as a String `create java action M.J (Subject: StringTemplate(Text))` - the form describe prints for a text-template parameter - wrote a String parameter, so a describe -> exec of, say, BusinessEvents.PublishBusinessEvent_V2 silently turned its template into a String and every call Studio Pro had written for it stopped matching (CE0115). Found while fixing ako/mxcli#1451, whose check-side template validation reads the declared type from the script. Cause: astDataTypeToJavaActionParamType had no case for ast.TypeStringTemplate and returned its default StringType. The Java source generator mapped the type to java.lang.String, pinned by a golden (JA_Tpl) that had been measured on an action this bug created with String parameters. Fix: StringTemplate(Text|Sql) converts to a StringTemplateParameterType with that grammar, and the generated .java declares it as com.mendix.systemwideinterfaces.javaactions.parameters.IStringTemplate, fully qualified - what mxbuild generates (BusinessEvents 3.12's source). Verification: TestParamType_StringTemplateIsATemplate and the JA_Tpl golden fail with the fix reverted. On a blank 11.13.0 app, mxcli's .java for a Sql + Text template action is byte-identical to the one an mxbuild --target=deploy build leaves, and a copy edited back to java.lang.String is rewritten to IStringTemplate by the build. describe prints StringTemplate(Sql)/StringTemplate(Text) for it. Co-Authored-By: Claude Opus 5.5 --- ...-template-parameter-written-as-string.json | 1 + .../java-actions/reference/writing-java.md | 2 +- mdl/executor/cmd_javaactions.go | 14 ++++++++ ..._javaactions_string_template_param_test.go | 32 +++++++++++++++++++ sdk/javaactions/mxbuild_golden_test.go | 6 +++- sdk/javaactions/source.go | 7 +++- .../testdata/mxbuild/JA_Tpl.java.golden | 8 ++--- 7 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-25-create-java-action-string-template-parameter-written-as-string.json create mode 100644 mdl/executor/cmd_javaactions_string_template_param_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-create-java-action-string-template-parameter-written-as-string.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-create-java-action-string-template-parameter-written-as-string.json new file mode 100644 index 000000000..add6c1402 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-create-java-action-string-template-parameter-written-as-string.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-25", "symptom": "`create java action M.J (Subject: StringTemplate(Text))` — the form describe prints for a text-template parameter — writes a String parameter; describe of the new action prints `Subject: String`. A describe -> exec of a module's action (BusinessEvents.PublishBusinessEvent_V2) silently turns its template parameter into a String. The generated .java typed it java.lang.String", "cause": "astDataTypeToJavaActionParamType had no case for ast.TypeStringTemplate and returned its default StringType; sdk/javaactions/source.go mapped StringTemplateParameterType to java.lang.String, pinned by a golden (JA_Tpl) that had itself been measured on an action this bug had created with String parameters", "file": "`mdl/executor/cmd_javaactions.go` (astDataTypeToJavaActionParamType), `sdk/javaactions/source.go` (javaParamType), golden `sdk/javaactions/testdata/mxbuild/JA_Tpl.java.golden`; test `mdl/executor/cmd_javaactions_string_template_param_test.go`", "insight": "**A golden is only as good as the model it was measured on**: JA_Tpl.java.golden recorded `java.lang.String Q` because mxbuild generated it for the String parameter mxcli had wrongly written. Check a golden's provenance before trusting it as the spec. The real type is `com.mendix.systemwideinterfaces.javaactions.parameters.IStringTemplate`, fully qualified and not imported — seen in the source mxbuild generated for BusinessEvents 3.12's PublishBusinessEvent_V2, and re-measured on 11.13.0: after a `--target=deploy` build mxcli's file is byte-identical to mxbuild's, and a file edited back to java.lang.String is rewritten to IStringTemplate by the build (the control that shows the build regenerates rather than keeps the file). Found while fixing ako/mxcli#1451, whose check-side template validation reads the declared type from the script and would otherwise disagree with exec", "refs": ["ako/mxcli#1451", "ako/mxcli#934"], "ce": ["CE0115"], "test": "TestParamType_StringTemplateIsATemplate, TestGenerateSource_RegeneratesWhatMxbuildWrote"} diff --git a/.claude/skills/mendix/java-actions/reference/writing-java.md b/.claude/skills/mendix/java-actions/reference/writing-java.md index 2a2104721..f0cbecad9 100644 --- a/.claude/skills/mendix/java-actions/reference/writing-java.md +++ b/.claude/skills/mendix/java-actions/reference/writing-java.md @@ -32,7 +32,7 @@ still compiles after a full build regenerates it. | `entity ` | `java.lang.String` | The entity's qualified name. | | `pEntity` | `IMendixObject` | No proxy: the type is only known at runtime. | | `list of pEntity` | `java.util.List` | | -| `stringtemplate(sql)` / `stringtemplate(text)` | `java.lang.String` | | +| `stringtemplate(sql)` / `stringtemplate(text)` | `com.mendix.systemwideinterfaces.javaactions.parameters.IStringTemplate` | Fully qualified, not imported. `replacePlaceholders(...)` fills the `{n}`; each `ITemplateParameter` carries its value and type. A call passes `'text {1}' with ({1} = …)` | | `Microflow` | `java.lang.String` | The microflow's qualified name. | Return types: an entity, `pEntity` or `System.FileDocument` return is diff --git a/mdl/executor/cmd_javaactions.go b/mdl/executor/cmd_javaactions.go index 581d48c9b..42ad49ab7 100644 --- a/mdl/executor/cmd_javaactions.go +++ b/mdl/executor/cmd_javaactions.go @@ -749,6 +749,20 @@ func astDataTypeToJavaActionParamType(dt ast.DataType) javaactions.CodeActionPar TypeName: "CodeActions$DateTimeType", }, } + case ast.TypeStringTemplate: + // `StringTemplate(Text)` / `StringTemplate(Sql)` — what describe prints + // for a text-template parameter. It fell to the default below and was + // written as a String: a describe -> exec round trip turned the + // parameter into a String, and every call Studio Pro had written for + // it (a template value) no longer matched — CE0115. + grammar := "Text" + if strings.EqualFold(dt.TemplateContext, "sql") { + grammar = "Sql" + } + return &javaactions.StringTemplateParameterType{ + BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, + Grammar: grammar, + } case ast.TypeEntity, ast.TypeEnumeration: // An unambiguous `ENUM`/`Enumeration(...)` type must serialize as an // enumeration parameter, not an entity reference (#680). A bare diff --git a/mdl/executor/cmd_javaactions_string_template_param_test.go b/mdl/executor/cmd_javaactions_string_template_param_test.go new file mode 100644 index 000000000..037a429ec --- /dev/null +++ b/mdl/executor/cmd_javaactions_string_template_param_test.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/sdk/javaactions" +) + +// `create java action M.J (S: StringTemplate(Text))` — the type describe prints +// for a text-template parameter, BusinessEvents.PublishBusinessEvent_V2's +// Subject among them — had no case in the converter and was written as a +// String parameter. A describe -> exec round trip of such an action turned the +// template into a String, after which every call of it built a template value +// for a String parameter (found while fixing ako/mxcli#1451). +func TestParamType_StringTemplateIsATemplate(t *testing.T) { + for _, c := range []struct{ context, grammar string }{ + {"Text", "Text"}, {"text", "Text"}, {"Sql", "Sql"}, {"SQL", "Sql"}, + } { + pt := astDataTypeToJavaActionParamType(ast.DataType{Kind: ast.TypeStringTemplate, TemplateContext: c.context}) + st, ok := pt.(*javaactions.StringTemplateParameterType) + if !ok { + t.Errorf("StringTemplate(%s) converted to %T, want *StringTemplateParameterType", c.context, pt) + continue + } + if st.Grammar != c.grammar { + t.Errorf("StringTemplate(%s) grammar = %q, want %q", c.context, st.Grammar, c.grammar) + } + } +} diff --git a/sdk/javaactions/mxbuild_golden_test.go b/sdk/javaactions/mxbuild_golden_test.go index 22ae78e9f..85a025252 100644 --- a/sdk/javaactions/mxbuild_golden_test.go +++ b/sdk/javaactions/mxbuild_golden_test.go @@ -53,7 +53,11 @@ var mxbuildCases = []mxbuildCase{ param("C", &EnumerationType{Enumeration: "MyFirstModule.Color"}), }, &EnumerationType{Enumeration: "MyFirstModule.Color"}}, // Its import list was emptied before the build: mxbuild appended the three - // it needs, sorted. + // it needs, sorted. Q and X were first measured as java.lang.String — but + // `create java action` then wrote StringTemplate(…) as a String parameter. + // For a real text template mxbuild declares IStringTemplate, fully + // qualified: re-measured on 11.13.0, where a build rewrites the String + // form back to it (ako/mxcli#1451). {"JA_Tpl", "return null;", []*JavaActionParameter{ param("Q", &StringTemplateParameterType{Grammar: "Sql"}), param("X", &StringTemplateParameterType{Grammar: "Text"}), diff --git a/sdk/javaactions/source.go b/sdk/javaactions/source.go index 32755065f..572be82a4 100644 --- a/sdk/javaactions/source.go +++ b/sdk/javaactions/source.go @@ -220,10 +220,15 @@ func javaParamType(t CodeActionParameterType) string { return "java.lang.Long" case *DecimalType: return "java.math.BigDecimal" - case *StringType, *StringTemplateParameterType, *MicroflowType, *EnumerationType, *EntityTypeParameterType: + case *StringType, *MicroflowType, *EnumerationType, *EntityTypeParameterType: // A microflow arrives as its qualified name, an entity type selector as // the entity's, an enumeration as its value's name. return "java.lang.String" + case *StringTemplateParameterType: + // What mxbuild declares for a text template (Text or Sql), fully + // qualified and not imported — measured on the source it generated for + // BusinessEvents 3.12's PublishBusinessEvent_V2.Subject. + return "com.mendix.systemwideinterfaces.javaactions.parameters.IStringTemplate" case *DateTimeType: return "java.util.Date" case *EntityType, *FileDocumentType, *TypeParameter: diff --git a/sdk/javaactions/testdata/mxbuild/JA_Tpl.java.golden b/sdk/javaactions/testdata/mxbuild/JA_Tpl.java.golden index bb5eaf79c..7baef18d5 100644 --- a/sdk/javaactions/testdata/mxbuild/JA_Tpl.java.golden +++ b/sdk/javaactions/testdata/mxbuild/JA_Tpl.java.golden @@ -15,13 +15,13 @@ import com.mendix.systemwideinterfaces.core.UserAction; public class JA_Tpl extends UserAction> { - private final java.lang.String Q; - private final java.lang.String X; + private final com.mendix.systemwideinterfaces.javaactions.parameters.IStringTemplate Q; + private final com.mendix.systemwideinterfaces.javaactions.parameters.IStringTemplate X; public JA_Tpl( IContext context, - java.lang.String _q, - java.lang.String _x + com.mendix.systemwideinterfaces.javaactions.parameters.IStringTemplate _q, + com.mendix.systemwideinterfaces.javaactions.parameters.IStringTemplate _x ) { super(context); From b57af11729ca6b67d428e58b5c9a23f9db55e402 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 11:24:32 +0000 Subject: [PATCH 11/13] fix: call java action writes a text-template or document parameter as an expression (ako/mxcli#1451) Mirror of mendixlabs/mxcli#1554. A Java action parameter of type StringTemplate(Text|Sql) - BusinessEvents.PublishBusinessEvent_V2's Subject - or a custom document - AgentEditorCommons' Agent - was bound as a Microflows$BasicCodeActionParameterValue holding the argument's source text. check --references and exec passed; mx check reported CE0115 at the call. describe java action showed the document parameter as `String`. Cause: addCallJavaActionAction gave only entity-type and Microflow parameters their own value; the two others fell to the expression default. The document parameter type read back as a String (codeActionParamTypeFromGen had no case), and MDL had no way to write a template's {n} arguments. Fix: - A text-template argument is written the way a log message is, `Subject = 'Order {1}' with ({1} = $Order/Number)` (grammar: templateParams on a call argument, refused outside call java action). It stores a Microflows$StringTemplateParameterValue: the text plus one Microflows$PrimitiveTypedTemplateArgument per placeholder, each typed from its expression (String/Integer/Decimal/Boolean/DateTime; an enumeration is refused). A plain string is the text; another expression is '{1}' with it as the argument (the ako/mxcli#1444 rule). An expression followed by `with (...)` is refused (MDL-JA02, no project needed). - A document parameter reads as CustomBlobDocumentType (describe: `Document(agenteditor.agent)`; written back unchanged on a rewrite) and its argument - `Agent = Module.Doc` or 'Module.Doc' - stores a Microflows$CustomBlobDocumentCodeActionParameterValue. check --references resolves agent-editor documents (missing: CE1613, wrong type: CE0115). - describe prints both re-executably; '{1}' with one non-literal argument prints as that expression, so describe -> exec reports Unchanged. No Studio Pro-authored example of either value exists in the repo's fixtures or on this machine. The shapes are the metamodel's, measured against mxbuild 11.13 by patching units: accepted as written; CE0117 when an argument's Type is not its expression's (absent Type = String; an enumeration takes none); CE0720 for a placeholder without an argument. Verification: the new tests fail with the builder, reader, writer, describe and reference-check changes stubbed out. mx check on 11.13.0: BusinessEvents 3.12 app, PublishBusinessEvent_V2 calls - pre-fix CE0115 x2, fixed 0 errors (plain, expression, and `with` with an Integer and [%CurrentDateTime%]); bug-test script on a blank app - 0 errors, control (fixed action, pre-fix microflow) CE0115 x2; synthetic document-parameter action - pre-fix shape CE0115, fixed 0 errors. describe -> exec: Unchanged microflow in all three. go vet and go test pass for the touched packages. Co-Authored-By: Claude Opus 5.5 --- ...ent-parameters-written-as-expressions.json | 1 + .../skills/mendix/business-events/SKILL.md | 18 +- .../write-microflows/reference/integration.md | 41 ++ cmd/mxcli/syntax/features_microflow.go | 13 +- docs/01-project/MDL_QUICK_REFERENCE.md | 2 + ...ction-1451-expression-with-params.fail.mdl | 17 + .../java-action-1451-template-parameter.mdl | 75 ++++ mdl/ast/ast_microflow.go | 4 + .../java_action_call_values_1451_test.go | 158 +++++++ mdl/backend/modelsdk/java_read.go | 11 + mdl/backend/modelsdk/java_write.go | 12 + .../modelsdk/microflow_read_actions.go | 19 + mdl/backend/modelsdk/microflow_write.go | 23 +- mdl/executor/cmd_microflows_builder_calls.go | 30 +- ...roflows_builder_java_template_1451_test.go | 302 +++++++++++++ mdl/executor/cmd_microflows_format_action.go | 31 +- mdl/executor/flow_built_match.go | 2 +- mdl/executor/java_action_call_values.go | 413 ++++++++++++++++++ mdl/executor/validate.go | 36 +- mdl/executor/validate_microflow.go | 4 + mdl/exprcheck/infer.go | 16 + mdl/grammar/domains/MDLMicroflow.g4 | 7 +- mdl/types/javaaction_types.go | 17 + mdl/visitor/visitor_argument_binding.go | 18 + .../visitor_java_action_template_1451_test.go | 57 +++ mdl/visitor/visitor_microflow_actions.go | 9 +- sdk/javaactions/javaactions.go | 1 + sdk/microflows/microflows_actions.go | 31 +- 28 files changed, 1347 insertions(+), 21 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-25-call-java-action-template-and-document-parameters-written-as-expressions.json create mode 100644 mdl-examples/bug-tests/java-action-1451-expression-with-params.fail.mdl create mode 100644 mdl-examples/bug-tests/java-action-1451-template-parameter.mdl create mode 100644 mdl/backend/modelsdk/java_action_call_values_1451_test.go create mode 100644 mdl/executor/cmd_microflows_builder_java_template_1451_test.go create mode 100644 mdl/executor/java_action_call_values.go create mode 100644 mdl/visitor/visitor_java_action_template_1451_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-call-java-action-template-and-document-parameters-written-as-expressions.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-call-java-action-template-and-document-parameters-written-as-expressions.json new file mode 100644 index 000000000..45fb26426 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-call-java-action-template-and-document-parameters-written-as-expressions.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-25", "symptom": "`call java action BusinessEvents.PublishBusinessEvent_V2(Subject = 'Order shipped', BusinessEvent = $Event)` and `call java action AgentEditorCommons.Agent_CallDocument_WithoutHistory(Agent = 'M.MyAgent', …)` pass `check --references` and exec; mx check reports CE0115 \"The arguments that are passed to Java action '…' do not match the expected parameters and need to be refreshed\" at each call. `describe java action` showed the agent parameter as `Agent: String not null`; `describe microflow` printed a template value as its text only", "cause": "addCallJavaActionAction (mdl/executor/cmd_microflows_builder_calls.go) gave only entity-type and Microflow parameters their own value; a text-template parameter (CodeActions$StringTemplateParameterType) and a custom-document parameter (CodeActions$CustomBlobDocumentParameterType) fell to the default Microflows$BasicCodeActionParameterValue with the argument's source text. The document parameter TYPE also read back as a String (codeActionParamTypeFromGen had no case for it), and MDL had no way to write a template's {n} arguments at all", "file": "`mdl/executor/java_action_call_values.go` (javaActionTemplateValue, templateArgumentType + flowExprTypes, javaActionDocumentValue, javaActionCallValueErrors, MDL-JA02), called from `addCallJavaActionAction` and `validate.go` (validateFlowBodyReferences); `mdl/backend/modelsdk/microflow_write.go` / `microflow_read_actions.go` (PrimitiveTypedTemplateArgument, CustomBlobDocumentCodeActionParameterValue), `java_read.go` / `java_write.go` (CustomBlobDocumentType), `cmd_microflows_format_action.go` (describeTypedTemplate); grammar `callArgument … templateParams?` + visitor; tests `mdl/executor/cmd_microflows_builder_java_template_1451_test.go`, `mdl/backend/modelsdk/java_action_call_values_1451_test.go`, `mdl/visitor/visitor_java_action_template_1451_test.go`; fixtures `mdl-examples/bug-tests/java-action-1451-template-parameter.mdl`, `java-action-1451-expression-with-params.fail.mdl`", "insight": "**No Studio Pro reference exists here** (searched every .mpr/.mxunit/.mpk on the machine: 0 StringTemplateParameterValue, 0 CustomBlobDocumentParameterType outside module definitions; the Studio Pro MCP tunnel was down), so the shapes come from generated/metamodel and were *measured* against mxbuild 11.13 by hand-patching the unit (a throwaway Go probe over the v1 Unit table). What the probes settled, none of it guessable: a TypedTemplate argument is Microflows$PrimitiveTypedTemplateArgument{Expression, Type}; **Type is checked against the expression** (CE0117 for String-typed Integer, Integer-typed String, and for any Type on an enumeration expression — an enum is not one of the five); an absent Type reads as String; {1} with no argument is CE0720; Long attributes take Integer; `[%CurrentDateTime%]` takes DateTime — and exprcheck's inferKind calls every token a String, so use its argKind (now InferArgumentKind), which the built-in-function check already special-cased. The runtime hands Type to Java (ITemplateParameter.getParameterType), so the type is semantics, not decoration. For the document value mxbuild reports a wrong-type document as CE0115 and a missing one as CE1613 — check --references resolves agent-editor documents for that reason. The REST-body rule (#1444) carries over: a non-literal is '{1}' with the expression; but **describe must print `$X`, not `'{1}' with ({1} = $X)`**, because the declared-match pass (flow_declared_match) cannot know which CallArgument is a template, so only the short form re-execs as 'Unchanged' — except when the one argument is itself a string literal, which would read back as the text. A pre-fix build is no control for a script-created action: `create java action … StringTemplate(Text)` wrote a String parameter (separate fix), so the control is fixed-build action + pre-fix microflow (CE0115 x2), and the BusinessEvents module (CE0115 per call -> 0 errors)", "refs": ["ako/mxcli#1451", "mendixlabs/mxcli#1554", "ako/mxcli#1444", "mendixlabs/mxcli#1210"], "ce": ["CE0115", "CE0117", "CE0720", "CE1613"], "rules": ["MDL-JA02"], "test": "TestCallJavaAction_TemplateParameterWithArguments, TestCallJavaAction_TemplateParameterPlainString, TestCallJavaAction_TemplateParameterExpression, TestCallJavaAction_DocumentParameter, TestCallJavaAction_TemplateAndDocumentRefusals, TestCallJavaAction_TemplateAndDocumentDescribeRoundTrip, TestValidate_JavaActionTemplateAndDocumentArguments, TestJavaActionCallToGen_TemplateAndDocumentValues, TestJavaActionCallFromGen_TemplateAndDocumentValuesRoundTrip, TestJavaActionParamType_CustomBlobDocumentRoundTrips, TestCallJavaActionTemplateArgument, TestTemplateArgumentRefusedOutsideCallJavaAction"} diff --git a/.claude/skills/mendix/business-events/SKILL.md b/.claude/skills/mendix/business-events/SKILL.md index f0527f64d..708778bde 100644 --- a/.claude/skills/mendix/business-events/SKILL.md +++ b/.claude/skills/mendix/business-events/SKILL.md @@ -15,8 +15,8 @@ description: "Define event-driven APIs over Kafka with Mendix business event ser | 102-117 | Message Definition Syntax | | 118-125 | Service Properties | | 126-132 | Operations | -| 133-169 | Publishing Events from Microflows | -| 170-182 | Checklist | +| 133-179 | Publishing Events from Microflows | +| 180-192 | Checklist | # Business Events @@ -143,10 +143,20 @@ create microflow Module.ACT_PublishCustomerChanged begin $event = create Module.PBE_CustomerChangedEvent (CustomerId = $CustomerId); commit $event; - call java action BusinessEvents.PublishBusinessEvent_V2(EventObject = $event); + call java action BusinessEvents.PublishBusinessEvent_V2( + Subject = 'Customer {1} changed' with ({1} = $CustomerId), + BusinessEvent = $event + ); end; ``` +`PublishBusinessEvent_V2` takes two parameters: `Subject`, a **text template** +(`StringTemplate(Text)`), and `BusinessEvent`, the event object. `Subject` is +written like a log message — a string, or `'… {1} …' with ({1} = …)`; each +argument must be a String, Integer, Decimal, Boolean or DateTime value (wrap an +enumeration in `getCaption(…)`). `describe java action` shows the parameters of +any action before you call it. + ### Available Java Actions (from BusinessEvents module) | Java Action | Description | @@ -164,7 +174,7 @@ end; 2. Create entities prefixed with `PBE_` that **extend `BusinessEvents.PublishedBusinessEvent`** 3. Entity attributes must **exactly match** the message attributes (no extra attributes on the entity) 4. In microflows, create an instance of the event entity, populate its attributes, commit it -5. Call `BusinessEvents.PublishBusinessEvent_V2` passing the entity instance +5. Call `BusinessEvents.PublishBusinessEvent_V2` with a `Subject` template and the entity instance as `BusinessEvent` 6. For subscribe operations, link a handler microflow in the service definition ## Checklist diff --git a/.claude/skills/mendix/write-microflows/reference/integration.md b/.claude/skills/mendix/write-microflows/reference/integration.md index 5736063fb..38d78b990 100644 --- a/.claude/skills/mendix/write-microflows/reference/integration.md +++ b/.claude/skills/mendix/write-microflows/reference/integration.md @@ -297,3 +297,44 @@ $Tool = call java action MCPServer.AddTool( `DESCRIBE JAVA ACTION` prints such a parameter's type as the bare word `Microflow` (`Nanoflow` for JavaScript actions), and that spelling is what `CREATE JAVA ACTION` accepts, so the round-trip is stable. + +## Text-Template and Document Java-Action Parameters + +Two more parameter types take something other than an expression. Check the +action's signature with `DESCRIBE JAVA ACTION` first. + +A **text template** — `StringTemplate(Text)` or `StringTemplate(Sql)`, such as +`BusinessEvents.PublishBusinessEvent_V2`'s `Subject` — is written like a log +message: a string is the template text, and `with ({1} = …)` fills its +placeholders. Any other expression becomes the template `'{1}'` with that +expression as its argument. + +```mdl +call java action BusinessEvents.PublishBusinessEvent_V2( + Subject = 'Order {1} shipped ({2} items)' with ({1} = $Order/Number, {2} = $Order/Lines), + BusinessEvent = $Event +); +``` + +Each argument is stored with the type of its result, and mxbuild checks the +two match (CE0117). The five types a template takes are String, Integer, +Decimal, Boolean and DateTime. An enumeration is refused, so pass +`getCaption($Order/Status)` or `getKey(…)` instead. + +A **document** parameter — `DESCRIBE JAVA ACTION` prints its type as +`Document()`, for example `Document(agenteditor.agent)` for +AgentEditorCommons' `Agent` — takes the document's qualified name, never an +expression: + +```mdl +$Response = call java action AgentEditorCommons.Agent_CallDocument_WithoutHistory( + Agent = MyModule.PaymentReminderAgent, + OptionalContextObject = $Invoice +); +``` + +`check --references` reports a missing agent-editor document; mxbuild reports +one as CE1613, and a document of another type as CE0115. `CREATE JAVA ACTION` +cannot declare a document parameter, so `describe` output for such an action +does not re-execute: the parser stops at `Document(…)` instead of rewriting +the parameter as a String. diff --git a/cmd/mxcli/syntax/features_microflow.go b/cmd/mxcli/syntax/features_microflow.go index be5bad060..ea0f13305 100644 --- a/cmd/mxcli/syntax/features_microflow.go +++ b/cmd/mxcli/syntax/features_microflow.go @@ -661,9 +661,18 @@ func init() { Keywords: []string{ "call microflow", "call java action", "invoke", "in queue", "queued call", "background execution", "sub-microflow", "java action", "parameter passing", "blank argument", "nothing", + "text template parameter", "stringtemplate", "document parameter", "agent parameter", }, - Syntax: "$Result = CALL MICROFLOW Module.Name (Param = value) [IN QUEUE Module.Queue];\n$Result = CALL JAVA ACTION Module.Name (Param = value) [IN QUEUE Module.Queue];", - Example: "$IsValid = CALL MICROFLOW MyModule.ValidateOrder (\n Order = $NewOrder\n);\n\n$Token = CALL JAVA ACTION MyModule.GenerateToken (\n UserId = $User/Id\n);\n\n-- Run the call on a task queue (background execution). The queue must\n-- exist; a queued CALL JAVA ACTION must return Nothing, or the build fails\n-- with CE7038.\nCALL MICROFLOW MyModule.ACT_Refresh () IN QUEUE MyModule.RefreshQueue;\nCALL JAVA ACTION MyModule.RefreshData (Url = $Url) IN QUEUE MyModule.RefreshQueue;\n\n-- A blank argument: the mapping is kept, its expression left empty, as\n-- Studio Pro stores an argument field left blank. Not the same as `empty`\n-- (a Mendix value); only CALL MICROFLOW and CALL NANOFLOW take it. Mendix\n-- reports it as CE0127 \"Missing argument\": it exists so DESCRIBE round-trips.\nCALL MICROFLOW MyModule.ACT_Log (Message = 'x', TraceId = nothing);", + Syntax: "$Result = CALL MICROFLOW Module.Name (Param = value) [IN QUEUE Module.Queue];\n$Result = CALL JAVA ACTION Module.Name (Param = value) [IN QUEUE Module.Queue];\n\n" + + "-- A Java action parameter of type StringTemplate(Text|Sql) takes a template, filled\n" + + "-- like a log message: Param = 'text {1}' WITH ({1} = expression, ...). A plain string\n" + + "-- is the text; any other expression is the template '{1}' with it as the argument,\n" + + "-- so an expression followed by WITH (...) is refused (MDL-JA02).\n" + + "-- Each argument is typed from its expression (String, Integer, Decimal, Boolean,\n" + + "-- DateTime); an enumeration is refused — pass getKey(...) or getCaption(...).\n" + + "-- A document parameter (DESCRIBE JAVA ACTION shows Document(), e.g. an\n" + + "-- agent-editor Agent) takes the document's name: Param = Module.Document.", + Example: "$IsValid = CALL MICROFLOW MyModule.ValidateOrder (\n Order = $NewOrder\n);\n\n$Token = CALL JAVA ACTION MyModule.GenerateToken (\n UserId = $User/Id\n);\n\n-- Run the call on a task queue (background execution). The queue must\n-- exist; a queued CALL JAVA ACTION must return Nothing, or the build fails\n-- with CE7038.\nCALL MICROFLOW MyModule.ACT_Refresh () IN QUEUE MyModule.RefreshQueue;\nCALL JAVA ACTION MyModule.RefreshData (Url = $Url) IN QUEUE MyModule.RefreshQueue;\n\n-- A blank argument: the mapping is kept, its expression left empty, as\n-- Studio Pro stores an argument field left blank. Not the same as `empty`\n-- (a Mendix value); only CALL MICROFLOW and CALL NANOFLOW take it. Mendix\n-- reports it as CE0127 \"Missing argument\": it exists so DESCRIBE round-trips.\nCALL MICROFLOW MyModule.ACT_Log (Message = 'x', TraceId = nothing);\n\n-- A text-template parameter, and a document parameter.\nCALL JAVA ACTION BusinessEvents.PublishBusinessEvent_V2 (\n Subject = 'Order {1} shipped' WITH ({1} = $Order/OrderNumber),\n BusinessEvent = $Event\n);\n$Response = CALL JAVA ACTION AgentEditorCommons.Agent_CallDocument_WithoutHistory (\n Agent = MyModule.PaymentReminderAgent, OptionalContextObject = $Invoice\n);", SeeAlso: []string{"java-action", "microflow.create", "queue", "microflow.workflow-actions"}, }) diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index 6484cbc44..b7d547672 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -1458,6 +1458,8 @@ Module.OrderResponse_CustomerInfo/Module.CustomerInfo as customer { | Drop Java action | `drop java action [if exists] Module.Name;` | Deletes MPR unit and .java source file | | Call from microflow | `$Result = call java action Module.Name(Param = value);` | Inside BEGIN...END | | Empty argument | `call java action Module.Name(Param = empty);` | Unbound code-action parameter preserved as empty mapping | +| Text-template argument | `call java action Module.Name(Subject = 'Order {1}' with ({1} = $Order/Number));` | For a `StringTemplate(Text\|Sql)` parameter. A plain string is the text; another expression is the template `'{1}'` with it as the argument (so an expression with `with (…)` is refused, MDL-JA02). Arguments are typed from their expression; an enumeration is refused (use `getKey`/`getCaption`) | +| Document argument | `call java action Module.Name(Agent = Module.MyAgent);` | For a document parameter (`describe java action` shows `Document()`, e.g. an agent-editor Agent): the document's name, never an expression | | Show JavaScript actions | `list javascript actions [in module];` | List all or filtered by module | | Describe JavaScript action | `describe javascript action Module.Name;` | Re-executable MDL with signature + body | | Create JavaScript action | `create [or modify] javascript action Module.Name [folder 'path'](params) returns type [platform Web] as $$ ... $$;` | Writes the unit + `javascriptsource//actions/.js`; OR MODIFY preserves UUID | diff --git a/mdl-examples/bug-tests/java-action-1451-expression-with-params.fail.mdl b/mdl-examples/bug-tests/java-action-1451-expression-with-params.fail.mdl new file mode 100644 index 000000000..d025bbb40 --- /dev/null +++ b/mdl-examples/bug-tests/java-action-1451-expression-with-params.fail.mdl @@ -0,0 +1,17 @@ +-- Negative test (ako/mxcli#1451): a `call java action` argument written as an +-- expression and followed by `with (…)` must be REFUSED (MDL-JA02). +-- +-- `with (…)` fills the {n} placeholders of a text-template parameter's text, +-- written as a string. An expression is not a template — it is stored as the +-- template '{1}' with the expression as its argument — so there is nothing +-- left to fill; and a parameter that is not a text template takes no `with` +-- at all. See java-action-1451-template-parameter.mdl for the forms that work. + +mdl 1; + +create module BugTest1451Fail; + +create microflow BugTest1451Fail.MF_Notify ($Number: String) +begin + call java action BugTest1451Fail.JA_Notify(Subject = $Number with ({1} = $Number)); +end; diff --git a/mdl-examples/bug-tests/java-action-1451-template-parameter.mdl b/mdl-examples/bug-tests/java-action-1451-template-parameter.mdl new file mode 100644 index 000000000..135af661c --- /dev/null +++ b/mdl-examples/bug-tests/java-action-1451-template-parameter.mdl @@ -0,0 +1,75 @@ +-- ako/mxcli#1451 (upstream mendixlabs/mxcli#1554): `call java action` wrote a +-- text-template parameter, and a document parameter, as a plain expression. +-- +-- A Java action parameter of type StringTemplate(Text) — such as +-- BusinessEvents.PublishBusinessEvent_V2's Subject — was stored as a +-- Microflows$BasicCodeActionParameterValue holding the argument's source text. +-- `check --references` passed and exec reported "Created microflow"; mx check +-- then failed with +-- +-- CE0115 "The arguments that are passed to Java action '…' do not match the +-- expected parameters and need to be refreshed." +-- +-- The same happened to a document parameter (AgentEditorCommons' +-- Agent_CallDocument_WithoutHistory.Agent, a custom agent-editor document), +-- which `describe java action` also showed as `Agent: String`. +-- +-- Fixed: a text-template parameter stores a Microflows$StringTemplateParameterValue +-- — the text with one typed argument per {n} placeholder — written the way a +-- log message is: +-- +-- Subject = 'Order {1} shipped' with ({1} = $Order/Number) +-- +-- A plain string is the text; any other expression is the template '{1}' with +-- the expression as its argument. Each argument is stored with the type of its +-- result (mxbuild checks it: CE0117 on a mismatch); an enumeration is refused, +-- and an expression followed by `with (…)` is refused (MDL-JA02). A document +-- parameter stores a Microflows$CustomBlobDocumentCodeActionParameterValue +-- naming the document — `Agent = MyModule.PaymentReminderAgent` — and +-- `describe java action` prints its type as Document(agenteditor.agent). +-- MDL cannot declare a document parameter, so that half is not in this script. +-- +-- Verify: exec on a blank app, then `mx check`: 0 errors on 11.13.0. Then +-- describe the microflow and re-exec it: "Unchanged microflow". +-- +-- A pre-fix build is no control on its own: it also wrote `StringTemplate(Text)` +-- as a String parameter (fixed with this issue), so its calls matched. The +-- control is this build for the module, entity and action, and a pre-fix build +-- for the microflow without the `with` call (it cannot parse one): mx check +-- then reports CE0115 at both calls. The real-world action, with the Business +-- Events module imported, goes from CE0115 per call to 0 errors: +-- +-- call java action BusinessEvents.PublishBusinessEvent_V2( +-- Subject = 'Order shipped: {1} items, {2}' with ({1} = $Event/Quantity, {2} = [%CurrentDateTime%]), +-- BusinessEvent = $Event); + +mdl 1; + +create module BugTest1451; + +create persistent entity BugTest1451.Shipment ( + Number: String(20), + Lines: Integer, + Total: Decimal, + ShippedOn: DateTime +); + +create java action BugTest1451.JA_Notify ( + Subject: StringTemplate(Text) not null, + Shipment: BugTest1451.Shipment +) returns Boolean +as $$ +return true; +$$; + +create microflow BugTest1451.MF_Notify ($Shipment: BugTest1451.Shipment) +begin + -- A plain string: the template's text, no arguments. + $Sent = call java action BugTest1451.JA_Notify(Subject = 'Shipment sent', Shipment = $Shipment); + -- Placeholders filled with ({n} = …); one argument of each type. + $Detailed = call java action BugTest1451.JA_Notify( + Subject = 'Shipment {1}: {2} lines, {3} total, shipped {4}' with ({1} = $Shipment/Number, {2} = $Shipment/Lines, {3} = $Shipment/Total, {4} = $Shipment/ShippedOn), + Shipment = $Shipment); + -- An expression: the template '{1}' with the expression as its argument. + $Short = call java action BugTest1451.JA_Notify(Subject = 'Shipment ' + $Shipment/Number, Shipment = $Shipment); +end; diff --git a/mdl/ast/ast_microflow.go b/mdl/ast/ast_microflow.go index 9cb92ee9d..fc09eb37e 100644 --- a/mdl/ast/ast_microflow.go +++ b/mdl/ast/ast_microflow.go @@ -737,6 +737,10 @@ func (s *LogStmt) isMicroflowStatement() {} type CallArgument struct { Name string // Parameter name Value Expression // Value expression + // TemplateParams fills the {n} placeholders of Value when it is the text of + // a Java action's text-template parameter: `Subject = 'Order {1}' with + // ({1} = $Order/Number)` (ako/mxcli#1451). Only call java action takes it. + TemplateParams []TemplateParam } // CallMicroflowStmt represents: [$Result =] CALL MICROFLOW Name (args) [ON ERROR ...] diff --git a/mdl/backend/modelsdk/java_action_call_values_1451_test.go b/mdl/backend/modelsdk/java_action_call_values_1451_test.go new file mode 100644 index 000000000..d71064709 --- /dev/null +++ b/mdl/backend/modelsdk/java_action_call_values_1451_test.go @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/modelsdk/codec" + genCa "github.com/mendixlabs/mxcli/modelsdk/gen/codeactions" + "github.com/mendixlabs/mxcli/sdk/javaactions" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// ako/mxcli#1451 (mendixlabs/mxcli#1554): a Java action parameter that is a +// text template (BusinessEvents.PublishBusinessEvent_V2's Subject) or a custom +// document (AgentEditorCommons' Agent) was written as an expression, and +// mxbuild reported CE0115. These pin the BSON the write path produces for the +// two values the builder now chooses, and that the read path gives them back. + +// encodedJavaCallValues encodes a Java action call and returns its mappings' +// Value documents, in order. +func encodedJavaCallValues(t *testing.T, action *microflows.JavaActionCallAction) []bson.Raw { + t.Helper() + encoded, err := (&codec.Encoder{}).Encode(microflowActionToGen(action)) + if err != nil { + t.Fatalf("encode: %v", err) + } + vals, err := bson.Raw(encoded).Lookup("ParameterMappings").Array().Values() + if err != nil { + t.Fatalf("ParameterMappings: %v", err) + } + var out []bson.Raw + for _, v := range vals { + if doc, ok := v.DocumentOK(); ok { // element 0 is the list marker + out = append(out, doc.Lookup("Value").Document()) + } + } + return out +} + +func templateAndDocumentCall() *microflows.JavaActionCallAction { + return µflows.JavaActionCallAction{ + JavaAction: "BusinessEvents.PublishBusinessEvent_V2", + ParameterMappings: []*microflows.JavaActionParameterMapping{ + { + Parameter: "BusinessEvents.PublishBusinessEvent_V2.Subject", + Value: µflows.StringTemplateParameterValue{TypedTemplate: µflows.TypedTemplate{ + Text: "Order {1} shipped, {2} items", + Arguments: []*microflows.TypedTemplateArgument{ + {Expression: "$Event/OrderNumber", Type: "String"}, + {Expression: "$Event/Quantity", Type: "Integer"}, + }, + }}, + }, + { + Parameter: "AgentEditorCommons.Agent_CallDocument_WithoutHistory.Agent", + Value: µflows.CustomBlobDocumentCodeActionParameterValue{CustomDocument: "MyModule.PaymentReminderAgent"}, + }, + }, + } +} + +func TestJavaActionCallToGen_TemplateAndDocumentValues(t *testing.T) { + values := encodedJavaCallValues(t, templateAndDocumentCall()) + if len(values) != 2 { + t.Fatalf("values = %d, want 2", len(values)) + } + + tmpl := values[0] + if got := tmpl.Lookup("$Type").StringValue(); got != "Microflows$StringTemplateParameterValue" { + t.Fatalf("Subject value $Type = %q, want Microflows$StringTemplateParameterValue (CE0115 otherwise)", got) + } + tt := tmpl.Lookup("TypedTemplate").Document() + if got := tt.Lookup("$Type").StringValue(); got != "Microflows$TypedTemplate" { + t.Errorf("TypedTemplate $Type = %q", got) + } + if got := tt.Lookup("Text").StringValue(); got != "Order {1} shipped, {2} items" { + t.Errorf("Text = %q", got) + } + args, err := tt.Lookup("Arguments").Array().Values() + if err != nil { + t.Fatalf("Arguments: %v", err) + } + if len(args) != 3 || args[0].Int32() != 2 { + t.Fatalf("Arguments = %v, want marker 2 then two arguments", args) + } + for i, want := range [][2]string{{"$Event/OrderNumber", "String"}, {"$Event/Quantity", "Integer"}} { + a := args[i+1].Document() + if got := a.Lookup("$Type").StringValue(); got != "Microflows$PrimitiveTypedTemplateArgument" { + t.Errorf("argument %d $Type = %q", i+1, got) + } + if got := a.Lookup("Expression").StringValue(); got != want[0] { + t.Errorf("argument %d Expression = %q, want %q", i+1, got, want[0]) + } + // mxbuild checks the expression against Type: CE0117 on a mismatch. + if got := a.Lookup("Type").StringValue(); got != want[1] { + t.Errorf("argument %d Type = %q, want %q", i+1, got, want[1]) + } + } + + doc := values[1] + if got := doc.Lookup("$Type").StringValue(); got != "Microflows$CustomBlobDocumentCodeActionParameterValue" { + t.Fatalf("Agent value $Type = %q, want Microflows$CustomBlobDocumentCodeActionParameterValue", got) + } + if got := doc.Lookup("CustomDocument").StringValue(); got != "MyModule.PaymentReminderAgent" { + t.Errorf("CustomDocument = %q", got) + } +} + +func TestJavaActionCallFromGen_TemplateAndDocumentValuesRoundTrip(t *testing.T) { + values := encodedJavaCallValues(t, templateAndDocumentCall()) + + tmpl, ok := codeActionParameterValueFromRaw(values[0]).(*microflows.StringTemplateParameterValue) + if !ok || tmpl.TypedTemplate == nil { + t.Fatalf("Subject read back as %T", codeActionParameterValueFromRaw(values[0])) + } + if len(tmpl.TypedTemplate.Arguments) != 2 { + t.Fatalf("arguments read back = %d, want 2 (describe would drop them)", len(tmpl.TypedTemplate.Arguments)) + } + if a := tmpl.TypedTemplate.Arguments[1]; a.Expression != "$Event/Quantity" || a.Type != "Integer" { + t.Errorf("argument 2 = %+v", a) + } + + doc, ok := codeActionParameterValueFromRaw(values[1]).(*microflows.CustomBlobDocumentCodeActionParameterValue) + if !ok || doc.CustomDocument != "MyModule.PaymentReminderAgent" { + t.Errorf("Agent read back as %#v", codeActionParameterValueFromRaw(values[1])) + } +} + +// The document parameter's TYPE: read as a String before, so describe printed +// `Agent: String` and the call builder took it for an expression parameter. +func TestJavaActionParamType_CustomBlobDocumentRoundTrips(t *testing.T) { + g := genCa.NewCustomBlobDocumentParameterType() + g.SetCustomDocumentTypeName("agenteditor.agent") + g.SetCustomDocumentReadableTypeName("Agent") + + got, ok := codeActionParamTypeFromGen(g).(*javaactions.CustomBlobDocumentType) + if !ok { + t.Fatalf("read as %T, want *CustomBlobDocumentType", codeActionParamTypeFromGen(g)) + } + if got.DocumentType != "agenteditor.agent" || got.ReadableName != "Agent" { + t.Errorf("read %+v", got) + } + if s := got.TypeString(); s != "Document(agenteditor.agent)" { + t.Errorf("TypeString = %q", s) + } + + back, ok := codeActionParamTypeToGen(got).(*genCa.CustomBlobDocumentParameterType) + if !ok { + t.Fatalf("written as %T, want *CustomBlobDocumentParameterType (a rewrite would turn it into a String)", + codeActionParamTypeToGen(got)) + } + if back.CustomDocumentTypeName() != "agenteditor.agent" || back.CustomDocumentReadableTypeName() != "Agent" { + t.Errorf("written %q / %q", back.CustomDocumentTypeName(), back.CustomDocumentReadableTypeName()) + } +} diff --git a/mdl/backend/modelsdk/java_read.go b/mdl/backend/modelsdk/java_read.go index 21fd0a8b9..185aaed3a 100644 --- a/mdl/backend/modelsdk/java_read.go +++ b/mdl/backend/modelsdk/java_read.go @@ -190,6 +190,17 @@ func codeActionParamTypeFromGen(el element.Element) javaactions.CodeActionParame e := &javaactions.EntityTypeParameterType{TypeParameterID: model.ID(t.TypeParameterRefID())} e.ID = model.ID(t.ID()) return e + // A document of a custom (extension-defined) type — AgentEditorCommons' + // `Agent` parameter. It fell to the default below and read back as a + // String, so describe printed `Agent: String` and a call bound it as an + // expression: CE0115 (ako/mxcli#1451). + case *genCa.CustomBlobDocumentParameterType: + c := &javaactions.CustomBlobDocumentType{ + DocumentType: t.CustomDocumentTypeName(), + ReadableName: t.CustomDocumentReadableTypeName(), + } + c.ID = model.ID(t.ID()) + return c default: return codeActionBasicFromGen(el) } diff --git a/mdl/backend/modelsdk/java_write.go b/mdl/backend/modelsdk/java_write.go index f1b5bb8cc..488f904ce 100644 --- a/mdl/backend/modelsdk/java_write.go +++ b/mdl/backend/modelsdk/java_write.go @@ -287,6 +287,18 @@ func codeActionParamTypeToGen(t javaactions.CodeActionParameterType) element.Ele assignID(s) s.SetGrammar(v.Grammar) return s + case *javaactions.CustomBlobDocumentType: + // Direct, like StringTemplate: the inverse of codeActionParamTypeFromGen. + // Without it a rewrite of the action turned the document parameter into + // a String, and every call of it failed CE0115 (ako/mxcli#1451). + c := genCa.NewCustomBlobDocumentParameterType() + if v.ID != "" { + c.SetID(element.ID(v.ID)) + } + assignID(c) + c.SetCustomDocumentTypeName(v.DocumentType) + c.SetCustomDocumentReadableTypeName(v.ReadableName) + return c case *javaactions.EntityTypeParameterType: e := genCa.NewEntityTypeParameterType() if v.ID != "" { diff --git a/mdl/backend/modelsdk/microflow_read_actions.go b/mdl/backend/modelsdk/microflow_read_actions.go index e7eddf3e7..ae365b52e 100644 --- a/mdl/backend/modelsdk/microflow_read_actions.go +++ b/mdl/backend/modelsdk/microflow_read_actions.go @@ -757,9 +757,28 @@ func codeActionParameterValueFromRaw(doc bson.Raw) microflows.CodeActionParamete if tt, ok := doc.Lookup("TypedTemplate").DocumentOK(); ok { t := µflows.TypedTemplate{Text: rawStr(tt, "Text")} t.ID = model.ID(rawStr(tt, "$ID")) + if arr, ok := tt.Lookup("Arguments").ArrayOK(); ok { + vals, _ := arr.Values() + for _, av := range vals { + ad, ok := av.DocumentOK() + if !ok { + continue // the list's leading marker + } + a := µflows.TypedTemplateArgument{ + Expression: rawStr(ad, "Expression"), + Type: rawStr(ad, "Type"), + } + a.ID = model.ID(rawStr(ad, "$ID")) + t.Arguments = append(t.Arguments, a) + } + } v.TypedTemplate = t } return v + case "Microflows$CustomBlobDocumentCodeActionParameterValue": + v := µflows.CustomBlobDocumentCodeActionParameterValue{CustomDocument: rawStr(doc, "CustomDocument")} + v.ID = id + return v case "Microflows$ExpressionBasedCodeActionParameterValue": v := µflows.ExpressionBasedCodeActionParameterValue{Expression: rawStr(doc, "Expression")} v.ID = id diff --git a/mdl/backend/modelsdk/microflow_write.go b/mdl/backend/modelsdk/microflow_write.go index b24e203d3..1a290cfbf 100644 --- a/mdl/backend/modelsdk/microflow_write.go +++ b/mdl/backend/modelsdk/microflow_write.go @@ -77,6 +77,7 @@ func init() { codec.RegisterTypeDefaults("Microflows$TypedTemplate", codec.TypeDefaults{ MandatoryListMarkers: map[string]int32{"Arguments": 2}, }) + codec.RegisterListMarker("Microflows$PrimitiveTypedTemplateArgument", 2) // EXECUTE DATABASE QUERY action: both mapping lists empty as marker 2; their // child mapping $Types use marker 2 too. codec.RegisterListMarker("DatabaseConnector$ConnectionParameterMapping", 2) @@ -1100,14 +1101,34 @@ func exportXmlActionToGen(a *microflows.ExportXmlAction) element.Element { func codeActionParameterValueToGen(v microflows.CodeActionParameterValue) element.Element { switch val := v.(type) { case *microflows.StringTemplateParameterValue: + // A text-template (or SQL-template) Java action parameter: the text with + // {n} placeholders plus one PrimitiveTypedTemplateArgument per + // placeholder, each an expression and its result type + // (ako/mxcli#1451; shape from the metamodel, accepted by mxbuild 11.13). g := newElem("Microflows$StringTemplateParameterValue", string(val.ID)) if val.TypedTemplate != nil { tt := newElem("Microflows$TypedTemplate", string(val.TypedTemplate.ID)) - addPartList(tt, "Arguments", nil) // empty (marker 2, registered in init) + var args []element.Element + for _, a := range val.TypedTemplate.Arguments { + if a == nil { + continue + } + ae := newElem("Microflows$PrimitiveTypedTemplateArgument", string(a.ID)) + addStr(ae, "Expression", a.Expression) + addStr(ae, "Type", a.Type) + args = append(args, ae) + } + addPartList(tt, "Arguments", args) // marker 2, registered in init addStr(tt, "Text", val.TypedTemplate.Text) addPart(g, "TypedTemplate", tt) } return g + case *microflows.CustomBlobDocumentCodeActionParameterValue: + // A document of a custom type (an agent-editor Agent), by name + // (ako/mxcli#1451). + g := newElem("Microflows$CustomBlobDocumentCodeActionParameterValue", string(val.ID)) + addStr(g, "CustomDocument", val.CustomDocument) + return g case *microflows.ExpressionBasedCodeActionParameterValue: g := newElem("Microflows$ExpressionBasedCodeActionParameterValue", string(val.ID)) addStr(g, "Expression", val.Expression) diff --git a/mdl/executor/cmd_microflows_builder_calls.go b/mdl/executor/cmd_microflows_builder_calls.go index cf0b3cd39..1eba05c95 100644 --- a/mdl/executor/cmd_microflows_builder_calls.go +++ b/mdl/executor/cmd_microflows_builder_calls.go @@ -315,6 +315,10 @@ func (fb *flowBuilder) addCallJavaActionAction(s *ast.CallJavaActionStmt) model. entityTypeParams := make(map[string]bool) microflowTypeParams := make(map[string]bool) resolvedBasicParams := make(map[string]bool) + // A text template and a custom document take neither an expression nor + // `empty` as Argument text; each has its own value (ako/mxcli#1451). + templateParams := make(map[string]bool) + documentParams := make(map[string]bool) if jaDef != nil { for _, p := range jaDef.Parameters { switch p.ParameterType.(type) { @@ -322,6 +326,10 @@ func (fb *flowBuilder) addCallJavaActionAction(s *ast.CallJavaActionStmt) model. entityTypeParams[p.Name] = true case *javaactions.MicroflowType: microflowTypeParams[p.Name] = true + case *javaactions.StringTemplateParameterType: + templateParams[p.Name] = true + case *javaactions.CustomBlobDocumentType: + documentParams[p.Name] = true default: resolvedBasicParams[p.Name] = true } @@ -337,7 +345,27 @@ func (fb *flowBuilder) addCallJavaActionAction(s *ast.CallJavaActionStmt) model. // Check if this parameter is typed to a type parameter (EntityTypeParameterType) var value microflows.CodeActionParameterValue - if entityTypeParams[arg.Name] { + if len(arg.TemplateParams) > 0 && !templateParams[arg.Name] { + if jaDef == nil { + fb.addError("call java action %s: parameter %s: the action's parameters could not be read, "+ + "so `with (…)` cannot be written as a text template", actionQN, arg.Name) + } else { + fb.addError("%s", notTemplateParamRefusal(actionQN, arg.Name)) + } + } + if templateParams[arg.Name] { + v, msg := fb.javaActionTemplateValue(actionQN, arg) + if msg != "" { + fb.addError("%s", msg) + } + value = v + } else if documentParams[arg.Name] { + v, msg := javaActionDocumentValue(actionQN, arg) + if msg != "" { + fb.addError("%s", msg) + } + value = v + } else if entityTypeParams[arg.Name] { // Entity type parameter: value is the entity qualified name, not the variable reference. // When the argument is a variable like $Email, resolve its entity type from varTypes. value = µflows.EntityTypeCodeActionParameterValue{ diff --git a/mdl/executor/cmd_microflows_builder_java_template_1451_test.go b/mdl/executor/cmd_microflows_builder_java_template_1451_test.go new file mode 100644 index 000000000..1a86d7893 --- /dev/null +++ b/mdl/executor/cmd_microflows_builder_java_template_1451_test.go @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/agenteditor" + "github.com/mendixlabs/mxcli/sdk/domainmodel" + "github.com/mendixlabs/mxcli/sdk/javaactions" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// ako/mxcli#1451 (mendixlabs/mxcli#1554): `call java action` wrote a +// text-template parameter (BusinessEvents.PublishBusinessEvent_V2's Subject, +// a CodeActions$StringTemplateParameterType) and a document parameter +// (AgentEditorCommons' Agent, a CodeActions$CustomBlobDocumentParameterType) +// as Microflows$BasicCodeActionParameterValue expressions. check and exec +// passed; mx check reported CE0115 "The arguments that are passed to Java +// action … do not match the expected parameters and need to be refreshed". + +// templateCallBackend is a project with one Java action taking a text +// template, an object and an agent document, and the entity the microflow's +// $Event holds. +func templateCallBackend() *mock.MockBackend { + mod := mkModule("MyModule") + ent := mkEntity(mod.ID, "OrderShipped") + ent.Attributes = []*domainmodel.Attribute{ + {Name: "OrderNumber", Type: &domainmodel.StringAttributeType{}}, + {Name: "Quantity", Type: &domainmodel.IntegerAttributeType{}}, + {Name: "Price", Type: &domainmodel.DecimalAttributeType{}}, + {Name: "Shipped", Type: &domainmodel.DateTimeAttributeType{}}, + {Name: "Status", Type: &domainmodel.EnumerationAttributeType{EnumerationRef: "MyModule.Status"}}, + } + dm := mkDomainModel(mod.ID, ent) + return &mock.MockBackend{ + GetModuleByNameFunc: func(name string) (*model.Module, error) { + if name == "MyModule" { + return mod, nil + } + return nil, nil + }, + GetDomainModelFunc: func(model.ID) (*domainmodel.DomainModel, error) { return dm, nil }, + ReadJavaActionByNameFunc: func(string) (*javaactions.JavaAction, error) { + return &javaactions.JavaAction{ + Name: "Publish", + Parameters: []*javaactions.JavaActionParameter{ + {Name: "Subject", ParameterType: &javaactions.StringTemplateParameterType{Grammar: "Text"}}, + {Name: "BusinessEvent", ParameterType: &javaactions.EntityType{Entity: "MyModule.OrderShipped"}}, + {Name: "Agent", ParameterType: &javaactions.CustomBlobDocumentType{DocumentType: "agenteditor.agent", ReadableName: "Agent"}}, + }, + }, nil + }, + } +} + +func templateCallBuilder() *flowBuilder { + return &flowBuilder{ + posX: 100, posY: 100, spacing: HorizontalSpacing, + backend: templateCallBackend(), + varTypes: map[string]string{"Event": "MyModule.OrderShipped"}, + declaredVars: map[string]string{"Count": "Integer"}, + } +} + +// buildJavaCall builds the first `call java action` of a microflow body. +func buildJavaCall(t *testing.T, fb *flowBuilder, call string) *microflows.JavaActionCallAction { + t.Helper() + body := parsedFlowBody(t, "create microflow MyModule.MF ($Event: MyModule.OrderShipped)\nbegin\n "+call+"\nend;") + for _, stmt := range body { + s, ok := stmt.(*ast.CallJavaActionStmt) + if !ok { + continue + } + id := fb.addCallJavaActionAction(s) + for _, obj := range fb.objects { + if obj.GetID() == id { + return obj.(*microflows.ActionActivity).Action.(*microflows.JavaActionCallAction) + } + } + } + t.Fatalf("no call java action in %q", call) + return nil +} + +func mappingValue(t *testing.T, a *microflows.JavaActionCallAction, param string) microflows.CodeActionParameterValue { + t.Helper() + for _, m := range a.ParameterMappings { + if strings.HasSuffix(m.Parameter, "."+param) { + return m.Value + } + } + t.Fatalf("no mapping for %s", param) + return nil +} + +func typedTemplate(t *testing.T, v microflows.CodeActionParameterValue) *microflows.TypedTemplate { + t.Helper() + st, ok := v.(*microflows.StringTemplateParameterValue) + if !ok || st.TypedTemplate == nil { + t.Fatalf("value = %#v, want a StringTemplateParameterValue — a BasicCodeActionParameterValue is CE0115", v) + } + return st.TypedTemplate +} + +func TestCallJavaAction_TemplateParameterWithArguments(t *testing.T) { + fb := templateCallBuilder() + a := buildJavaCall(t, fb, `call java action MyModule.Publish( + Subject = 'Order {1} shipped: {2} x {3} on {4}' with ({1} = $Event/OrderNumber, {2} = $Event/Quantity, {3} = $Event/Price, {4} = $Event/Shipped), + BusinessEvent = $Event);`) + if len(fb.errors) > 0 { + t.Fatalf("builder errors: %v", fb.errors) + } + tt := typedTemplate(t, mappingValue(t, a, "Subject")) + if tt.Text != "Order {1} shipped: {2} x {3} on {4}" { + t.Errorf("Text = %q", tt.Text) + } + want := [][2]string{ + {"$Event/OrderNumber", "String"}, {"$Event/Quantity", "Integer"}, + {"$Event/Price", "Decimal"}, {"$Event/Shipped", "DateTime"}, + } + if len(tt.Arguments) != len(want) { + t.Fatalf("arguments = %d, want %d", len(tt.Arguments), len(want)) + } + for i, w := range want { + if got := tt.Arguments[i]; got.Expression != w[0] || got.Type != w[1] { + // The Type is checked by mxbuild against the expression: CE0117. + t.Errorf("{%d} = %q (%s), want %q (%s)", i+1, got.Expression, got.Type, w[0], w[1]) + } + } + // The object parameter next to it keeps its expression. + if b, ok := mappingValue(t, a, "BusinessEvent").(*microflows.BasicCodeActionParameterValue); !ok || b.Argument != "$Event" { + t.Errorf("BusinessEvent = %#v", mappingValue(t, a, "BusinessEvent")) + } +} + +// The issue's own repro: a plain string is the template's text. +func TestCallJavaAction_TemplateParameterPlainString(t *testing.T) { + fb := templateCallBuilder() + a := buildJavaCall(t, fb, `call java action MyModule.Publish(Subject = 'Order shipped', BusinessEvent = $Event);`) + tt := typedTemplate(t, mappingValue(t, a, "Subject")) + if tt.Text != "Order shipped" || len(tt.Arguments) != 0 { + t.Errorf("template = %q with %d arguments, want 'Order shipped' with none", tt.Text, len(tt.Arguments)) + } +} + +// An expression is the template '{1}' with the expression as its argument, +// as a REST call body (ako/mxcli#1444) — not its source text. +func TestCallJavaAction_TemplateParameterExpression(t *testing.T) { + fb := templateCallBuilder() + a := buildJavaCall(t, fb, `call java action MyModule.Publish(Subject = 'Order ' + $Event/OrderNumber, BusinessEvent = $Event);`) + tt := typedTemplate(t, mappingValue(t, a, "Subject")) + if tt.Text != "{1}" || len(tt.Arguments) != 1 { + t.Fatalf("template = %q with %d arguments, want '{1}' with one", tt.Text, len(tt.Arguments)) + } + if got := tt.Arguments[0]; got.Expression != "'Order ' + $Event/OrderNumber" || got.Type != "String" { + t.Errorf("{1} = %+v", got) + } + + a = buildJavaCall(t, templateCallBuilder(), `call java action MyModule.Publish(Subject = $Count, BusinessEvent = $Event);`) + if got := typedTemplate(t, mappingValue(t, a, "Subject")).Arguments[0].Type; got != "Integer" { + t.Errorf("{1} = $Count has Type %q, want Integer (a declared Integer variable)", got) + } + // A date token is a DateTime, not the String exprcheck calls every token + // (measured: mxbuild reports CE0117 for [%CurrentDateTime%] typed String). + a = buildJavaCall(t, templateCallBuilder(), `call java action MyModule.Publish(Subject = 'At {1}' with ({1} = [%CurrentDateTime%]), BusinessEvent = $Event);`) + if got := typedTemplate(t, mappingValue(t, a, "Subject")).Arguments[0].Type; got != "DateTime" { + t.Errorf("{1} = [%%CurrentDateTime%%] has Type %q, want DateTime", got) + } +} + +func TestCallJavaAction_DocumentParameter(t *testing.T) { + for _, arg := range []string{"MyModule.PaymentReminderAgent", "'MyModule.PaymentReminderAgent'"} { + fb := templateCallBuilder() + a := buildJavaCall(t, fb, `call java action MyModule.Publish(Subject = 'x', Agent = `+arg+`);`) + if len(fb.errors) > 0 { + t.Fatalf("Agent = %s: builder errors: %v", arg, fb.errors) + } + v, ok := mappingValue(t, a, "Agent").(*microflows.CustomBlobDocumentCodeActionParameterValue) + if !ok { + t.Fatalf("Agent = %s: value = %#v, want a CustomBlobDocumentCodeActionParameterValue — an expression is CE0115", + arg, mappingValue(t, a, "Agent")) + } + if v.CustomDocument != "MyModule.PaymentReminderAgent" { + t.Errorf("Agent = %s: CustomDocument = %q", arg, v.CustomDocument) + } + } +} + +func TestCallJavaAction_TemplateAndDocumentRefusals(t *testing.T) { + cases := []struct { + name, call, want string + }{ + {"expression with parameters", + `call java action MyModule.Publish(Subject = $Event/OrderNumber with ({1} = $Event/Quantity), BusinessEvent = $Event);`, + "is an expression, so `with (…)` has no template"}, + {"with on a parameter that is no template", + `call java action MyModule.Publish(Subject = 'x', BusinessEvent = 'a {1}' with ({1} = $Event/OrderNumber));`, + "BusinessEvent is not a text template"}, + {"enumeration argument", + `call java action MyModule.Publish(Subject = 'Now {1}' with ({1} = $Event/Status), BusinessEvent = $Event);`, + "is an enumeration"}, + {"gap in the placeholders", + `call java action MyModule.Publish(Subject = 'a {2}' with ({2} = $Event/OrderNumber), BusinessEvent = $Event);`, + "has no {1} argument"}, + {"document given as an expression", + `call java action MyModule.Publish(Subject = 'x', Agent = $Event/OrderNumber);`, + "Agent takes a document"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + fb := templateCallBuilder() + buildJavaCall(t, fb, c.call) + if !strings.Contains(strings.Join(fb.errors, "\n"), c.want) { + t.Errorf("errors = %v, want one containing %q", fb.errors, c.want) + } + }) + } +} + +// describe prints each value in a form that re-executes to the same document. +func TestCallJavaAction_TemplateAndDocumentDescribeRoundTrip(t *testing.T) { + calls := []string{ + `call java action MyModule.Publish(Subject = 'Order {1} shipped: {2}' with ({1} = $Event/OrderNumber, {2} = $Event/Quantity), BusinessEvent = $Event, Agent = 'MyModule.PaymentReminderAgent');`, + `call java action MyModule.Publish(Subject = 'Order shipped', BusinessEvent = $Event);`, + `call java action MyModule.Publish(Subject = $Event/OrderNumber, BusinessEvent = $Event);`, + // '{1}' holding a string literal is not the literal: it stays a template. + `call java action MyModule.Publish(Subject = '{1}' with ({1} = 'x'), BusinessEvent = $Event);`, + } + for _, call := range calls { + ctx, _ := newMockCtx(t) + first := buildJavaCall(t, templateCallBuilder(), call) + described := formatAction(ctx, first, nil, nil) + if described != call { + t.Errorf("describe printed\n %s\nfor\n %s", described, call) + } + second := buildJavaCall(t, templateCallBuilder(), described) + for i, m := range first.ParameterMappings { + a, b := describeValueKey(m.Value), describeValueKey(second.ParameterMappings[i].Value) + if a != b { + t.Errorf("%s: re-executed as %s, built %s", m.Parameter, b, a) + } + } + } +} + +// describeValueKey is a parameter value without its IDs, for comparing two +// builds. +func describeValueKey(v microflows.CodeActionParameterValue) string { + switch x := v.(type) { + case *microflows.StringTemplateParameterValue: + var b strings.Builder + b.WriteString("template " + x.TypedTemplate.Text) + for _, a := range x.TypedTemplate.Arguments { + b.WriteString(" | " + a.Expression + " : " + a.Type) + } + return b.String() + case *microflows.CustomBlobDocumentCodeActionParameterValue: + return "document " + x.CustomDocument + case *microflows.BasicCodeActionParameterValue: + return "basic " + x.Argument + } + return "other" +} + +// check --references gives the builder's verdict, and resolves the document: +// a missing one is CE1613 at build time. +func TestValidate_JavaActionTemplateAndDocumentArguments(t *testing.T) { + mb := templateCallBackend() + mod, _ := mb.GetModuleByNameFunc("MyModule") + mb.IsConnectedFunc = func() bool { return true } + mb.ListModulesFunc = func() ([]*model.Module, error) { return []*model.Module{mod}, nil } + mb.ListJavaActionsFunc = func() ([]*types.JavaAction, error) { + return []*types.JavaAction{{Name: "Publish", ContainerID: mod.ID}}, nil + } + mb.ListAgentEditorAgentsFunc = func() ([]*agenteditor.Agent, error) { + return []*agenteditor.Agent{{ContainerID: mod.ID, Name: "PaymentReminderAgent"}}, nil + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(mkHierarchy(mod))) + + errs := func(call string) string { + body := parsedFlowBody(t, "create microflow MyModule.MF ($Event: MyModule.OrderShipped)\nbegin\n "+call+"\nend;") + return strings.Join(validateFlowBodyReferences(ctx, body, newScriptContext()), "\n") + } + if got := errs(`call java action MyModule.Publish(Subject = 'Order {1}' with ({1} = $Event/OrderNumber), BusinessEvent = $Event, Agent = MyModule.PaymentReminderAgent);`); got != "" { + t.Errorf("valid call: unexpected reference errors: %s", got) + } + if got := errs(`call java action MyModule.Publish(Subject = 'x', Agent = MyModule.NoSuchAgent);`); !strings.Contains(got, "MyModule.NoSuchAgent") { + t.Errorf("missing agent: want a reference error naming it, got %q", got) + } + if got := errs(`call java action MyModule.Publish(Subject = $Event/OrderNumber with ({1} = 1));`); !strings.Contains(got, "is an expression") { + t.Errorf("expression with parameters: got %q", got) + } + if got := errs(`call java action MyModule.Publish(BusinessEvent = 'a {1}' with ({1} = 1));`); !strings.Contains(got, "not a text template") { + t.Errorf("with on an object parameter: got %q", got) + } +} diff --git a/mdl/executor/cmd_microflows_format_action.go b/mdl/executor/cmd_microflows_format_action.go index 920d3e0ce..812497a89 100644 --- a/mdl/executor/cmd_microflows_format_action.go +++ b/mdl/executor/cmd_microflows_format_action.go @@ -853,8 +853,12 @@ func formatAction( valueStr := "" switch v := pm.Value.(type) { case *microflows.StringTemplateParameterValue: - if v.TypedTemplate != nil { - valueStr = mdlQuote(ctx, v.TypedTemplate.Text) + valueStr = describeTypedTemplate(ctx, v.TypedTemplate) + case *microflows.CustomBlobDocumentCodeActionParameterValue: + if v.CustomDocument != "" { + valueStr = mdlQuote(ctx, v.CustomDocument) + } else { + valueStr = "empty" } case *microflows.ExpressionBasedCodeActionParameterValue: valueStr = describeExpr(ctx, v.Expression) @@ -2416,6 +2420,29 @@ func mdlAggregateKeyword(fn microflows.AggregateFunction) (string, bool) { } } +// describeTypedTemplate renders a Java action's text-template argument +// (ako/mxcli#1451): the text, with its arguments as ` with ({1} = …)`. The +// template '{1}' holding one expression is that expression — what the builder +// writes for one — unless the expression is itself a string literal, which +// would read back as the template's text. +func describeTypedTemplate(ctx *ExecContext, tt *microflows.TypedTemplate) string { + if tt == nil { + return "''" + } + args := make([]string, 0, len(tt.Arguments)) + for _, a := range tt.Arguments { + if a != nil { + args = append(args, a.Expression) + } + } + if tt.Text == "{1}" && len(args) == 1 { + if _, isText := singleStringLiteral(args[0]); !isText { + return describeExpr(ctx, args[0]) + } + } + return mdlQuote(ctx, tt.Text) + templateArgsClause(ctx, args) +} + // templateArgsClause renders a text template's arguments as ` with ({1} = a, // {2} = b)` — R4's one text-template form (ako/mxcli#751) — or "" when there // are none. diff --git a/mdl/executor/flow_built_match.go b/mdl/executor/flow_built_match.go index d968c04a0..3aa549729 100644 --- a/mdl/executor/flow_built_match.go +++ b/mdl/executor/flow_built_match.go @@ -316,7 +316,7 @@ var expressionFields = map[reflect.Type]map[string]bool{ reflect.TypeOf(microflows.NanoflowCallParameterMapping{}): {"Argument": true}, reflect.TypeOf(microflows.BasicCodeActionParameterValue{}): {"Argument": true}, reflect.TypeOf(microflows.ExpressionBasedCodeActionParameterValue{}): {"Expression": true}, - reflect.TypeOf(microflows.TypedTemplate{}): {"Arguments": true}, + reflect.TypeOf(microflows.TypedTemplateArgument{}): {"Expression": true}, reflect.TypeOf(microflows.ShowMessageAction{}): {"TemplateParameters": true}, reflect.TypeOf(microflows.ValidationFeedbackAction{}): {"TemplateParameters": true}, reflect.TypeOf(microflows.LogMessageAction{}): {"TemplateParameters": true}, diff --git a/mdl/executor/java_action_call_values.go b/mdl/executor/java_action_call_values.go new file mode 100644 index 000000000..acf03f5de --- /dev/null +++ b/mdl/executor/java_action_call_values.go @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: Apache-2.0 + +// The values of a `call java action` parameter that is not an expression. +// +// A Java action parameter is mostly bound to an expression +// (Microflows$BasicCodeActionParameterValue). Two parameter types take +// something else, and mxcli wrote an expression for both, which mxbuild +// rejects with CE0115 "The arguments that are passed to Java action … do not +// match the expected parameters" (ako/mxcli#1451, mendixlabs/mxcli#1554): +// +// - a text template, CodeActions$StringTemplateParameterType +// (BusinessEvents.PublishBusinessEvent_V2's Subject), stores a +// Microflows$StringTemplateParameterValue: a TypedTemplate with the text and +// one Microflows$PrimitiveTypedTemplateArgument per {n} placeholder, each +// an expression and the type of its result. MDL spells it the way log and +// show message spell a template: +// +// Subject = 'Order {1} shipped' with ({1} = $Order/Number) +// +// A string literal is the text; any other expression is the template '{1}' +// with the expression as its one argument, as for a REST call body +// (restCallTemplate, ako/mxcli#1444). +// +// - a document of a custom type, CodeActions$CustomBlobDocumentParameterType +// (AgentEditorCommons' Agent), stores a +// Microflows$CustomBlobDocumentCodeActionParameterValue naming the document. +// Like a Microflow parameter (#1210) it takes a name, never an expression. +// +// No Studio Pro-authored document of either value exists in the repo's fixtures +// or on this machine; the shapes are the metamodel's (generated/metamodel, +// Microflows$TypedTemplate / PrimitiveTypedTemplateArgument / +// CustomBlobDocumentCodeActionParameterValue), and were measured against +// mxbuild 11.13: it accepts both, reports CE0117 when an argument's Type is not +// its expression's type (an absent Type reads as String, and an enumeration is +// accepted as none of the five), CE0720 for a placeholder with no argument, +// CE0115 for a document of the wrong type and CE1613 for a missing one. +package executor + +import ( + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/exprcheck" + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// javaActionTemplateValue builds the value of a text-template parameter from +// its argument. The second result is why the argument cannot be written, or "". +func (fb *flowBuilder) javaActionTemplateValue(actionQN string, arg ast.CallArgument) (*microflows.StringTemplateParameterValue, string) { + value := µflows.StringTemplateParameterValue{ + BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, + TypedTemplate: µflows.TypedTemplate{BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}}, + } + if isEmptyJavaActionArgument(arg.Value) && len(arg.TemplateParams) == 0 { + return value, "" // an empty template: no text, no arguments + } + if msg := templateArgumentShapeRefusal(actionQN, arg); msg != "" { + return value, msg + } + text, params := restCallTemplate(javaActionArgumentValue(arg.Value), arg.TemplateParams) + s, _ := restTemplateText(text) + value.TypedTemplate.Text = s + for i, p := range orderedTemplateParams(params) { + if p == nil { + return value, fmt.Sprintf("call java action %s: parameter %s: the template has no {%d} argument "+ + "in with (…); number the arguments {1}, {2}, … without gaps", actionQN, arg.Name, i+1) + } + typ, problem := fb.templateArgumentType(p.Value) + if problem != "" { + return value, fmt.Sprintf("call java action %s: parameter %s: {%d} = %s %s", + actionQN, arg.Name, i+1, fb.exprToString(p.Value), problem) + } + value.TypedTemplate.Arguments = append(value.TypedTemplate.Arguments, µflows.TypedTemplateArgument{ + BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, + Expression: fb.exprToString(p.Value), + Type: typ, + }) + } + return value, "" +} + +// templateArgumentShapeRefusal returns why a text-template argument cannot be +// written, judged from the statement alone, or "": an expression that also +// states `with (…)` has no {n} to fill. The flow builder refuses the call with +// it and reference validation reports it, so the two cannot disagree. +func templateArgumentShapeRefusal(actionQN string, arg ast.CallArgument) string { + if len(arg.TemplateParams) == 0 { + return "" + } + if _, ok := restTemplateText(javaActionArgumentValue(arg.Value)); ok { + return "" + } + return fmt.Sprintf("call java action %s: parameter %s is an expression, so `with (…)` has no template to "+ + "fill — the arguments fill the {n} placeholders of a text written as a string. Write %s = '…{1}…' "+ + "with ({1} = …), or %s = without `with (…)`", actionQN, arg.Name, arg.Name, arg.Name) +} + +// notTemplateParamRefusal is the refusal for `with (…)` on a parameter that is +// not a text template. +func notTemplateParamRefusal(actionQN, param string) string { + return fmt.Sprintf("call java action %s: parameter %s is not a text template, so `with (…)` has nothing "+ + "to fill; bind it to an expression", actionQN, param) +} + +// orderedTemplateParams lays params out by placeholder index: element i is +// {i+1}, nil where no parameter states it. +func orderedTemplateParams(params []ast.TemplateParam) []*ast.TemplateParam { + maxIndex := 0 + for _, p := range params { + if p.Index > maxIndex { + maxIndex = p.Index + } + } + out := make([]*ast.TemplateParam, maxIndex) + for i := range params { + if p := ¶ms[i]; p.Index > 0 { + out[p.Index-1] = p + } + } + return out +} + +// templateArgumentType returns the Type a typed-template argument stores for +// expr — the type of its result, one of the five a typed template takes — or, +// as the second result, why expr cannot be an argument. mxbuild checks the +// expression against the stored Type (CE0117), so a guess is not good enough +// where the type is known; where it cannot be told it is String, which is +// also what an absent Type means. +func (fb *flowBuilder) templateArgumentType(expr ast.Expression) (string, string) { + src := fb.exprToString(expr) + kinds := flowExprTypes{fb: fb} + kind := exprcheck.InferArgumentKind(src, exprcheck.Context{Scope: kinds, Entities: kinds, Catalog: kinds}) + switch kind { + case exprcheck.KindInteger, exprcheck.KindLong: + return "Integer", "" + case exprcheck.KindDecimal: + return "Decimal", "" + case exprcheck.KindBoolean: + return "Boolean", "" + case exprcheck.KindDateTime: + return "DateTime", "" + case exprcheck.KindEnumeration: + return "", "is an enumeration, which a text template cannot take — " + + "pass getKey(…) or getCaption(…) of it" + case exprcheck.KindObject, exprcheck.KindList, exprcheck.KindBinary: + return "", "is not a String, Integer, Decimal, Boolean or DateTime value, " + + "the five a text template takes" + } + return "String", "" +} + +// flowExprTypes types a flow expression from what the flow builder knows: its +// declared primitive variables, its object variables, and the project's +// domain models through the backend. It is exprcheck's Scope, EntityScope and +// CatalogReader at once; what it cannot answer stays unknown. +type flowExprTypes struct{ fb *flowBuilder } + +var ( + _ exprcheck.Scope = flowExprTypes{} + _ exprcheck.EntityScope = flowExprTypes{} + _ exprcheck.CatalogReader = flowExprTypes{} +) + +func (t flowExprTypes) Lookup(name string) (exprcheck.TypeKind, bool) { + typ, ok := t.fb.declaredVars[strings.TrimPrefix(name, "$")] + if !ok { + return exprcheck.KindUnknown, false + } + switch typ { + case "String", "HashedString", "StringTemplate": + return exprcheck.KindString, true + case "Integer", "AutoNumber": + return exprcheck.KindInteger, true + case "Long": + return exprcheck.KindLong, true + case "Decimal": + return exprcheck.KindDecimal, true + case "Boolean": + return exprcheck.KindBoolean, true + case "DateTime", "Date": + return exprcheck.KindDateTime, true + case "Binary": + return exprcheck.KindBinary, true + case "Enumeration": + return exprcheck.KindEnumeration, true + } + return exprcheck.KindUnknown, false +} + +func (t flowExprTypes) VariableEntity(name string) (string, bool) { + typ := t.fb.varTypes[strings.TrimPrefix(name, "$")] + if typ == "" || strings.HasPrefix(typ, "List of ") { + return "", false + } + return typ, true +} + +func (t flowExprTypes) AssociationTarget(assocQN, fromEntityQN string) (string, bool) { + module, name, ok := strings.Cut(assocQN, ".") + if !ok { + return "", false + } + info := t.fb.lookupAssociation(module, name) + if info == nil { + return "", false + } + if info.childEntityQN == fromEntityQN { + return info.parentEntityQN, info.parentEntityQN != "" + } + return info.childEntityQN, info.childEntityQN != "" +} + +func (t flowExprTypes) AttributeKind(entityQN, attrName string) (exprcheck.TypeKind, bool) { + attr := t.fb.findAttribute(entityQN, attrName) + if attr == nil { + return exprcheck.KindUnknown, false + } + switch attr.Type.(type) { + case *domainmodel.StringAttributeType, *domainmodel.HashedStringAttributeType: + return exprcheck.KindString, true + case *domainmodel.IntegerAttributeType, *domainmodel.AutoNumberAttributeType: + return exprcheck.KindInteger, true + case *domainmodel.LongAttributeType: + return exprcheck.KindLong, true + case *domainmodel.DecimalAttributeType: + return exprcheck.KindDecimal, true + case *domainmodel.BooleanAttributeType: + return exprcheck.KindBoolean, true + case *domainmodel.DateTimeAttributeType, *domainmodel.DateAttributeType: + return exprcheck.KindDateTime, true + case *domainmodel.EnumerationAttributeType: + return exprcheck.KindEnumeration, true + case *domainmodel.BinaryAttributeType: + return exprcheck.KindBinary, true + } + return exprcheck.KindUnknown, false +} + +func (t flowExprTypes) AttributeEnumQN(entityQN, attrName string) (string, bool) { + if attr := t.fb.findAttribute(entityQN, attrName); attr != nil { + if et, ok := attr.Type.(*domainmodel.EnumerationAttributeType); ok { + return et.EnumerationRef, et.EnumerationRef != "" + } + } + return "", false +} + +func (flowExprTypes) EnumCases(string) ([]string, bool) { return nil, false } +func (flowExprTypes) MicroflowReturn(string) (exprcheck.TypeKind, bool) { + return exprcheck.KindUnknown, false +} +func (flowExprTypes) MicroflowParam(string, string) (exprcheck.TypeKind, bool) { + return exprcheck.KindUnknown, false +} + +// findAttribute finds attrName on entityQN or on one of its generalizations. +func (fb *flowBuilder) findAttribute(entityQN, attrName string) *domainmodel.Attribute { + if fb == nil || fb.backend == nil { + return nil + } + seen := map[string]bool{} + for current := entityQN; current != "" && !seen[current]; { + seen[current] = true + module, name, ok := strings.Cut(current, ".") + if !ok { + return nil + } + mod, err := fb.backend.GetModuleByName(module) + if err != nil || mod == nil { + return nil + } + dm, err := fb.backend.GetDomainModel(mod.ID) + if err != nil || dm == nil { + return nil + } + entity := dm.FindEntityByName(name) + if entity == nil { + return nil + } + for _, attr := range entity.Attributes { + if attr != nil && attr.Name == attrName { + return attr + } + } + current = entity.GeneralizationRef + } + return nil +} + +// javaActionDocumentValue builds the value of a custom-document parameter: +// the document's qualified name. `empty` leaves it unset. +func javaActionDocumentValue(actionQN string, arg ast.CallArgument) (*microflows.CustomBlobDocumentCodeActionParameterValue, string) { + value := µflows.CustomBlobDocumentCodeActionParameterValue{ + BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, + } + if isEmptyJavaActionArgument(arg.Value) { + return value, "" + } + name, msg := documentParamArgName(actionQN, arg.Name, arg.Value) + value.CustomDocument = name + return value, msg +} + +// documentParamArgName returns the document a custom-document parameter's +// argument names — `Module.Doc` or 'Module.Doc' — or why it names none. The +// flow builder refuses the call with it and reference validation reports it. +func documentParamArgName(action, param string, v ast.Expression) (string, string) { + v = javaActionArgumentValue(v) + switch e := v.(type) { + case *ast.QualifiedNameExpr: + return e.QualifiedName.String(), "" + case *ast.LiteralExpr: + if s, ok := e.Value.(string); ok && e.Kind == ast.LiteralString && microflowIdentifierRe.MatchString(s) { + return s, "" + } + } + return "", fmt.Sprintf("call java action %s: parameter %s takes a document, named as Module.Document, "+ + "not %s — Mendix stores a reference to the document, so there is no expression form", + action, param, expressionToString(v)) +} + +// customDocumentExists reports whether module.name is a document of the custom +// type docType, and whether that could be judged at all. +func customDocumentExists(ctx *ExecContext, docType, qn string) (exists, judged bool) { + module, name, ok := strings.Cut(qn, ".") + if !ok { + return false, false + } + switch docType { + case "agenteditor.agent": + return findAgentEditorAgent(ctx, module, name) != nil, true + case "agenteditor.model": + return findAgentEditorModel(ctx, module, name) != nil, true + case "agenteditor.knowledgebase": + return findAgentEditorKnowledgeBase(ctx, module, name) != nil, true + case "agenteditor.consumedMCPService": + return findAgentEditorConsumedMCPService(ctx, module, name) != nil, true + } + return false, false +} + +// templateTypedParams names the parameters a CREATE JAVA ACTION declares as a +// text template (`StringTemplate(Text)` / `StringTemplate(Sql)`). +func templateTypedParams(params []ast.JavaActionParam) map[string]bool { + out := map[string]bool{} + for _, p := range params { + if p.Type.Kind == ast.TypeStringTemplate { + out[p.Name] = true + } + } + return out +} + +// javaActionCallValueErrors is reference validation's copy of the flow +// builder's refusals for one call's text-template and document arguments, +// given which of the action's parameters are templates and which documents +// (name -> CustomDocumentTypeName). A document argument is also resolved: a +// missing one is CE1613 at build time, one of another type CE0115. Documents +// the script creates count as present. +func javaActionCallValueErrors(ctx *ExecContext, sc *scriptContext, ref codeActionCallRef, + templates map[string]bool, documents map[string]string) []string { + var out []string + for _, a := range ref.args { + switch { + case templates[a.Name]: + if msg := templateArgumentShapeRefusal(ref.name, a); msg != "" { + out = append(out, msg) + } + case documents[a.Name] != "": + if isEmptyJavaActionArgument(a.Value) { + continue + } + name, msg := documentParamArgName(ref.name, a.Name, a.Value) + if msg != "" { + out = append(out, msg) + continue + } + docType := documents[a.Name] + if sc != nil && sc.customDocuments[docType+" "+name] { + continue + } + if exists, judged := customDocumentExists(ctx, docType, name); judged && !exists { + out = append(out, fmt.Sprintf("call java action %s: parameter %s: there is no %s document %s "+ + "(a missing document is CE1613 at build time, one of another type CE0115)", + ref.name, a.Name, docType, name)) + } + default: + if len(a.TemplateParams) > 0 { + out = append(out, notTemplateParamRefusal(ref.name, a.Name)) + } + } + } + return out +} + +// checkJavaActionTemplateArgs is MDL-JA02: an argument written as an +// expression followed by `with (…)`. Whatever the parameter is, that cannot be +// written — a text template takes the parameters on a string, and any other +// parameter takes none — so it needs no project to judge; the builder refuses +// it with the same function. +func (v *microflowValidator) checkJavaActionTemplateArgs(stmt *ast.CallJavaActionStmt) { + for _, arg := range stmt.Arguments { + if msg := templateArgumentShapeRefusal(stmt.ActionName.String(), arg); msg != "" { + v.addViolation("MDL-JA02", linter.SeverityError, msg, + "See `mxcli syntax microflow.call` for text-template arguments") + } + } +} diff --git a/mdl/executor/validate.go b/mdl/executor/validate.go index d297bb4d0..ce5ab2d6c 100644 --- a/mdl/executor/validate.go +++ b/mdl/executor/validate.go @@ -63,6 +63,12 @@ type scriptContext struct { // (Module.Action -> parameter name). Such a parameter takes a microflow // NAME, never an expression (mendixlabs/mxcli#1210). javaActionFlowParams map[string]map[string]bool + // Text-template parameters of the Java actions the script creates + // (Module.Action -> parameter name), and the agent-editor documents it + // creates (CustomDocumentTypeName + " " + Module.Name): what a call's + // template and document arguments are checked against (ako/mxcli#1451). + javaActionTemplateParams map[string]map[string]bool + customDocuments map[string]bool // Entities the script declares, and those it alters, for resolving the // bare members of a retrieve constraint on an entity the project does not @@ -145,9 +151,11 @@ func newScriptContext() *scriptContext { imageCollections: make(map[string]bool), images: make(map[string]bool), - javaActionFlowParams: make(map[string]map[string]bool), - entityDecls: make(map[string]*ast.CreateEntityStmt), - alteredEntities: make(map[string]bool), + javaActionFlowParams: make(map[string]map[string]bool), + javaActionTemplateParams: make(map[string]map[string]bool), + customDocuments: make(map[string]bool), + entityDecls: make(map[string]*ast.CreateEntityStmt), + alteredEntities: make(map[string]bool), javaActions: make(map[string][]string), javaScriptActions: make(map[string][]string), @@ -347,7 +355,16 @@ func (sc *scriptContext) collectSingle(stmt ast.Statement) { if s.Name.Module != "" { sc.javaActions[s.Name.String()] = codeActionParamNames(s.Parameters) sc.javaActionFlowParams[s.Name.String()] = microflowTypedParams(s.Parameters) - } + sc.javaActionTemplateParams[s.Name.String()] = templateTypedParams(s.Parameters) + } + case *ast.CreateAgentStmt: + sc.customDocuments["agenteditor.agent "+s.Name.String()] = true + case *ast.CreateModelStmt: + sc.customDocuments["agenteditor.model "+s.Name.String()] = true + case *ast.CreateKnowledgeBaseStmt: + sc.customDocuments["agenteditor.knowledgebase "+s.Name.String()] = true + case *ast.CreateConsumedMCPServiceStmt: + sc.customDocuments["agenteditor.consumedMCPService "+s.Name.String()] = true case *ast.CreateQueueStmt: if s.Name.Module != "" { sc.queues[strings.ToLower(s.Name.String())] = true @@ -1516,6 +1533,7 @@ func validateFlowBodyReferences(ctx *ExecContext, body []ast.MicroflowStatement, if declared, inScript := sc.javaActions[ref.name]; inScript { errors = append(errors, validateCodeActionParams("java action", ref, declared)...) errors = append(errors, microflowParamArgErrors(ref, sc.javaActionFlowParams[ref.name])...) + errors = append(errors, javaActionCallValueErrors(ctx, sc, ref, sc.javaActionTemplateParams[ref.name], nil)...) continue } if !known[ref.name] { @@ -1525,14 +1543,22 @@ func validateFlowBodyReferences(ctx *ExecContext, body []ast.MicroflowStatement, if ja, err := ctx.Backend.ReadJavaActionByName(ref.name); err == nil && ja != nil { declared := []string{} // read: an action with no parameters is known to have none flowParams := map[string]bool{} + templates := map[string]bool{} + documents := map[string]string{} for _, p := range ja.Parameters { declared = append(declared, p.Name) - if _, isFlow := p.ParameterType.(*javaactions.MicroflowType); isFlow { + switch pt := p.ParameterType.(type) { + case *javaactions.MicroflowType: flowParams[p.Name] = true + case *javaactions.StringTemplateParameterType: + templates[p.Name] = true + case *javaactions.CustomBlobDocumentType: + documents[p.Name] = pt.DocumentType } } errors = append(errors, validateCodeActionParams("java action", ref, declared)...) errors = append(errors, microflowParamArgErrors(ref, flowParams)...) + errors = append(errors, javaActionCallValueErrors(ctx, sc, ref, templates, documents)...) } } } diff --git a/mdl/executor/validate_microflow.go b/mdl/executor/validate_microflow.go index 281feea6e..e50c5d5ff 100644 --- a/mdl/executor/validate_microflow.go +++ b/mdl/executor/validate_microflow.go @@ -448,6 +448,10 @@ func (v *microflowValidator) walkBody(body []ast.MicroflowStatement) { // ako/mxcli#1444: an expression URL or body with `with (…)`. // Same function exec calls. v.checkRestCallTemplates(stmt) + case *ast.CallJavaActionStmt: + // ako/mxcli#1451: an expression argument with `with (…)`. Same + // function exec calls. + v.checkJavaActionTemplateArgs(stmt) case *ast.CallWebServiceStmt: // MDL-SOAP01: a call stores ONE request body, so arguments and a send // mapping are alternatives. Same function exec calls. diff --git a/mdl/exprcheck/infer.go b/mdl/exprcheck/infer.go index 5d04cbfe9..d42b1a1c4 100644 --- a/mdl/exprcheck/infer.go +++ b/mdl/exprcheck/infer.go @@ -27,6 +27,22 @@ func InferSourceKind(src string, vars map[string]TypeKind) TypeKind { return inferKind(root, ctx) } +// InferArgumentKind types src the way a built-in function's argument is typed +// (argKind): with a full Context — an EntityScope and a CatalogReader type +// attribute paths (`$Order/Number`), which a bare variable scope cannot — and +// with [%CurrentDateTime%] and the [%BeginOf…%]/[%EndOf…%] tokens as the +// DateTimes they are. Anything it cannot type is KindUnknown. +func InferArgumentKind(src string, ctx Context) TypeKind { + if strings.TrimSpace(src) == "" { + return KindUnknown + } + root, _ := (&parserImpl{}).Parse(src, ctx) + if root == nil { + return KindUnknown + } + return argKind(root, ctx) +} + // BuiltinCallHints parses src and returns the built-in signature hints — E006 // (wrong argument count) and E005 (an argument of the wrong kind) — for every // built-in call in it, typing variables from vars (names without the '$'). diff --git a/mdl/grammar/domains/MDLMicroflow.g4 b/mdl/grammar/domains/MDLMicroflow.g4 index 866dcfdcc..a2a437100 100644 --- a/mdl/grammar/domains/MDLMicroflow.g4 +++ b/mdl/grammar/domains/MDLMicroflow.g4 @@ -754,8 +754,13 @@ callArgumentList // which writes no mapping at all. Only call microflow / call nanoflow take it // (the visitor refuses it elsewhere). NOTHING is listed before expression so // the keyword wins over reading `nothing` as a name. +// +// `Param = 'Order {1} shipped' with ({1} = $Order/Number)` fills a Java +// action's text-template parameter (StringTemplate(Text|Sql)) — the `with` +// form of log and show message. Only call java action takes it (the visitor +// refuses it elsewhere; ako/mxcli#1451). callArgument - : (VARIABLE /* @alias MDL-DEPR006 */ | parameterName) EQUALS (NOTHING | expression) + : (VARIABLE /* @alias MDL-DEPR006 */ | parameterName) EQUALS (NOTHING | expression templateParams?) | parameterName COLON /* @alias MDL-DEPR007 */ expression | expression // positional: refused by the visitor ; diff --git a/mdl/types/javaaction_types.go b/mdl/types/javaaction_types.go index d566b73d1..6a5159a56 100644 --- a/mdl/types/javaaction_types.go +++ b/mdl/types/javaaction_types.go @@ -204,6 +204,23 @@ func (s StringTemplateParameterType) TypeString() string { return "StringTemplate" } +// CustomBlobDocumentType is a parameter whose value is a document of a custom +// (extension-defined) document type — CodeActions$CustomBlobDocumentParameterType, +// e.g. AgentEditorCommons' `Agent` parameter, of type "agenteditor.agent". A +// call passes the document by name, never an expression. Read as a String, it +// described as `String` and a call bound it as an expression (CE0115, +// ako/mxcli#1451). +type CustomBlobDocumentType struct { + model.BaseElement + DocumentType string `json:"documentType,omitempty"` // CustomDocumentTypeName, e.g. "agenteditor.agent" + ReadableName string `json:"readableName,omitempty"` // CustomDocumentReadableTypeName, e.g. "Agent" +} + +func (CustomBlobDocumentType) isCodeActionParameterType() {} +func (c CustomBlobDocumentType) TypeString() string { + return "Document(" + c.DocumentType + ")" +} + // FileDocumentType represents a file document type. type FileDocumentType struct { model.BaseElement diff --git a/mdl/visitor/visitor_argument_binding.go b/mdl/visitor/visitor_argument_binding.go index 847696f9a..32309f232 100644 --- a/mdl/visitor/visitor_argument_binding.go +++ b/mdl/visitor/visitor_argument_binding.go @@ -44,9 +44,27 @@ func (b *Builder) ExitCallArgument(ctx *parser.CallArgumentContext) { "call nanoflow; bind the parameter to a value, or to `empty`", tok.GetLine(), tok.GetColumn())) } + if tp := ctx.TemplateParams(); tp != nil && !isJavaActionCallArgument(ctx) { + tok := tp.GetStart() + b.addError(fmt.Errorf("line %d:%d `with ({1} = …)` fills a text-template parameter, which only "+ + "call java action has; bind this parameter to an expression", + tok.GetLine(), tok.GetColumn())) + } b.recordDollarArgument(ctx.VARIABLE()) } +// isJavaActionCallArgument reports whether a call argument belongs to a call +// java action — the one call whose parameters can be text templates +// (StringTemplate(Text|Sql)), filled with `with ({1} = …)` (ako/mxcli#1451). +func isJavaActionCallArgument(ctx *parser.CallArgumentContext) bool { + list := ctx.GetParent() + if list == nil { + return false + } + _, ok := list.GetParent().(*parser.CallJavaActionStatementContext) + return ok +} + // isFlowCallArgument reports whether a call argument belongs to a call // microflow or call nanoflow — the two calls whose parameter mappings store a // bare expression, where a blank one (`Argument: ""`) is a state Studio Pro diff --git a/mdl/visitor/visitor_java_action_template_1451_test.go b/mdl/visitor/visitor_java_action_template_1451_test.go new file mode 100644 index 000000000..0c670232c --- /dev/null +++ b/mdl/visitor/visitor_java_action_template_1451_test.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// ako/mxcli#1451: a Java action's text-template parameter is filled the way +// log and show message fill a template — `Subject = 'Order {1}' with ({1} = +// $Order/Number)`. The `with` belongs to that one argument. +func TestCallJavaActionTemplateArgument(t *testing.T) { + prog, errs := Build(`create microflow M.F ($O: M.E) +begin + call java action M.J(Subject = 'Order {1} of {2}' with ({1} = $O/Number, {2} = $O/Total), Event = $O); +end;`) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + call := prog.Statements[0].(*ast.CreateMicroflowStmt).Body[0].(*ast.CallJavaActionStmt) + if len(call.Arguments) != 2 { + t.Fatalf("arguments = %d, want 2", len(call.Arguments)) + } + subject := call.Arguments[0] + if subject.Name != "Subject" || len(subject.TemplateParams) != 2 { + t.Fatalf("Subject = %+v, want two template parameters", subject) + } + if lit, ok := subject.Value.(*ast.LiteralExpr); !ok || lit.Value != "Order {1} of {2}" { + t.Errorf("Subject text = %#v", subject.Value) + } + if p := subject.TemplateParams[1]; p.Index != 2 || p.Value == nil { + t.Errorf("{2} = %+v", p) + } + if len(call.Arguments[1].TemplateParams) != 0 { + t.Errorf("Event took the template's parameters: %+v", call.Arguments[1]) + } +} + +// Only a Java action has text-template parameters. +func TestTemplateArgumentRefusedOutsideCallJavaAction(t *testing.T) { + for _, call := range []string{ + "call microflow M.G(Text = 'a {1}' with ({1} = $O));", + "call javascript action M.JS(Text = 'a {1}' with ({1} = $O));", + } { + _, errs := Build("create microflow M.F ($O: M.E) begin " + call + " end;") + var msgs []string + for _, e := range errs { + msgs = append(msgs, e.Error()) + } + if !strings.Contains(strings.Join(msgs, "\n"), "only call java action has") { + t.Errorf("%s: errors = %v, want the text-template refusal", call, msgs) + } + } +} diff --git a/mdl/visitor/visitor_microflow_actions.go b/mdl/visitor/visitor_microflow_actions.go index b53f19a6b..67e9f4349 100644 --- a/mdl/visitor/visitor_microflow_actions.go +++ b/mdl/visitor/visitor_microflow_actions.go @@ -734,7 +734,14 @@ func buildCallArgumentList(ctx parser.ICallArgumentListContext) []ast.CallArgume ca.Value = &ast.BlankExpr{} } else if expr := arg.Expression(); expr != nil { value := buildSourceExpression(expr) - ca.Value = appendExpressionListTrailingWhitespace(listCtx, nextParserRuleContext(allArgs, i), expr, value) + if tp := arg.TemplateParams(); tp != nil { + // The template text ends at `with`, so there is no trailing + // whitespace before `)` to keep. + ca.Value = value + ca.TemplateParams = buildTemplateParams(tp) + } else { + ca.Value = appendExpressionListTrailingWhitespace(listCtx, nextParserRuleContext(allArgs, i), expr, value) + } } args = append(args, ca) diff --git a/sdk/javaactions/javaactions.go b/sdk/javaactions/javaactions.go index 114128d5c..3089cda53 100644 --- a/sdk/javaactions/javaactions.go +++ b/sdk/javaactions/javaactions.go @@ -78,6 +78,7 @@ type ( EntityType = types.EntityType ListType = types.ListType StringTemplateParameterType = types.StringTemplateParameterType + CustomBlobDocumentType = types.CustomBlobDocumentType FileDocumentType = types.FileDocumentType EnumerationType = types.EnumerationType MicroflowType = types.MicroflowType diff --git a/sdk/microflows/microflows_actions.go b/sdk/microflows/microflows_actions.go index b833362ff..f752c725f 100644 --- a/sdk/microflows/microflows_actions.go +++ b/sdk/microflows/microflows_actions.go @@ -812,13 +812,38 @@ type StringTemplateParameterValue struct { func (StringTemplateParameterValue) isCodeActionParameterValue() {} -// TypedTemplate represents a typed template with arguments. +// TypedTemplate represents a typed template with arguments: the value of a +// Java action parameter of type StringTemplate (Text or Sql). Text holds {1}, +// {2}, … placeholders, one argument per placeholder. type TypedTemplate struct { model.BaseElement - Arguments []string `json:"arguments,omitempty"` - Text string `json:"text,omitempty"` + Arguments []*TypedTemplateArgument `json:"arguments,omitempty"` + Text string `json:"text,omitempty"` } +// TypedTemplateArgument is one {n} argument of a TypedTemplate +// (Microflows$PrimitiveTypedTemplateArgument): an expression and the type of +// its result, one of String, Integer, Decimal, Boolean, DateTime. mxbuild +// checks the expression against Type (CE0117 on a mismatch, and an absent Type +// reads as String); the runtime hands Type to the Java action with the value +// (ITemplateParameter.getParameterType). +type TypedTemplateArgument struct { + model.BaseElement + Expression string `json:"expression,omitempty"` + Type string `json:"type,omitempty"` +} + +// CustomBlobDocumentCodeActionParameterValue is the document passed to a Java +// action parameter of a custom document type (CodeActions$ +// CustomBlobDocumentParameterType, e.g. an agent-editor Agent): a by-name +// reference to the document, never an expression. +type CustomBlobDocumentCodeActionParameterValue struct { + model.BaseElement + CustomDocument string `json:"customDocument,omitempty"` // BY_NAME_REFERENCE: qualified document name +} + +func (CustomBlobDocumentCodeActionParameterValue) isCodeActionParameterValue() {} + // ExpressionBasedCodeActionParameterValue is an expression-based parameter value. type ExpressionBasedCodeActionParameterValue struct { model.BaseElement From 5ab2d3d4c30124137393e62df573cf416d3db7a7 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 11:35:23 +0000 Subject: [PATCH 12/13] fix: create javascript action accepts a StringTemplate type (MDL-JSA01) A text template (StringTemplate(Text|Sql)) is a Java action type only, but the grammar reads it for JavaScript actions too and both create handlers share one type converter. On main, `create javascript action M.JSA (S: StringTemplate(Text))` was silently written as a String parameter (describe: `S: String`); after the Java-action StringTemplate fix on this branch it was written as a CodeActions$StringTemplateParameterType into the JavaScript action. mx check 11.13.0 reports 0 errors for both, so nothing told the author. A StringTemplate return type was written as Boolean. Fix: unsupportedJavaScriptActionTypes is the one decision. exec refuses the statement with it before anything is written, and check reports it as MDL-JSA01 in the no-project tier (validate_program.go), so exec's pre-flight refuses it too. Parameters and the return type are both covered. Document(...) has no MDL spelling, so the parser already refuses it for either kind of action. Verification: TestCreateJavaScriptAction_StringTemplateIsRefusedAtExec and TestCreateJavaScriptAction_StringTemplateIsRefusedAtCheck fail against a stubbed validator (patch revert) and pass with the fix. Blank 11.13.0 app: main and the pre-refusal branch both exec the script at mx check 0 errors (main as String parameters); now check, exec and exec --no-check all refuse it and nothing is written. Fixture mdl-examples/bug-tests/javascript-action-1451-string-template.fail.mdl. go vet and go test ./mdl/executor/ ./mdl/conformance/ pass. Co-Authored-By: Claude Opus 5.5 --- ...on-string-template-parameter-accepted.json | 1 + ...cript-action-1451-string-template.fail.mdl | 17 ++++ ...javascript_actions_string_template_test.go | 77 +++++++++++++++++++ mdl/executor/cmd_javascript_actions_write.go | 4 + .../validate_javascript_action_types.go | 77 +++++++++++++++++++ mdl/executor/validate_program.go | 4 + 6 files changed, 180 insertions(+) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-25-javascript-action-string-template-parameter-accepted.json create mode 100644 mdl-examples/bug-tests/javascript-action-1451-string-template.fail.mdl create mode 100644 mdl/executor/cmd_javascript_actions_string_template_test.go create mode 100644 mdl/executor/validate_javascript_action_types.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-javascript-action-string-template-parameter-accepted.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-javascript-action-string-template-parameter-accepted.json new file mode 100644 index 000000000..3ecfe6333 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-javascript-action-string-template-parameter-accepted.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-25", "symptom": "`create javascript action M.JSA (S: StringTemplate(Text))` passes check and exec. On main the parameter is silently written as a String (describe: `S: String`); after the Java-action StringTemplate fix it was written as a CodeActions$StringTemplateParameterType into the JavaScript action. mx check 11.13 reports 0 errors for both", "cause": "The grammar's dataType accepts StringTemplate(...) for Java and JavaScript actions alike, and both create handlers share astDataTypeToJavaActionParamType; nothing said a text template is a Java action type only", "file": "`mdl/executor/validate_javascript_action_types.go` (`unsupportedJavaScriptActionTypes`, the one decision; `validateJavaScriptActionTypes` = MDL-JSA01 in the no-project tier via validate_program.go); exec refuses in `execCreateJavaScriptAction`; test `mdl/executor/cmd_javascript_actions_string_template_test.go`; fixture `mdl-examples/bug-tests/javascript-action-1451-string-template.fail.mdl`", "insight": "**Fixing a shared converter widens what every caller accepts.** Teaching astDataTypeToJavaActionParamType StringTemplate for Java actions also made JavaScript actions write one — the twin of the twin-builder lesson (#1137) in the other direction: when a converter is shared, ask which caller must refuse the new case. mx check is no gate here (0 errors on main and on the branch), so the refusal has to be mxcli's; it needs no project, so it lives in the no-project tier and exec's pre-flight refuses it before anything is written. Document(...) has no MDL spelling, so the parser already refuses it for either kind of action. The return type is refused too: the return converter's default turned StringTemplate into Boolean", "refs": ["ako/mxcli#1451"], "rules": ["MDL-JSA01"], "test": "TestCreateJavaScriptAction_StringTemplateIsRefusedAtExec, TestCreateJavaScriptAction_StringTemplateIsRefusedAtCheck"} diff --git a/mdl-examples/bug-tests/javascript-action-1451-string-template.fail.mdl b/mdl-examples/bug-tests/javascript-action-1451-string-template.fail.mdl new file mode 100644 index 000000000..0f63061cf --- /dev/null +++ b/mdl-examples/bug-tests/javascript-action-1451-string-template.fail.mdl @@ -0,0 +1,17 @@ +-- Negative test (found with ako/mxcli#1451): a JavaScript action parameter or +-- return type of StringTemplate(Text|Sql) must be REFUSED (MDL-JSA01). +-- +-- A text template is a Java action type only. The grammar reads the type for +-- both kinds of action and they share one converter, so main wrote it as a +-- String parameter without a word (describe printed `S: String`), and after +-- the Java-action fix it was written as a template into a JavaScript action. +-- mxbuild 11.13 reports 0 errors for both, so only mxcli can refuse it. + +mdl 1; + +create module BugTestJsa1451; + +create javascript action BugTestJsa1451.JSA_Tpl (S: StringTemplate(Text)) returns Boolean +as $$ +return true; +$$; diff --git a/mdl/executor/cmd_javascript_actions_string_template_test.go b/mdl/executor/cmd_javascript_actions_string_template_test.go new file mode 100644 index 000000000..72165375f --- /dev/null +++ b/mdl/executor/cmd_javascript_actions_string_template_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// A JavaScript action has no text-template parameter type — Studio Pro offers +// it for Java actions only. Main silently wrote `StringTemplate(Text)` on a +// JavaScript action as a String (describe then printed `S: String`); once the +// shared converter learned StringTemplate for Java actions it wrote a +// CodeActions$StringTemplateParameterType into a JavaScript action instead. +// mxbuild 11.13 accepts both at 0 errors, so neither tells the author. Both +// check and exec now refuse it, through validateJavaScriptActionTypes. + +const jsTemplateScript = `create javascript action MyFirstModule.JSA_Tpl ( + S: StringTemplate(Text), + Q: StringTemplate(Sql) +) returns Boolean +as $$ +return true; +$$;` + +func TestCreateJavaScriptAction_StringTemplateIsRefusedAtExec(t *testing.T) { + p := newCodeActionProject() + ctx, _ := newMockCtx(t, withBackend(p.mb), withHierarchy(p.h)) + + err := execCreateJavaScriptAction(ctx, parseOne[*ast.CreateJavaScriptActionStmt](t, jsTemplateScript)) + if err == nil { + t.Fatal("expected a refusal: a JavaScript action has no StringTemplate parameter type") + } + for _, want := range []string{"parameter S", "parameter Q", "StringTemplate(Text)", "StringTemplate(Sql)", "Java action"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal %q does not mention %q", err.Error(), want) + } + } + if p.jsa != nil { + t.Error("the action was written despite the refusal") + } +} + +func TestCreateJavaScriptAction_StringTemplateIsRefusedAtCheck(t *testing.T) { + prog, errs := visitor.Build(jsTemplateScript + ` +create javascript action MyFirstModule.JSA_Ret () returns StringTemplate(Text) +as $$ +return null; +$$; +create javascript action MyFirstModule.JSA_Ok (S: String) returns String +as $$ +return S; +$$;`) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + var got []string + for _, stmt := range prog.Statements { + if s, ok := stmt.(*ast.CreateJavaScriptActionStmt); ok { + for _, v := range validateJavaScriptActionTypes(s) { + if v.RuleID != "MDL-JSA01" { + t.Errorf("rule = %s, want MDL-JSA01", v.RuleID) + } + got = append(got, v.Message) + } + } + } + if len(got) != 3 { + t.Fatalf("violations = %d (%v), want 3: S and Q of JSA_Tpl, the return type of JSA_Ret", len(got), got) + } + if !strings.Contains(got[2], "JSA_Ret") || !strings.Contains(got[2], "return type") { + t.Errorf("return-type violation = %q", got[2]) + } +} diff --git a/mdl/executor/cmd_javascript_actions_write.go b/mdl/executor/cmd_javascript_actions_write.go index 029098553..8d340c9d5 100644 --- a/mdl/executor/cmd_javascript_actions_write.go +++ b/mdl/executor/cmd_javascript_actions_write.go @@ -20,6 +20,10 @@ func execCreateJavaScriptAction(ctx *ExecContext, s *ast.CreateJavaScriptActionS if !ctx.ConnectedForWrite() { return mdlerrors.NewNotConnectedWrite() } + // A text template is a Java action type only (MDL-JSA01). + if bad := unsupportedJavaScriptActionTypes(s); len(bad) > 0 { + return mdlerrors.NewValidationf("%s", strings.Join(bad, "\n")) + } h, err := getHierarchy(ctx) if err != nil { diff --git a/mdl/executor/validate_javascript_action_types.go b/mdl/executor/validate_javascript_action_types.go new file mode 100644 index 000000000..73bb247b3 --- /dev/null +++ b/mdl/executor/validate_javascript_action_types.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// A JavaScript action has no text-template type: StringTemplate(Text) and +// StringTemplate(Sql) are Java action parameter types (Studio Pro offers them +// nowhere else, and a nanoflow has no template value to pass). The grammar +// reads the type for both kinds of action and they share one converter, so +// `create javascript action` took it without complaint — main wrote it as a +// String parameter (describe printed `S: String`), and once the converter +// learned StringTemplate for Java actions it wrote a +// CodeActions$StringTemplateParameterType into the JavaScript action instead. +// mxbuild 11.13 accepts either at 0 errors, so nothing told the author. +// +// unsupportedJavaScriptActionTypes is the one decision: exec refuses the +// statement with it before anything is written, and check reports it as +// MDL-JSA01 (validateJavaScriptActionTypes), with no project needed. A +// document type (Document(…)) has no MDL spelling at all, so the parser +// already refuses it. +func unsupportedJavaScriptActionTypes(s *ast.CreateJavaScriptActionStmt) []string { + refuse := func(where string, dt ast.DataType) string { + if dt.Kind != ast.TypeStringTemplate { + return "" + } + return fmt.Sprintf("javascript action %s: %s is typed %s, which only a Java action has — "+ + "use String for a JavaScript action, or make it a Java action to take a text template", + s.Name.String(), where, codeActionTemplateSpelling(dt)) + } + var out []string + for _, p := range s.Parameters { + if msg := refuse("parameter "+p.Name, p.Type); msg != "" { + out = append(out, msg) + } + } + if msg := refuse("the return type", s.ReturnType); msg != "" { + out = append(out, msg) + } + return out +} + +// codeActionTemplateSpelling is how a StringTemplate type is written: +// StringTemplate(Text) or StringTemplate(Sql). +func codeActionTemplateSpelling(dt ast.DataType) string { + grammar := "Text" + if strings.EqualFold(dt.TemplateContext, "sql") { + grammar = "Sql" + } + return "StringTemplate(" + grammar + ")" +} + +// validateJavaScriptActionTypes is MDL-JSA01: exec's refusal of a type a +// JavaScript action cannot have, reported at check time. +func validateJavaScriptActionTypes(s *ast.CreateJavaScriptActionStmt) []linter.Violation { + var out []linter.Violation + for _, msg := range unsupportedJavaScriptActionTypes(s) { + out = append(out, linter.Violation{ + RuleID: "MDL-JSA01", + Severity: linter.SeverityError, + Location: linter.Location{ + Module: s.Name.Module, + DocumentType: "javascript action", + DocumentName: s.Name.Name, + }, + Message: msg, + Suggestion: "Declare the parameter as String; text templates are a Java action type", + }) + } + return out +} diff --git a/mdl/executor/validate_program.go b/mdl/executor/validate_program.go index 320917b0d..37763e18f 100644 --- a/mdl/executor/validate_program.go +++ b/mdl/executor/validate_program.go @@ -109,6 +109,10 @@ func ValidateProgram(prog *ast.Program, projectPath string) []linter.Violation { if jaStmt, ok := stmt.(*ast.CreateJavaActionStmt); ok { violations = append(violations, validateJavaActionTypeNames(jaStmt)...) } + // A text template is a Java action type only (MDL-JSA01). + if jsStmt, ok := stmt.(*ast.CreateJavaScriptActionStmt); ok { + violations = append(violations, validateJavaScriptActionTypes(jsStmt)...) + } // Check microflow body for common issues if mfStmt, ok := stmt.(*ast.CreateMicroflowStmt); ok { violations = append(violations, validateMicroflowWith(mfStmt, voids)...) From a919a9950d5b4ba740acb1ff80ea64eee1824933 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 25 Oct 2026 11:45:13 +0000 Subject: [PATCH 13/13] fix: create java action writes an unreturnable type as Boolean (MDL-JA03) `create java action M.J () returns StringTemplate(Text)` passed check and exec and was written as a Boolean action; so were `returns Binary`, `returns AutoNumber` and `returns entity `, for Java and JavaScript actions alike. Found while fixing ako/mxcli#1451. Cause: astDataTypeToJavaActionReturnType ended in a default that turned every kind it did not handle into Boolean. The deliberate default (no `returns` clause: Boolean, Studio Pro's default for a new action) and the fall-through shared one branch. Measured: a Java action cannot return a text template. generated/metamodel types JavaAction.ActionReturnType (storage JavaReturnType) as CodeActions$Type, whose implementers do not include StringTemplateParameterType; a unit patched to carry one makes mx check 11.13.0 fail to load the project ("Type Mendix.Modeler.CodeActions.StringTemplateParameterType does not contain a constructor with a parameter of type Mendix.Modeler.JavaActions.JavaAction"). main's Boolean form checks at 0 errors - the declared type was simply lost. Fix: the converter returns an error for any kind it does not handle; only an omitted `returns` is Boolean. Both create handlers refuse with it before anything is written. check reports the same verdict as MDL-JA03 by calling the converter through javaActionReturnTypeError (type parameters resolved first, as exec does), so the two cannot disagree; for a JavaScript action a StringTemplate return stays reported once, as MDL-JSA01. Verification: TestCreateJavaAction_UnreturnableTypeIsRefusedAtExec and TestValidate_UnreturnableTypeIsReportedAtCheck fail against the reverted fix (patch revert) and pass with it; the control TestCreateJavaAction_OmittedReturnTypeIsBoolean passes both ways. On an 11.13.0 app, check and exec --no-check refuse the statement and nothing is written. Fixture java-action-1451-return-string-template.fail.mdl. go vet and go test ./mdl/executor/ ./cmd/mxcli/syntax/ ./mdl/conformance/ pass. Co-Authored-By: Claude Opus 5.5 --- ...va-action-return-type-default-boolean.json | 1 + cmd/mxcli/syntax/features_integration.go | 2 +- ...ction-1451-return-string-template.fail.mdl | 20 ++++ mdl/executor/cmd_javaactions.go | 43 +++++---- mdl/executor/cmd_javaactions_enum_680_test.go | 5 +- .../cmd_javaactions_return_type_test.go | 94 +++++++++++++++++++ mdl/executor/cmd_javascript_actions_write.go | 4 +- .../validate_code_action_return_type.go | 84 +++++++++++++++++ .../validate_javascript_action_types.go | 7 ++ mdl/executor/validate_program.go | 2 + 10 files changed, 242 insertions(+), 20 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-25-java-action-return-type-default-boolean.json create mode 100644 mdl-examples/bug-tests/java-action-1451-return-string-template.fail.mdl create mode 100644 mdl/executor/cmd_javaactions_return_type_test.go create mode 100644 mdl/executor/validate_code_action_return_type.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-java-action-return-type-default-boolean.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-java-action-return-type-default-boolean.json new file mode 100644 index 000000000..86640f4d7 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-25-java-action-return-type-default-boolean.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-25", "symptom": "`create java action M.J () returns StringTemplate(Text)` passes check and exec and writes a Boolean action (describe: `returns Boolean`); so did `returns Binary`, `returns AutoNumber` and `returns entity `, for Java and JavaScript actions alike", "cause": "astDataTypeToJavaActionReturnType ended in `default: // Default to Boolean type (most common for Java actions)`, so every kind it did not handle became a CodeActions$BooleanType. The intended default (no `returns` clause, Studio Pro's Boolean) and the unhandled kinds shared one branch", "file": "`mdl/executor/cmd_javaactions.go` (astDataTypeToJavaActionReturnType now returns an error; TypeUnknown = omitted is the only Boolean default), `mdl/executor/validate_code_action_return_type.go` (codeActionReturnTypeRefusal, javaActionReturnTypeError = check's call into the same converter, MDL-JA03), wired in validate_program.go and validateJavaScriptActionTypes; test `mdl/executor/cmd_javaactions_return_type_test.go`; fixture `mdl-examples/bug-tests/java-action-1451-return-string-template.fail.mdl`", "insight": "**A converter's `default:` that returns a plausible value is a silent write** (cf. the read-side lie in the #1210 finding). Split the deliberate default from the fall-through: here the deliberate one is TypeUnknown (no `returns`), everything else must error. Make check call the converter itself (javaActionReturnTypeError mirrors exec's type-parameter shortcuts, then asks the converter) rather than keep a second list of supported kinds that can drift. Whether a StringTemplate return is possible was measured, not assumed: generated/metamodel types JavaAction.ActionReturnType (storage JavaReturnType) as CodeActions$Type, whose implementers exclude StringTemplateParameterType, and a v1 unit patched to carry one makes `mx check` 11.13 fail to LOAD the project (InvalidOperationException: no constructor of StringTemplateParameterType taking a JavaAction) — while describe, through codeActionReturnTypeFromGen's own String default, printed `returns String` for it", "refs": ["ako/mxcli#1451"], "rules": ["MDL-JA03"], "test": "TestCreateJavaAction_UnreturnableTypeIsRefusedAtExec, TestCreateJavaAction_OmittedReturnTypeIsBoolean, TestValidate_UnreturnableTypeIsReportedAtCheck"} diff --git a/cmd/mxcli/syntax/features_integration.go b/cmd/mxcli/syntax/features_integration.go index d1b7cae13..5bb461078 100644 --- a/cmd/mxcli/syntax/features_integration.go +++ b/cmd/mxcli/syntax/features_integration.go @@ -657,7 +657,7 @@ DESCRIBE DATABASE CONNECTION Ops.Erp;`, "create java action", "or modify java action", "type parameter", "entity parameter", "exposed as", "returns", "generics", "drop java action", }, - Syntax: "CREATE [OR MODIFY] JAVA ACTION Module.Name[] [FOLDER 'path'](\n [/** description */] Param: Type [NOT NULL],\n EntityType: ENTITY NOT NULL,\n Obj: pEntity\n) RETURNS ReturnType [AS $ReturnName]\n[EXPOSED AS 'Label' IN 'Category'\n [ICON 'icon.png'] [ICON DARK 'icon-dark.png']\n [IMAGE 'image.png'] [IMAGE DARK 'image-dark.png']]\n[NOT EXPOSED]\nAS $$\n// Java code — AS $$ ... $$ is mandatory, cannot be omitted\n$$;\n\nOR MODIFY: updates signature/body in-place, preserves UUID.\n\nEXPOSED AS puts the action in Studio Pro's toolbox. The icon is a 64x64 PNG\nand the image a 256x192 PNG, read from disk relative to the .mdl file's own directory.\nAn OMITTED clause preserves what is stored — including bitmaps set in Studio\nPro — so removing an entry is NOT EXPOSED, and clearing one bitmap is\nDROP ICON / DROP IMAGE [DARK].\n\nA parameter's description is a /** ... */ doc comment before it, as on an\nentity attribute; AS $ReturnName names the return value (the default name of\nthe variable a microflow call stores it in), as on a microflow. Leaving either\nout keeps what is stored; a new action gets no description and ReturnValueName.\n\nType parameters: an ENTITY parameter declares T. A type parameter no\nENTITY parameter names is declared in the header, Module.Name; an\nunqualified type that is neither is refused (MDL-JA01), since it would be\nwritten as an entity with no module (CE1613).\n\nA bare Module.Name type is an entity or an enumeration, whichever the project\n(or an earlier statement of the script) has; ENUM Module.Name says enumeration\nexplicitly. A name that is neither is refused by check --references and exec,\nsince it would be written as an entity that does not exist (CE1613).", + Syntax: "CREATE [OR MODIFY] JAVA ACTION Module.Name[] [FOLDER 'path'](\n [/** description */] Param: Type [NOT NULL],\n EntityType: ENTITY NOT NULL,\n Obj: pEntity\n) RETURNS ReturnType [AS $ReturnName]\n[EXPOSED AS 'Label' IN 'Category'\n [ICON 'icon.png'] [ICON DARK 'icon-dark.png']\n [IMAGE 'image.png'] [IMAGE DARK 'image-dark.png']]\n[NOT EXPOSED]\nAS $$\n// Java code — AS $$ ... $$ is mandatory, cannot be omitted\n$$;\n\nOR MODIFY: updates signature/body in-place, preserves UUID.\n\nEXPOSED AS puts the action in Studio Pro's toolbox. The icon is a 64x64 PNG\nand the image a 256x192 PNG, read from disk relative to the .mdl file's own directory.\nAn OMITTED clause preserves what is stored — including bitmaps set in Studio\nPro — so removing an entry is NOT EXPOSED, and clearing one bitmap is\nDROP ICON / DROP IMAGE [DARK].\n\nA parameter's description is a /** ... */ doc comment before it, as on an\nentity attribute; AS $ReturnName names the return value (the default name of\nthe variable a microflow call stores it in), as on a microflow. Leaving either\nout keeps what is stored; a new action gets no description and ReturnValueName.\n\nType parameters: an ENTITY parameter declares T. A type parameter no\nENTITY parameter names is declared in the header, Module.Name; an\nunqualified type that is neither is refused (MDL-JA01), since it would be\nwritten as an entity with no module (CE1613).\n\nA bare Module.Name type is an entity or an enumeration, whichever the project\n(or an earlier statement of the script) has; ENUM Module.Name says enumeration\nexplicitly. A name that is neither is refused by check --references and exec,\nsince it would be written as an entity that does not exist (CE1613).\n\nRETURNS takes Boolean, Integer, Long, Decimal, String, DateTime, an enumeration,\nan entity, a list, a type parameter or Void; leaving it out returns Boolean.\nAny other type is refused (MDL-JA03) — StringTemplate(...) is a parameter\ntype only.", Example: "mdl 1;\nCREATE JAVA ACTION Utils.FormatCurrency(\n /** The amount to format. */\n Amount: Decimal NOT NULL\n) RETURNS String AS $FormattedAmount\nEXPOSED AS 'Format Currency' IN 'Formatting'\n ICON 'assets/currency-64.png'\n IMAGE 'assets/currency-256.png'\nAS $$\nreturn String.format(\"%.2f\", Amount);\n$$;\n\n-- Generic entity validator with type parameter\nCREATE JAVA ACTION Utils.IsValid(\n EntityType: ENTITY NOT NULL,\n Obj: pEntity NOT NULL\n) RETURNS Boolean\nAS $$\nreturn Obj != null;\n$$;\n\n-- Type parameter with no entity selector: the argument's entity binds it\nCREATE JAVA ACTION Utils.Describe(\n Obj: T NOT NULL\n) RETURNS String\nAS $$\nreturn Obj.toString();\n$$;\n\n-- Idempotent update (preserves UUID)\nCREATE OR MODIFY JAVA ACTION Utils.FormatCurrency(\n Amount: Decimal NOT NULL,\n Decimals: Integer NOT NULL\n) RETURNS String\nAS $$\nreturn String.format(\"%.\" + Decimals + \"f\", Amount);\n$$;", SeeAlso: []string{"java-action", "javascript-action"}, }) diff --git a/mdl-examples/bug-tests/java-action-1451-return-string-template.fail.mdl b/mdl-examples/bug-tests/java-action-1451-return-string-template.fail.mdl new file mode 100644 index 000000000..94236a4cd --- /dev/null +++ b/mdl-examples/bug-tests/java-action-1451-return-string-template.fail.mdl @@ -0,0 +1,20 @@ +-- Negative test (found with ako/mxcli#1451): a Java action that returns a +-- StringTemplate must be REFUSED (MDL-JA03). +-- +-- A text template is a parameter type only: a Java action's return type is a +-- CodeActions$Type, and CodeActions$StringTemplateParameterType is not one. +-- A unit patched to return one does not load in mxbuild 11.13 ("Type +-- Mendix.Modeler.CodeActions.StringTemplateParameterType does not contain a +-- constructor with a parameter of type Mendix.Modeler.JavaActions.JavaAction"). +-- main wrote this statement as a Boolean action without a word: the return +-- converter's default turned every type it did not handle into Boolean. That +-- default is now an error; only an omitted `returns` is Boolean. + +mdl 1; + +create module BugTestJaRet1451; + +create java action BugTestJaRet1451.JA_Subject (Code: String) returns StringTemplate(Text) +as $$ +return null; +$$; diff --git a/mdl/executor/cmd_javaactions.go b/mdl/executor/cmd_javaactions.go index 42ad49ab7..ea98f08f6 100644 --- a/mdl/executor/cmd_javaactions.go +++ b/mdl/executor/cmd_javaactions.go @@ -657,7 +657,9 @@ func execCreateJavaAction(ctx *ExecContext, s *ast.CreateJavaActionStmt) error { } else if l := listOfTypeParameter(returnType, typeParamNameToID); l != nil { ja.ReturnType = l } else { - ja.ReturnType = astDataTypeToJavaActionReturnType(returnType) + if ja.ReturnType, err = astDataTypeToJavaActionReturnType(returnType); err != nil { + return mdlerrors.NewValidationf("java action %s: %v", s.Name.String(), err) + } } // Fold the EXPOSED AS clause onto the stored toolbox entry rather than @@ -840,8 +842,13 @@ func bareDataTypeName(dt ast.DataType) string { return "" } -// astDataTypeToJavaActionReturnType converts an AST DataType to a Java action return type. -func astDataTypeToJavaActionReturnType(dt ast.DataType) javaactions.CodeActionReturnType { +// astDataTypeToJavaActionReturnType converts an AST DataType to a Java (or +// JavaScript) action return type. A type it does not handle is an error: the +// default used to be Boolean, so `returns StringTemplate(Text)` — which a code +// action cannot return at all — was written as Boolean without a word. Only an +// omitted `returns` (TypeUnknown) is Boolean, Studio Pro's default for a new +// action. check reports the same error as MDL-JA03 (validateJavaActionReturnType). +func astDataTypeToJavaActionReturnType(dt ast.DataType) (javaactions.CodeActionReturnType, error) { switch dt.Kind { case ast.TypeVoid: return &javaactions.VoidType{ @@ -849,49 +856,49 @@ func astDataTypeToJavaActionReturnType(dt ast.DataType) javaactions.CodeActionRe ID: model.ID(types.GenerateID()), TypeName: "CodeActions$VoidType", }, - } + }, nil case ast.TypeBoolean: return &javaactions.BooleanType{ BaseElement: model.BaseElement{ ID: model.ID(types.GenerateID()), TypeName: "CodeActions$BooleanType", }, - } + }, nil case ast.TypeInteger: return &javaactions.IntegerType{ BaseElement: model.BaseElement{ ID: model.ID(types.GenerateID()), TypeName: "CodeActions$IntegerType", }, - } + }, nil case ast.TypeLong: return &javaactions.LongType{ BaseElement: model.BaseElement{ ID: model.ID(types.GenerateID()), TypeName: "CodeActions$LongType", }, - } + }, nil case ast.TypeDecimal: return &javaactions.DecimalType{ BaseElement: model.BaseElement{ ID: model.ID(types.GenerateID()), TypeName: "CodeActions$DecimalType", }, - } + }, nil case ast.TypeString: return &javaactions.StringType{ BaseElement: model.BaseElement{ ID: model.ID(types.GenerateID()), TypeName: "CodeActions$StringType", }, - } + }, nil case ast.TypeDateTime, ast.TypeDate: return &javaactions.DateTimeType{ BaseElement: model.BaseElement{ ID: model.ID(types.GenerateID()), TypeName: "CodeActions$DateTimeType", }, - } + }, nil case ast.TypeEntity, ast.TypeEnumeration: // An unambiguous `ENUM`/`Enumeration(...)` return type serializes as an // enumeration, not an entity reference (#680). A bare qualified name @@ -905,7 +912,7 @@ func astDataTypeToJavaActionReturnType(dt ast.DataType) javaactions.CodeActionRe TypeName: "CodeActions$EnumerationType", }, Enumeration: dt.EnumRef.Module + "." + dt.EnumRef.Name, - } + }, nil } entityName := "" if dt.EntityRef != nil { @@ -919,7 +926,7 @@ func astDataTypeToJavaActionReturnType(dt ast.DataType) javaactions.CodeActionRe ID: model.ID(types.GenerateID()), TypeName: "CodeActions$VoidType", }, - } + }, nil } return &javaactions.EntityType{ BaseElement: model.BaseElement{ @@ -927,7 +934,7 @@ func astDataTypeToJavaActionReturnType(dt ast.DataType) javaactions.CodeActionRe TypeName: "CodeActions$EntityType", }, Entity: entityName, - } + }, nil case ast.TypeListOf: entityName := "" if dt.EntityRef != nil { @@ -939,15 +946,17 @@ func astDataTypeToJavaActionReturnType(dt ast.DataType) javaactions.CodeActionRe TypeName: "CodeActions$ListType", }, Entity: entityName, - } - default: - // Default to Boolean type (most common for Java actions) + }, nil + case ast.TypeUnknown: + // No `returns` clause: Boolean, what Studio Pro gives a new action. return &javaactions.BooleanType{ BaseElement: model.BaseElement{ ID: model.ID(types.GenerateID()), TypeName: "CodeActions$BooleanType", }, - } + }, nil + default: + return nil, codeActionReturnTypeRefusal(dt) } } diff --git a/mdl/executor/cmd_javaactions_enum_680_test.go b/mdl/executor/cmd_javaactions_enum_680_test.go index 5afba301e..63017e1ca 100644 --- a/mdl/executor/cmd_javaactions_enum_680_test.go +++ b/mdl/executor/cmd_javaactions_enum_680_test.go @@ -39,7 +39,10 @@ func TestParamType_ExplicitEnum_IsEnumeration(t *testing.T) { } func TestReturnType_ExplicitEnum_IsEnumeration(t *testing.T) { - rt := astDataTypeToJavaActionReturnType(enumDataType("M", "Status")) + rt, err := astDataTypeToJavaActionReturnType(enumDataType("M", "Status")) + if err != nil { + t.Fatal(err) + } et, ok := rt.(*javaactions.EnumerationType) if !ok { t.Fatalf("expected *EnumerationType, got %T", rt) diff --git a/mdl/executor/cmd_javaactions_return_type_test.go b/mdl/executor/cmd_javaactions_return_type_test.go new file mode 100644 index 000000000..afacafa52 --- /dev/null +++ b/mdl/executor/cmd_javaactions_return_type_test.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/sdk/javaactions" +) + +// `create java action M.J () returns StringTemplate(Text)` passed check and +// exec and was written with a Boolean return: the return-type converter's +// default turned every kind it did not handle into Boolean, silently. A Java +// action cannot return a text template at all — its return type is a +// CodeActions$Type, of which StringTemplateParameterType is not one, and a +// unit patched to carry one does not load in mxbuild 11.13 ("Type +// Mendix.Modeler.CodeActions.StringTemplateParameterType does not contain a +// constructor with a parameter of type Mendix.Modeler.JavaActions.JavaAction"). +// The same default swallowed Binary, the Auto* types and `entity `. + +func javaReturnScript(ret string) string { + return "create java action MyFirstModule.JA_Ret (S: String) " + ret + "\nas $$\nreturn null;\n$$;" +} + +func TestCreateJavaAction_UnreturnableTypeIsRefusedAtExec(t *testing.T) { + for _, c := range []struct{ ret, want string }{ + {"returns StringTemplate(Text)", "StringTemplate(Text)"}, + {"returns StringTemplate(Sql)", "StringTemplate(Sql)"}, + {"returns Binary", "Binary"}, + {"returns AutoNumber", "AutoNumber"}, + {"returns entity ", "entity "}, + } { + t.Run(c.ret, func(t *testing.T) { + p := newCodeActionProject() + ctx, _ := newMockCtx(t, withBackend(p.mb), withHierarchy(p.h)) + err := execCreateJavaAction(ctx, parseOne[*ast.CreateJavaActionStmt](t, javaReturnScript(c.ret))) + if err == nil || !strings.Contains(err.Error(), c.want) || !strings.Contains(err.Error(), "return") { + t.Fatalf("err = %v, want a refusal naming the return type %s", err, c.want) + } + if p.ja != nil { + t.Errorf("written with return type %T despite the refusal", p.ja.ReturnType) + } + }) + } +} + +// The control: a statement with no `returns` still gets Boolean, Studio Pro's +// default for a new Java action — that one default is deliberate. +func TestCreateJavaAction_OmittedReturnTypeIsBoolean(t *testing.T) { + p := newCodeActionProject() + ctx, _ := newMockCtx(t, withBackend(p.mb), withHierarchy(p.h)) + assertNoError(t, execCreateJavaAction(ctx, parseOne[*ast.CreateJavaActionStmt](t, javaReturnScript("")))) + if _, ok := p.ja.ReturnType.(*javaactions.BooleanType); !ok { + t.Errorf("omitted return type written as %T, want *BooleanType", p.ja.ReturnType) + } +} + +func TestValidate_UnreturnableTypeIsReportedAtCheck(t *testing.T) { + prog, errs := visitor.Build(javaReturnScript("returns StringTemplate(Text)") + "\n" + + strings.Replace(javaReturnScript("returns String"), "JA_Ret", "JA_Ok", 1) + "\n" + + "create javascript action MyFirstModule.JSA_Ret () returns Binary\nas $$\nreturn null;\n$$;\n" + + "create javascript action MyFirstModule.JSA_Tpl () returns StringTemplate(Text)\nas $$\nreturn null;\n$$;") + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + rules := map[string][]string{} + for _, stmt := range prog.Statements { + switch s := stmt.(type) { + case *ast.CreateJavaActionStmt: + for _, v := range validateJavaActionReturnType(s) { + rules[s.Name.Name] = append(rules[s.Name.Name], v.RuleID) + } + case *ast.CreateJavaScriptActionStmt: + for _, v := range validateJavaScriptActionTypes(s) { + rules[s.Name.Name] = append(rules[s.Name.Name], v.RuleID) + } + } + } + want := map[string]string{ + "JA_Ret": "MDL-JA03", + "JA_Ok": "", + "JSA_Ret": "MDL-JA03", + "JSA_Tpl": "MDL-JSA01", // reported once, by the JavaScript rule + } + for name, rule := range want { + got := strings.Join(rules[name], ",") + if got != rule { + t.Errorf("%s: rules = %q, want %q", name, got, rule) + } + } +} diff --git a/mdl/executor/cmd_javascript_actions_write.go b/mdl/executor/cmd_javascript_actions_write.go index 8d340c9d5..d57c7c034 100644 --- a/mdl/executor/cmd_javascript_actions_write.go +++ b/mdl/executor/cmd_javascript_actions_write.go @@ -208,7 +208,9 @@ func execCreateJavaScriptAction(ctx *ExecContext, s *ast.CreateJavaScriptActionS // the JavaScript twin of the Java action's #1183. jsa.ReturnType = l } else { - jsa.ReturnType = astDataTypeToJavaActionReturnType(returnType) + if jsa.ReturnType, err = astDataTypeToJavaActionReturnType(returnType); err != nil { + return mdlerrors.NewValidationf("javascript action %s: %v", s.Name.String(), err) + } } if jsa.MicroflowActionInfo, err = mergeMicroflowActionInfo(ctx, diff --git a/mdl/executor/validate_code_action_return_type.go b/mdl/executor/validate_code_action_return_type.go new file mode 100644 index 000000000..5669f07ae --- /dev/null +++ b/mdl/executor/validate_code_action_return_type.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// A Java or JavaScript action's return type is a CodeActions$Type: Boolean, +// Integer/Long, Decimal, String, DateTime, an enumeration, an entity, a list, +// a type parameter, or Void. The grammar reads any data type after `returns`, +// and the return-type converter turned every kind it did not handle into +// Boolean — so `returns StringTemplate(Text)` passed check and exec and built a +// Boolean action. A text template is a parameter type only +// (CodeActions$StringTemplateParameterType is not a CodeActions$Type), and a +// unit patched to return one does not load in mxbuild 11.13: "Type +// Mendix.Modeler.CodeActions.StringTemplateParameterType does not contain a +// constructor with a parameter of type Mendix.Modeler.JavaActions.JavaAction". + +// codeActionReturnTypeRefusal is the error the return-type converter gives for +// a kind it does not handle. +func codeActionReturnTypeRefusal(dt ast.DataType) error { + switch dt.Kind { + case ast.TypeStringTemplate: + return fmt.Errorf("the return type is %s, which is a parameter type only — a Java action cannot "+ + "return a text template; return String", codeActionTemplateSpelling(dt)) + case ast.TypeEntityTypeParam: + return fmt.Errorf("the return type is entity <%s>, which is a parameter type only (it selects an "+ + "entity) — return the type parameter %s, or a list of it", dt.TypeParamName, dt.TypeParamName) + } + return fmt.Errorf("the return type is %s, which a Java or JavaScript action cannot return — return "+ + "Boolean, Integer, Long, Decimal, String, DateTime, an enumeration, an entity, a list, a type "+ + "parameter or Void", dt.Kind.String()) +} + +// javaActionReturnTypeError is exec's verdict on a return type, for check: a +// type parameter, or a list of one, is resolved before the converter, as in +// execCreateJavaAction and execCreateJavaScriptAction; anything else is +// whatever the converter says. One decision, so the two cannot disagree. +func javaActionReturnTypeError(dt ast.DataType, typeParams []string) error { + declared := make(map[string]bool, len(typeParams)) + for _, tp := range typeParams { + declared[tp] = true + } + if isTypeParamRef(dt, declared) { + return nil + } + if dt.Kind == ast.TypeListOf && dt.EntityRef != nil && dt.EntityRef.Module == "" && declared[dt.EntityRef.Name] { + return nil + } + _, err := astDataTypeToJavaActionReturnType(dt) + return err +} + +// codeActionReturnTypeViolation is MDL-JA03 for one action, or nil. +func codeActionReturnTypeViolation(kind string, name ast.QualifiedName, dt ast.DataType, typeParams []string) *linter.Violation { + err := javaActionReturnTypeError(dt, typeParams) + if err == nil { + return nil + } + return &linter.Violation{ + RuleID: "MDL-JA03", + Severity: linter.SeverityError, + Location: linter.Location{ + Module: name.Module, + DocumentType: kind, + DocumentName: name.Name, + }, + Message: fmt.Sprintf("%s '%s': %v", kind, name.String(), err), + Suggestion: "See `mxcli syntax java-action.create` for the types an action returns", + } +} + +// validateJavaActionReturnType is MDL-JA03 for a Java action: exec's refusal of +// a return type the converter does not handle, reported at check time. +func validateJavaActionReturnType(s *ast.CreateJavaActionStmt) []linter.Violation { + if v := codeActionReturnTypeViolation("java action", s.Name, s.ReturnType, s.TypeParameters); v != nil { + return []linter.Violation{*v} + } + return nil +} diff --git a/mdl/executor/validate_javascript_action_types.go b/mdl/executor/validate_javascript_action_types.go index 73bb247b3..037ef36fc 100644 --- a/mdl/executor/validate_javascript_action_types.go +++ b/mdl/executor/validate_javascript_action_types.go @@ -73,5 +73,12 @@ func validateJavaScriptActionTypes(s *ast.CreateJavaScriptActionStmt) []linter.V Suggestion: "Declare the parameter as String; text templates are a Java action type", }) } + // Any other type the return converter does not handle (MDL-JA03). A + // StringTemplate return is reported once, above. + if s.ReturnType.Kind != ast.TypeStringTemplate { + if v := codeActionReturnTypeViolation("javascript action", s.Name, s.ReturnType, s.TypeParameters); v != nil { + out = append(out, *v) + } + } return out } diff --git a/mdl/executor/validate_program.go b/mdl/executor/validate_program.go index 37763e18f..55a93ba89 100644 --- a/mdl/executor/validate_program.go +++ b/mdl/executor/validate_program.go @@ -108,6 +108,8 @@ func ValidateProgram(prog *ast.Program, projectPath string) []linter.Violation { // at build time (MDL-JA01, mendixlabs/mxcli#1404). if jaStmt, ok := stmt.(*ast.CreateJavaActionStmt); ok { violations = append(violations, validateJavaActionTypeNames(jaStmt)...) + // A return type no action can have (MDL-JA03). + violations = append(violations, validateJavaActionReturnType(jaStmt)...) } // A text template is a Java action type only (MDL-JSA01). if jsStmt, ok := stmt.(*ast.CreateJavaScriptActionStmt); ok {