From 0a7ec6f05794df631b79668ac21740bb41217705 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 21 Oct 2026 20:41:03 -0400 Subject: [PATCH 1/6] perf(hermitcrab): share trailing affix-template slot batches --- .../AnalysisAffixTemplateRule.cs | 38 ++- .../AnalysisAffixTemplatesRule.cs | 145 ++++++++++ .../AnalysisStratumRule.cs | 7 +- .../AnalysisAffixTemplatesRuleTests.cs | 259 ++++++++++++++++++ 4 files changed, 432 insertions(+), 17 deletions(-) create mode 100644 src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs create mode 100644 tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs index 9044a8488..ba963745a 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs @@ -30,13 +30,34 @@ public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template) public IEnumerable Apply(Word input) { - if (!_morpher.RuleSelector(_template)) + if (!CanApply(input)) return Enumerable.Empty(); + Word inWord = input.Clone(); + inWord.Freeze(); + + var output = new HashSet(FreezableEqualityComparer.Default); + if (_morpher.MaxDegreeOfParallelism == 1) + ApplySlots(inWord, _rules.Count - 1, output); + else + ParallelApplySlots(inWord, output); + + return output; + } + + internal AffixTemplate Template => _template; + + internal IReadOnlyList> SlotRules => _rules; + + internal bool CanApply(Word input) + { + if (!_morpher.RuleSelector(_template)) + return false; + // A check only: adding the template's required features here would make SyntacticFeatureStruct // depend on whether a template was un-applied, which Word.ValueEquals assumes cannot happen. if (!input.SyntacticFeatureStruct.IsUnifiable(_template.RequiredSyntacticFeatureStruct)) - return Enumerable.Empty(); + return false; if (_morpher.TraceManager.IsTracing) _morpher.TraceManager.BeginUnapplyTemplate(_template, input); @@ -58,19 +79,10 @@ public IEnumerable Apply(Word input) FailureReason.NonPartialRuleProhibitedAfterFinalTemplate ); } - return Enumerable.Empty(); + return false; } - Word inWord = input.Clone(); - inWord.Freeze(); - - var output = new HashSet(FreezableEqualityComparer.Default); - if (_morpher.MaxDegreeOfParallelism == 1) - ApplySlots(inWord, _rules.Count - 1, output); - else - ParallelApplySlots(inWord, output); - - return output; + return true; } private void ApplySlots(Word inWord, int index, HashSet output) diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs new file mode 100644 index 000000000..cbd08578f --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs @@ -0,0 +1,145 @@ +using System.Collections.Generic; +using System.Linq; +using SIL.Machine.Annotations; +using SIL.Machine.Rules; +using SIL.ObjectModel; + +namespace SIL.Machine.Morphology.HermitCrab +{ + internal class AnalysisAffixTemplatesRule : RuleBatch + { + private readonly Morpher _morpher; + private readonly AnalysisAffixTemplateRule[] _templates; + private readonly bool[][] _slotOptionality; + private readonly SlotNode _root = new SlotNode(); + private readonly bool _hasSharedSlots; + + public AnalysisAffixTemplatesRule(Morpher morpher, IEnumerable> rules) + : base(rules, false, FreezableEqualityComparer.Default) + { + _morpher = morpher; + _templates = Rules.OfType().ToArray(); + if (_templates.Length != Rules.Count) + return; + + _slotOptionality = _templates.Select(t => t.Template.Slots.Select(s => s.Optional).ToArray()).ToArray(); + for (int t = 0; t < _templates.Length; t++) + { + AnalysisAffixTemplateRule template = _templates[t]; + SlotNode node = _root; + for (int s = template.Template.Slots.Count - 1; s >= 0; s--) + { + AffixTemplateSlot slot = template.Template.Slots[s]; + SlotNode child = node.Children.FirstOrDefault(n => n.Matches(slot)); + if (child == null) + { + child = new SlotNode + { + Slot = slot, + Optional = slot.Optional, + Rule = template.SlotRules[s], + }; + node.Children.Add(child); + } + else + { + _hasSharedSlots = true; + } + child.Templates.Add(t); + node = child; + } + node.Terminals.Add(t); + } + } + + public override IEnumerable Apply(Word input) + { + // A configured limit must stop before later templates are evaluated. + if ( + !_hasSharedSlots + || _morpher.TraceManager.IsTracing + || _morpher.MaxDegreeOfParallelism != 1 + || MaxAlternatives > 0 + || !SlotsUnchanged() + ) + return base.Apply(input); + + var outputs = new HashSet[_templates.Length]; + for (int t = 0; t < _templates.Length; t++) + { + if (_templates[t].CanApply(input)) + outputs[t] = new HashSet(Comparer); + } + + if (outputs.Any(o => o != null)) + { + Word inWord = input.Clone(); + inWord.Freeze(); + Walk(_root, inWord, outputs); + } + + // Template order determines which equal word survives and the order seen by analysis folding. + var output = new HashSet(Comparer); + foreach (HashSet templateOutput in outputs) + { + if (templateOutput != null) + output.UnionWith(templateOutput); + if (MaxAlternatives > 0 && output.Count > MaxAlternatives) + throw new MaxAlternativesExceededException("MaxAlternatives exceeded"); + } + return output; + } + + private bool SlotsUnchanged() + { + for (int t = 0; t < _templates.Length; t++) + { + IList slots = _templates[t].Template.Slots; + bool[] optionality = _slotOptionality[t]; + if (slots.Count != optionality.Length) + return false; + for (int s = 0; s < slots.Count; s++) + { + if (slots[s].Optional != optionality[s]) + return false; + } + } + return true; + } + + private static void Walk(SlotNode node, Word input, HashSet[] outputs) + { + foreach (int t in node.Terminals) + outputs[t]?.Add(input); + + foreach (SlotNode child in node.Children) + { + if (!child.Templates.Any(t => outputs[t] != null)) + continue; + + foreach (Word word in child.Rule.Apply(input)) + Walk(child, word, outputs); + if (child.Optional) + Walk(child, input, outputs); + } + } + + private sealed class SlotNode + { + public AffixTemplateSlot Slot { get; set; } + public bool Optional { get; set; } + public IRule Rule { get; set; } + public List Children { get; } = new List(); + public List Templates { get; } = new List(); + public List Terminals { get; } = new List(); + + public bool Matches(AffixTemplateSlot slot) + { + if (Optional != slot.Optional || Slot.Rules.Count != slot.Rules.Count) + return false; + + return Slot.Rules.Zip(slot.Rules, (a, b) => ReferenceEquals(a, b)).All(equal => equal); + } + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs index ef48637ac..811471d74 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs @@ -25,10 +25,9 @@ public AnalysisStratumRule(Morpher morpher, Stratum stratum) _prulesRule = new LinearRuleCascade( stratum.PhonologicalRules.Select(prule => CompilePhonologicalRule(prule, morpher)).Reverse() ); - _templatesRule = new RuleBatch( - stratum.AffixTemplates.Select(template => CompileAffixTemplate(template, morpher)), - false, - FreezableEqualityComparer.Default + _templatesRule = new AnalysisAffixTemplatesRule( + morpher, + stratum.AffixTemplates.Select(template => CompileAffixTemplate(template, morpher)) ); _mrulesRule = null; IEnumerable> mrules = stratum diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs new file mode 100644 index 000000000..9d91572ee --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs @@ -0,0 +1,259 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; +using SIL.Machine.Rules; +using SIL.ObjectModel; + +namespace SIL.Machine.Morphology.HermitCrab; + +[TestFixture] +public class AnalysisAffixTemplatesRuleTests : HermitCrabTestBase +{ + [TestCase(0)] + [TestCase(1)] + [TestCase(2)] + [TestCase(3)] + public void SharedWalkMatchesPerTemplateWalkInOrder(int variant) + { + var d = Suffix("d", "d"); + var dd = Suffix("dd", "dd"); + var s = Suffix("s", "s"); + AffixTemplateSlot Tail(bool optional = false) => new(d, dd) { Optional = optional }; + AffixTemplateSlot Inner() => new(s) { Optional = (variant & 1) == 0 }; + AffixTemplate[] templates = + [ + Template(Inner(), Tail()), + Template(Tail()), + Template(Inner(), Tail(true)), + Template(new AffixTemplateSlot(d) { Optional = true }, Inner(), Tail()), + Template(new AffixTemplateSlot(s), Tail()), + Template(), + Template(Inner(), Tail()), + Template(new AffixTemplateSlot(dd, d)), + Template(new AffixTemplateSlot()), + ]; + templates[4].RequiredSyntacticFeatureStruct = Pos("N"); + templates[3].IsFinal = false; + if ((variant & 2) != 0) + Array.Reverse(templates); + + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + RuleSelector = rule => variant != 3 || !ReferenceEquals(rule, templates[1]), + }; + IRule[] rules = templates.Select(t => t.CompileAnalysisRule(morpher)).ToArray(); + var shared = new AnalysisAffixTemplatesRule(morpher, rules); + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default); + + int compared = 0; + foreach (string text in Inputs(4)) + { + foreach (FeatureStruct fs in new[] { FeatureStruct.New().Value, Pos("N"), Pos("V") }) + { + foreach ( + FinalTemplateState finalState in new[] { FinalTemplateState.None, FinalTemplateState.NonTemplate } + ) + { + Word input = Input(text); + input.SyntacticFeatureStruct = fs; + input.FinalTemplateState = finalState; + input.Freeze(); + + Word[] expected = own.Apply(input).ToArray(); + Word[] actual = shared.Apply(input).ToArray(); + Assert.That(actual, Is.EqualTo(expected).Using(FreezableEqualityComparer.Default), text); + Assert.That( + actual.Select(AnalysisStateKey.PinAndKey), + Is.EqualTo(expected.Select(AnalysisStateKey.PinAndKey)), + text + ); + compared += actual.Length; + } + } + } + Assert.That(compared, Is.GreaterThan(1000)); + } + + [Test] + public void UntracedBatteryRunsSharedSlotBatchOnce() + { + var d = Suffix("d", "d"); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + IRule[] rules = new[] + { + Template(new AffixTemplateSlot(d)), + Template(new AffixTemplateSlot(d)), + } + .Select(t => t.CompileAnalysisRule(morpher)) + .ToArray(); + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default); + var shared = new AnalysisAffixTemplatesRule(morpher, rules); + Word input = Input("sagd"); + input.Freeze(); + + Word[] expected = own.Apply(input).ToArray(); + Assert.That(d.ApplyCount, Is.EqualTo(2)); + d.ApplyCount = 0; + Assert.That(shared.Apply(input), Is.EqualTo(expected).Using(FreezableEqualityComparer.Default)); + Assert.That(d.ApplyCount, Is.EqualTo(1), "two identical tails must execute one slot batch"); + _ = shared.Apply(input).ToArray(); + Assert.That(d.ApplyCount, Is.EqualTo(2), "each battery call must execute its own batch"); + } + + [TestCase(false, 2)] + [TestCase(true, 1)] + public void TracedAndParallelBatteriesKeepPerTemplateWalk(bool tracing, int degree) + { + var d = Suffix("d", "d"); + var traceManager = new TraceManager { IsTracing = tracing }; + var morpher = new Morpher(traceManager, Language, maxDegreeOfParallelism: degree); + var shared = new AnalysisAffixTemplatesRule( + morpher, + new[] { Template(new AffixTemplateSlot(d)), Template(new AffixTemplateSlot(d)) }.Select(t => + t.CompileAnalysisRule(morpher) + ) + ); + Word input = Input("sagd"); + input.Freeze(); + if (tracing) + traceManager.AnalyzeWord(Language, input); + _ = shared.Apply(input).ToArray(); + Assert.That(d.ApplyCount, Is.EqualTo(2)); + } + + [Test] + public void SharedWalkPreservesBatteryAlternativeLimit() + { + var d = Suffix("d", "d"); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + IRule[] rules = new[] + { + Template(new AffixTemplateSlot(d) { Optional = true }), + Template(new AffixTemplateSlot(d) { Optional = true }), + } + .Select(t => t.CompileAnalysisRule(morpher)) + .ToArray(); + var shared = new AnalysisAffixTemplatesRule(morpher, rules) { MaxAlternatives = 1 }; + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default) + { + MaxAlternatives = 1, + }; + Word input = Input("sagd"); + input.Freeze(); + Assert.Throws(() => own.Apply(input).ToArray()); + Assert.Throws(() => shared.Apply(input).ToArray()); + } + + [Test] + public void AlternativeLimitStopsBeforeLaterTemplateAdmission() + { + var d = Suffix("d", "d"); + AffixTemplate[] templates = + [ + Template(new AffixTemplateSlot(d) { Optional = true }), + Template(new AffixTemplateSlot(d) { Optional = true }), + Template(new AffixTemplateSlot(Suffix("s", "s"))), + ]; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + RuleSelector = rule => + !ReferenceEquals(rule, templates[2]) + ? true + : throw new InvalidOperationException("later template must not be reached"), + }; + IRule[] rules = templates.Select(t => t.CompileAnalysisRule(morpher)).ToArray(); + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default) + { + MaxAlternatives = 1, + }; + var shared = new AnalysisAffixTemplatesRule(morpher, rules) { MaxAlternatives = 1 }; + Word input = Input("sagd"); + input.Freeze(); + + Assert.Throws(() => own.Apply(input).ToArray()); + Assert.Throws(() => shared.Apply(input).ToArray()); + } + + [TestCase(false, 1)] + [TestCase(true, 0)] + public void ChangedSlotOptionalityPreservesPerTemplateWalk(bool optional, int changedTemplate) + { + var d = Suffix("d", "d"); + AffixTemplate[] templates = + [ + Template(new AffixTemplateSlot(d) { Optional = optional }), + Template(new AffixTemplateSlot(d) { Optional = optional }), + ]; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1); + IRule[] rules = templates.Select(t => t.CompileAnalysisRule(morpher)).ToArray(); + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default); + var shared = new AnalysisAffixTemplatesRule(morpher, rules); + templates[changedTemplate].Slots[0].Optional = !optional; + Word input = Input("sag"); + input.Freeze(); + + Assert.That(shared.Apply(input), Is.EqualTo(own.Apply(input)).Using(FreezableEqualityComparer.Default)); + } + + private Word Input(string text) => new(Morphophonemic, Table3.Segment(text)); + + private FeatureStruct Pos(string symbol) => FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol(symbol).Value; + + private static AffixTemplate Template(params AffixTemplateSlot[] slots) + { + var template = new AffixTemplate(); + foreach (AffixTemplateSlot slot in slots) + template.Slots.Add(slot); + return template; + } + + private CountingSuffix Suffix(string name, string suffix) + { + var rule = new CountingSuffix { Name = name, MaxApplicationCount = 3 }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern + .New("stem") + .Annotation(FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value) + .OneOrMore.Value, + }, + Rhs = { new CopyFromInput("stem"), new InsertSegments(Table3, suffix) }, + } + ); + return rule; + } + + private static IEnumerable Inputs(int maxLength) + { + string[] frontier = [""]; + yield return ""; + for (int length = 1; length <= maxLength; length++) + { + frontier = frontier.SelectMany(w => new[] { "s", "a", "d" }.Select(c => w + c)).ToArray(); + foreach (string word in frontier) + yield return word; + } + } + + private sealed class CountingSuffix : AffixProcessRule + { + public int ApplyCount; + + public override IRule CompileAnalysisRule(Morpher morpher) => + new CountingRule(this, base.CompileAnalysisRule(morpher)); + + private sealed class CountingRule(CountingSuffix owner, IRule rule) : IRule + { + public IEnumerable Apply(Word input) + { + Interlocked.Increment(ref owner.ApplyCount); + return rule.Apply(input); + } + } + } +} From c07febe5b1767bbb3a00dd06ae3fe9b0f8abe0d5 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Wed, 21 Oct 2026 20:56:32 -0400 Subject: [PATCH 2/6] Address review: drop the unreachable limit check, explain the fallback guard, avoid per-node closures The shared walk is only taken when MaxAlternatives is 0, so its limit check could never fire; the guard comment now names all five fallback conditions; Walk checks admitted templates with a loop instead of a capturing lambda. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../AnalysisAffixTemplatesRule.cs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs index cbd08578f..ff92b8fa6 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs @@ -54,7 +54,9 @@ public AnalysisAffixTemplatesRule(Morpher morpher, IEnumerable Apply(Word input) { - // A configured limit must stop before later templates are evaluated. + // The original walk is required when: no tail is shared; tracing (ApplySlots pairs per-template + // Begin/End events); parallel slots (a separate breadth-first walk); a configured alternative limit + // (it must stop before later templates run); or slot optionality changed since the trie was built. if ( !_hasSharedSlots || _morpher.TraceManager.IsTracing @@ -84,8 +86,6 @@ public override IEnumerable Apply(Word input) { if (templateOutput != null) output.UnionWith(templateOutput); - if (MaxAlternatives > 0 && output.Count > MaxAlternatives) - throw new MaxAlternativesExceededException("MaxAlternatives exceeded"); } return output; } @@ -114,7 +114,7 @@ private static void Walk(SlotNode node, Word input, HashSet[] outputs) foreach (SlotNode child in node.Children) { - if (!child.Templates.Any(t => outputs[t] != null)) + if (!AnyAdmitted(child.Templates, outputs)) continue; foreach (Word word in child.Rule.Apply(input)) @@ -124,6 +124,16 @@ private static void Walk(SlotNode node, Word input, HashSet[] outputs) } } + private static bool AnyAdmitted(List templates, HashSet[] outputs) + { + for (int i = 0; i < templates.Count; i++) + { + if (outputs[templates[i]] != null) + return true; + } + return false; + } + private sealed class SlotNode { public AffixTemplateSlot Slot { get; set; } From 1c51b2cf9163a39d1651ae1690f0f5d15b182626 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 23 Oct 2026 07:05:16 -0400 Subject: [PATCH 3/6] Shorten the shared-slot fallback comment to the hygiene budget Co-Authored-By: Claude Opus 5.5 --- .../AnalysisAffixTemplatesRule.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs index ff92b8fa6..58b55e194 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs @@ -54,9 +54,8 @@ public AnalysisAffixTemplatesRule(Morpher morpher, IEnumerable Apply(Word input) { - // The original walk is required when: no tail is shared; tracing (ApplySlots pairs per-template - // Begin/End events); parallel slots (a separate breadth-first walk); a configured alternative limit - // (it must stop before later templates run); or slot optionality changed since the trie was built. + // Use the original walk when no tail is shared, when tracing (per-template Begin/End events), for + // parallel slots, under an alternative limit, or if slot optionality changed after the trie was built. if ( !_hasSharedSlots || _morpher.TraceManager.IsTracing From 17a6894ee6222da1ef91068ff1ba0008fa5aef6c Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 23 Oct 2026 07:09:21 -0400 Subject: [PATCH 4/6] Compare shared-walk outputs with AnalysisMergeKey #522 removed AnalysisStateKey along with the analysis-cascade memo and kept the same identity as AnalysisStratumRule.AnalysisMergeKey, so the shared-walk equivalence test now compares outputs under that key. Co-Authored-By: Claude Opus 5.5 --- .../AnalysisAffixTemplatesRuleTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs index 9d91572ee..275d27325 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs @@ -65,8 +65,8 @@ public void SharedWalkMatchesPerTemplateWalkInOrder(int variant) Word[] actual = shared.Apply(input).ToArray(); Assert.That(actual, Is.EqualTo(expected).Using(FreezableEqualityComparer.Default), text); Assert.That( - actual.Select(AnalysisStateKey.PinAndKey), - Is.EqualTo(expected.Select(AnalysisStateKey.PinAndKey)), + actual.Select(w => new AnalysisStratumRule.AnalysisMergeKey(w)), + Is.EqualTo(expected.Select(w => new AnalysisStratumRule.AnalysisMergeKey(w))), text ); compared += actual.Length; From 4724e4d148f8ef20b03719d54dbfb191baf63f3d Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 27 Oct 2026 21:02:39 -0400 Subject: [PATCH 5/6] Prune affix template partials whose required slot cannot find its material An analysis template partial now stops as soon as a required slot can no longer find the segments its affix inserts, instead of paying for every optional slot before the required one. The test is used only while every slot walked since is a pure copy-and-insert rule, so parses do not change. A count-only shadow mode and a null-by-default check observer let a harness verify soundness and cost on real grammars. Closes #561 Co-Authored-By: Claude Opus 5.5 --- .../AnalysisAffixTemplateRule.cs | 114 ++++- .../Morpher.cs | 20 + .../RequiredSlotMaterial.cs | 264 ++++++++++++ .../AnalysisAffixTemplateRuleTests.cs | 403 ++++++++++++++++++ 4 files changed, 790 insertions(+), 11 deletions(-) create mode 100644 src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs index ba963745a..f467f934d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs @@ -14,6 +14,7 @@ internal class AnalysisAffixTemplateRule : IRule private readonly Morpher _morpher; private readonly AffixTemplate _template; private readonly List> _rules; + private readonly RequiredSlotMaterial _lookahead; public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template) { @@ -26,6 +27,7 @@ public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template) FreezableEqualityComparer.Default )) ); + _lookahead = new RequiredSlotMaterial(template); } public IEnumerable Apply(Word input) @@ -33,14 +35,22 @@ public IEnumerable Apply(Word input) if (!CanApply(input)) return Enumerable.Empty(); + int[] fired = null; + if (RequiredSlotCannotUnapply(input, _rules.Count - 1, ref fired)) + { + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate(_template, input, false, FailureReason.None); + return Enumerable.Empty(); + } + Word inWord = input.Clone(); inWord.Freeze(); var output = new HashSet(FreezableEqualityComparer.Default); if (_morpher.MaxDegreeOfParallelism == 1) - ApplySlots(inWord, _rules.Count - 1, output); + ApplySlots(inWord, _rules.Count - 1, output, fired); else - ParallelApplySlots(inWord, output); + ParallelApplySlots(inWord, output, fired); return output; } @@ -85,12 +95,76 @@ internal bool CanApply(Word input) return true; } - private void ApplySlots(Word inWord, int index, HashSet output) + // Testing runs from the next slot to unapply and stops after an impure slot, which makes the test + // for any slot behind it unsound. Shadow mode prunes nothing and carries the fired slots in fired. + internal bool RequiredSlotCannotUnapply(Word word, int lastSlot, ref int[] fired) + { + RequiredSlotLookaheadMode mode = _morpher.RequiredSlotLookahead; + if (mode == RequiredSlotLookaheadMode.Off) + return false; + + long unifications = 0; + bool cannot = false; + for (int j = lastSlot; j >= 0; j--) + { + if ( + !_template.Slots[j].Optional + && !IsFired(fired, j) + && _lookahead.CannotUnapply(j, word.Shape, ref unifications) + ) + { + if (mode == RequiredSlotLookaheadMode.On) + { + cannot = true; + break; + } + fired = fired == null ? new[] { j } : fired.Append(j).ToArray(); + _morpher.RequiredSlotShadow?.Invoke("fired", _template, j); + } + if (!_lookahead.IsPure(j)) + break; + } + if (unifications != 0) + _morpher.RequiredSlotCheckObserver?.Invoke(unifications); + return cannot; + } + + internal static bool IsFired(int[] fired, int slot) + { + return fired != null && Array.IndexOf(fired, slot) >= 0; + } + + internal void ReportShadow(string kind, int[] fired, int slot) + { + if (fired == null || _morpher.RequiredSlotShadow == null) + return; + if (slot < 0) + { + foreach (int j in fired) + _morpher.RequiredSlotShadow(kind, _template, j); + } + else if (IsFired(fired, slot)) + { + _morpher.RequiredSlotShadow(kind, _template, slot); + } + } + + private void ApplySlots(Word inWord, int index, HashSet output, int[] fired) { for (int i = index; i >= 0; i--) { foreach (Word outWord in _rules[i].Apply(inWord)) - ApplySlots(outWord, i - 1, output); + { + ReportShadow("violated", fired, i); + int[] outFired = fired; + if (RequiredSlotCannotUnapply(outWord, i - 1, ref outFired)) + { + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate(_template, outWord, false, FailureReason.None); + continue; + } + ApplySlots(outWord, i - 1, output, outFired); + } if (!_template.Slots[i].Optional) { @@ -102,16 +176,17 @@ private void ApplySlots(Word inWord, int index, HashSet output) if (_morpher.TraceManager.IsTracing) _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true, FailureReason.None); + ReportShadow("completed-under-fired", fired, -1); output.Add(inWord); } - private void ParallelApplySlots(Word inWord, HashSet output) + private void ParallelApplySlots(Word inWord, HashSet output, int[] fired) { ParallelOptions parallelOptions = _morpher.CreateParallelOptions(); var outStack = new ConcurrentStack(); - var from = new ConcurrentStack>(); - from.Push(Tuple.Create(inWord, _rules.Count - 1)); - var to = new ConcurrentStack>(); + var from = new ConcurrentStack>(); + from.Push(Tuple.Create(inWord, _rules.Count - 1, fired)); + var to = new ConcurrentStack>(); while (!from.IsEmpty) { to.Clear(); @@ -123,9 +198,25 @@ private void ParallelApplySlots(Word inWord, HashSet output) bool add = true; for (int i = work.Item2; i >= 0; i--) { - Tuple[] workItems = _rules[i] + int next = i - 1; + Tuple[] workItems = _rules[i] .Apply(work.Item1) - .Select(res => Tuple.Create(res, i - 1)) + .Select(res => + { + ReportShadow("violated", work.Item3, i); + int[] resFired = work.Item3; + if (!RequiredSlotCannotUnapply(res, next, ref resFired)) + return Tuple.Create(res, next, resFired); + if (_morpher.TraceManager.IsTracing) + _morpher.TraceManager.EndUnapplyTemplate( + _template, + res, + false, + FailureReason.None + ); + return null; + }) + .Where(item => item != null) .ToArray(); if (workItems.Length > 0) to.PushRange(workItems); @@ -153,11 +244,12 @@ private void ParallelApplySlots(Word inWord, HashSet output) true, FailureReason.None ); + ReportShadow("completed-under-fired", work.Item3, -1); outStack.Push(work.Item1); } } ); - ConcurrentStack> temp = from; + ConcurrentStack> temp = from; from = to; to = temp; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index d0675509e..ca906f91e 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -83,6 +83,26 @@ public ITraceManager TraceManager public int DeletionReapplications { get; set; } + /// + /// Whether affix template walks drop partials in which a required slot can no longer unapply because + /// the material it inserts is absent. See . + /// + internal RequiredSlotLookaheadMode RequiredSlotLookahead { get; set; } = RequiredSlotLookaheadMode.On; + + /// + /// Called in with a kind and the template and slot it is about: + /// "fired" (the test is true for a required slot), "violated" (a rule of a fired slot then unapplied under + /// that partial) and "completed-under-fired" (the template walk completed with a fired slot on its path). + /// Any "violated" or "completed-under-fired" is a soundness failure. + /// + internal Action RequiredSlotShadow { get; set; } + + /// + /// Called after each required-slot check that unified, with the number of unifications it performed. + /// Null unless a diagnostic sets it; it may be called from several threads. + /// + internal Action RequiredSlotCheckObserver { get; set; } + public int MaxStemCount { get; set; } /// diff --git a/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs new file mode 100644 index 000000000..27e60caa6 --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs @@ -0,0 +1,264 @@ +using System.Collections.Generic; +using System.Linq; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab +{ + /// + /// How uses . + /// + internal enum RequiredSlotLookaheadMode + { + /// The look-ahead is not evaluated. + Off, + + /// The look-ahead is evaluated and reported to Morpher.RequiredSlotShadow, never pruning. + Shadow, + + /// The look-ahead prunes template partials. + On, + } + + /// + /// A necessary condition for a required affix template slot to unapply later in a template walk: some + /// allomorph of some rule in the slot must have every inserted segment unify with a segment of the shape. + /// + /// + /// The tables are built once from the template and depend only on the grammar. + /// + /// Soundness. The analysis pattern of an allomorph holds one top-level constraint per inserted segment + /// (, ), + /// and the rule can only unapply if each constraint unifies with a distinct segment node of the shape. + /// A rule is pure when every right-hand side action copies an input part or inserts material, and every + /// left-hand part is copied by a that must capture a segment, or has no + /// segment to restore. Then only copies input + /// nodes and never untruncates a segment. Unification can only narrow the feature structure of a copied + /// node, so a constraint that unifies with no node of the shape unifies with no node of the shape after + /// any number of pure unapplications. Inside one template walk only the rules of that template's slots + /// run, and the stratum's phonological rules ran before the walk. The test for slot j is therefore used + /// only when every rule in the slots still to be walked after j is pure. Optional nodes count as present + /// because the matcher can match them. Anything that cannot be summarised (a null affix, a constraint with + /// variables, an unknown rule type) leaves the slot unprunable. + /// + internal class RequiredSlotMaterial + { + private readonly SlotSummary[] _slots; + + public RequiredSlotMaterial(AffixTemplate template) + { + _slots = template.Slots.Select(Summarize).ToArray(); + } + + /// + /// Whether every rule of the slot is pure, so that unapplying the slot leaves the segments needed by + /// slots before it no easier to find. + /// + public bool IsPure(int slot) + { + return _slots[slot].IsPure; + } + + /// + /// True if no allomorph of the slot can unapply to a word with this shape, provided that every slot + /// walked between this word and the slot is pure. + /// + /// The slot to test. + /// The shape of the partial. + /// Incremented by the feature structure unifications performed. + public bool CannotUnapply(int slot, Shape shape, ref long unifications) + { + SlotSummary summary = _slots[slot]; + if (summary.MayAlwaysUnapply) + return false; + + // The constraints of a slot repeat across its allomorphs, so each distinct one is looked up once. + ulong known = 0; + ulong present = 0; + foreach (int[] requirements in summary.Allomorphs) + { + bool all = true; + foreach (int index in requirements) + { + bool found; + ulong bit = index < 64 ? 1UL << index : 0; + if ((known & bit) != 0) + { + found = (present & bit) != 0; + } + else + { + found = IsPresent(summary.Distinct[index], shape, ref unifications); + known |= bit; + if (found) + present |= bit; + } + if (!found) + { + all = false; + break; + } + } + if (all) + return false; + } + return true; + } + + private static bool IsPresent(FeatureStruct required, Shape shape, ref long unifications) + { + foreach (ShapeNode node in shape) + { + if (node.Type() != HCFeatureSystem.Segment) + continue; + + unifications++; + if (node.Annotation.FeatureStruct.IsUnifiable(required, false, null)) + return true; + } + return false; + } + + private static SlotSummary Summarize(AffixTemplateSlot slot) + { + var summary = new SlotSummary(); + var allomorphs = new List(); + foreach (MorphemicMorphologicalRule rule in slot.Rules) + { + IList ruleAllomorphs; + switch (rule) + { + case AffixProcessRule affixRule: + ruleAllomorphs = affixRule.Allomorphs; + break; + case RealizationalAffixProcessRule realizationalRule: + ruleAllomorphs = realizationalRule.Allomorphs; + break; + default: + summary.MayAlwaysUnapply = true; + summary.IsPure = false; + continue; + } + + foreach (AffixProcessAllomorph allomorph in ruleAllomorphs) + { + if (!IsPure(allomorph)) + summary.IsPure = false; + + FeatureStruct[] requirements = GetRequirements(allomorph); + if (requirements.Length == 0) + summary.MayAlwaysUnapply = true; + else + allomorphs.Add(requirements); + } + } + var distinct = new List(); + summary.Allomorphs = allomorphs + .Select(requirements => + requirements + .Select(fs => + { + int index = distinct.FindIndex(d => d.ValueEquals(fs)); + if (index < 0) + { + distinct.Add(fs); + index = distinct.Count - 1; + } + return index; + }) + .ToArray() + ) + .ToArray(); + summary.Distinct = distinct.ToArray(); + return summary; + } + + private static FeatureStruct[] GetRequirements(AffixProcessAllomorph allomorph) + { + var requirements = new List(); + foreach (MorphologicalOutputAction action in allomorph.Rhs) + { + if (action is InsertSegments insertSegments) + { + foreach (ShapeNode node in insertSegments.Segments.Shape) + { + if (node.Type() != HCFeatureSystem.Boundary) + AddRequirement(requirements, node.Annotation.FeatureStruct); + } + } + else if (action is InsertSimpleContext insertContext) + { + AddRequirement(requirements, insertContext.SimpleContext.FeatureStruct); + } + } + return requirements.ToArray(); + } + + // A constraint with variables is bound by earlier matches, and one that is not a segment is never + // matched against the segments the matcher considers, so neither can be tested here. + private static void AddRequirement(List requirements, FeatureStruct fs) + { + if (fs.HasVariables || (FeatureSymbol)fs.GetValue(HCFeatureSystem.Type) != HCFeatureSystem.Segment) + return; + + FeatureStruct copy = fs.Clone(); + copy.Freeze(); + requirements.Add(copy); + } + + private static bool IsPure(AffixProcessAllomorph allomorph) + { + var copied = new HashSet(); + foreach (MorphologicalOutputAction action in allomorph.Rhs) + { + if (action is CopyFromInput) + copied.Add(action.PartName); + else if (!(action is InsertSegments || action is InsertSimpleContext)) + return false; + } + + foreach (Pattern part in allomorph.Lhs) + { + if (!HasSegmentConstraint(part)) + continue; + if (!copied.Contains(part.Name) || MinSegments(part) == 0) + return false; + } + return true; + } + + private static bool HasSegmentConstraint(PatternNode node) + { + if (node is Constraint constraint) + return constraint.Type() == HCFeatureSystem.Segment; + return node.Children.Any(HasSegmentConstraint); + } + + // The fewest segments a successful match of the node must consume. A part matching none is captured + // as a null range, and GenerateShape then untruncates it, which adds segments that are not copies. + private static int MinSegments(PatternNode node) + { + switch (node) + { + case Constraint constraint: + return constraint.Type() == HCFeatureSystem.Segment ? 1 : 0; + case Quantifier quantifier: + return quantifier.MinOccur * quantifier.Children.Sum(MinSegments); + case Alternation alternation: + return alternation.Children.Count == 0 ? 0 : alternation.Children.Min(MinSegments); + default: + return node.Children.Sum(MinSegments); + } + } + + private class SlotSummary + { + public bool IsPure = true; + public bool MayAlwaysUnapply; + public int[][] Allomorphs; + public FeatureStruct[] Distinct; + } + } +} diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs index ed5cabc36..ee799323a 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs @@ -69,6 +69,409 @@ public void Apply_SlotLeftEmpty_LeavesTheSameStateAsNotEnteringTheTemplate() }); } + // Slots are listed in synthesis order and unapplied from the last to the first, so slot 0 is the suffix + // closest to the stem and a word ends with the material of its highest slot. + private static AffixTemplateSlot Slot(bool optional, params MorphemicMorphologicalRule[] rules) => + new(rules) { Optional = optional }; + + private static AffixTemplate TemplateOf(params AffixTemplateSlot[] slots) + { + var template = new AffixTemplate { Name = "lookahead_template" }; + foreach (AffixTemplateSlot slot in slots) + template.Slots.Add(slot); + return template; + } + + // The analysis of a rule that deletes a final segment in synthesis: it adds the segment back, so the + // unapplication produces a shape that is not made of copies of the input. + private AffixProcessRule SegmentRestoringRule(string name, string restored) + { + var rule = new AffixProcessRule + { + Id = name, + Name = name, + Gloss = name, + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern.New("2").Annotation(Character(Table3, restored)).Value, + }, + Rhs = { new CopyFromInput("1") }, + } + ); + return rule; + } + + private AffixProcessRule SuffixRule(string name, string insert) => + Suffix(name, FeatureStruct.New().Value, FeatureStruct.New().Value, Table3, insert); + + private Word Input(string form, bool lastNodeOptional = false) + { + var input = new Word(Morphophonemic, Morphophonemic.CharacterDefinitionTable.Segment(form)); + if (lastNodeOptional) + input.Shape.Last.Annotation.Optional = true; + return input; + } + + private sealed record WalkResult(List Parses, int RuleAttempts); + + private WalkResult Walk( + AffixTemplate template, + Word input, + RequiredSlotLookaheadMode mode, + int maxDegreeOfParallelism = 1, + bool tracing = false + ) + { + int attempts = 0; + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = mode, + RuleSelector = rule => + { + if (rule is AffixProcessRule) + System.Threading.Interlocked.Increment(ref attempts); + return true; + }, + }; + TraceManager.IsTracing = tracing; + if (tracing) + TraceManager.AnalyzeWord(Language, input); + input.Freeze(); + IRule rule = template.CompileAnalysisRule(morpher); + List parses = rule.Apply(input) + .Select(w => w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToList(); + return new WalkResult(parses, attempts); + } + + private (WalkResult Off, WalkResult On) WalkBoth(AffixTemplate template, string form, bool lastNodeOptional = false) + { + WalkResult off = Walk(template, Input(form, lastNodeOptional), RequiredSlotLookaheadMode.Off); + WalkResult on = Walk(template, Input(form, lastNodeOptional), RequiredSlotLookaheadMode.On); + Assert.That(on.Parses, Is.EqualTo(off.Parses), "the look-ahead must not change the parses"); + return (off, on); + } + + // stem + required -ti + optional -u/-f/-v + optional -z + private AffixTemplate RequiredInnerSuffixTemplate() => + TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SuffixRule("u", "u"), SuffixRule("f", "f"), SuffixRule("v", "v")), + Slot(true, SuffixRule("z", "z")) + ); + + [Test] + public void Apply_RequiredSlotLiteralAbsent_PrunesBeforeTryingTheOptionalSlots() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagufz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0), "no rule can supply -ti, so no slot rule is attempted"); + }); + } + + [Test] + public void Apply_RequiredSlotLiteralPresent_DoesNotPrune() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagtiuz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_RequiredSlotLiteralLostAfterAnOptionalSlot_PrunesTheInnerPartial() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + // -z unapplies, which leaves "sagtuu"; no node can still be the i of -ti + (WalkResult off, WalkResult on) = WalkBoth(template, "sagttz"); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Empty); + Assert.That(on.RuleAttempts, Is.LessThan(off.RuleAttempts)); + }); + } + + [Test] + public void Apply_ImpureSlotBetween_DoesNotPruneForTheInnerRequiredSlot() + { + // the middle slot restores the i that -ti needs, so "sagt" is a stem plus -ti once it has run + AffixTemplate template = TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SegmentRestoringRule("restore_i", "i")) + ); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagt"); + + Assert.That(on.Parses, Is.Not.Empty, "the parse that needs the restored segment must survive"); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + } + + [Test] + public void Apply_SeveralRequiredSlots_PrunesWhicheverLiteralIsAbsent() + { + // -u is required as well as -ti; the word has neither + AffixTemplate template = TemplateOf( + Slot(false, SuffixRule("ti", "ti")), + Slot(true, SuffixRule("z", "z")), + Slot(false, SuffixRule("u", "u")) + ); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagfz"); + + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + } + + [Test] + public void Apply_OptionalNodeThatUnifiesWithTheLiteral_CountsAsPresent() + { + AffixTemplate template = TemplateOf(Slot(false, SuffixRule("ti", "ti"))); + + // "sagti" where the i stands for a deleted segment: the matcher may match or skip an optional node + (WalkResult off, WalkResult on) = WalkBoth(template, "sagti", lastNodeOptional: true); + + Assert.Multiple(() => + { + Assert.That(off.Parses, Is.Not.Empty); + Assert.That(on.Parses, Is.EqualTo(off.Parses)); + }); + } + + [Test] + public void Apply_RequiredSlotWithNullAffix_IsNeverPruned() + { + AffixProcessRule withNull = SuffixRule("ti_or_null", "ti"); + withNull.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1") }, + } + ); + AffixTemplate template = TemplateOf(Slot(false, withNull), Slot(true, SuffixRule("z", "z"))); + + (WalkResult off, WalkResult on) = WalkBoth(template, "sagz"); + + Assert.Multiple(() => + { + Assert.That(on.Parses, Is.Not.Empty); + Assert.That(on.RuleAttempts, Is.EqualTo(off.RuleAttempts)); + }); + } + + // An optional slot whose rule inserts "z" after the copied parts; the word never contains the t and i + // that the required inner slot inserts, so a pure rule lets the walk prune at entry, before the optional rule is tried, and an impure one does not. + private (WalkResult Off, WalkResult On) WalkWithOptionalSlot( + Pattern[] lhs, + MorphologicalOutputAction[] rhs + ) + { + var rule = new AffixProcessRule + { + Id = "z_slot", + Name = "z_slot", + Gloss = "z_slot", + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + var allomorph = new AffixProcessAllomorph(); + foreach (Pattern part in lhs) + allomorph.Lhs.Add(part); + foreach (MorphologicalOutputAction action in rhs) + allomorph.Rhs.Add(action); + allomorph.Rhs.Add(new InsertSegments(Table3, "z")); + rule.Allomorphs.Add(allomorph); + AffixTemplate template = TemplateOf(Slot(false, SuffixRule("ti", "ti")), Slot(true, rule)); + return WalkBoth(template, "sagz"); + } + + [Test] + public void Apply_CopiedPartThatMayMatchNothing_MakesTheSlotImpure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] { Pattern.New("1").Annotation(Any).ZeroOrMore.Value }, + new MorphologicalOutputAction[] { new CopyFromInput("1") } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_ModifyFromInputInASlot_MakesTheSlotImpure() + { + FeatureStruct voiced = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("vd+") + .Value; + + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] { Pattern.New("1").Annotation(Any).OneOrMore.Value }, + new MorphologicalOutputAction[] + { + new ModifyFromInput("1", voiced, System.Array.Empty()), + } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_CopiedAlternationOfSegments_KeepsTheSlotPure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern + .New("2") + .Annotation(Character(Table3, "i")) + .Or.Annotation(Character(Table3, "u")) + .Value, + }, + new MorphologicalOutputAction[] { new CopyFromInput("1"), new CopyFromInput("2") } + ); + + Assert.That(off.RuleAttempts, Is.GreaterThan(0)); + Assert.That(on.RuleAttempts, Is.EqualTo(0)); + } + + [Test] + public void Apply_CopiedAlternationWithAnOptionalBranch_MakesTheSlotImpure() + { + (WalkResult off, WalkResult on) = WalkWithOptionalSlot( + new[] + { + Pattern.New("1").Annotation(Any).OneOrMore.Value, + Pattern + .New("2") + .Annotation(Character(Table3, "i")) + .Optional.Or.Annotation(Character(Table3, "u")) + .Value, + }, + new MorphologicalOutputAction[] { new CopyFromInput("1"), new CopyFromInput("2") } + ); + + Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry"); + } + + [Test] + public void Apply_SlotOptionalityChangedAfterCompiling_IsReadLive() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + var morpher = new Morpher(TraceManager, Language, 1) { RequiredSlotLookahead = RequiredSlotLookaheadMode.On }; + IRule rule = template.CompileAnalysisRule(morpher); + Word input = Input("sagu"); + input.Freeze(); + + Assert.That(rule.Apply(input), Is.Empty, "slot 0 is required and -ti is absent"); + + template.Slots[0].Optional = true; + Assert.That(rule.Apply(input), Is.Not.Empty, "an optional slot is not tested"); + + template.Slots[0].Optional = false; + Assert.That(rule.Apply(input), Is.Empty); + } + + [TestCase(0)] + [TestCase(4)] + public void Apply_Parallel_ReturnsTheSameParsesAsSequential(int maxDegreeOfParallelism) + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + + foreach (string form in new[] { "sagtiuz", "sagufz", "sagttz", "sagtif" }) + { + WalkResult sequential = Walk(template, Input(form), RequiredSlotLookaheadMode.On); + WalkResult parallel = Walk(template, Input(form), RequiredSlotLookaheadMode.On, maxDegreeOfParallelism); + WalkResult parallelOff = Walk(template, Input(form), RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + Assert.That(parallel.Parses, Is.EqualTo(sequential.Parses), form); + Assert.That(parallel.Parses, Is.EqualTo(parallelOff.Parses), form); + } + } + + [Test] + public void Apply_Tracing_ShowsAPrunedTemplateAsAFailedOutput() + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + Word input = Input("sagufz"); + + Walk(template, input, RequiredSlotLookaheadMode.On, tracing: true); + TraceManager.IsTracing = false; + + var root = (Trace)input.CurrentTrace; + Trace[] children = root.Children.ToArray(); + Assert.Multiple(() => + { + Assert.That( + children.Select(t => t.Type), + Is.EqualTo(new[] { TraceType.TemplateAnalysisInput, TraceType.TemplateAnalysisOutput }) + ); + Assert.That(children[1].Output, Is.Null, "the partial is reported as not unapplied"); + }); + } + + [TestCase(1)] + [TestCase(0)] + public void Apply_Shadow_ReportsFiredPointsAndNeverViolatesOrChangesTheParses(int maxDegreeOfParallelism) + { + AffixTemplate template = RequiredInnerSuffixTemplate(); + var events = new List(); + + foreach (string form in new[] { "sagtiuz", "sagufz", "sagttz", "sagtif", "sagu" }) + { + WalkResult off = Walk(template, Input(form), RequiredSlotLookaheadMode.Off, maxDegreeOfParallelism); + Word input = Input(form); + input.Freeze(); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism) + { + RequiredSlotLookahead = RequiredSlotLookaheadMode.Shadow, + RequiredSlotShadow = (kind, t, slot) => + { + lock (events) + events.Add(kind); + }, + }; + TraceManager.IsTracing = false; + string[] parses = template + .CompileAnalysisRule(morpher) + .Apply(input) + .Select(w => + w.ToString() + " " + string.Join(",", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss)) + ) + .OrderBy(p => p, System.StringComparer.Ordinal) + .ToArray(); + Assert.That(parses, Is.EqualTo(off.Parses), form); + } + + Assert.Multiple(() => + { + Assert.That(events.Count(e => e == "fired"), Is.GreaterThan(0)); + Assert.That(events.Count(e => e == "violated"), Is.EqualTo(0)); + Assert.That(events.Count(e => e == "completed-under-fired"), Is.EqualTo(0)); + }); + } + private AffixTemplate VerbTemplate(FeatureStruct required) { var template = new AffixTemplate { Name = "verb_template", RequiredSyntacticFeatureStruct = required }; From fcb65d7b5dc5be083ed1123cbea5b0c130a9ec60 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 27 Oct 2026 21:12:49 -0400 Subject: [PATCH 6/6] Run the required-slot look-ahead on the shared trailing-slot template walk Each template under a trie node gets its own look-ahead and a dead template stops collecting below it while its siblings go on. Closes #566 Co-Authored-By: Claude Opus 5.5 --- .../AnalysisAffixTemplatesRule.cs | 85 +++++++++++-- .../AnalysisAffixTemplateRuleTests.cs | 6 +- .../AnalysisAffixTemplatesRuleTests.cs | 116 +++++++++++++++++- 3 files changed, 195 insertions(+), 12 deletions(-) diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs index 58b55e194..5103c13cd 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplatesRule.cs @@ -22,6 +22,7 @@ public AnalysisAffixTemplatesRule(Morpher morpher, IEnumerable t.Template.Slots.Select(s => s.Optional).ToArray()).ToArray(); for (int t = 0; t < _templates.Length; t++) { @@ -38,6 +39,7 @@ public AnalysisAffixTemplatesRule(Morpher morpher, IEnumerable Apply(Word input) { Word inWord = input.Clone(); inWord.Freeze(); - Walk(_root, inWord, outputs); + Walk(_root, inWord, new WalkState(_templates, outputs, _morpher), null); } // Template order determines which equal word survives and the order seen by analysis folding. @@ -106,37 +108,104 @@ private bool SlotsUnchanged() return true; } - private static void Walk(SlotNode node, Word input, HashSet[] outputs) + // A template that fails the look-ahead on a word is dead below it while its siblings go on. + // Marks are set on entry and undone on return, so the mask is one bool per template per call. + private void Walk(SlotNode node, Word input, WalkState state, int[][] fired) { + List marked = null; + if (state.Mode != RequiredSlotLookaheadMode.Off) + { + foreach (int t in node.Templates) + { + if (state.Outputs[t] == null || state.Dead[t]) + continue; + AnalysisAffixTemplateRule template = _templates[t]; + int[] templateFired = fired?[t]; + if ( + !template.RequiredSlotCannotUnapply( + input, + template.Template.Slots.Count - 1 - node.Depth, + ref templateFired + ) + ) + { + if (!ReferenceEquals(templateFired, fired?[t])) + { + fired = fired == null ? new int[_templates.Length][] : (int[][])fired.Clone(); + fired[t] = templateFired; + } + continue; + } + state.Dead[t] = true; + if (marked == null) + marked = new List(); + marked.Add(t); + } + } + foreach (int t in node.Terminals) - outputs[t]?.Add(input); + { + if (state.Outputs[t] == null || state.Dead[t]) + continue; + _templates[t].ReportShadow("completed-under-fired", fired?[t], -1); + state.Outputs[t].Add(input); + } foreach (SlotNode child in node.Children) { - if (!AnyAdmitted(child.Templates, outputs)) + if (!AnyLive(child.Templates, state)) continue; foreach (Word word in child.Rule.Apply(input)) - Walk(child, word, outputs); + { + if (fired != null) + { + foreach (int t in child.Templates) + _templates[t] + .ReportShadow("violated", fired[t], _templates[t].SlotRules.Count - child.Depth); + } + Walk(child, word, state, fired); + } if (child.Optional) - Walk(child, input, outputs); + Walk(child, input, state, fired); + } + + if (marked != null) + { + foreach (int t in marked) + state.Dead[t] = false; } } - private static bool AnyAdmitted(List templates, HashSet[] outputs) + private static bool AnyLive(List templates, WalkState state) { for (int i = 0; i < templates.Count; i++) { - if (outputs[templates[i]] != null) + if (state.Outputs[templates[i]] != null && !state.Dead[templates[i]]) return true; } return false; } + private sealed class WalkState + { + public WalkState(AnalysisAffixTemplateRule[] templates, HashSet[] outputs, Morpher morpher) + { + Outputs = outputs; + Dead = new bool[templates.Length]; + Mode = morpher.RequiredSlotLookahead; + } + + public HashSet[] Outputs { get; } + public bool[] Dead { get; } + public RequiredSlotLookaheadMode Mode { get; } + } + private sealed class SlotNode { public AffixTemplateSlot Slot { get; set; } public bool Optional { get; set; } + public int Depth { get; set; } public IRule Rule { get; set; } public List Children { get; } = new List(); public List Templates { get; } = new List(); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs index ee799323a..439f0b4d1 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs @@ -1,4 +1,4 @@ -using NUnit.Framework; +using NUnit.Framework; using SIL.Machine.Annotations; using SIL.Machine.FeatureModel; using SIL.Machine.Matching; @@ -280,8 +280,8 @@ public void Apply_RequiredSlotWithNullAffix_IsNeverPruned() }); } - // An optional slot whose rule inserts "z" after the copied parts; the word never contains the t and i - // that the required inner slot inserts, so a pure rule lets the walk prune at entry, before the optional rule is tried, and an impure one does not. + // The word lacks the t and i that the required inner slot inserts, so a pure optional rule that inserts "z" + // lets the walk prune at entry, and an impure one does not. private (WalkResult Off, WalkResult On) WalkWithOptionalSlot( Pattern[] lhs, MorphologicalOutputAction[] rhs diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs index 275d27325..d3b1c82a9 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplatesRuleTests.cs @@ -1,4 +1,4 @@ -using NUnit.Framework; +using NUnit.Framework; using SIL.Machine.Annotations; using SIL.Machine.FeatureModel; using SIL.Machine.Matching; @@ -197,6 +197,120 @@ public void ChangedSlotOptionalityPreservesPerTemplateWalk(bool optional, int ch Assert.That(shared.Apply(input), Is.EqualTo(own.Apply(input)).Using(FreezableEqualityComparer.Default)); } + private sealed record LookaheadRun(string[] Output, int InnerA, int InnerS, int Tail, List Shadow); + + // Two templates share the tail slot "ad". Template A needs an inner "a" suffix, which the tail's unapplication + // consumes from "sad"; template B has an optional inner "s" slot and still yields its outputs. + private LookaheadRun RunSharedTail(RequiredSlotLookaheadMode mode, string text, bool innerAIsOptional = false) + { + CountingSuffix a = Suffix("a", "a"); + CountingSuffix s = Suffix("s", "s"); + CountingSuffix tail = Suffix("ad", "ad"); + var shadow = new List(); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + RequiredSlotLookahead = mode, + RequiredSlotShadow = (kind, template, slot) => shadow.Add(kind), + }; + IRule[] rules = new[] + { + Template(new AffixTemplateSlot(a) { Optional = innerAIsOptional }, new AffixTemplateSlot(tail)), + Template(new AffixTemplateSlot(s) { Optional = true }, new AffixTemplateSlot(tail)), + } + .Select(t => t.CompileAnalysisRule(morpher)) + .ToArray(); + var shared = new AnalysisAffixTemplatesRule(morpher, rules); + Word input = Input(text); + input.Freeze(); + string[] output = shared.Apply(input).Select(Signature).ToArray(); + return new LookaheadRun(output, a.ApplyCount, s.ApplyCount, tail.ApplyCount, shadow); + } + + [Test] + public void LookaheadPrunesOnlyTheTemplateWhoseRequiredSlotIsGone() + { + LookaheadRun off = RunSharedTail(RequiredSlotLookaheadMode.Off, "sad"); + LookaheadRun on = RunSharedTail(RequiredSlotLookaheadMode.On, "sad"); + + Assert.That(off.InnerA, Is.EqualTo(1)); + Assert.That(on.InnerA, Is.EqualTo(0), "the required inner suffix cannot unapply once the tail took the a"); + Assert.That(on.Tail, Is.EqualTo(off.Tail), "the shared tail still runs once"); + Assert.That(on.InnerS, Is.EqualTo(off.InnerS), "the other template is unaffected"); + Assert.That(off.Output, Is.Not.Empty); + Assert.That(on.Output, Is.EqualTo(off.Output)); + } + + [Test] + public void LookaheadKeepsEveryAttemptWhenBothTemplatesAreFeasible() + { + LookaheadRun off = RunSharedTail(RequiredSlotLookaheadMode.Off, "saad"); + LookaheadRun on = RunSharedTail(RequiredSlotLookaheadMode.On, "saad"); + + Assert.That((on.InnerA, on.InnerS, on.Tail), Is.EqualTo((off.InnerA, off.InnerS, off.Tail))); + Assert.That(on.Output, Is.EqualTo(off.Output)); + } + + [TestCase("sad", false)] + [TestCase("saad", false)] + [TestCase("sad", true)] + [TestCase("dd", false)] + public void ShadowLookaheadOnTheSharedWalkNeverViolatesOrChangesTheOutput(string text, bool optional) + { + LookaheadRun off = RunSharedTail(RequiredSlotLookaheadMode.Off, text, optional); + LookaheadRun shadow = RunSharedTail(RequiredSlotLookaheadMode.Shadow, text, optional); + + Assert.That(shadow.Output, Is.EqualTo(off.Output)); + Assert.That((shadow.InnerA, shadow.InnerS, shadow.Tail), Is.EqualTo((off.InnerA, off.InnerS, off.Tail))); + Assert.That(shadow.Shadow, Does.Not.Contain("violated")); + Assert.That(shadow.Shadow, Does.Not.Contain("completed-under-fired")); + if (text == "sad" && !optional) + Assert.That(shadow.Shadow, Does.Contain("fired")); + } + + [Test] + public void LookaheadSharedWalkMatchesPerTemplateWalkOnEveryShortInput() + { + int compared = 0; + foreach (string text in Inputs(5)) + { + foreach ( + RequiredSlotLookaheadMode mode in new[] { RequiredSlotLookaheadMode.Off, RequiredSlotLookaheadMode.On } + ) + { + CountingSuffix a = Suffix("a", "a"); + CountingSuffix s = Suffix("s", "s"); + CountingSuffix tail = Suffix("ad", "ad"); + var morpher = new Morpher(TraceManager, Language, maxDegreeOfParallelism: 1) + { + RequiredSlotLookahead = mode, + }; + IRule[] rules = new[] + { + Template(new AffixTemplateSlot(a), new AffixTemplateSlot(tail)), + Template( + new AffixTemplateSlot(s) { Optional = true }, + new AffixTemplateSlot(a), + new AffixTemplateSlot(tail) + ), + Template(new AffixTemplateSlot(s), new AffixTemplateSlot(tail)), + } + .Select(t => t.CompileAnalysisRule(morpher)) + .ToArray(); + var own = new RuleBatch(rules, false, FreezableEqualityComparer.Default); + var shared = new AnalysisAffixTemplatesRule(morpher, rules); + Word input = Input(text); + input.Freeze(); + Word[] expected = own.Apply(input).ToArray(); + Word[] actual = shared.Apply(input).ToArray(); + Assert.That(actual, Is.EqualTo(expected).Using(FreezableEqualityComparer.Default), text); + compared += actual.Length; + } + } + Assert.That(compared, Is.GreaterThan(10)); + } + + private static string Signature(Word w) => w + " " + string.Join(",", w.MorphologicalRules.Select(r => r.Name)); + private Word Input(string text) => new(Morphophonemic, Table3.Segment(text)); private FeatureStruct Pos(string symbol) => FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol(symbol).Value;